From fb834c9758aad2342e511c63b7174aac57b09780 Mon Sep 17 00:00:00 2001 From: Baz Date: Thu, 16 Dec 2021 17:50:01 +0200 Subject: [PATCH] =?UTF-8?q?=F0=9F=90=9B=20Source=20Shopify:=20fix=20for=20?= =?UTF-8?q?mismatched=20number=20of=20tables=20during=20normalization=20(#?= =?UTF-8?q?8597)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../9da77001-af33-4bcd-be46-6252bf9342b9.json | 2 +- .../resources/seed/source_definitions.yaml | 2 +- .../src/main/resources/seed/source_specs.yaml | 4 +- .../connectors/source-shopify/Dockerfile | 2 +- .../integration_tests/abnormal_state.json | 4 +- .../integration_tests/configured_catalog.json | 4 +- .../integration_tests/state.json | 2 +- .../schemas/abandoned_checkouts.json | 342 +---- .../source_shopify/schemas/draft_orders.json | 1239 ++++++----------- .../source_shopify/schemas/metafields.json | 2 +- ...orders_refunds.json => order_refunds.json} | 73 + .../{orders_risks.json => order_risks.json} | 0 .../source_shopify/schemas/orders.json | 52 +- .../source_shopify/schemas/shop.json | 2 +- .../source-shopify/source_shopify/source.py | 75 +- .../source-shopify/source_shopify/spec.json | 4 +- .../source_shopify/transform.py | 6 + docs/integrations/sources/shopify.md | 1 + 18 files changed, 634 insertions(+), 1182 deletions(-) rename airbyte-integrations/connectors/source-shopify/source_shopify/schemas/{orders_refunds.json => order_refunds.json} (86%) rename airbyte-integrations/connectors/source-shopify/source_shopify/schemas/{orders_risks.json => order_risks.json} (100%) diff --git a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json index 724874f83d2dc..a3a19d62e1239 100644 --- a/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json +++ b/airbyte-config/init/src/main/resources/config/STANDARD_SOURCE_DEFINITION/9da77001-af33-4bcd-be46-6252bf9342b9.json @@ -2,7 +2,7 @@ "sourceDefinitionId": "9da77001-af33-4bcd-be46-6252bf9342b9", "name": "Shopify", "dockerRepository": "airbyte/source-shopify", - "dockerImageTag": "0.1.25", + "dockerImageTag": "0.1.26", "documentationUrl": "https://docs.airbyte.io/integrations/sources/shopify", "icon": "shopify.svg" } diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 9235f813eaa08..d20114daf54d8 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -595,7 +595,7 @@ - name: Shopify sourceDefinitionId: 9da77001-af33-4bcd-be46-6252bf9342b9 dockerRepository: airbyte/source-shopify - dockerImageTag: 0.1.25 + dockerImageTag: 0.1.26 documentationUrl: https://docs.airbyte.io/integrations/sources/shopify icon: shopify.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index e40d78ae7ba90..205f1fb0c6ca4 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -5957,7 +5957,7 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] -- dockerImage: "airbyte/source-shopify:0.1.25" +- dockerImage: "airbyte/source-shopify:0.1.26" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/shopify" connectionSpecification: @@ -5990,6 +5990,7 @@ - "client_id" - "client_secret" - "access_token" + - "auth_method" properties: auth_method: type: "string" @@ -6014,6 +6015,7 @@ type: "object" required: - "api_password" + - "auth_method" properties: auth_method: type: "string" diff --git a/airbyte-integrations/connectors/source-shopify/Dockerfile b/airbyte-integrations/connectors/source-shopify/Dockerfile index 50a92a81ee45b..48a52243bbf4d 100644 --- a/airbyte-integrations/connectors/source-shopify/Dockerfile +++ b/airbyte-integrations/connectors/source-shopify/Dockerfile @@ -28,5 +28,5 @@ COPY source_shopify ./source_shopify ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.25 +LABEL io.airbyte.version=0.1.26 LABEL io.airbyte.name=airbyte/source-shopify diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json index 3274886656c38..e4a7d9adcbb2f 100644 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/abnormal_state.json @@ -23,10 +23,10 @@ "custom_collections": { "updated_at": "2024-07-19T07:01:37-07:00" }, - "orders_refunds": { + "order_refunds": { "created_at": "2024-07-19T06:41:47-07:00" }, - "orders_risks": { + "order_risks": { "id": 9991307599038 }, "transactions": { diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json index 8631f030275b3..c9cc07b41ad14 100644 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/configured_catalog.json @@ -98,7 +98,7 @@ }, { "stream": { - "name": "orders_refunds", + "name": "order_refunds", "json_schema": {}, "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, @@ -110,7 +110,7 @@ }, { "stream": { - "name": "orders_risks", + "name": "order_risks", "json_schema": {}, "supported_sync_modes": ["incremental", "full_refresh"], "source_defined_cursor": true, diff --git a/airbyte-integrations/connectors/source-shopify/integration_tests/state.json b/airbyte-integrations/connectors/source-shopify/integration_tests/state.json index 059da172e51f8..5c9988b0f2830 100644 --- a/airbyte-integrations/connectors/source-shopify/integration_tests/state.json +++ b/airbyte-integrations/connectors/source-shopify/integration_tests/state.json @@ -23,7 +23,7 @@ "custom_collections": { "updated_at": "2021-08-18T02:39:34-07:00" }, - "orders_refunds": { + "order_refunds": { "created_at": "2021-09-09T02:57:43-07:00" }, "orders_risks": { diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/abandoned_checkouts.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/abandoned_checkouts.json index 8b07ab63fbd75..6e2c0e6cfc522 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/abandoned_checkouts.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/abandoned_checkouts.json @@ -93,8 +93,7 @@ "type": ["null", "string"] }, "amount": { - "type": ["null", "number"], - "multipleOf": 1e-10 + "type": ["null", "string"] }, "code": { "type": ["null", "string"] @@ -212,357 +211,38 @@ "line_items": { "items": { "properties": { - "applied_discounts": { - "type": ["null", "array"], - "items": { - "type": ["null", "string"] - } - }, - "total_discount_set": { - "type": ["null", "object"], - "properties": { - "shop_money": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "number"] - }, - "currency_code": { - "type": ["null", "string"] - } - } - }, - "presentment_money": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "number"] - }, - "currency_code": { - "type": ["null", "string"] - } - } - } - } - }, - "pre_tax_price_set": { - "properties": { - "shop_money": { - "properties": { - "currency_code": { - "type": ["null", "string"] - }, - "amount": { - "type": ["null", "number"] - } - }, - "type": ["null", "object"] - }, - "presentment_money": { - "properties": { - "currency_code": { - "type": ["null", "string"] - }, - "amount": { - "type": ["null", "number"] - } - }, - "type": ["null", "object"] - } - }, - "type": ["null", "object"] - }, - "price_set": { - "type": ["null", "object"], - "properties": { - "shop_money": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "number"] - }, - "currency_code": { - "type": ["null", "string"] - } - } - }, - "presentment_money": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "number"] - }, - "currency_code": { - "type": ["null", "string"] - } - } - } - } + "sku": { + "type": ["null", "string"] }, "grams": { - "type": ["null", "integer"] - }, - "compare_at_price": { "type": ["null", "number"] }, - "destination_location_id": { - "type": ["null", "integer"] + "price": { + "type": ["null", "string"] }, - "key": { + "title": { "type": ["null", "string"] }, - "line_price": { - "type": ["null", "number"] + "vendor": { + "type": ["null", "string"] }, - "origin_location_id": { + "quantity": { "type": ["null", "integer"] }, - "applied_discount": { + "product_id": { "type": ["null", "integer"] }, - "fulfillable_quantity": { + "variant_id": { "type": ["null", "integer"] }, "variant_title": { "type": ["null", "string"] }, - "properties": { - "anyOf": [ - { - "items": { - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - }, - { - "type": ["null", "object"], - "additionalProperties": true - } - ] - }, - "tax_code": { - "type": ["null", "string"] - }, - "discount_allocations": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "discount_application_index": { - "type": ["null", "integer"] - }, - "amount_set": { - "type": ["null", "object"], - "properties": { - "shop_money": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "string"] - }, - "currency_code": { - "type": ["null", "string"] - } - } - }, - "presentment_money": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "string"] - }, - "currency_code": { - "type": ["null", "string"] - } - } - } - } - }, - "amount": { - "type": ["null", "number"] - } - } - } - }, - "admin_graphql_api_id": { - "type": ["null", "string"] - }, - "pre_tax_price": { - "type": ["null", "number"] - }, - "sku": { - "type": ["null", "string"] - }, - "product_exists": { - "type": ["null", "boolean"] - }, - "total_discount": { - "type": ["null", "number"], - "multipleOf": 1e-10 - }, - "name": { - "type": ["null", "string"] - }, - "fulfillment_status": { - "type": ["null", "string"] - }, - "gift_card": { - "type": ["null", "boolean"] - }, - "id": { - "type": ["null", "integer", "string"] - }, - "taxable": { - "type": ["null", "boolean"] - }, - "vendor": { - "type": ["null", "string"] - }, - "tax_lines": { - "items": { - "properties": { - "price_set": { - "type": ["null", "object"], - "properties": { - "shop_money": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "number"] - }, - "currency_code": { - "type": ["null", "string"] - } - } - }, - "presentment_money": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "number"] - }, - "currency_code": { - "type": ["null", "string"] - } - } - } - } - }, - "price": { - "type": ["null", "number"] - }, - "title": { - "type": ["null", "string"] - }, - "rate": { - "type": ["null", "number"] - }, - "compare_at": { - "type": ["null", "number"] - }, - "position": { - "type": ["null", "integer"] - }, - "source": { - "type": ["null", "string"] - }, - "zone": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - }, - "origin_location": { - "properties": { - "country_code": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "address1": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "address2": { - "type": ["null", "string"] - }, - "province_code": { - "type": ["null", "string"] - }, - "zip": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "price": { - "type": ["null", "number"] - }, "requires_shipping": { "type": ["null", "boolean"] }, "fulfillment_service": { "type": ["null", "string"] - }, - "variant_inventory_management": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "destination_location": { - "properties": { - "country_code": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "address1": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "id": { - "type": ["null", "integer"] - }, - "address2": { - "type": ["null", "string"] - }, - "province_code": { - "type": ["null", "string"] - }, - "zip": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "quantity": { - "type": ["null", "integer"] - }, - "product_id": { - "type": ["null", "integer"] - }, - "variant_id": { - "type": ["null", "integer"] } }, "type": ["null", "object"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/draft_orders.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/draft_orders.json index 83aaa02386cb7..4a6c86820999f 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/draft_orders.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/draft_orders.json @@ -4,653 +4,87 @@ "id": { "type": ["null", "integer"] }, - "order_id": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "customer": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "integer"] - }, - "email": { - "type": ["null", "string"] - }, - "accepts_marketing": { - "type": ["null", "boolean"] - }, - "created_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "updated_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "first_name": { - "type": ["null", "string"] - }, - "last_name": { - "type": ["null", "string"] - }, - "orders_count": { - "type": ["null", "integer"] - }, - "state": { - "type": ["null", "string"] - }, - "total_spent": { - "type": ["null", "number"] - }, - "last_order_id": { - "type": ["null", "integer"] - }, - "note": { - "type": ["null", "string"] - }, - "verified_email": { - "type": ["null", "boolean"] - }, - "multipass_identifier": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "tags": { - "type": ["null", "string"] - }, - "last_order_name": { - "type": ["null", "string"] - }, - "currency": { - "type": ["null", "string"] - }, - "accepts_marketing_updated_at": { - "type": ["null", "string"] - }, - "marketing_opt_in_level": { - "type": ["null", "string"] - }, - "admin_graphql_api_id": { - "type": ["null", "string"] - }, - "default_address": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "integer"] - }, - "customer_id": { - "type": ["null", "integer"] - }, - "first_name": { - "type": ["null", "string"] - }, - "last_name": { - "type": ["null", "string"] - }, - "company": { - "type": ["null", "string"] - }, - "address1": { - "type": ["null", "string"] - }, - "address2": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "province": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "zip": { - "type": ["null", "string"] - }, - "phone": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "province_code": { - "type": ["null", "string"] - }, - "country_code": { - "type": ["null", "string"] - }, - "country_name": { - "type": ["null", "string"] - }, - "default": { - "type": ["null", "boolean"] - } - } - } - } - }, - "shipping_address": { - "properties": { - "phone": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "address1": { - "type": ["null", "string"] - }, - "longitude": { - "type": ["null", "number"] - }, - "address2": { - "type": ["null", "string"] - }, - "last_name": { - "type": ["null", "string"] - }, - "first_name": { - "type": ["null", "string"] - }, - "province": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "company": { - "type": ["null", "string"] - }, - "latitude": { - "type": ["null", "number"] - }, - "country_code": { - "type": ["null", "string"] - }, - "province_code": { - "type": ["null", "string"] - }, - "zip": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "billing_address": { - "properties": { - "phone": { - "type": ["null", "string"] - }, - "country": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "address1": { - "type": ["null", "string"] - }, - "longitude": { - "type": ["null", "number"] - }, - "address2": { - "type": ["null", "string"] - }, - "last_name": { - "type": ["null", "string"] - }, - "first_name": { - "type": ["null", "string"] - }, - "province": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "company": { - "type": ["null", "string"] - }, - "latitude": { - "type": ["null", "number"] - }, - "country_code": { - "type": ["null", "string"] - }, - "province_code": { - "type": ["null", "string"] - }, - "zip": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "note": { - "type": ["null", "string"] - }, - "note_attributes": { - "items": { - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - }, - "email": { - "type": ["null", "string"] - }, - "currency": { - "type": ["null", "string"] - }, - "invoice_sent_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "invoice_url": { - "type": ["null", "string"] - }, - "line_items": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "integer"] - }, - "admin_graphql_api_id": { - "type": ["null", "string"] - }, - "destination_location": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "integer"] - }, - "country_code": { - "type": ["null", "string"] - }, - "province_code": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "address1": { - "type": ["null", "string"] - }, - "address2": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "zip": { - "type": ["null", "string"] - } - } - }, - "fulfillable_quantity": { - "type": ["null", "integer"] - }, - "fulfillment_service": { - "type": ["null", "string"] - }, - "fulfillment_status": { - "type": ["null", "string"] - }, - "gift_card": { - "type": ["null", "boolean"] - }, - "grams": { - "type": ["null", "integer"] - }, - "name": { - "type": ["null", "string"] - }, - "origin_location": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "integer"] - }, - "country_code": { - "type": ["null", "string"] - }, - "province_code": { - "type": ["null", "string"] - }, - "name": { - "type": ["null", "string"] - }, - "address1": { - "type": ["null", "string"] - }, - "address2": { - "type": ["null", "string"] - }, - "city": { - "type": ["null", "string"] - }, - "zip": { - "type": ["null", "string"] - } - } - }, - "price": { - "type": ["null", "number"] - }, - "price_set": { - "type": ["null", "object"], - "properties": { - "shop_money": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "number"] - }, - "currency_code": { - "type": ["null", "string"] - } - } - }, - "presentment_money": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "number"] - }, - "currency_code": { - "type": ["null", "string"] - } - } - } - } - }, - "product_exists": { - "type": ["null", "boolean"] - }, - "product_id": { - "type": ["null", "integer"] - }, - "properties": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "name": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - } - } - }, - "quantity": { - "type": ["null", "integer"] - }, - "requires_shipping": { - "type": ["null", "boolean"] - }, - "sku": { - "type": ["null", "string"] - }, - "taxable": { - "type": ["null", "boolean"] - }, - "title": { - "type": ["null", "string"] - }, - "total_discount": { - "type": ["null", "number"] - }, - "total_discount_set": { - "type": ["null", "object"], - "properties": { - "shop_money": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "number"] - }, - "currency_code": { - "type": ["null", "string"] - } - } - }, - "presentment_money": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "number"] - }, - "currency_code": { - "type": ["null", "string"] - } - } - } - } - }, - "variant_id": { - "type": ["null", "integer"] - }, - "variant_inventory_management": { - "type": ["null", "string"] - }, - "variant_title": { - "type": ["null", "string"] - }, - "vendor": { - "type": ["null", "string"] - }, - "tax_lines": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "price": { - "type": ["null", "number"] - }, - "price_set": { - "type": ["null", "object"], - "properties": { - "shop_money": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "number"] - }, - "currency_code": { - "type": ["null", "string"] - } - } - }, - "presentment_money": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "number"] - }, - "currency_code": { - "type": ["null", "string"] - } - } - } - } - }, - "rate": { - "type": ["null", "number"] - }, - "title": { - "type": ["null", "string"] - } - } - } - }, - "duties": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "harmonized_system_code": { - "type": ["null", "string"] - }, - "country_code_of_origin": { - "type": ["null", "string"] - }, - "shop_money": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "string"] - }, - "currency_code": { - "type": ["null", "string"] - } - } - }, - "presentment_money": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "string"] - }, - "currency_code": { - "type": ["null", "string"] - } - } - }, - "tax_lines": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "title": { - "type": ["null", "string"] - }, - "price": { - "type": ["null", "string"] - }, - "rate": { - "type": ["null", "number"] - }, - "price_set": { - "type": ["null", "object"], - "properties": { - "shop_money": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "string"] - }, - "currency_code": { - "type": ["null", "string"] - } - } - }, - "presentment_money": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "string"] - }, - "currency_code": { - "type": ["null", "string"] - } - } - } - } - }, - "channel_liable": { - "type": ["null", "boolean"] - } - } - } - }, - "admin_graphql_api_id": { - "type": ["null", "string"] - } - } - } - }, - "discount_allocations": { - "type": ["null", "array"], - "items": { - "type": ["null", "object"], - "properties": { - "id": { - "type": ["null", "string"] - }, - "amount": { - "type": ["null", "string"] - }, - "description": { - "type": ["null", "string"] - }, - "created_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "discount_application_index": { - "type": ["null", "number"] - }, - "amount_set": { - "type": ["null", "object"], - "properties": { - "shop_money": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "string"] - }, - "currency_code": { - "type": ["null", "string"] - } - } - }, - "presentment_money": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "string"] - }, - "currency_code": { - "type": ["null", "string"] - } - } - } - } - }, - "application_type": { - "type": ["null", "string"] - } - } - } - } - } - } - }, - "shipping_line": { + "note": { + "type": ["null", "string"] + }, + "email": { + "type": ["null", "string"] + }, + "taxes_included": { + "type": ["null", "boolean"] + }, + "currency": { + "type": ["null", "string"] + }, + "invoice_sent_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "tax_exempt": { + "type": ["null", "boolean"] + }, + "completed_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "name": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "line_items": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], "properties": { + "id": { + "type": ["null", "integer"] + }, + "variant_id": { + "type": ["null", "integer"] + }, + "product_id": { + "type": ["null", "integer"] + }, + "title": { + "type": ["null", "string"] + }, + "variant_title": { + "type": ["null", "string"] + }, + "sku": { + "type": ["null", "string"] + }, + "vendor": { + "type": ["null", "string"] + }, + "quantity": { + "type": ["null", "integer"] + }, + "requires_shipping": { + "type": ["null", "boolean"] + }, + "taxable": { + "type": ["null", "boolean"] + }, + "gift_card": { + "type": ["null", "boolean"] + }, + "fulfillment_service": { + "type": ["null", "string"] + }, + "grams": { + "type": ["null", "number"] + }, "tax_lines": { "type": ["null", "array"], "items": { @@ -695,35 +129,211 @@ } } }, - "phone": { - "type": ["null", "string"] + "applied_discount": { + "type": ["null", "object"], + "properties": { + "description": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + }, + "value_type": { + "type": ["null", "string"] + } + } + }, + "name": { + "type": ["null", "string"] + }, + "properties": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } + }, + "custom": { + "type": ["null", "boolean"] + }, + "price": { + "type": ["null", "number"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + } + } + } + }, + "shipping_address": { + "properties": { + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "longitude": { + "type": ["null", "number"] + }, + "address2": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "latitude": { + "type": ["null", "number"] + }, + "country_code": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "billing_address": { + "properties": { + "phone": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "longitude": { + "type": ["null", "number"] + }, + "address2": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "first_name": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "latitude": { + "type": ["null", "number"] + }, + "country_code": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "invoice_url": { + "type": ["null", "string"] + }, + "applied_discount": { + "type": ["null", "object"], + "properties": { + "description": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + }, + "title": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + }, + "value_type": { + "type": ["null", "string"] + } + } + }, + "order_id": { + "type": ["null", "integer"] + }, + "shipping_line": { + "properties": { + "price": { + "type": ["null", "number"] + }, + "title": { + "type": ["null", "string"] + }, + "custom": { + "type": ["null", "boolean"] + }, + "handle": { + "type": ["null", "string"] + } + }, + "type": ["null", "object"] + }, + "tax_lines": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "price": { + "type": ["null", "number"] + }, + "rate": { + "type": ["null", "number"] }, - "discounted_price_set": { - "type": ["null", "object"], - "properties": { - "shop_money": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "number"] - }, - "currency_code": { - "type": ["null", "string"] - } - } - }, - "presentment_money": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "number"] - }, - "currency_code": { - "type": ["null", "string"] - } - } - } - } + "title": { + "type": ["null", "string"] }, "price_set": { "type": ["null", "object"], @@ -751,154 +361,173 @@ } } } - }, - "price": { - "type": ["null", "number"] - }, - "title": { - "type": ["null", "string"] - }, - "discount_allocations": { - "items": { - "properties": { - "discount_application_index": { - "type": ["null", "integer"] - }, - "amount": { - "type": ["null", "number"] - } - }, - "type": ["null", "object"] - }, - "type": ["null", "array"] - }, - "delivery_category": { - "type": ["null", "string"] - }, - "discounted_price": { - "type": ["null", "number"] - }, - "code": { - "type": ["null", "string"] - }, - "requested_fulfillment_service_id": { - "type": ["null", "string"] - }, - "carrier_identifier": { + } + } + } + }, + "tags": { + "type": ["null", "string"] + }, + "note_attributes": { + "items": { + "properties": { + "name": { "type": ["null", "string"] }, - "id": { - "type": ["null", "integer"] - }, - "source": { + "value": { "type": ["null", "string"] } }, "type": ["null", "object"] }, - "tags": { - "type": ["null", "string"] - }, - "tax_exemptions": { - "type": ["null", "boolean"] - }, - "tax_lines": { - "type": ["null", "array"], - "items": { + "type": ["null", "array"] + }, + "total_price": { + "type": ["null", "string"] + }, + "subtotal_price": { + "type": ["null", "string"] + }, + "total_tax": { + "type": ["null", "string"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "customer": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "email": { + "type": ["null", "string"] + }, + "accepts_marketing": { + "type": ["null", "boolean"] + }, + "created_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "updated_at": { + "type": ["null", "string"], + "format": "date-time" + }, + "first_name": { + "type": ["null", "string"] + }, + "last_name": { + "type": ["null", "string"] + }, + "orders_count": { + "type": ["null", "integer"] + }, + "state": { + "type": ["null", "string"] + }, + "total_spent": { + "type": ["null", "number"] + }, + "last_order_id": { + "type": ["null", "integer"] + }, + "note": { + "type": ["null", "string"] + }, + "verified_email": { + "type": ["null", "boolean"] + }, + "multipass_identifier": { + "type": ["null", "string"] + }, + "tax_exempt": { + "type": ["null", "boolean"] + }, + "phone": { + "type": ["null", "string"] + }, + "tags": { + "type": ["null", "string"] + }, + "last_order_name": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "accepts_marketing_updated_at": { + "type": ["null", "string"] + }, + "marketing_opt_in_level": { + "type": ["null", "string"] + }, + "tax_exemptions": { + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "default_address": { "type": ["null", "object"], "properties": { - "price": { - "type": ["null", "number"] + "id": { + "type": ["null", "integer"] }, - "rate": { - "type": ["null", "number"] + "customer_id": { + "type": ["null", "integer"] }, - "title": { + "first_name": { "type": ["null", "string"] }, - "price_set": { - "type": ["null", "object"], - "properties": { - "shop_money": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "number"] - }, - "currency_code": { - "type": ["null", "string"] - } - } - }, - "presentment_money": { - "type": ["null", "object"], - "properties": { - "amount": { - "type": ["null", "number"] - }, - "currency_code": { - "type": ["null", "string"] - } - } - } - } + "last_name": { + "type": ["null", "string"] + }, + "company": { + "type": ["null", "string"] + }, + "address1": { + "type": ["null", "string"] + }, + "address2": { + "type": ["null", "string"] + }, + "city": { + "type": ["null", "string"] + }, + "province": { + "type": ["null", "string"] + }, + "country": { + "type": ["null", "string"] + }, + "zip": { + "type": ["null", "string"] + }, + "phone": { + "type": ["null", "string"] + }, + "name": { + "type": ["null", "string"] + }, + "province_code": { + "type": ["null", "string"] + }, + "country_code": { + "type": ["null", "string"] + }, + "country_name": { + "type": ["null", "string"] + }, + "default": { + "type": ["null", "boolean"] } } } - }, - "applied_discount": { - "type": ["null", "object"], - "properties": { - "description": { - "type": ["null", "string"] - }, - "value": { - "type": ["null", "string"] - }, - "title": { - "type": ["null", "string"] - }, - "amount": { - "type": ["null", "string"] - }, - "value_type": { - "type": ["null", "string"] - } - } - }, - "taxes_included": { - "type": ["null", "boolean"] - }, - "total_price": { - "type": ["null", "number"] - }, - "completed_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "created_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "updated_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "status": { - "type": ["null", "string"] } - }, - "created_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "updated_at": { - "type": ["null", "string"], - "format": "date-time" - }, - "completed_at": { - "type": ["null", "string"], - "format": "date-time" } } } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafields.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafields.json index 0c0c9b9c02ff7..4277aeb4351a1 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafields.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/metafields.json @@ -29,7 +29,7 @@ "type": ["null", "string"] }, "value": { - "type": ["null", "number", "string", "integer", "boolean"] + "type": ["null", "string"] }, "updated_at": { "type": ["null", "string"], diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders_refunds.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_refunds.json similarity index 86% rename from airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders_refunds.json rename to airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_refunds.json index e852ef31d321b..608a420a46917 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders_refunds.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_refunds.json @@ -388,6 +388,79 @@ }, "type": ["null", "object"] } + }, + "transactions": { + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "id": { + "type": ["null", "integer"] + }, + "admin_graphql_api_id": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + }, + "authorization": { + "type": ["null", "string"] + }, + "created_at": { + "type": ["null", "string"] + }, + "currency": { + "type": ["null", "string"] + }, + "device_id": { + "type": ["null", "integer"] + }, + "error_code": { + "type": ["null", "string"] + }, + "gateway": { + "type": ["null", "string"] + }, + "kind": { + "type": ["null", "string"] + }, + "location_id": { + "type": ["null", "integer"] + }, + "message": { + "type": ["null", "string"] + }, + "order_id": { + "type": ["null", "integer"] + }, + "parent_id": { + "type": ["null", "integer"] + }, + "processed_at": { + "type": ["null", "string"] + }, + "receipt": { + "type": ["null", "object"], + "properties": { + "paid_amount": { + "type": ["null", "string"] + } + } + }, + "source_name": { + "type": ["null", "string"] + }, + "status": { + "type": ["null", "string"] + }, + "test": { + "type": ["null", "boolean"] + }, + "user_id": { + "type": ["null", "integer"] + } + } + } } } } diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders_risks.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_risks.json similarity index 100% rename from airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders_risks.json rename to airbyte-integrations/connectors/source-shopify/source_shopify/schemas/order_risks.json diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders.json index 70c3b987923c6..6a95e33402666 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/orders.json @@ -202,7 +202,21 @@ "type": ["null", "string"] }, "discount_codes": { - "type": ["null", "array"] + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "code": { + "type": ["null", "string"] + }, + "amount": { + "type": ["null", "string"] + }, + "type": { + "type": ["null", "string"] + } + } + } }, "email": { "type": ["null", "string"] @@ -818,13 +832,19 @@ "type": ["null", "string"] }, "tracking_numbers": { - "type": ["null", "array"] + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, "tracking_url": { "type": ["null", "string"] }, "tracking_urls": { - "type": ["null", "array"] + "type": ["null", "array"], + "items": { + "type": ["null", "string"] + } }, "updated_at": { "type": ["null", "string"], @@ -957,7 +977,18 @@ "type": ["null", "integer"] }, "properties": { - "type": ["null", "array"] + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } }, "quantity": { "type": ["null", "integer"] @@ -1336,7 +1367,18 @@ "type": ["null", "integer"] }, "properties": { - "type": ["null", "array"] + "type": ["null", "array"], + "items": { + "type": ["null", "object"], + "properties": { + "name": { + "type": ["null", "string"] + }, + "value": { + "type": ["null", "string"] + } + } + } }, "quantity": { "type": ["null", "integer"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/shop.json b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/shop.json index 60613accc4729..b4eb4fb815b25 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/shop.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/schemas/shop.json @@ -1,5 +1,5 @@ { - "type": "object", + "type": ["null", "object"], "properties": { "address1": { "type": ["null", "string"] diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py index 3591fe3bf898f..96c222ad8a94f 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/source.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/source.py @@ -63,9 +63,14 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp # transform method was implemented according to issue 4841 # Shopify API returns price fields as a string and it should be converted to number # this solution designed to convert string into number, but in future can be modified for general purpose - for record in records: - yield self._transformer.transform(record) - + if isinstance(records, dict): + # for cases when we have a single record as dict + yield self._transformer.transform(records) + else: + # for other cases + for record in records: + yield self._transformer.transform(record) + @property @abstractmethod def data_field(self) -> str: @@ -141,12 +146,17 @@ class ChildSubstream(IncrementalShopifyStream): :: @ parent_stream_class - defines the parent stream object to read from :: @ slice_key - defines the name of the property in stream slices dict. - :: @ record_field_name - the name of the field inside of parent stream record. Default is `id`. + :: @ nested_record - the name of the field inside of parent stream record. Default is `id`. + :: @ nested_record_field_name - the name of the field inside of nested_record. + :: @ nested_substream - the name of the nested entity inside of parent stream, helps to reduce the number of + API Calls, if present, see `OrderRefunds` stream for more. """ parent_stream_class: object = None slice_key: str = None - record_field_name: str = "id" + nested_record: str = "id" + nested_record_field_name: str = None + nested_substream = None def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> MutableMapping[str, Any]: params = {"limit": self.limit} @@ -157,14 +167,21 @@ def request_params(self, next_page_token: Mapping[str, Any] = None, **kwargs) -> def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: """ Reading the parent stream for slices with structure: - EXAMPLE: for given record_field_name as `id` of Orders, + EXAMPLE: for given nested_record as `id` of Orders, Output: [ {slice_key: 123}, {slice_key: 456}, ..., {slice_key: 999} ] """ parent_stream = self.parent_stream_class(self.config) parent_stream_state = stream_state_cache.cached_state.get(parent_stream.name) for record in parent_stream.read_records(stream_state=parent_stream_state, **kwargs): - yield {self.slice_key: record[self.record_field_name]} + # to limit the number of API Calls and reduce the time of data fetch, + # we can pull the ready data for child_substream, if nested data is present, + # and corresponds to the data of child_substream we need. + if self.nested_substream: + if record.get(self.nested_substream): + yield {self.slice_key: record[self.nested_record]} + else: + yield {self.slice_key: record[self.nested_record]} def read_records( self, @@ -174,7 +191,13 @@ def read_records( ) -> Iterable[Mapping[str, Any]]: """Reading child streams records for each `id`""" - self.logger.info(f"Reading {self.name} for {self.slice_key}: {stream_slice.get(self.slice_key)}") + slice_data = stream_slice.get(self.slice_key) + # sometimes the stream_slice.get(self.slice_key) has the list of records, + # to avoid data exposition inside the logs, we should get the data we need correctly out of stream_slice. + if isinstance(slice_data, list) and self.nested_record_field_name is not None and len(slice_data) > 0: + slice_data = slice_data[0].get(self.nested_record_field_name) + + self.logger.info(f"Reading {self.name} for {self.slice_key}: {slice_data}") records = super().read_records(stream_slice=stream_slice, **kwargs) yield from self.filter_records_newer_than_state(stream_state=stream_state, records_slice=records) @@ -256,20 +279,22 @@ def request_params( return params -class OrdersRefunds(ChildSubstream): +class OrderRefunds(ChildSubstream): parent_stream_class: object = Orders slice_key = "order_id" data_field = "refunds" cursor_field = "created_at" + # we pull out the records that we already know has the refunds data from Orders object + nested_substream = "refunds" def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: order_id = stream_slice["order_id"] return f"orders/{order_id}/{self.data_field}.json" -class OrdersRisks(ChildSubstream): +class OrderRisks(ChildSubstream): parent_stream_class: object = Orders slice_key = "order_id" @@ -364,12 +389,13 @@ class InventoryItems(ChildSubstream): parent_stream_class: object = Products slice_key = "id" - record_field_name = "variants" - + nested_record = "variants" + nested_record_field_name = "inventory_item_id" data_field = "inventory_items" def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: - ids = ",".join(str(x["inventory_item_id"]) for x in stream_slice[self.slice_key]) + + ids = ",".join(str(x[self.nested_record_field_name]) for x in stream_slice[self.slice_key]) return f"inventory_items.json?ids={ids}" @@ -405,12 +431,6 @@ def path(self, stream_slice: Mapping[str, Any] = None, **kwargs) -> str: class Shop(ShopifyStream): data_field = "shop" - @limiter.balance_rate_limit() - def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: - json_response = response.json() - record = json_response.get(self.data_field, []) if self.data_field is not None else json_response - return [record] - def path(self, **kwargs) -> str: return f"{self.data_field}.json" @@ -421,14 +441,13 @@ def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> """ Testing connection availability for the connector. """ - auth = ShopifyAuthenticator(config).get_auth_header() - api_version = "2021-07" # Latest Stable Release - url = f"https://{config['shop']}.myshopify.com/admin/api/{api_version}/shop.json" - + config["authenticator"] = ShopifyAuthenticator(config) try: - session = requests.get(url, headers=auth) - session.raise_for_status() - return True, None + responce = list(Shop(config).read_records(sync_mode=None)) + # check for the shop_id is present in the responce + shop_id = responce[0].get("id") + if shop_id is not None: + return True, None except requests.exceptions.RequestException as e: return False, e @@ -449,8 +468,8 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: Metafields(config), CustomCollections(config), Collects(config), - OrdersRefunds(config), - OrdersRisks(config), + OrderRefunds(config), + OrderRisks(config), Transactions(config), Pages(config), PriceRules(config), diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/spec.json b/airbyte-integrations/connectors/source-shopify/source_shopify/spec.json index fc40724706713..7f7f785fab0a9 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/spec.json +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/spec.json @@ -24,7 +24,7 @@ { "type": "object", "title": "OAuth2.0", - "required": ["client_id", "client_secret", "access_token"], + "required": ["client_id", "client_secret", "access_token", "auth_method"], "properties": { "auth_method": { "type": "string", @@ -53,7 +53,7 @@ { "title": "API Password", "type": "object", - "required": ["api_password"], + "required": ["api_password", "auth_method"], "properties": { "auth_method": { "type": "string", diff --git a/airbyte-integrations/connectors/source-shopify/source_shopify/transform.py b/airbyte-integrations/connectors/source-shopify/source_shopify/transform.py index f73583333cac8..8398d2096a8cb 100644 --- a/airbyte-integrations/connectors/source-shopify/source_shopify/transform.py +++ b/airbyte-integrations/connectors/source-shopify/source_shopify/transform.py @@ -64,6 +64,10 @@ def _first_non_null_type(schema_types: List[str]) -> str: def _transform_number(value: Any): return Decimal(value) + @staticmethod + def _transform_string(value: Any): + return str(value) + def _transform_array(self, array: List[Any], item_properties: Mapping[str, Any]): # iterate over items in array, compare schema types and convert if necessary. for index, record in enumerate(array): @@ -92,6 +96,8 @@ def transform(self, field: Any, schema: Mapping[str, Any] = None) -> Iterable[Mu if not any(field_json_type in schema_types for field_json_type in field_json_types): if schema_type == "number": return self._transform_number(field) + if schema_type == "string": + return self._transform_string(field) if schema_type == "object": properties = schema.get("properties", {}) return self._transform_object(field, properties) diff --git a/docs/integrations/sources/shopify.md b/docs/integrations/sources/shopify.md index 0dc8bc745e12e..e86be77f4995b 100644 --- a/docs/integrations/sources/shopify.md +++ b/docs/integrations/sources/shopify.md @@ -101,6 +101,7 @@ This connector support both: `OAuth 2.0` and `API PASSWORD` (for private applica | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.26 | 2021-12-14 | [8597](https://github.com/airbytehq/airbyte/pull/8597) | Fix `mismatched number of tables` for base-normalization, increased performance of `order_refunds` stream | | 0.1.25 | 2021-12-02 | [8297](https://github.com/airbytehq/airbyte/pull/8297) | Added Shop stream | | 0.1.24 | 2021-11-30 | [7783](https://github.com/airbytehq/airbyte/pull/7783) | Reviewed and corrected schemas for all streams | | 0.1.23 | 2021-11-15 | [7973](https://github.com/airbytehq/airbyte/pull/7973) | Added `InventoryItems` |