From dbc4f0d47e75165849b6a52eed9a38a5bdccf3ce Mon Sep 17 00:00:00 2001 From: monai Date: Tue, 16 Nov 2021 12:15:53 +0200 Subject: [PATCH 1/7] Add stream ProcessedOrderDetails --- .../schemas/processed_order_details.json | 827 ++++++++++++++++++ .../source_linnworks/source.py | 3 +- .../source_linnworks/streams.py | 36 +- 3 files changed, 864 insertions(+), 2 deletions(-) create mode 100644 airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/processed_order_details.json diff --git a/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/processed_order_details.json b/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/processed_order_details.json new file mode 100644 index 000000000000..62572a7ee920 --- /dev/null +++ b/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/processed_order_details.json @@ -0,0 +1,827 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "type": "object", + "properties": { + "OrderId": { + "type": "string", + "description": "Order ID (pkOrderId)" + }, + "NumOrderId": { + "type": "integer", + "description": "Linnworks order number" + }, + "Processed": { + "type": "boolean", + "description": "If order is processed" + }, + "ProcessedDateTime": { + "type": ["null", "string"], + "format": "date-time", + "description": "Date and time when order was processed" + }, + "FulfilmentLocationId": { + "type": "string", + "description": "Location ID" + }, + "GeneralInfo": { + "type": "object", + "description": "General information about order", + "additionalProperties": false, + "properties": { + "Status": { + "type": "integer", + "description": "Order Status (0 = UNPAID, 1 = PAID, 2 = RETURN, 3 = PENDING, 4 = RESEND)" + }, + "LabelPrinted": { + "type": "boolean", + "description": "Is label printed" + }, + "LabelError": { + "type": "string", + "description": "Is there a label error" + }, + "InvoicePrinted": { + "type": "boolean", + "description": "Is invoice printed" + }, + "PickListPrinted": { + "type": "boolean", + "description": "Is pick list printed" + }, + "IsRuleRun": { + "type": "boolean", + "description": "If rules engine rule ran on an order" + }, + "Notes": { + "type": "integer", + "description": "Quantity of order notes" + }, + "PartShipped": { + "type": "boolean", + "description": "If order partly shipped" + }, + "Marker": { + "type": ["null", "integer"], + "description": "Order marker (0 = NOT TAG, 1 = Tag 1, 2 = Tag 2, 3 = Tag 3, 4 = Tag 4, 5 = Tag 5, 6 = Tag 6, 7 = Parked)" + }, + "IsParked": { + "type": "boolean", + "description": "Is the order parked?" + }, + "Identifiers": { + "type": "array", + "description": "Order identifiers. [Prime | Scheduled]", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "IdentifierId": { + "type": "integer", + "description": "Internal identifier id. Use to update image and name." + }, + "IsCustom": { + "type": "boolean", + "description": "Is the tag user or system defined?" + }, + "ImageId": { + "type": "string" + }, + "ImageUrl": { + "type": "string" + }, + "Tag": { + "type": "string", + "description": "Internal tag for identification purposes" + }, + "Name": { + "type": "string", + "description": "Name displayed where the tag is used" + } + } + } + }, + "ReferenceNum": { + "type": "string", + "description": "Order reference number (Channel defined)" + }, + "SecondaryReference": { + "type": "string", + "description": "An additional reference number for the orderr (Used by some channels)" + }, + "ExternalReferenceNum": { + "type": "string", + "description": "This is an additional reference number from the sales channel, typically used by eBay" + }, + "ReceivedDate": { + "type": "string", + "format": "date-time", + "description": "The date and time at which the order was placed on the sales channel" + }, + "Source": { + "type": "string", + "description": "Order ChannelName/Source (e.g. EBAY)" + }, + "SubSource": { + "type": "string", + "description": "Order Subsource (e.g. EBAY1)" + }, + "SiteCode": { + "type": "string", + "description": "SiteCode used to differentiate between different sites from a single channel (eg. Amazon UK, Amazon US, Amazon FR...)" + }, + "HoldOrCancel": { + "type": "boolean", + "description": "This shows whether the order has been marked as on hold, for processed orders if the order has been cancelled OnHold = 1" + }, + "DespatchByDate": { + "type": "string", + "format": "date-time", + "description": "Despatch by Date" + }, + "ScheduledDelivery": { + "type": "object", + "description": "Scheduled delivery dates. Take priority over despatch by date", + "additionalProperties": false, + "properties": { + "From": { + "type": "string", + "format": "date-time" + }, + "To": { + "type": "string", + "format": "date-time" + } + } + }, + "HasScheduledDelivery": { + "type": "boolean" + }, + "Location": { + "type": "string", + "description": "Order location ID" + }, + "NumItems": { + "type": "integer", + "description": "Quantity of order items" + }, + "PickwaveIds": { + "type": "array", + "description": "All related Pickwave Ids", + "items": { + "type": "integer" + } + }, + "StockAllocationType": { + "type": ["null", "string"] + } + } + }, + "ShippingInfo": { + "type": "object", + "description": "Order shipping information", + "additionalProperties": false, + "properties": { + "Vendor": { + "type": "string", + "description": "Courier name (e.g. Royal Mail)" + }, + "PostalServiceId": { + "type": "string", + "description": "Postal service ID" + }, + "PostalServiceName": { + "type": "string", + "description": "Postal service name (e.g. Next day delivery)" + }, + "TotalWeight": { + "type": "number", + "description": "Order total weight" + }, + "ItemWeight": { + "type": "number", + "description": "If order is processed" + }, + "PackageCategoryId": { + "type": "string", + "description": "Package category ID" + }, + "PackageCategory": { + "type": "string", + "description": "Package category name" + }, + "PackageTypeId": { + "type": ["null", "string"], + "description": "Package type ID" + }, + "PackageType": { + "type": "string", + "description": "Package type name" + }, + "PostageCost": { + "type": "number", + "description": "Order postage cost" + }, + "PostageCostExTax": { + "type": "number", + "description": "Order postage cost excluding tax" + }, + "TrackingNumber": { + "type": "string", + "description": "Order tracking number provided by courier" + }, + "ManualAdjust": { + "type": "boolean", + "description": "If there is an adjustment to shipping cost was made" + } + } + }, + "CustomerInfo": { + "type": "object", + "description": "Order Customer information (Name, email etc)", + "additionalProperties": false, + "properties": { + "ChannelBuyerName": { + "type": "string", + "description": "Username of customer (Comes from channel)" + }, + "Address": { + "$ref": "#/$defs/customer_address", + "description": "Customer address" + }, + "BillingAddress": { + "$ref": "#/$defs/customer_address", + "description": "Customer billing address" + } + } + }, + "TotalsInfo": { + "type": "object", + "description": "Order totals information", + "additionalProperties": false, + "properties": { + "pkOrderId": { + "type": "string", + "description": "Order Id" + }, + "Subtotal": { + "type": "number", + "description": "Order subtotal" + }, + "PostageCost": { + "type": "number", + "description": "Order postage cost" + }, + "PostageCostExTax": { + "type": "number", + "description": "Order postage cost ex. tax" + }, + "Tax": { + "type": "number", + "description": "Tax" + }, + "TotalCharge": { + "type": "number", + "description": "Total charge" + }, + "PaymentMethod": { + "type": "string", + "description": "Payment method" + }, + "PaymentMethodId": { + "type": "string", + "description": "Payment method ID" + }, + "ProfitMargin": { + "type": "number", + "description": "Profit margin" + }, + "TotalDiscount": { + "type": "number", + "description": "Total discount applied to the order" + }, + "Currency": { + "type": "string", + "description": "Order currency" + }, + "CountryTaxRate": { + "type": "number", + "description": "Country tax rate" + }, + "ConversionRate": { + "type": "number", + "description": "Currency conversion rate. Set at point of save by the currency" + } + } + }, + "ExtendedProperties": { + "type": "array", + "description": "Extended properties of an order", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "RowId": { + "type": "string", + "description": "Record row ID" + }, + "Name": { + "type": "string", + "description": "Extended property name" + }, + "Value": { + "type": "string", + "description": "Extended property value" + }, + "Type": { + "type": "string", + "description": "Extended property type" + } + } + } + }, + "FolderName": { + "type": "array", + "description": "Folder names assigned to an order", + "items": { + "type": "string" + } + }, + "Items": { + "type": "array", + "description": "List of order items", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "ItemId": { + "type": "string", + "description": "Stock Item ID" + }, + "ItemNumber": { + "type": "string", + "description": "Item number as on channel" + }, + "SKU": { + "type": "string", + "description": "Product SKU" + }, + "ItemSource": { + "type": "string", + "description": "Item source / channel name" + }, + "Title": { + "type": "string", + "description": "Item title" + }, + "Quantity": { + "type": "integer", + "description": "Quantity" + }, + "CategoryId": { + "type": "string" + }, + "CategoryName": { + "type": "string", + "description": "Product category" + }, + "CompositeAvailablity": { + "type": ["null", "integer"], + "description": "Composite availability" + }, + "StockLevelsSpecified": { + "type": "boolean", + "description": "If stock level specified" + }, + "OnOrder": { + "type": "integer", + "description": "Level due in purchase orders" + }, + "OnPurchaseOrder": { + "type": "object", + "description": "Purchase order bound to this item", + "additionalProperties": false, + "properties": { + "pkPurchaseItemId": { + "type": "string", + "description": "Primary key of the bound" + }, + "Rowid": { + "type": "string" + }, + "pkPurchaseId": { + "type": "string" + }, + "ExternalInvoiceNumber": { + "type": "string" + }, + "fkSupplierId": { + "type": "string" + }, + "DateOfDelivery": { + "type": "string", + "format": "date-time" + }, + "QuotedDeliveryDate": { + "type": "string", + "format": "date-time" + }, + "SupplierName": { + "type": "string" + }, + "fkLocationId": { + "type": "string" + } + } + }, + "InOrderBook": { + "type": ["null", "integer"], + "description": "Quantity currently in open orders" + }, + "Level": { + "type": "integer", + "description": "Current stock level" + }, + "MinimumLevel": { + "type": [ + "null", + "integer" + ], + "description": "Minimum level" + }, + "AvailableStock": { + "type": "integer", + "description": "Currently available stock level (Level-InOrderBook)" + }, + "PricePerUnit": { + "type": "number", + "description": "Unit price" + }, + "UnitCost": { + "type": "number", + "description": "Unit cost" + }, + "DespatchStockUnitCost": { + "type": "number", + "description": "Despatch stock unit cost" + }, + "Discount": { + "type": "number", + "description": "Percentage (0%, 10%, 20%, etc...)" + }, + "Tax": { + "type": "number", + "description": "Actual tax value on an item" + }, + "TaxRate": { + "type": "number", + "description": "Tax rate" + }, + "Cost": { + "type": "number", + "description": "Total item cost (exc tax)" + }, + "CostIncTax": { + "type": "number", + "description": "Total item cost (inc tax)" + }, + "CompositeSubItems": { + "$comment": "It should be \"$ref\": \"#/properties/Items\" but Airbyte doesn't support recursive $refs.", + "type": "array", + "items": { + "type": "object" + } + }, + "IsService": { + "type": "boolean", + "description": "if item is a service" + }, + "SalesTax": { + "type": "number", + "description": "Sales Tax" + }, + "TaxCostInclusive": { + "type": "boolean", + "description": "If tax is included in a cost" + }, + "PartShipped": { + "type": "boolean", + "description": "If order is partly shipped" + }, + "Weight": { + "type": "number", + "description": "Order weight" + }, + "BarcodeNumber": { + "type": "string", + "description": "Product barcode" + }, + "Market": { + "type": "integer", + "description": "Market" + }, + "ChannelSKU": { + "type": "string", + "description": "Channel product SKU" + }, + "ChannelTitle": { + "type": "string", + "description": "Channel product title" + }, + "DiscountValue": { + "type": "number" + }, + "HasImage": { + "type": "boolean", + "description": "If item got an image" + }, + "ImageId": { + "type": [ + "null", + "string" + ], + "description": "Image ID" + }, + "AdditionalInfo": { + "type": "array", + "description": "List of order item options", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "pkOptionId": { + "type": "string", + "description": "Option ID" + }, + "Property": { + "type": "string", + "description": "Option property" + }, + "Value": { + "type": "string", + "description": "Value of the option" + } + } + } + }, + "StockLevelIndicator": { + "type": "integer", + "description": "Stock level indicator" + }, + "ShippingCost": { + "type": "number", + "description": "If batch number scan required" + }, + "PartShippedQty": { + "type": "integer", + "description": "ShippingCost" + }, + "ItemName": { + "type": "string", + "description": "PartShippedQty" + }, + "BatchNumberScanRequired": { + "type": "boolean", + "description": "ItemName" + }, + "SerialNumberScanRequired": { + "type": "boolean", + "description": "If serial number scan required" + }, + "BinRack": { + "type": "string", + "description": "Binrack location" + }, + "BinRacks": { + "type": "array", + "description": "List of BinRacks used for OrderItem", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "Quantity": { + "type": "integer", + "description": "Quantity for BinRack per Location" + }, + "BinRack": { + "type": "string", + "description": "BinRack" + }, + "Location": { + "type": "string", + "description": "LocationId of the BinRack" + }, + "BatchId": { + "type": ["null", "integer"], + "description": "If the item is batched, identifies the batch number" + }, + "OrderItemBatchId": { + "type": [ + "null", + "integer" + ], + "description": "If the item is batched, identifies the unique order item batch row" + } + } + } + }, + "InventoryTrackingType": { + "type": "integer", + "description": "Identifies whether the item has a sell by date or other defined order in which inventory is to be sold" + }, + "isBatchedStockItem": { + "type": "boolean", + "description": "If item has batches" + }, + "IsWarehouseManaged": { + "type": "boolean" + }, + "IsUnlinked": { + "type": "boolean" + }, + "ParentItemId": { + "type": "string" + }, + "StockItemIntId": { + "type": "integer" + }, + "Boxes": { + "type": "array", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "BoxId": { + "type": "integer", + "description": "Unique box id." + }, + "StockItemIntId": { + "type": "integer" + }, + "BoxName": { + "type": "string", + "description": "Box name max 16 characters" + }, + "Width": { + "type": "number", + "description": "Width of the box" + }, + "Height": { + "type": "number", + "description": "Height of the box" + }, + "Length": { + "type": "number", + "description": "Depth of the box" + }, + "Weight": { + "type": "number", + "description": "Total weight of the box." + }, + "ValuePercentage": { + "type": "number", + "description": "Value break down percentage" + }, + "Barcode": { + "type": "string", + "description": "Box barcode, max 64 characters." + }, + "PackagingTypeId": { + "type": "string", + "description": "Packaging type id" + }, + "LogicalDelete": { + "type": "boolean", + "description": "IsDeleted flag." + } + } + } + }, + "RowId": { + "type": "string", + "description": "Record row ID" + }, + "OrderId": { + "type": "string", + "description": "Order ID (pkOrderID)" + }, + "StockItemId": { + "type": "string", + "description": "Stock Item ID" + }, + "StockId": { + "type": "string" + } + } + } + }, + "Notes": { + "type": "array", + "description": "List of order notes", + "items": { + "type": "object", + "additionalProperties": false, + "properties": { + "OrderNoteId": { + "type": "string", + "description": "Order note ID" + }, + "OrderId": { + "type": "string", + "description": "Order Id" + }, + "NoteDate": { + "type": "string", + "format": "date-time", + "description": "Date and time when note was added" + }, + "Internal": { + "type": "boolean", + "description": "order note type (Internal or External)" + }, + "Note": { + "type": "string", + "description": "Note's text" + }, + "CreatedBy": { + "type": "string", + "description": "User that created note" + }, + "NoteTypeId": { + "type": ["null", "string"] + } + } + } + }, + "PaidDateTime": { + "type": [ + "null", + "string" + ], + "format": "date-time", + "description": "Date and time when the order was marked as paid" + }, + "TaxId": { + "type": "string", + "description": "Buyer's tax number." + } + }, + "$defs": { + "customer_address": { + "type": "object", + "additionalProperties": false, + "properties": { + "EmailAddress": { + "type": "string", + "description": "Customer's email address." + }, + "Address1": { + "type": "string", + "description": "First line of customer address." + }, + "Address2": { + "type": "string", + "description": "Second line of customer address." + }, + "Address3": { + "type": "string", + "description": "Third line of customer address." + }, + "Town": { + "type": "string", + "description": "Customer's town." + }, + "Region": { + "type": "string", + "description": "Customer's region." + }, + "PostCode": { + "type": "string", + "description": "Customer's postcode." + }, + "Country": { + "type": "string", + "description": "Customer's country." + }, + "Continent": { + "type": "string", + "description": "Customer's continent" + }, + "FullName": { + "type": "string", + "description": "Customer's first and second name." + }, + "Company": { + "type": "string", + "description": "Customer's company name." + }, + "PhoneNumber": { + "type": "string", + "description": "Customer's telephone number." + }, + "CountryId": { + "type": "string" + } + } + } + } +} diff --git a/airbyte-integrations/connectors/source-linnworks/source_linnworks/source.py b/airbyte-integrations/connectors/source-linnworks/source_linnworks/source.py index 81469c31f194..e12abed735ae 100644 --- a/airbyte-integrations/connectors/source-linnworks/source_linnworks/source.py +++ b/airbyte-integrations/connectors/source-linnworks/source_linnworks/source.py @@ -11,7 +11,7 @@ from airbyte_cdk.sources.streams import Stream from airbyte_cdk.sources.streams.http.requests_native_auth import Oauth2Authenticator -from .streams import ProcessedOrders, StockItems, StockLocations +from .streams import ProcessedOrderDetails, ProcessedOrders, StockItems, StockLocations class LinnworksAuthenticator(Oauth2Authenticator): @@ -108,4 +108,5 @@ def streams(self, config: Mapping[str, Any]) -> List[Stream]: StockLocations(authenticator=auth), StockItems(authenticator=auth), ProcessedOrders(authenticator=auth, start_date=config["start_date"]), + ProcessedOrderDetails(authenticator=auth, start_date=config["start_date"]), ] diff --git a/airbyte-integrations/connectors/source-linnworks/source_linnworks/streams.py b/airbyte-integrations/connectors/source-linnworks/source_linnworks/streams.py index 73a00bec7429..9a37f3e5ccd1 100644 --- a/airbyte-integrations/connectors/source-linnworks/source_linnworks/streams.py +++ b/airbyte-integrations/connectors/source-linnworks/source_linnworks/streams.py @@ -10,7 +10,7 @@ import pendulum import requests from airbyte_cdk.models.airbyte_protocol import SyncMode -from airbyte_cdk.sources.streams.http import HttpStream +from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.streams.http.auth.core import HttpAuthenticator from requests.auth import AuthBase @@ -187,6 +187,7 @@ class ProcessedOrders(LinnworksGenericPagedResult, IncrementalLinnworksStream): primary_key = "nOrderId" cursor_field = "dReceivedDate" page_size = 500 + use_cache = True def path(self, **kwargs) -> str: return "/api/ProcessedOrders/SearchProcessedOrders" @@ -237,3 +238,36 @@ def paged_result(self, response: requests.Response) -> Mapping[str, Any]: def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: for record in self.paged_result(response)["Data"]: yield record + + +class ProcessedOrderDetails(HttpSubStream, LinnworksStream): + # https://apps.linnworks.net/Api/Method/Orders-GetOrdersById + # Response: List https://apps.linnworks.net/Api/Class/linnworks-spa-commondata-OrderManagement-ClassBase-OrderDetails + # Allows 250 calls per minute + primary_key = "NumOrderId" + page_size = 100 + + def __init__(self, **kwargs): + super().__init__(ProcessedOrders(**kwargs), **kwargs) + + def path(self, **kwargs) -> str: + return "/api/Orders/GetOrdersById" + + + def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: + buffer = [] + for slice in HttpSubStream.stream_slices(self, **kwargs): + buffer.append(slice["parent"]["pkOrderID"]) + if len(buffer) == self.page_size: + yield buffer + buffer = [] + if len(buffer) > 0: + yield buffer + + + def request_body_data( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + return { + "pkOrderIds": json.dumps(stream_slice, separators=(",", ":")), + } From 3202ca61f81a3691cf93b800170e233cef8cb9e8 Mon Sep 17 00:00:00 2001 From: monai Date: Mon, 22 Nov 2021 14:17:32 +0200 Subject: [PATCH 2/7] Fix cache --- .../source_linnworks/streams.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-linnworks/source_linnworks/streams.py b/airbyte-integrations/connectors/source-linnworks/source_linnworks/streams.py index 9a37f3e5ccd1..72958bba3c79 100644 --- a/airbyte-integrations/connectors/source-linnworks/source_linnworks/streams.py +++ b/airbyte-integrations/connectors/source-linnworks/source_linnworks/streams.py @@ -3,16 +3,19 @@ # import json +import os from abc import ABC, abstractmethod from typing import Any, Iterable, List, Mapping, MutableMapping, Optional, Union from urllib.parse import parse_qsl, urlparse import pendulum import requests +import vcr from airbyte_cdk.models.airbyte_protocol import SyncMode from airbyte_cdk.sources.streams.http import HttpStream, HttpSubStream from airbyte_cdk.sources.streams.http.auth.core import HttpAuthenticator from requests.auth import AuthBase +from vcr.cassette import Cassette class LinnworksStream(HttpStream, ABC): @@ -239,6 +242,19 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp for record in self.paged_result(response)["Data"]: yield record + def request_cache(self) -> Cassette: + try: + os.remove(self.cache_filename) + except FileNotFoundError: + pass + + return vcr.use_cassette( + self.cache_filename, + record_mode="new_episodes", + serializer="yaml", + match_on=["method", "scheme", "host", "port", "path", "query", "body"], + ) + class ProcessedOrderDetails(HttpSubStream, LinnworksStream): # https://apps.linnworks.net/Api/Method/Orders-GetOrdersById @@ -253,7 +269,6 @@ def __init__(self, **kwargs): def path(self, **kwargs) -> str: return "/api/Orders/GetOrdersById" - def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: buffer = [] for slice in HttpSubStream.stream_slices(self, **kwargs): @@ -264,7 +279,6 @@ def stream_slices(self, **kwargs) -> Iterable[Optional[Mapping[str, any]]]: if len(buffer) > 0: yield buffer - def request_body_data( self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, any] = None, next_page_token: Mapping[str, Any] = None ) -> MutableMapping[str, Any]: From 4b5db0abc7a767dd0748bb4cc896c7efd1136341 Mon Sep 17 00:00:00 2001 From: monai Date: Mon, 22 Nov 2021 14:17:50 +0200 Subject: [PATCH 3/7] Run gradlew format --- .../schemas/processed_order_details.json | 20 ++++--------------- 1 file changed, 4 insertions(+), 16 deletions(-) diff --git a/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/processed_order_details.json b/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/processed_order_details.json index 62572a7ee920..f9f611c57309 100644 --- a/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/processed_order_details.json +++ b/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/processed_order_details.json @@ -442,10 +442,7 @@ "description": "Current stock level" }, "MinimumLevel": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "description": "Minimum level" }, "AvailableStock": { @@ -535,10 +532,7 @@ "description": "If item got an image" }, "ImageId": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "description": "Image ID" }, "AdditionalInfo": { @@ -615,10 +609,7 @@ "description": "If the item is batched, identifies the batch number" }, "OrderItemBatchId": { - "type": [ - "null", - "integer" - ], + "type": ["null", "integer"], "description": "If the item is batched, identifies the unique order item batch row" } } @@ -753,10 +744,7 @@ } }, "PaidDateTime": { - "type": [ - "null", - "string" - ], + "type": ["null", "string"], "format": "date-time", "description": "Date and time when the order was marked as paid" }, From 5303b0b848732f3c79643e465bee8d3c79808b0b Mon Sep 17 00:00:00 2001 From: monai Date: Mon, 22 Nov 2021 17:30:52 +0200 Subject: [PATCH 4/7] Remove $def/$ref to support basic normalization --- .../schemas/processed_order_details.json | 175 +++++++++++------- 1 file changed, 112 insertions(+), 63 deletions(-) diff --git a/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/processed_order_details.json b/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/processed_order_details.json index f9f611c57309..a5afaaf4ca60 100644 --- a/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/processed_order_details.json +++ b/airbyte-integrations/connectors/source-linnworks/source_linnworks/schemas/processed_order_details.json @@ -245,12 +245,120 @@ "description": "Username of customer (Comes from channel)" }, "Address": { - "$ref": "#/$defs/customer_address", - "description": "Customer address" + "type": "object", + "description": "Customer address", + "additionalProperties": false, + "properties": { + "EmailAddress": { + "type": "string", + "description": "Customer's email address." + }, + "Address1": { + "type": "string", + "description": "First line of customer address." + }, + "Address2": { + "type": "string", + "description": "Second line of customer address." + }, + "Address3": { + "type": "string", + "description": "Third line of customer address." + }, + "Town": { + "type": "string", + "description": "Customer's town." + }, + "Region": { + "type": "string", + "description": "Customer's region." + }, + "PostCode": { + "type": "string", + "description": "Customer's postcode." + }, + "Country": { + "type": "string", + "description": "Customer's country." + }, + "Continent": { + "type": "string", + "description": "Customer's continent" + }, + "FullName": { + "type": "string", + "description": "Customer's first and second name." + }, + "Company": { + "type": "string", + "description": "Customer's company name." + }, + "PhoneNumber": { + "type": "string", + "description": "Customer's telephone number." + }, + "CountryId": { + "type": "string" + } + } }, "BillingAddress": { - "$ref": "#/$defs/customer_address", - "description": "Customer billing address" + "type": "object", + "description": "Customer billing address", + "additionalProperties": false, + "properties": { + "EmailAddress": { + "type": "string", + "description": "Customer's email address." + }, + "Address1": { + "type": "string", + "description": "First line of customer address." + }, + "Address2": { + "type": "string", + "description": "Second line of customer address." + }, + "Address3": { + "type": "string", + "description": "Third line of customer address." + }, + "Town": { + "type": "string", + "description": "Customer's town." + }, + "Region": { + "type": "string", + "description": "Customer's region." + }, + "PostCode": { + "type": "string", + "description": "Customer's postcode." + }, + "Country": { + "type": "string", + "description": "Customer's country." + }, + "Continent": { + "type": "string", + "description": "Customer's continent" + }, + "FullName": { + "type": "string", + "description": "Customer's first and second name." + }, + "Company": { + "type": "string", + "description": "Customer's company name." + }, + "PhoneNumber": { + "type": "string", + "description": "Customer's telephone number." + }, + "CountryId": { + "type": "string" + } + } } } }, @@ -752,64 +860,5 @@ "type": "string", "description": "Buyer's tax number." } - }, - "$defs": { - "customer_address": { - "type": "object", - "additionalProperties": false, - "properties": { - "EmailAddress": { - "type": "string", - "description": "Customer's email address." - }, - "Address1": { - "type": "string", - "description": "First line of customer address." - }, - "Address2": { - "type": "string", - "description": "Second line of customer address." - }, - "Address3": { - "type": "string", - "description": "Third line of customer address." - }, - "Town": { - "type": "string", - "description": "Customer's town." - }, - "Region": { - "type": "string", - "description": "Customer's region." - }, - "PostCode": { - "type": "string", - "description": "Customer's postcode." - }, - "Country": { - "type": "string", - "description": "Customer's country." - }, - "Continent": { - "type": "string", - "description": "Customer's continent" - }, - "FullName": { - "type": "string", - "description": "Customer's first and second name." - }, - "Company": { - "type": "string", - "description": "Customer's company name." - }, - "PhoneNumber": { - "type": "string", - "description": "Customer's telephone number." - }, - "CountryId": { - "type": "string" - } - } - } } } From 64a5701d122d995cffb569e7ffb260a422eb50eb Mon Sep 17 00:00:00 2001 From: monai Date: Tue, 23 Nov 2021 12:07:54 +0200 Subject: [PATCH 5/7] Add request_cache test --- .../unit_tests/test_incremental_streams.py | 22 +++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/airbyte-integrations/connectors/source-linnworks/unit_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-linnworks/unit_tests/test_incremental_streams.py index 1c64338ec458..febe8764fccc 100644 --- a/airbyte-integrations/connectors/source-linnworks/unit_tests/test_incremental_streams.py +++ b/airbyte-integrations/connectors/source-linnworks/unit_tests/test_incremental_streams.py @@ -4,10 +4,13 @@ import json +import os +from unittest.mock import MagicMock import pendulum import pytest import requests +import vcr from source_linnworks.streams import IncrementalLinnworksStream, ProcessedOrders @@ -191,3 +194,22 @@ def test_processed_orders_parse_response(patch_incremental_base_class, requests_ with pytest.raises(KeyError, match="'Data'"): list(stream.parse_response(bad_response)) + + +def test_processed_orders_request_cache(patch_incremental_base_class, mocker): + remove = MagicMock() + use_cassette = MagicMock() + + mocker.patch.object(os, "remove", remove) + mocker.patch.object(vcr, "use_cassette", use_cassette) + + stream = ProcessedOrders() + stream.request_cache() + + remove.assert_called_with(stream.cache_filename) + use_cassette.assert_called_with( + stream.cache_filename, + record_mode="new_episodes", + serializer="yaml", + match_on=["method", "scheme", "host", "port", "path", "query", "body"], + ) From 105e3904b6e706e6f847d0180cdf9650e6aeb2ed Mon Sep 17 00:00:00 2001 From: monai Date: Tue, 23 Nov 2021 16:24:55 +0200 Subject: [PATCH 6/7] Add ProcessedOrderDetails tests --- .../unit_tests/test_source.py | 2 +- .../unit_tests/test_streams.py | 30 ++++++++++++++++++- 2 files changed, 30 insertions(+), 2 deletions(-) diff --git a/airbyte-integrations/connectors/source-linnworks/unit_tests/test_source.py b/airbyte-integrations/connectors/source-linnworks/unit_tests/test_source.py index dec58d6d1002..fa02eef83b80 100644 --- a/airbyte-integrations/connectors/source-linnworks/unit_tests/test_source.py +++ b/airbyte-integrations/connectors/source-linnworks/unit_tests/test_source.py @@ -103,5 +103,5 @@ def test_streams(mocker): source = SourceLinnworks() config_mock = MagicMock() streams = source.streams(config_mock) - expected_streams_number = 3 + expected_streams_number = 4 assert len(streams) == expected_streams_number diff --git a/airbyte-integrations/connectors/source-linnworks/unit_tests/test_streams.py b/airbyte-integrations/connectors/source-linnworks/unit_tests/test_streams.py index 0406704233b7..7356be8d33eb 100644 --- a/airbyte-integrations/connectors/source-linnworks/unit_tests/test_streams.py +++ b/airbyte-integrations/connectors/source-linnworks/unit_tests/test_streams.py @@ -7,7 +7,7 @@ import pytest import requests from airbyte_cdk.models.airbyte_protocol import SyncMode -from source_linnworks.streams import LinnworksStream, Location, StockItems, StockLocations +from source_linnworks.streams import LinnworksStream, Location, ProcessedOrderDetails, ProcessedOrders, StockItems, StockLocations @pytest.fixture @@ -141,3 +141,31 @@ def test_stock_items_request_params(mocker, requests_mock, next_page_token, expe assert ("NextPageTokenKey" in params) == expected if next_page_token: assert next_page_token.items() <= params.items() + + +@pytest.mark.parametrize( + ("count"), + [ + (5), + (205), + ], +) +def test_processed_order_details_stream_slices(patch_base_class, mocker, count): + parent_records = [{"pkOrderID": str(n)} for n in range(count)] + + mocker.patch.object(ProcessedOrders, "stream_slices", MagicMock(return_value=[{}])) + mocker.patch.object(ProcessedOrders, "read_records", MagicMock(return_value=parent_records)) + + stream = ProcessedOrderDetails() + expected_slices = [[str(m) for m in range(count)[i : i + stream.page_size]] for i in range(0, count, stream.page_size)] + + stream_slices = stream.stream_slices(sync_mode=SyncMode.full_refresh) + + assert list(stream_slices) == list(expected_slices) + + +def test_processed_order_details_request_body_data(patch_base_class): + stream = ProcessedOrderDetails() + request_body_data = stream.request_body_data(None, ["abc", "def", "ghi"]) + + assert request_body_data == {"pkOrderIds": '["abc","def","ghi"]'} From bcfe66c65910eca92a2029f7a23f83da0b1e1e81 Mon Sep 17 00:00:00 2001 From: monai Date: Tue, 23 Nov 2021 16:27:27 +0200 Subject: [PATCH 7/7] Update documentation --- docs/integrations/sources/linnworks.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/integrations/sources/linnworks.md b/docs/integrations/sources/linnworks.md index 3e0e3f7e39d8..ee58246696c9 100644 --- a/docs/integrations/sources/linnworks.md +++ b/docs/integrations/sources/linnworks.md @@ -13,6 +13,7 @@ This Source is capable of syncing the following data as streams: * [StockLocations](https://apps.linnworks.net/Api/Method/Inventory-GetStockLocations) * [StockItems](https://apps.linnworks.net//Api/Method/Stock-GetStockItemsFull) * [ProcessedOrders](https://apps.linnworks.net/Api/Method/ProcessedOrders-SearchProcessedOrders) +* [ProcessedOrderDetails](https://apps.linnworks.net/Api/Method/Orders-GetOrdersById) ### Data type mapping @@ -52,5 +53,6 @@ Authentication credentials can be obtained on developer portal section Applicati | Version | Date | Pull Request | Subject | | :--- | :--- | :--- | :--- | +| 0.1.2 | 2021-11-23 | [8177](https://github.com/airbytehq/airbyte/pull/8177) | Source Linnworks: add stream ProcessedOrderDetails | | 0.1.0 | 2021-11-09 | [7588](https://github.com/airbytehq/airbyte/pull/7588) | New Source: Linnworks |