diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index 2b26dca2f72e..20bbc5a82f21 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -344,7 +344,7 @@ - name: LinkedIn Ads sourceDefinitionId: 137ece28-5434-455c-8f34-69dc3782f451 dockerRepository: airbyte/source-linkedin-ads - dockerImageTag: 0.1.2 + dockerImageTag: 0.1.4 documentationUrl: https://docs.airbyte.io/integrations/sources/linkedin-ads icon: linkedin.svg sourceType: api diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index 5defb3b220d4..fd52f050cfc8 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -3298,7 +3298,7 @@ - - "client_secret" - - "refresh_token" oauthFlowOutputParameters: [] -- dockerImage: "airbyte/source-linkedin-ads:0.1.2" +- dockerImage: "airbyte/source-linkedin-ads:0.1.4" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/linkedin-ads" connectionSpecification: @@ -3307,8 +3307,7 @@ type: "object" required: - "start_date" - - "access_token" - additionalProperties: false + additionalProperties: true properties: start_date: type: "string" @@ -3318,11 +3317,6 @@ \ not be replicated." examples: - "2021-05-17" - access_token: - type: "string" - title: "Access Token" - description: "The token value ganerated using Auth Code" - airbyte_secret: true account_ids: title: "Account IDs" type: "array" @@ -3331,9 +3325,59 @@ items: type: "integer" default: [] + credentials: + title: "Authorization Method" + type: "object" + oneOf: + - type: "object" + title: "oAuth2.0" + required: + - "client_id" + - "client_secret" + - "refresh_token" + properties: + auth_method: + type: "string" + const: "oAuth2.0" + client_id: + type: "string" + description: "The API ID of the Gitlab developer application." + airbyte_secret: true + client_secret: + type: "string" + description: "The API Secret the Gitlab developer application." + airbyte_secret: true + refresh_token: + type: "string" + description: "The key to refresh the expired access_token." + airbyte_secret: true + - title: "Access Token" + type: "object" + required: + - "access_token" + properties: + auth_method: + type: "string" + const: "access_token" + access_token: + type: "string" + title: "Access Token" + description: "The token value ganerated using Auth Code" + airbyte_secret: true supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] + authSpecification: + auth_type: "oauth2.0" + oauth2Specification: + rootObject: + - "credentials" + - "0" + oauthFlowInitParameters: + - - "client_id" + - - "client_secret" + oauthFlowOutputParameters: + - - "refresh_token" - dockerImage: "airbyte/source-linnworks:0.1.3" spec: documentationUrl: "https://docsurl.com" diff --git a/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile b/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile index a742a5e3c73c..043d4ab15db7 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile +++ b/airbyte-integrations/connectors/source-linkedin-ads/Dockerfile @@ -33,5 +33,5 @@ COPY source_linkedin_ads ./source_linkedin_ads ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] -LABEL io.airbyte.version=0.1.3 +LABEL io.airbyte.version=0.1.4 LABEL io.airbyte.name=airbyte/source-linkedin-ads diff --git a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py index 3c2ed1f0740e..1223cbee363b 100644 --- a/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py +++ b/airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py @@ -67,6 +67,18 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp """ yield from transform_data(response.json().get("elements")) + def should_retry(self, response: requests.Response) -> bool: + if response.status_code == 429: + error_message = ( + f"Stream {self.name}: LinkedIn API requests are rate limited. " + f"Rate limits specify the maximum number of API calls that can be made in a 24 hour period. " + f"These limits reset at midnight UTC every day. " + f"You can find more information here https://docs.airbyte.io/integrations/sources/linkedin-ads. " + f"Also quotas and usage are here: https://www.linkedin.com/developers/apps." + ) + self.logger.error(error_message) + return super().should_retry(response) + class Accounts(LinkedinAdsStream): """ diff --git a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py index f021af5b5474..73217066eee6 100644 --- a/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py +++ b/airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py @@ -357,7 +357,7 @@ def get_updated_state( class EngageSchema(MixpanelStream): """ Engage helper stream for dynamic schema extraction. - :: reqs_per_hour_limit: int - property is set to the value of 1 million, + :: reqs_per_hour_limit: int - property is set to the value of 1 million, to get the sleep time close to the zero, while generating dynamic schema. When `reqs_per_hour_limit = 0` - it means we skip this limits. """ @@ -634,7 +634,7 @@ def process_response(self, response: requests.Response, **kwargs) -> Iterable[Ma class ExportSchema(MixpanelStream): """ Export helper stream for dynamic schema extraction. - :: reqs_per_hour_limit: int - property is set to the value of 1 million, + :: reqs_per_hour_limit: int - property is set to the value of 1 million, to get the sleep time close to the zero, while generating dynamic schema. When `reqs_per_hour_limit = 0` - it means we skip this limits. """ diff --git a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/csv_parser.py b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/csv_parser.py index 0d69b4240c47..3e781e9724a6 100644 --- a/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/csv_parser.py +++ b/airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/csv_parser.py @@ -131,7 +131,6 @@ def _get_schema_dict(self, file, infer_schema_process): # TODO Rename this here and in `_get_schema_dict` def _get_schema_dict_without_inference(self, file): self.logger.debug("infer_datatypes is False, skipping infer_schema") - encoding = self.format.encoding delimiter = self.format.delimiter quote_char = self.format.quote_char reader = csv.reader([six.ensure_text(file.readline())], delimiter=delimiter, quotechar=quote_char) diff --git a/airbyte-integrations/connectors/source-s3/unit_tests/abstract_test_parser.py b/airbyte-integrations/connectors/source-s3/unit_tests/abstract_test_parser.py index fc191e7fae5a..9b4b6c711510 100644 --- a/airbyte-integrations/connectors/source-s3/unit_tests/abstract_test_parser.py +++ b/airbyte-integrations/connectors/source-s3/unit_tests/abstract_test_parser.py @@ -2,8 +2,7 @@ # Copyright (c) 2021 Airbyte, Inc., all rights reserved. # -from abc import ABC, abstractmethod -from typing import Any, List, Mapping +from abc import ABC import pytest from airbyte_cdk import AirbyteLogger diff --git a/docs/integrations/sources/linkedin-ads.md b/docs/integrations/sources/linkedin-ads.md index 88c3537bed02..2b1054c3af0b 100644 --- a/docs/integrations/sources/linkedin-ads.md +++ b/docs/integrations/sources/linkedin-ads.md @@ -47,7 +47,7 @@ This Source is capable of syncing the following data as streams: ### Performance considerations -There are official Rate Limits for LinkedIn Ads API Usage, [more information here](https://docs.microsoft.com/en-us/linkedin/shared/api-guide/concepts/rate-limits?context=linkedin/marketing/context). Rate limited requests will receive a 429 response. In rare cases, LinkedIn may also return a 429 response as part of infrastructure protection. API service will return to normal automatically. In such cases you will receive the next error message: +There are official Rate Limits for LinkedIn Ads API Usage, [more information here](https://docs.microsoft.com/en-us/linkedin/shared/api-guide/concepts/rate-limits?context=linkedin/marketing/context). Rate limited requests will receive a 429 response. Rate limits specify the maximum number of API calls that can be made in a 24 hour period. These limits reset at midnight UTC every day. In rare cases, LinkedIn may also return a 429 response as part of infrastructure protection. API service will return to normal automatically. In such cases you will receive the next error message: ```text "Caught retryable error ' or null' after tries. Waiting seconds then retrying..." @@ -59,7 +59,7 @@ This is expected when the connector hits the 429 - Rate Limit Exceeded HTTP Erro "Max try rate limit exceded..." ``` -After 5 unsuccessful attempts - the connector will stop the sync operation. In such cases check your Rate Limits [on this page](https://www.linkedin.com/developers/apps) > Choose you app > Analytics +After 5 unsuccessful attempts - the connector will stop the sync operation. In such cases check your Rate Limits [on this page](https://www.linkedin.com/developers/apps) > Choose you app > Analytics. ## Getting started The API user account should be assigned the following permissions for the API endpoints: @@ -132,6 +132,7 @@ The source LinkedIn supports the oAuth2 protocol. Everyone can use it directly v | Version | Date | Pull Request | Subject | | :------ | :--------- | :----------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------- | +| 0.1.4 | 2021-12-02 | [8382](https://github.com/airbytehq/airbyte/pull/8382) | Modify log message in rate-limit cases | | 0.1.3 | 2021-11-11 | [7839](https://github.com/airbytehq/airbyte/pull/7839) | Added oauth support | | 0.1.2 | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies | | 0.1.1 | 2021-10-02 | [6610](https://github.com/airbytehq/airbyte/pull/6610) | Fix for `Campaigns/targetingCriteria` transformation, coerced `Creatives/variables/values` to string by default |