🐛 Source LinkedIn Ads: hands 429 response status code

airbyte-config/init/src/main/resources/seed/source_definitions.yaml

@@ -344,7 +344,7 @@
 - name: LinkedIn Ads
   sourceDefinitionId: 137ece28-5434-455c-8f34-69dc3782f451
   dockerRepository: airbyte/source-linkedin-ads
-  dockerImageTag: 0.1.2
+  dockerImageTag: 0.1.4
   documentationUrl: https://docs.airbyte.io/integrations/sources/linkedin-ads
   icon: linkedin.svg
   sourceType: api

airbyte-config/init/src/main/resources/seed/source_specs.yaml

@@ -3298,7 +3298,7 @@
         - - "client_secret"
         - - "refresh_token"
         oauthFlowOutputParameters: []
-- dockerImage: "airbyte/source-linkedin-ads:0.1.2"
+- dockerImage: "airbyte/source-linkedin-ads:0.1.4"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/linkedin-ads"
     connectionSpecification:
@@ -3307,8 +3307,7 @@
       type: "object"
       required:
       - "start_date"
-      - "access_token"
-      additionalProperties: false
+      additionalProperties: true
       properties:
         start_date:
           type: "string"
@@ -3318,11 +3317,6 @@
             \ not be replicated."
           examples:
           - "2021-05-17"
-        access_token:
-          type: "string"
-          title: "Access Token"
-          description: "The token value ganerated using Auth Code"
-          airbyte_secret: true
         account_ids:
           title: "Account IDs"
           type: "array"
@@ -3331,9 +3325,59 @@
           items:
             type: "integer"
           default: []
+        credentials:
+          title: "Authorization Method"
+          type: "object"
+          oneOf:
+          - type: "object"
+            title: "oAuth2.0"
+            required:
+            - "client_id"
+            - "client_secret"
+            - "refresh_token"
+            properties:
+              auth_method:
+                type: "string"
+                const: "oAuth2.0"
+              client_id:
+                type: "string"
+                description: "The API ID of the Gitlab developer application."
+                airbyte_secret: true
+              client_secret:
+                type: "string"
+                description: "The API Secret the Gitlab developer application."
+                airbyte_secret: true
+              refresh_token:
+                type: "string"
+                description: "The key to refresh the expired access_token."
+                airbyte_secret: true
+          - title: "Access Token"
+            type: "object"
+            required:
+            - "access_token"
+            properties:
+              auth_method:
+                type: "string"
+                const: "access_token"
+              access_token:
+                type: "string"
+                title: "Access Token"
+                description: "The token value ganerated using Auth Code"
+                airbyte_secret: true
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
+    authSpecification:
+      auth_type: "oauth2.0"
+      oauth2Specification:
+        rootObject:
+        - "credentials"
+        - "0"
+        oauthFlowInitParameters:
+        - - "client_id"
+        - - "client_secret"
+        oauthFlowOutputParameters:
+        - - "refresh_token"
 - dockerImage: "airbyte/source-linnworks:0.1.3"
   spec:
     documentationUrl: "https://docsurl.com"

airbyte-integrations/connectors/source-linkedin-ads/Dockerfile

@@ -33,5 +33,5 @@ COPY source_linkedin_ads ./source_linkedin_ads
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.1.3
+LABEL io.airbyte.version=0.1.4
 LABEL io.airbyte.name=airbyte/source-linkedin-ads

airbyte-integrations/connectors/source-linkedin-ads/source_linkedin_ads/source.py

@@ -67,6 +67,18 @@ def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapp
         """
         yield from transform_data(response.json().get("elements"))
 
+    def should_retry(self, response: requests.Response) -> bool:
+        if response.status_code == 429:
+            error_message = (
+                f"Stream {self.name}: LinkedIn API requests are rate limited. "
+                f"Rate limits specify the maximum number of API calls that can be made in a 24 hour period. "
+                f"These limits reset at midnight UTC every day. "
+                f"You can find more information here https://docs.airbyte.io/integrations/sources/linkedin-ads. "
+                f"Also quotas and usage are here: https://www.linkedin.com/developers/apps."
+            )
+            self.logger.error(error_message)
+        return super().should_retry(response)
+
 
 class Accounts(LinkedinAdsStream):
     """

airbyte-integrations/connectors/source-mixpanel/source_mixpanel/source.py

@@ -357,7 +357,7 @@ def get_updated_state(
 class EngageSchema(MixpanelStream):
     """
     Engage helper stream for dynamic schema extraction.
-    :: reqs_per_hour_limit: int - property is set to the value of 1 million, 
+    :: reqs_per_hour_limit: int - property is set to the value of 1 million,
        to get the sleep time close to the zero, while generating dynamic schema.
        When `reqs_per_hour_limit = 0` - it means we skip this limits.
     """
@@ -634,7 +634,7 @@ def process_response(self, response: requests.Response, **kwargs) -> Iterable[Ma
 class ExportSchema(MixpanelStream):
     """
     Export helper stream for dynamic schema extraction.
-    :: reqs_per_hour_limit: int - property is set to the value of 1 million, 
+    :: reqs_per_hour_limit: int - property is set to the value of 1 million,
        to get the sleep time close to the zero, while generating dynamic schema.
        When `reqs_per_hour_limit = 0` - it means we skip this limits.
     """

airbyte-integrations/connectors/source-s3/source_s3/source_files_abstract/formats/csv_parser.py

@@ -131,7 +131,6 @@ def _get_schema_dict(self, file, infer_schema_process):
     # TODO Rename this here and in `_get_schema_dict`
     def _get_schema_dict_without_inference(self, file):
         self.logger.debug("infer_datatypes is False, skipping infer_schema")
-        encoding = self.format.encoding
         delimiter = self.format.delimiter
         quote_char = self.format.quote_char
         reader = csv.reader([six.ensure_text(file.readline())], delimiter=delimiter, quotechar=quote_char)

airbyte-integrations/connectors/source-s3/unit_tests/abstract_test_parser.py

@@ -2,8 +2,7 @@
 # Copyright (c) 2021 Airbyte, Inc., all rights reserved.
 #
 
-from abc import ABC, abstractmethod
-from typing import Any, List, Mapping
+from abc import ABC
 
 import pytest
 from airbyte_cdk import AirbyteLogger

docs/integrations/sources/linkedin-ads.md

@@ -47,7 +47,7 @@ This Source is capable of syncing the following data as streams:
 
 ### Performance considerations
 
-There are official Rate Limits for LinkedIn Ads API Usage, [more information here](https://docs.microsoft.com/en-us/linkedin/shared/api-guide/concepts/rate-limits?context=linkedin/marketing/context). Rate limited requests will receive a 429 response. In rare cases, LinkedIn may also return a 429 response as part of infrastructure protection. API service will return to normal automatically. In such cases you will receive the next error message:
+There are official Rate Limits for LinkedIn Ads API Usage, [more information here](https://docs.microsoft.com/en-us/linkedin/shared/api-guide/concepts/rate-limits?context=linkedin/marketing/context). Rate limited requests will receive a 429 response. Rate limits specify the maximum number of API calls that can be made in a 24 hour period. These limits reset at midnight UTC every day. In rare cases, LinkedIn may also return a 429 response as part of infrastructure protection. API service will return to normal automatically. In such cases you will receive the next error message:
 
 ```text
 "Caught retryable error '<some_error> or null' after <some_number> tries. Waiting <some_number> seconds then retrying..."
@@ -59,7 +59,7 @@ This is expected when the connector hits the 429 - Rate Limit Exceeded HTTP Erro
 "Max try rate limit exceded..."
 ```
 
-After 5 unsuccessful attempts - the connector will stop the sync operation. In such cases check your Rate Limits [on this page](https://www.linkedin.com/developers/apps) &gt; Choose you app &gt; Analytics
+After 5 unsuccessful attempts - the connector will stop the sync operation. In such cases check your Rate Limits [on this page](https://www.linkedin.com/developers/apps) &gt; Choose you app &gt; Analytics. 
 
 ## Getting started
 The API user account should be assigned the following permissions for the API endpoints:
@@ -132,6 +132,7 @@ The source LinkedIn supports the oAuth2 protocol. Everyone can use it directly v
 
 | Version | Date       | Pull Request                                           | Subject                                                                                                           |
 | :------ | :--------- | :----------------------------------------------------- | :---------------------------------------------------------------------------------------------------------------- |
+| 0.1.4   | 2021-12-02 | [8382](https://github.com/airbytehq/airbyte/pull/8382) | Modify log message in rate-limit cases                                                                            |
 | 0.1.3   | 2021-11-11 | [7839](https://github.com/airbytehq/airbyte/pull/7839) | Added oauth support                                                                                               |
 | 0.1.2   | 2021-11-08 | [7499](https://github.com/airbytehq/airbyte/pull/7499) | Remove base-python dependencies                                                                                   |
 | 0.1.1   | 2021-10-02 | [6610](https://github.com/airbytehq/airbyte/pull/6610) | Fix for  `Campaigns/targetingCriteria` transformation, coerced  `Creatives/variables/values` to string by default |