Source Facebook Marketing: Add 80000 as a rate-limiting error code. (#…

…3996) * Fix typo in Facebook rate limit error code (should be 80000, not 8000). * Use error code 80000 in the unit test. * Make the sample error more realistic by using a real error message. * In the Facebook connector, handle rate-limiting error cases where the "call_count" field is not provided. * remove typo * update CHANGELOG.md and bump version Co-authored-by: Eugene Kulak <[email protected]>
airbytehq · Jun 9, 2021 · 88939d2 · 88939d2
1 parent 38dcb6f
commit 88939d2
Show file tree

Hide file tree

Showing 6 changed files with 19 additions and 8 deletions.
diff --git a/...ain/resources/config/STANDARD_SOURCE_DEFINITION/e7778cfc-e97c-4458-9ecb-b4f2bba8946c.json b/...ain/resources/config/STANDARD_SOURCE_DEFINITION/e7778cfc-e97c-4458-9ecb-b4f2bba8946c.json
@@ -2,7 +2,7 @@
   "sourceDefinitionId": "e7778cfc-e97c-4458-9ecb-b4f2bba8946c",
   "name": "Facebook Marketing",
   "dockerRepository": "airbyte/source-facebook-marketing",
-  "dockerImageTag": "0.2.7",
+  "dockerImageTag": "0.2.8",
   "documentationUrl": "https://hub.docker.com/r/airbyte/source-facebook-marketing",
   "icon": "facebook.svg"
 }
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -106,7 +106,7 @@
 - sourceDefinitionId: e7778cfc-e97c-4458-9ecb-b4f2bba8946c
   name: Facebook Marketing
   dockerRepository: airbyte/source-facebook-marketing
-  dockerImageTag: 0.2.7
+  dockerImageTag: 0.2.8
   documentationUrl: https://hub.docker.com/r/airbyte/source-facebook-marketing
   icon: facebook.svg
 - sourceDefinitionId: 36c891d9-4bd9-43ac-bad2-10e12756272c

diff --git a/airbyte-integrations/connectors/source-facebook-marketing/CHANGELOG.md b/airbyte-integrations/connectors/source-facebook-marketing/CHANGELOG.md
@@ -1,5 +1,8 @@
 # Changelog
 
+## 0.2.8
+Add 80000 as a rate-limiting error code (#3996) 
+
 ## 0.2.7
 Add missing fields to AdInsights streams (#3693) 
 

diff --git a/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile b/airbyte-integrations/connectors/source-facebook-marketing/Dockerfile
@@ -11,5 +11,5 @@ RUN pip install .
 
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.2.7
+LABEL io.airbyte.version=0.2.8
 LABEL io.airbyte.name=airbyte/source-facebook-marketing
diff --git a/...egrations/connectors/source-facebook-marketing/source_facebook_marketing/client/common.py b/...egrations/connectors/source-facebook-marketing/source_facebook_marketing/client/common.py
@@ -32,8 +32,10 @@
 from airbyte_cdk.entrypoint import logger  # FIXME (Eugene K): register logger as standard python logger
 from facebook_business.exceptions import FacebookRequestError
 
+# The Facebook API error codes indicating rate-limiting are listed at
+# https://developers.facebook.com/docs/graph-api/overview/rate-limiting/
+FACEBOOK_RATE_LIMIT_ERROR_CODES = (4, 17, 32, 613, 80000, 80001, 80002, 80003, 80004, 80005, 80006, 80008)
 FACEBOOK_UNKNOWN_ERROR_CODE = 99
-FACEBOOK_API_CALL_LIMIT_ERROR_CODES = (4, 17, 32, 613, 8000, 80001, 80002, 80003, 80004, 80005, 80006, 80008)
 DEFAULT_SLEEP_INTERVAL = pendulum.Interval(minutes=1)
 
 
@@ -57,13 +59,13 @@ def handle_call_rate_response(exc: FacebookRequestError) -> bool:
     if platform_header:
         platform_header = json.loads(platform_header)
         call_count = platform_header.get("call_count") or platform_header.get("acc_id_util_pct")
-        if call_count > 99:
+        if call_count and call_count > 99:
             logger.info(f"Reached platform call limit: {exc}")
 
     buc_header = exc.http_headers().get("x-business-use-case-usage")
     buc_header = json.loads(buc_header) if buc_header else {}
     for business_object_id, stats in buc_header.items():
-        if stats["call_count"] > 99:
+        if stats.get("call_count", 0) > 99:
             logger.info(f"Reached call limit on {stats['type']}: {exc}")
             pause_time = max(pause_time, stats["estimated_time_to_regain_access"])
     logger.info(f"Sleeping for {pause_time.total_seconds()} seconds")
@@ -80,7 +82,7 @@ def log_retry_attempt(details):
 
     def should_retry_api_error(exc):
         if isinstance(exc, FacebookRequestError):
-            if exc.api_error_code() in FACEBOOK_API_CALL_LIMIT_ERROR_CODES:
+            if exc.api_error_code() in FACEBOOK_RATE_LIMIT_ERROR_CODES:
                 return handle_call_rate_response(exc)
             return exc.api_transient_error() or exc.api_error_subcode() == FACEBOOK_UNKNOWN_ERROR_CODE
         return True

diff --git a/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py b/airbyte-integrations/connectors/source-facebook-marketing/unit_tests/test_client.py
@@ -56,7 +56,13 @@ def client_fixture(some_config, requests_mock, fb_account_response):
 
 @pytest.fixture(name="fb_call_rate_response")
 def fb_call_rate_response_fixture():
-    error = {"message": "(#32) Page request limit reached", "type": "OAuthException", "code": 32, "fbtrace_id": "Fz54k3GZrio"}
+    error = {
+        "message": "(#80000) There have been too many calls from this ad-account. Wait a bit and try again. For more info, please refer to https://developers.facebook.com/docs/graph-api/overview/rate-limiting.",
+        "type": "OAuthException",
+        "code": 80000,
+        "error_subcode": 2446079,
+        "fbtrace_id": "Fz54k3GZrio"
+    }
 
     headers = {"x-app-usage": json.dumps({"call_count": 28, "total_time": 25, "total_cputime": 25})}