🎉Source Google Analytics Data Api: code refactoring (#15229)

* Source: google analytics data api refactoring * google-analytics-data-api.md Signed-off-by: Sergey Chvalyuk <[email protected]> * updated spec: added custom reports field * doclint Signed-off-by: Sergey Chvalyuk <[email protected]> * auto-bump connector version [ci skip] Signed-off-by: Sergey Chvalyuk <[email protected]> Co-authored-by: Sergey Chvalyuk <[email protected]> Co-authored-by: Octavia Squidington III <[email protected]>
airbytehq · Aug 17, 2022 · e40ee3f · e40ee3f
1 parent f4adef7
commit e40ee3f
Show file tree

Hide file tree

Showing 22 changed files with 1,202 additions and 305 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -363,7 +363,7 @@
 - name: Google Analytics Data API
   sourceDefinitionId: 3cc2eafd-84aa-4dca-93af-322d9dfeec1a
   dockerRepository: airbyte/source-google-analytics-data-api
-  dockerImageTag: 0.0.2
+  dockerImageTag: 0.0.3
   documentationUrl: https://docs.airbyte.com/integrations/sources/google-analytics-v4
   icon: google-analytics.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -3191,7 +3191,7 @@
         oauthFlowOutputParameters:
         - - "access_token"
         - - "refresh_token"
-- dockerImage: "airbyte/source-google-analytics-data-api:0.0.2"
+- dockerImage: "airbyte/source-google-analytics-data-api:0.0.3"
   spec:
     documentationUrl: "https://docs.airbyte.com/integrations/sources/google-analytics-v4"
     connectionSpecification:
@@ -3200,55 +3200,124 @@
       type: "object"
       required:
       - "property_id"
-      - "json_credentials"
-      - "report_name"
-      - "dimensions"
-      - "metrics"
       - "date_ranges_start_date"
-      - "date_ranges_end_date"
+      additionalProperties: true
       properties:
         property_id:
           type: "string"
           title: "Property ID"
           description: "A Google Analytics GA4 property identifier whose events are\
             \ tracked. Specified in the URL path and not the body"
           order: 1
-        json_credentials:
-          type: "string"
-          title: "JSON Credentials"
-          description: "The JSON key of the Service Account to use for authorization"
-          airbyte_secret: true
-          order: 2
-        report_name:
-          type: "string"
-          title: "Report Name"
-          description: "The report name"
-          order: 3
-        dimensions:
-          type: "string"
-          title: "Dimensions"
-          description: "Comma seprated report dimensions https://developers.google.com/analytics/devguides/reporting/data/v1/api-schema#dimensions"
-          order: 4
-        metrics:
-          type: "string"
-          title: "Metrics"
-          description: "Comma seprated report metrics https://developers.google.com/analytics/devguides/reporting/data/v1/api-schema#metrics"
-          order: 5
+        credentials:
+          order: 0
+          type: "object"
+          title: "Credentials"
+          description: "Credentials for the service"
+          oneOf:
+          - title: "Authenticate via Google (Oauth)"
+            type: "object"
+            required:
+            - "client_id"
+            - "client_secret"
+            - "refresh_token"
+            properties:
+              auth_type:
+                type: "string"
+                const: "Client"
+                order: 0
+              client_id:
+                title: "Client ID"
+                type: "string"
+                description: "The Client ID of your Google Analytics developer application."
+                airbyte_secret: true
+                order: 1
+              client_secret:
+                title: "Client Secret"
+                type: "string"
+                description: "The Client Secret of your Google Analytics developer\
+                  \ application."
+                airbyte_secret: true
+                order: 2
+              refresh_token:
+                title: "Refresh Token"
+                type: "string"
+                description: "The token for obtaining a new access token."
+                airbyte_secret: true
+                order: 3
+              access_token:
+                title: "Access Token (Optional)"
+                type: "string"
+                description: "Access Token for making authenticated requests."
+                airbyte_secret: true
+                order: 4
+          - type: "object"
+            title: "Service Account Key Authentication"
+            required:
+            - "credentials_json"
+            properties:
+              auth_type:
+                type: "string"
+                const: "Service"
+                order: 0
+              credentials_json:
+                title: "Service Account JSON Key"
+                type: "string"
+                description: "The JSON key of the service account to use for authorization"
+                examples:
+                - "{ \"type\": \"service_account\", \"project_id\": YOUR_PROJECT_ID,\
+                  \ \"private_key_id\": YOUR_PRIVATE_KEY, ... }"
+                airbyte_secret: true
         date_ranges_start_date:
           type: "string"
           title: "Date Range Start Date"
           description: "The start date. One of the values Ndaysago, yesterday, today\
             \ or in the format YYYY-MM-DD"
-          order: 6
-        date_ranges_end_date:
+          order: 2
+        custom_reports:
+          order: 3
           type: "string"
-          title: "Date Range End Date"
-          description: "The end date. One of the values Ndaysago, yesterday, today\
-            \ or in the format YYYY-MM-DD"
-          order: 7
+          title: "Custom Reports (Optional)"
+          description: "A JSON array describing the custom reports you want to sync\
+            \ from Google Analytics. See <a href=\"https://docs.airbyte.io/integrations/sources/google-analytics-v4#data-processing-latency\"\
+            >the docs</a> for more information about the exact format you can use\
+            \ to fill out this field."
+        window_in_days:
+          type: "integer"
+          title: "Data request time increment in days (Optional)"
+          description: "The time increment used by the connector when requesting data\
+            \ from the Google Analytics API. More information is available in the\
+            \ <a href=\"https://docs.airbyte.com/integrations/sources/google-analytics-v4/#sampling-in-reports\"\
+            >the docs</a>. The bigger this value is, the faster the sync will be,\
+            \ but the more likely that sampling will be applied to your data, potentially\
+            \ causing inaccuracies in the returned results. We recommend setting this\
+            \ to 1 unless you have a hard requirement to make the sync faster at the\
+            \ expense of accuracy. The minimum allowed value for this field is 1,\
+            \ and the maximum is 364. "
+          examples:
+          - 30
+          - 60
+          - 90
+          - 120
+          - 200
+          - 364
+          default: 1
+          order: 4
     supportsNormalization: false
     supportsDBT: false
     supported_destination_sync_modes: []
+    authSpecification:
+      auth_type: "oauth2.0"
+      oauth2Specification:
+        rootObject:
+        - "credentials"
+        - "0"
+        oauthFlowInitParameters:
+        - - "client_id"
+        - - "client_secret"
+        oauthFlowOutputParameters:
+        - - "access_token"
+        - - "refresh_token"
 - dockerImage: "airbyte/source-google-directory:0.1.9"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/google-directory"

diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/Dockerfile b/airbyte-integrations/connectors/source-google-analytics-data-api/Dockerfile
@@ -28,5 +28,5 @@ COPY source_google_analytics_data_api ./source_google_analytics_data_api
 ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
-LABEL io.airbyte.version=0.0.2
+LABEL io.airbyte.version=0.0.3
 LABEL io.airbyte.name=airbyte/source-google-analytics-data-api
diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/Makefile b/airbyte-integrations/connectors/source-google-analytics-data-api/Makefile
@@ -0,0 +1,19 @@
+docker_image := airbyte/$(notdir $(CURDIR)):dev
+
+run-build:
+	docker build . -t ${docker_image}
+
+spec:
+	@docker run --rm $(docker_image) spec | jq
+
+check:
+	@docker run --rm -v $(PWD)/secrets:/secrets $(docker_image) check --config /secrets/config.json | jq
+
+discover:
+	@docker run --rm -v $(PWD)/secrets:/secrets $(docker_image) discover --config /secrets/config.json | jq
+
+read:
+	@docker run --rm -v $(PWD)/secrets:/secrets -v $(PWD)/integration_tests:/integration_tests $(docker_image) read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json | jq
+
+unittest-local:
+	@python -m pytest unit_tests
diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/acceptance-test-config.yml b/airbyte-integrations/connectors/source-google-analytics-data-api/acceptance-test-config.yml
@@ -11,10 +11,21 @@ tests:
       status: "failed"
   discovery:
     - config_path: "secrets/config.json"
+      backward_compatibility_tests_config:
+        disable_for_version: "0.0.2"
   basic_read:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
       empty_streams: []
   full_refresh:
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog.json"
+      ignored_fields:
+        "daily_active_users": ["uuid"]
+        "weekly_active_users": ["uuid"]
+        "four_weekly_active_users": ["uuid"]
+        "devices": ["uuid"]
+        "locations": ["uuid"]
+        "pages": ["uuid"]
+        "traffic_sources": ["uuid"]
+        "website_overview": ["uuid"]
diff --git a/...ons/connectors/source-google-analytics-data-api/integration_tests/configured_catalog.json b/...ons/connectors/source-google-analytics-data-api/integration_tests/configured_catalog.json
@@ -2,13 +2,90 @@
   "streams": [
     {
       "stream": {
-        "name": "Analytics Report",
+        "name": "daily_active_users",
         "json_schema": {},
-        "supported_sync_modes": ["full_refresh"],
+        "supported_sync_modes": ["incremental"],
         "source_defined_cursor": false,
-        "default_cursor_field": ["column_name"]
+        "default_cursor_field": ["date"]
       },
-      "sync_mode": "full_refresh",
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "weekly_active_users",
+        "json_schema": {},
+        "supported_sync_modes": ["incremental"],
+        "source_defined_cursor": false,
+        "default_cursor_field": ["date"]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "four_weekly_active_users",
+        "json_schema": {},
+        "supported_sync_modes": ["incremental"],
+        "source_defined_cursor": false,
+        "default_cursor_field": ["date"]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "devices",
+        "json_schema": {},
+        "supported_sync_modes": ["incremental"],
+        "source_defined_cursor": false,
+        "default_cursor_field": ["date"]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "locations",
+        "json_schema": {},
+        "supported_sync_modes": ["incremental"],
+        "source_defined_cursor": false,
+        "default_cursor_field": ["date"]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "pages",
+        "json_schema": {},
+        "supported_sync_modes": ["incremental"],
+        "source_defined_cursor": false,
+        "default_cursor_field": ["date"]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "traffic_sources",
+        "json_schema": {},
+        "supported_sync_modes": ["incremental"],
+        "source_defined_cursor": false,
+        "default_cursor_field": ["date"]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "overwrite"
+    },
+    {
+      "stream": {
+        "name": "website_overview",
+        "json_schema": {},
+        "supported_sync_modes": ["incremental"],
+        "source_defined_cursor": false,
+        "default_cursor_field": ["date"]
+      },
+      "sync_mode": "incremental",
       "destination_sync_mode": "overwrite"
     }
   ]

diff --git a/airbyte-integrations/connectors/source-google-analytics-data-api/setup.py b/airbyte-integrations/connectors/source-google-analytics-data-api/setup.py
@@ -5,7 +5,7 @@
 
 from setuptools import find_packages, setup
 
-MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "google-analytics-data==0.11.2"]
+MAIN_REQUIREMENTS = ["airbyte-cdk~=0.1", "google-analytics-data==0.11.2", "PyJWT==2.4.0", "cryptography==37.0.4", "requests==2.28.1"]
 
 TEST_REQUIREMENTS = [
     "pytest~=6.1",