🎉 Source mixpanel: Service Accounts added (#16915)

Signed-off-by: Sergey Chvalyuk <[email protected]>
airbytehq · Sep 26, 2022 · e3553fe · e3553fe
1 parent a9630e3
commit e3553fe
Show file tree

Hide file tree

Showing 23 changed files with 425 additions and 176 deletions.
diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml
@@ -623,7 +623,7 @@
 - name: Mixpanel
   sourceDefinitionId: 12928b32-bf0a-4f1e-964f-07e12e37153a
   dockerRepository: airbyte/source-mixpanel
-  dockerImageTag: 0.1.23
+  dockerImageTag: 0.1.24
   documentationUrl: https://docs.airbyte.io/integrations/sources/mixpanel
   icon: mixpanel.svg
   sourceType: api

diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml
@@ -6223,33 +6223,77 @@
               path_in_connector_config:
               - "credentials"
               - "client_secret"
-- dockerImage: "airbyte/source-mixpanel:0.1.23"
+- dockerImage: "airbyte/source-mixpanel:0.1.24"
   spec:
     documentationUrl: "https://docs.airbyte.io/integrations/sources/mixpanel"
     connectionSpecification:
       $schema: "http://json-schema.org/draft-07/schema#"
       title: "Source Mixpanel Spec"
       type: "object"
-      required:
-      - "api_secret"
       properties:
-        api_secret:
+        credentials:
+          title: "Authentication *"
+          description: "Choose how to authenticate to Mixpanel"
+          type: "object"
           order: 0
-          title: "Project Secret"
-          type: "string"
-          description: "Mixpanel project secret. See the <a href=\"https://developer.mixpanel.com/reference/project-secret#managing-a-projects-secret\"\
+          oneOf:
+          - type: "object"
+            title: "Service Account"
+            required:
+            - "username"
+            - "secret"
+            properties:
+              option_title:
+                type: "string"
+                const: "Service Account"
+                order: 0
+              username:
+                order: 1
+                title: "Username"
+                type: "string"
+                description: "Mixpanel Service Account Username. See the <a href=\"\
+                  https://developer.mixpanel.com/reference/service-accounts\">docs</a>\
+                  \ for more information on how to obtain this."
+              secret:
+                order: 2
+                title: "Secret"
+                type: "string"
+                description: "Mixpanel Service Account Secret. See the <a href=\"\
+                  https://developer.mixpanel.com/reference/service-accounts\">docs</a>\
+                  \ for more information on how to obtain this."
+                airbyte_secret: true
+          - type: "object"
+            title: "Project Secret"
+            required:
+            - "api_secret"
+            properties:
+              option_title:
+                type: "string"
+                const: "Project Secret"
+                order: 0
+              api_secret:
+                order: 1
+                title: "Project Secret"
+                type: "string"
+                description: "Mixpanel project secret. See the <a href=\"https://developer.mixpanel.com/reference/project-secret#managing-a-projects-secret\"\
+                  >docs</a> for more information on how to obtain this."
+                airbyte_secret: true
+        project_id:
+          order: 1
+          title: "Project ID"
+          description: "Your project ID number. See the <a href=\"https://help.mixpanel.com/hc/en-us/articles/115004490503-Project-Settings#project-id\"\
             >docs</a> for more information on how to obtain this."
-          airbyte_secret: true
+          type: "integer"
         attribution_window:
-          order: 1
+          order: 2
           title: "Attribution Window"
           type: "integer"
           description: " A period of time for attributing results to ads and the lookback\
             \ period after those actions occur during which ad results are counted.\
             \ Default attribution window is 5 days."
           default: 5
         project_timezone:
-          order: 2
+          order: 3
           title: "Project Timezone"
           type: "string"
           description: "Time zone in which integer date times are stored. The project\
@@ -6260,15 +6304,15 @@
           - "US/Pacific"
           - "UTC"
         select_properties_by_default:
-          order: 3
+          order: 4
           title: "Select Properties By Default"
           type: "boolean"
           description: "Setting this config parameter to TRUE ensures that new properties\
             \ on events and engage records are captured. Otherwise new properties\
             \ will be ignored."
           default: true
         start_date:
-          order: 4
+          order: 5
           title: "Start Date"
           type: "string"
           description: "The date in the format YYYY-MM-DD. Any data before this date\
@@ -6278,7 +6322,7 @@
           - "2021-11-16"
           pattern: "^$|^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)?$"
         end_date:
-          order: 5
+          order: 6
           title: "End Date"
           type: "string"
           description: "The date in the format YYYY-MM-DD. Any data after this date\
@@ -6287,7 +6331,7 @@
           - "2021-11-16"
           pattern: "^$|^[0-9]{4}-[0-9]{2}-[0-9]{2}(T[0-9]{2}:[0-9]{2}:[0-9]{2}Z)?$"
         region:
-          order: 6
+          order: 7
           title: "Region"
           description: "The region of mixpanel domain instance either US or EU."
           type: "string"
@@ -6296,12 +6340,13 @@
           - "EU"
           default: "US"
         date_window_size:
-          order: 7
+          order: 8
           title: "Date slicing window"
           description: "Defines window size in days, that used to slice through data.\
             \ You can reduce it, if amount of data in each window is too big for your\
             \ environment."
           type: "integer"
+          minimum: 1
           default: 30
     supportsNormalization: false
     supportsDBT: false

diff --git a/airbyte-integrations/connectors/source-mixpanel/Dockerfile b/airbyte-integrations/connectors/source-mixpanel/Dockerfile
@@ -13,5 +13,5 @@ ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py"
 ENTRYPOINT ["python", "/airbyte/integration_code/main.py"]
 
 
-LABEL io.airbyte.version=0.1.23
+LABEL io.airbyte.version=0.1.24
 LABEL io.airbyte.name=airbyte/source-mixpanel
diff --git a/airbyte-integrations/connectors/source-mixpanel/acceptance-test-config.yml b/airbyte-integrations/connectors/source-mixpanel/acceptance-test-config.yml
@@ -4,13 +4,21 @@ connector_image: airbyte/source-mixpanel:dev
 tests:
   spec:
     - spec_path: "source_mixpanel/spec.json"
+      backward_compatibility_tests_config:
+        disable_for_version: "0.1.23"
   connection:
+    - config_path: "secrets/config_old.json"
+      status: "succeed"
+    - config_path: "secrets/config_project_secret.json"
+      status: "succeed"
     - config_path: "secrets/config.json"
       status: "succeed"
     - config_path: "integration_tests/invalid_config.json"
       status: "failed"
   discovery:
     - config_path: "secrets/config.json"
+      backward_compatibility_tests_config:
+        disable_for_version: "0.1.23"
       timeout_seconds: 60
   basic_read:
     - config_path: "secrets/config.json"
@@ -22,16 +30,13 @@ tests:
       configured_catalog_path: "integration_tests/configured_catalog.json"
       timeout_seconds: 3600
   incremental:
-    # incremental streams Funnels, Revenue, Export
-    # Funnels - fails because it has complex state, like {'funnel_idX': {'date': 'dateX'}}
-    # Export - fails because it could return a few previous records for the date of previous sync
     - config_path: "secrets/config.json"
       configured_catalog_path: "integration_tests/configured_catalog_incremental.json"
-      # Test is skipped because requests fails when start_date is in the future
-      # Incremental streams Funnels, Revenue always return data for any valid date
-      # future_state_path: "integration_tests/abnormal_state.json"
+      future_state_path: "integration_tests/abnormal_state.json"
       cursor_paths:
+        cohorts: ["created"]
+        export: ["time"]
+        funnels: ["8901755", "date"]
         revenue: ["date"]
-        export: ["date"]
+        cohort_members": ["last_seen"]
       timeout_seconds: 3600
-
diff --git a/airbyte-integrations/connectors/source-mixpanel/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-mixpanel/integration_tests/abnormal_state.json
@@ -1,5 +1,19 @@
 {
   "funnels": {
-    "date": "2022-07-01"
+    "8901755": {
+      "date": "2030-01-01"
+    }
+  },
+  "revenue": {
+    "date": "2030-01-01"
+  },
+  "cohorts": {
+    "created": "2030-01-01 00:00:00"
+  },
+  "export": {
+    "time": "2030-01-01T00:00:00Z"
+  },
+  "cohort_members": {
+    "last_seen": "2030-01-01T00:00:00"
   }
 }
diff --git a/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog.json b/airbyte-integrations/connectors/source-mixpanel/integration_tests/configured_catalog.json
@@ -7,7 +7,7 @@
         "supported_sync_modes": ["full_refresh", "incremental"],
         "default_cursor_field": ["date"]
       },
-      "sync_mode": "incremental",
+      "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite"
     },
     {
@@ -35,14 +35,14 @@
         "supported_sync_modes": ["full_refresh", "incremental"],
         "default_cursor_field": ["time"]
       },
-      "sync_mode": "incremental",
+      "sync_mode": "full_refresh",
       "destination_sync_mode": "append"
     },
     {
       "stream": {
         "name": "cohorts",
         "json_schema": {},
-        "supported_sync_modes": ["full_refresh"]
+        "supported_sync_modes": ["full_refresh", "incremental"]
       },
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite"
@@ -51,7 +51,7 @@
       "stream": {
         "name": "cohort_members",
         "json_schema": {},
-        "supported_sync_modes": ["full_refresh"]
+        "supported_sync_modes": ["full_refresh", "incremental"]
       },
       "sync_mode": "full_refresh",
       "destination_sync_mode": "overwrite"
@@ -63,7 +63,7 @@
         "supported_sync_modes": ["full_refresh", "incremental"],
         "default_cursor_field": ["date"]
       },
-      "sync_mode": "incremental",
+      "sync_mode": "full_refresh",
       "destination_sync_mode": "append"
     }
   ]

diff --git a/...grations/connectors/source-mixpanel/integration_tests/configured_catalog_incremental.json b/...grations/connectors/source-mixpanel/integration_tests/configured_catalog_incremental.json
@@ -1,14 +1,64 @@
 {
   "streams": [
+    {
+      "stream": {
+        "name": "funnels",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["date"],
+        "source_defined_primary_key": [["funnel_id"], ["date"]]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "append"
+    },
     {
       "stream": {
         "name": "revenue",
         "json_schema": {},
         "supported_sync_modes": ["full_refresh", "incremental"],
-        "default_cursor_field": ["date"]
+        "source_defined_cursor": true,
+        "default_cursor_field": ["date"],
+        "source_defined_primary_key": [["date"]]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "append"
+    },
+    {
+      "stream": {
+        "name": "cohorts",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["created"],
+        "source_defined_primary_key": [["id"]]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "append"
+    },
+    {
+      "stream": {
+        "name": "export",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": true,
+        "default_cursor_field": ["time"]
+      },
+      "sync_mode": "incremental",
+      "destination_sync_mode": "append"
+    },
+    {
+      "stream": {
+        "name": "cohort_members",
+        "json_schema": {},
+        "supported_sync_modes": ["full_refresh", "incremental"],
+        "source_defined_cursor": false,
+        "default_cursor_field": null,
+        "source_defined_primary_key": [["distinct_id"]]
       },
       "sync_mode": "incremental",
-      "destination_sync_mode": "overwrite"
+      "destination_sync_mode": "append",
+      "cursor_field": ["last_seen"]
     }
   ]
 }