From 611ef03631e31b8e9e5c00d22c78bda503496ebe Mon Sep 17 00:00:00 2001 From: jsrcodes Date: Thu, 11 Aug 2022 16:24:40 -0600 Subject: [PATCH] =?UTF-8?q?=F0=9F=8E=89=20New=20source:=20Freshcaller=20[R?= =?UTF-8?q?eopened]=20(#14759)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Add source: Freshcaller * Formatted * fix connector backward compatibility * fix unused files * add seed file * add changelog * auto-bump connector version [ci skip] Co-authored-by: Harshith Mullapudi Co-authored-by: marcosmarxm Co-authored-by: Octavia Squidington III --- .../resources/seed/source_definitions.yaml | 7 + .../src/main/resources/seed/source_specs.yaml | 47 ++ .../source-freshcaller/.dockerignore | 7 + .../connectors/source-freshcaller/Dockerfile | 39 ++ .../connectors/source-freshcaller/README.md | 133 ++++++ .../acceptance-test-config.yml | 22 + .../acceptance-test-docker.sh | 16 + .../source-freshcaller/build.gradle | 13 + .../integration_tests/__init__.py | 0 .../integration_tests/abnormal_state.json | 8 + .../integration_tests/acceptance.py | 14 + .../integration_tests/catalog.json | 40 ++ .../configured_catalog_full_refresh.json | 40 ++ .../configured_catalog_incremental.json | 30 ++ .../integration_tests/invalid_config.json | 5 + .../test_incremental_streams.py | 126 ++++++ .../connectors/source-freshcaller/main.py | 13 + .../source-freshcaller/requirements.txt | 2 + .../sample_files/config.json | 5 + .../sample_files/configured_catalog.json | 407 ++++++++++++++++++ .../configured_catalog_calls.json | 177 ++++++++ .../configured_catalog_full_refresh.json | 40 ++ .../configured_catalog_incremental.json | 30 ++ .../configured_catalog_metrics.json | 124 ++++++ .../connectors/source-freshcaller/setup.py | 31 ++ .../source_freshcaller/__init__.py | 3 + .../schemas/call_metrics.json | 109 +++++ .../source_freshcaller/schemas/calls.json | 162 +++++++ .../source_freshcaller/schemas/teams.json | 30 ++ .../source_freshcaller/schemas/users.json | 62 +++ .../source_freshcaller/source.py | 46 ++ .../source_freshcaller/spec.json | 49 +++ .../source_freshcaller/streams.py | 202 +++++++++ .../unit_tests/test_source.py | 28 ++ docs/integrations/sources/freshcaller.md | 43 ++ 35 files changed, 2110 insertions(+) create mode 100644 airbyte-integrations/connectors/source-freshcaller/.dockerignore create mode 100644 airbyte-integrations/connectors/source-freshcaller/Dockerfile create mode 100644 airbyte-integrations/connectors/source-freshcaller/README.md create mode 100644 airbyte-integrations/connectors/source-freshcaller/acceptance-test-config.yml create mode 100644 airbyte-integrations/connectors/source-freshcaller/acceptance-test-docker.sh create mode 100644 airbyte-integrations/connectors/source-freshcaller/build.gradle create mode 100644 airbyte-integrations/connectors/source-freshcaller/integration_tests/__init__.py create mode 100644 airbyte-integrations/connectors/source-freshcaller/integration_tests/abnormal_state.json create mode 100644 airbyte-integrations/connectors/source-freshcaller/integration_tests/acceptance.py create mode 100644 airbyte-integrations/connectors/source-freshcaller/integration_tests/catalog.json create mode 100644 airbyte-integrations/connectors/source-freshcaller/integration_tests/configured_catalog_full_refresh.json create mode 100644 airbyte-integrations/connectors/source-freshcaller/integration_tests/configured_catalog_incremental.json create mode 100644 airbyte-integrations/connectors/source-freshcaller/integration_tests/invalid_config.json create mode 100644 airbyte-integrations/connectors/source-freshcaller/integration_tests/test_incremental_streams.py create mode 100644 airbyte-integrations/connectors/source-freshcaller/main.py create mode 100644 airbyte-integrations/connectors/source-freshcaller/requirements.txt create mode 100644 airbyte-integrations/connectors/source-freshcaller/sample_files/config.json create mode 100644 airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog.json create mode 100644 airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_calls.json create mode 100644 airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_full_refresh.json create mode 100644 airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_incremental.json create mode 100644 airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_metrics.json create mode 100644 airbyte-integrations/connectors/source-freshcaller/setup.py create mode 100644 airbyte-integrations/connectors/source-freshcaller/source_freshcaller/__init__.py create mode 100644 airbyte-integrations/connectors/source-freshcaller/source_freshcaller/schemas/call_metrics.json create mode 100644 airbyte-integrations/connectors/source-freshcaller/source_freshcaller/schemas/calls.json create mode 100644 airbyte-integrations/connectors/source-freshcaller/source_freshcaller/schemas/teams.json create mode 100644 airbyte-integrations/connectors/source-freshcaller/source_freshcaller/schemas/users.json create mode 100644 airbyte-integrations/connectors/source-freshcaller/source_freshcaller/source.py create mode 100644 airbyte-integrations/connectors/source-freshcaller/source_freshcaller/spec.json create mode 100644 airbyte-integrations/connectors/source-freshcaller/source_freshcaller/streams.py create mode 100644 airbyte-integrations/connectors/source-freshcaller/unit_tests/test_source.py create mode 100644 docs/integrations/sources/freshcaller.md diff --git a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml index b3261c03942a..b4b7f194811e 100644 --- a/airbyte-config/init/src/main/resources/seed/source_definitions.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_definitions.yaml @@ -276,6 +276,13 @@ icon: file.svg sourceType: file releaseStage: alpha +- name: Freshcaller + sourceDefinitionId: 8a5d48f6-03bb-4038-a942-a8d3f175cca3 + dockerRepository: airbyte/source-freshcaller + dockerImageTag: 0.1.0 + documentationUrl: https://docs.airbyte.io/integrations/sources/freshcaller + sourceType: api + releaseStage: alpha - name: Freshdesk sourceDefinitionId: ec4b9503-13cb-48ab-a4ab-6ade4be46567 dockerRepository: airbyte/source-freshdesk diff --git a/airbyte-config/init/src/main/resources/seed/source_specs.yaml b/airbyte-config/init/src/main/resources/seed/source_specs.yaml index fc1d6e3b44fc..5415cb41587f 100644 --- a/airbyte-config/init/src/main/resources/seed/source_specs.yaml +++ b/airbyte-config/init/src/main/resources/seed/source_specs.yaml @@ -2487,6 +2487,53 @@ supportsNormalization: false supportsDBT: false supported_destination_sync_modes: [] +- dockerImage: "airbyte/source-freshcaller:0.1.0" + spec: + documentationUrl: "https://docs.airbyte.io/integrations/sources/freshcaller" + connectionSpecification: + $schema: "http://json-schema.org/draft-07/schema#" + title: "Freshcaller Spec" + type: "object" + required: + - "domain" + - "api_key" + - "start_date" + additionalProperties: true + properties: + domain: + type: "string" + title: "Domain for Freshcaller account" + description: "Used to construct Base URL for the Freshcaller APIs" + examples: + - "snaptravel" + api_key: + type: "string" + title: "API Key" + description: "Freshcaller API Key. See the docs for more information on how to obtain this key." + airbyte_secret: true + requests_per_minute: + title: "Requests per minute" + type: "integer" + description: "The number of requests per minute that this source allowed\ + \ to use. There is a rate limit of 50 requests per minute per app per\ + \ account." + start_date: + title: "Start Date" + description: "UTC date and time. Any data created after this date will be\ + \ replicated." + format: "date-time" + pattern: "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$" + examples: + - "2022-01-01T12:00:00Z" + sync_lag_minutes: + title: "Lag in minutes for each sync" + type: "integer" + description: "Lag in minutes for each sync, i.e., at time T, data for the\ + \ time range [prev_sync_time, T-30] will be fetched" + supportsNormalization: false + supportsDBT: false + supported_destination_sync_modes: [] - dockerImage: "airbyte/source-freshdesk:0.3.3" spec: documentationUrl: "https://docs.airbyte.io/integrations/sources/freshdesk" diff --git a/airbyte-integrations/connectors/source-freshcaller/.dockerignore b/airbyte-integrations/connectors/source-freshcaller/.dockerignore new file mode 100644 index 000000000000..9dcc6f34cdf5 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/.dockerignore @@ -0,0 +1,7 @@ +* +!Dockerfile +!Dockerfile.test +!main.py +!source_freshcaller +!setup.py +!secrets diff --git a/airbyte-integrations/connectors/source-freshcaller/Dockerfile b/airbyte-integrations/connectors/source-freshcaller/Dockerfile new file mode 100644 index 000000000000..f9bc7c0fca2a --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/Dockerfile @@ -0,0 +1,39 @@ +FROM python:3.9.11-alpine3.15 as base + +# build and load all requirements +FROM base as builder +WORKDIR /airbyte/integration_code + +# upgrade pip to the latest version +RUN apk --no-cache upgrade \ + && pip install --upgrade pip \ + && apk --no-cache add tzdata build-base + + +COPY setup.py ./ +# install necessary packages to a temporary folder +RUN pip install --prefix=/install . + +# build a clean environment +FROM base +WORKDIR /airbyte/integration_code + +# copy all loaded and built libraries to a pure basic image +COPY --from=builder /install /usr/local +# add default timezone settings +COPY --from=builder /usr/share/zoneinfo/Etc/UTC /etc/localtime +RUN echo "Etc/UTC" > /etc/timezone + +# bash is installed for more convenient debugging. +RUN apk --no-cache add bash + +# copy payload code only +COPY main.py ./ +COPY source_freshcaller ./source_freshcaller + + +ENV AIRBYTE_ENTRYPOINT "python /airbyte/integration_code/main.py" +ENTRYPOINT ["python", "/airbyte/integration_code/main.py"] + +LABEL io.airbyte.version=0.1.0 +LABEL io.airbyte.name=airbyte/source-freshcaller diff --git a/airbyte-integrations/connectors/source-freshcaller/README.md b/airbyte-integrations/connectors/source-freshcaller/README.md new file mode 100644 index 000000000000..0ea467b8aa97 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/README.md @@ -0,0 +1,133 @@ +# Freshcaller Source + +This is the repository for the Freshcaller source connector, written in Python. +For information about how to use this connector within Airbyte, see [the documentation](https://docs.airbyte.io/integrations/sources/freshcaller). + +## Local development + +### Prerequisites +**To iterate on this connector, make sure to complete this prerequisites section.** + +#### Minimum Python version required `= 3.7.0` + +#### Build & Activate Virtual Environment and install dependencies +From this connector directory, create a virtual environment: +``` +python -m venv .venv +``` + +This will generate a virtualenv for this module in `.venv/`. Make sure this venv is active in your +development environment of choice. To activate it from the terminal, run: +``` +source .venv/bin/activate +pip install -r requirements.txt +pip install '.[tests]' +``` +If you are in an IDE, follow your IDE's instructions to activate the virtualenv. + +Note that while we are installing dependencies from `requirements.txt`, you should only edit `setup.py` for your dependencies. `requirements.txt` is +used for editable installs (`pip install -e`) to pull in Python dependencies from the monorepo and will call `setup.py`. +If this is mumbo jumbo to you, don't worry about it, just put your deps in `setup.py` but install using `pip install -r requirements.txt` and everything +should work as you expect. + +#### Building via Gradle +You can also build the connector in Gradle. This is typically used in CI and not needed for your development workflow. + +To build using Gradle, from the Airbyte repository root, run: +``` +./gradlew :airbyte-integrations:connectors:source-freshcaller:build +``` + +#### Create credentials +**If you are a community contributor**, follow the instructions in the [documentation](https://docs.airbyte.io/integrations/sources/freshcaller) +to generate the necessary credentials. Then create a file `secrets/config.json` conforming to the `source_freshcaller/spec.json` file. +Note that any directory named `secrets` is gitignored across the entire Airbyte repo, so there is no danger of accidentally checking in sensitive information. +See `integration_tests/sample_config.json` for a sample config file. + +**If you are an Airbyte core member**, copy the credentials in Lastpass under the secret name `source freshcaller test creds` +and place them into `secrets/config.json`. + +### Locally running the connector +``` +python main.py spec +python main.py check --config secrets/config.json +python main.py discover --config secrets/config.json +python main.py read --config secrets/config.json --catalog integration_tests/configured_catalog.json +``` + +### Locally running the connector docker image + +#### Build +First, make sure you build the latest Docker image: +``` +docker build . -t airbyte/source-freshcaller:dev +``` + +You can also build the connector image via Gradle: +``` +./gradlew :airbyte-integrations:connectors:source-freshcaller:airbyteDocker +``` +When building via Gradle, the docker image name and tag, respectively, are the values of the `io.airbyte.name` and `io.airbyte.version` `LABEL`s in +the Dockerfile. + +#### Run +Then run any of the connector commands as follows: +``` +docker run --rm airbyte/source-freshcaller:dev spec +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-freshcaller:dev check --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets airbyte/source-freshcaller:dev discover --config /secrets/config.json +docker run --rm -v $(pwd)/secrets:/secrets -v $(pwd)/integration_tests:/integration_tests airbyte/source-freshcaller:dev read --config /secrets/config.json --catalog /integration_tests/configured_catalog.json +``` +## Testing +Make sure to familiarize yourself with [pytest test discovery](https://docs.pytest.org/en/latest/goodpractices.html#test-discovery) to know how your test files and methods should be named. +First install test dependencies into your virtual environment: +``` +pip install .[tests] +``` +### Unit Tests +To run unit tests locally, from the connector directory run: +``` +python -m pytest unit_tests +``` + +### Integration Tests +There are two types of integration tests: Acceptance Tests (Airbyte's test suite for all source connectors) and custom integration tests (which are specific to this connector). +#### Custom Integration tests +Place custom tests inside `integration_tests/` folder, then, from the connector root, run +``` +python -m pytest integration_tests +``` +#### Acceptance Tests +Customize `acceptance-test-config.yml` file to configure tests. See [Source Acceptance Tests](https://docs.airbyte.io/connector-development/testing-connectors/source-acceptance-tests-reference) for more information. +If your connector requires to create or destroy resources for use during acceptance tests create fixtures for it and place them inside integration_tests/acceptance.py. +To run your integration tests with acceptance tests, from the connector root, run +``` +docker build . --no-cache -t airbyte/source-freshcaller:dev \ +&& python -m pytest -p source_acceptance_test.plugin +``` +To run your integration tests with docker + +### Using gradle to run tests +All commands should be run from airbyte project root. +To run unit tests: +``` +./gradlew :airbyte-integrations:connectors:source-freshcaller:unitTest +``` +To run acceptance and custom integration tests: +``` +./gradlew :airbyte-integrations:connectors:source-freshcaller:integrationTest +``` + +## Dependency Management +All of your dependencies should go in `setup.py`, NOT `requirements.txt`. The requirements file is only used to connect internal Airbyte dependencies in the monorepo for local development. +We split dependencies between two groups, dependencies that are: +* required for your connector to work need to go to `MAIN_REQUIREMENTS` list. +* required for the testing need to go to `TEST_REQUIREMENTS` list + +### Publishing a new version of the connector +You've checked out the repo, implemented a million dollar feature, and you're ready to share your changes with the world. Now what? +1. Make sure your changes are passing unit and integration tests. +1. Bump the connector version in `Dockerfile` -- just increment the value of the `LABEL io.airbyte.version` appropriately (we use [SemVer](https://semver.org/)). +1. Create a Pull Request. +1. Pat yourself on the back for being an awesome contributor. +1. Someone from Airbyte will take a look at your PR and iterate with you to merge it into master. diff --git a/airbyte-integrations/connectors/source-freshcaller/acceptance-test-config.yml b/airbyte-integrations/connectors/source-freshcaller/acceptance-test-config.yml new file mode 100644 index 000000000000..a3d462b325a9 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/acceptance-test-config.yml @@ -0,0 +1,22 @@ +connector_image: airbyte/source-freshcaller:dev +tests: + spec: + - spec_path: "source_freshcaller/spec.json" + connection: + - config_path: "secrets/config.json" + status: "succeed" + - config_path: "integration_tests/invalid_config.json" + status: "failed" + discovery: + - config_path: "secrets/config.json" + basic_read: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog_full_refresh.json" + empty_streams: ["teams"] + incremental: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog_incremental.json" + future_state_path: "integration_tests/abnormal_state.json" + full_refresh: + - config_path: "secrets/config.json" + configured_catalog_path: "integration_tests/configured_catalog_full_refresh.json" diff --git a/airbyte-integrations/connectors/source-freshcaller/acceptance-test-docker.sh b/airbyte-integrations/connectors/source-freshcaller/acceptance-test-docker.sh new file mode 100644 index 000000000000..e4d8b1cef896 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/acceptance-test-docker.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env sh + +# Build latest connector image +docker build . -t $(cat acceptance-test-config.yml | grep "connector_image" | head -n 1 | cut -d: -f2) + +# Pull latest acctest image +docker pull airbyte/source-acceptance-test:latest + +# Run +docker run --rm -it \ + -v /var/run/docker.sock:/var/run/docker.sock \ + -v /tmp:/tmp \ + -v $(pwd):/test_input \ + airbyte/source-acceptance-test \ + --acceptance-test-config /test_input + diff --git a/airbyte-integrations/connectors/source-freshcaller/build.gradle b/airbyte-integrations/connectors/source-freshcaller/build.gradle new file mode 100644 index 000000000000..e09a198b7e9d --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/build.gradle @@ -0,0 +1,13 @@ +plugins { + id 'airbyte-python' + id 'airbyte-docker' + id 'airbyte-source-acceptance-test' +} + +airbytePython { + moduleDirectory 'source_freshcaller' +} + +dependencies { + implementation files(project(':airbyte-integrations:bases:source-acceptance-test').airbyteDocker.outputs) +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-freshcaller/integration_tests/__init__.py b/airbyte-integrations/connectors/source-freshcaller/integration_tests/__init__.py new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/airbyte-integrations/connectors/source-freshcaller/integration_tests/abnormal_state.json b/airbyte-integrations/connectors/source-freshcaller/integration_tests/abnormal_state.json new file mode 100644 index 000000000000..dbc7bff5fef0 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/integration_tests/abnormal_state.json @@ -0,0 +1,8 @@ +{ + "call_metrics": { + "created_time": "2099-03-16T05:33:40.987000+00:00" + }, + "calls": { + "created_time": "2099-03-16T05:33:40.823000+00:00" + } +} diff --git a/airbyte-integrations/connectors/source-freshcaller/integration_tests/acceptance.py b/airbyte-integrations/connectors/source-freshcaller/integration_tests/acceptance.py new file mode 100644 index 000000000000..950b53b59d41 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/integration_tests/acceptance.py @@ -0,0 +1,14 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import pytest + +pytest_plugins = ("source_acceptance_test.plugin",) + + +@pytest.fixture(scope="session", autouse=True) +def connector_setup(): + """This fixture is a placeholder for external resources that acceptance test might require.""" + yield diff --git a/airbyte-integrations/connectors/source-freshcaller/integration_tests/catalog.json b/airbyte-integrations/connectors/source-freshcaller/integration_tests/catalog.json new file mode 100644 index 000000000000..e634015fb51f --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/integration_tests/catalog.json @@ -0,0 +1,40 @@ +{ + "streams": [ + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "teams", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "calls", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh" + }, + { + "stream": { + "name": "call_metrics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh" + } + ] +} diff --git a/airbyte-integrations/connectors/source-freshcaller/integration_tests/configured_catalog_full_refresh.json b/airbyte-integrations/connectors/source-freshcaller/integration_tests/configured_catalog_full_refresh.json new file mode 100644 index 000000000000..e634015fb51f --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/integration_tests/configured_catalog_full_refresh.json @@ -0,0 +1,40 @@ +{ + "streams": [ + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "teams", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "calls", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh" + }, + { + "stream": { + "name": "call_metrics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh" + } + ] +} diff --git a/airbyte-integrations/connectors/source-freshcaller/integration_tests/configured_catalog_incremental.json b/airbyte-integrations/connectors/source-freshcaller/integration_tests/configured_catalog_incremental.json new file mode 100644 index 000000000000..0d038391a8c1 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/integration_tests/configured_catalog_incremental.json @@ -0,0 +1,30 @@ +{ + "streams": [ + { + "stream": { + "name": "calls", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created_time"], + "source_defined_primary_key": [["id"]] + }, + "destination_sync_mode": "append", + "cursor_field": ["created_time"], + "sync_mode": "incremental" + }, + { + "stream": { + "name": "call_metrics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["created_time"], + "source_defined_primary_key": [["id"]] + }, + "destination_sync_mode": "append", + "cursor_field": ["created_time"], + "sync_mode": "incremental" + } + ] +} diff --git a/airbyte-integrations/connectors/source-freshcaller/integration_tests/invalid_config.json b/airbyte-integrations/connectors/source-freshcaller/integration_tests/invalid_config.json new file mode 100644 index 000000000000..3e0559353558 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/integration_tests/invalid_config.json @@ -0,0 +1,5 @@ +{ + "api_key": "incorrect_key", + "start_date": "2021-08-02T00:00:00Z", + "domain": "incorrect_domain" +} diff --git a/airbyte-integrations/connectors/source-freshcaller/integration_tests/test_incremental_streams.py b/airbyte-integrations/connectors/source-freshcaller/integration_tests/test_incremental_streams.py new file mode 100644 index 000000000000..7e548eb000c5 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/integration_tests/test_incremental_streams.py @@ -0,0 +1,126 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from unittest.mock import MagicMock + +import pendulum +import pytest +from airbyte_cdk.models import SyncMode +from source_freshcaller.streams import APIIncrementalFreshcallerStream, CallMetrics, Calls + + +@pytest.fixture +def patch_incremental_base_class(mocker): + # Mock abstract methods to enable instantiating abstract class + mocker.patch.object(APIIncrementalFreshcallerStream, "cursor_field", "created_time") + mocker.patch.object(APIIncrementalFreshcallerStream, "__abstractmethods__", set()) + + +@pytest.fixture +def args(): + return {"authenticator": None, "config": {"api_key": "", "domain": "airbyte", "start_date": "2021-01-01T00:00:00.000Z"}} + + +@pytest.fixture +def stream(patch_incremental_base_class, args): + return APIIncrementalFreshcallerStream(**args) + + +@pytest.fixture +def call_metrics_stream(args): + return CallMetrics(**args) + + +@pytest.fixture +def calls_stream(args): + return Calls(**args) + + +@pytest.fixture +def streams_dict(calls_stream, call_metrics_stream): + return {"calls_stream": calls_stream, "call_metrics_stream": call_metrics_stream} + + +@pytest.mark.parametrize("fixture_name, expected", [("calls_stream", "created_time"), ("call_metrics_stream", "created_time")]) +def test_cursor_field(streams_dict, fixture_name, expected): + stream = streams_dict[fixture_name] + assert stream.cursor_field == expected + + +@pytest.mark.parametrize("fixture_name", [("calls_stream"), ("call_metrics_stream")]) +def test_get_updated_state(streams_dict, fixture_name): + stream = streams_dict[fixture_name] + inputs = { + "current_stream_state": {"created_time": "2021-10-10T00:00:00.00Z"}, + "latest_record": {"created_time": "2021-10-20T00:00:00.00Z"}, + } + state = stream.get_updated_state(**inputs) + assert state["created_time"] == pendulum.parse("2021-10-20T00:00:00.00Z") + + inputs = {"current_stream_state": state, "latest_record": {"created_time": "2021-10-30T00:00:00.00Z"}} + state = stream.get_updated_state(**inputs) + assert state["created_time"] == pendulum.parse("2021-10-30T00:00:00.00Z") + + +@pytest.mark.parametrize("fixture_name", [("calls_stream"), ("call_metrics_stream")]) +def test_get_updated_state_2(streams_dict, fixture_name): + stream = streams_dict[fixture_name] + current_stream_state = {"created_time": pendulum.now().add(days=-40)} + inputs = {"sync_mode": SyncMode.incremental, "cursor_field": [], "stream_state": current_stream_state} + # The number of slices should be total lookback days by window_in_days i.e., 40 / 5 = 8 + assert len(stream.stream_slices(**inputs)) == 8 + + +def test_end_of_stream_state(calls_stream, requests_mock): + stream = calls_stream + requests_mock.get( + "https://airbyte.freshcaller.com/api/v1/calls?per_page=1000", + json={ + "calls": [{"created_time": "2021-10-30T00:00:00.00Z"}, {"created_time": "2021-10-29T00:00:00.00Z"}], + "meta": {"total_pages": 40, "current": 40}, + }, + ) + + state = {"created_time": "2021-10-01T00:00:00.00Z"} + sync_mode = SyncMode.incremental + last_state = None + for idx, app_slice in enumerate(stream.stream_slices(state, **MagicMock())): + for record in stream.read_records(sync_mode=sync_mode, stream_slice=app_slice): + state = stream.get_updated_state(state, record) + last_state = state["created_time"] + assert last_state == pendulum.parse("2021-10-30T00:00:00.00Z") + + +@pytest.mark.parametrize("fixture_name", [("calls_stream"), ("call_metrics_stream")]) +def test_supports_incremental(mocker, streams_dict, fixture_name): + stream = streams_dict[fixture_name] + mocker.patch.object(APIIncrementalFreshcallerStream, "cursor_field", "dummy_field") + assert stream.supports_incremental + + +@pytest.mark.parametrize("fixture_name", [("calls_stream"), ("call_metrics_stream")]) +def test_source_defined_cursor(mocker, streams_dict, fixture_name): + stream = streams_dict[fixture_name] + assert stream.source_defined_cursor + + +@pytest.mark.parametrize("fixture_name", [("calls_stream"), ("call_metrics_stream")]) +def test_stream_checkpoint_interval(mocker, streams_dict, fixture_name): + stream = streams_dict[fixture_name] + expected_checkpoint_interval = None + assert stream.state_checkpoint_interval == expected_checkpoint_interval + + +@pytest.mark.parametrize("fixture_name", [("calls_stream"), ("call_metrics_stream")]) +def test_request_params(mocker, streams_dict, fixture_name): + stream = streams_dict[fixture_name] + inputs = { + "stream_state": {}, + "next_page_token": {"page": "5"}, + "stream_slice": {"by_time[from]": "2022-03-04 18:27:40", "by_time[to]": "2022-03-09 18:27:39"}, + } + expected_request_params = {"per_page": 1000, "page": "5", "by_time[from]": "2022-03-04 18:27:40", "by_time[to]": "2022-03-09 18:27:39"} + if stream.path() == "calls": + expected_request_params.update({"has_ancestry": "true"}) + assert stream.request_params(**inputs) == expected_request_params diff --git a/airbyte-integrations/connectors/source-freshcaller/main.py b/airbyte-integrations/connectors/source-freshcaller/main.py new file mode 100644 index 000000000000..9a9d1d2de2fb --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/main.py @@ -0,0 +1,13 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +import sys + +from airbyte_cdk.entrypoint import launch +from source_freshcaller import SourceFreshcaller + +if __name__ == "__main__": + source = SourceFreshcaller() + launch(source, sys.argv[1:]) diff --git a/airbyte-integrations/connectors/source-freshcaller/requirements.txt b/airbyte-integrations/connectors/source-freshcaller/requirements.txt new file mode 100644 index 000000000000..0411042aa091 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/requirements.txt @@ -0,0 +1,2 @@ +-e ../../bases/source-acceptance-test +-e . diff --git a/airbyte-integrations/connectors/source-freshcaller/sample_files/config.json b/airbyte-integrations/connectors/source-freshcaller/sample_files/config.json new file mode 100644 index 000000000000..8c560859a49f --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/sample_files/config.json @@ -0,0 +1,5 @@ +{ + "api_key": "YOUR_FRESHCALLER_API_KEY", + "start_date": "2021-08-02T00:00:00Z", + "domain": "FRESHCALLER_ACCOUNT_DOMAIN_NAME" +} diff --git a/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog.json b/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog.json new file mode 100644 index 000000000000..c9226aa5bdc0 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog.json @@ -0,0 +1,407 @@ +{ + "streams": [ + { + "stream": { + "name": "users", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["integer", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "email": { + "type": ["string", "null"] + }, + "phone": { + "type": ["string", "null"] + }, + "status": { + "type": ["integer", "null"] + }, + "preference": { + "type": ["integer", "null"] + }, + "mobile_app_preference": { + "type": ["integer", "null"] + }, + "last_call_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "last_seen_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "confirmed": { + "type": ["boolean", "null"] + }, + "language": { + "type": ["string", "null"] + }, + "time_zone": { + "type": ["string", "null"] + }, + "deleted": { + "type": ["boolean", "null"] + }, + "role": { + "type": ["string", "null"] + }, + "teams": { + "items": { + "properties": { + "id": { + "type": ["integer", "null"] + } + }, + "type": "object" + }, + "type": ["array", "null"] + } + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]], + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh" + }, + { + "stream": { + "name": "teams", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["integer", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "description": { + "type": ["string", "null"] + }, + "users": { + "items": { + "properties": { + "id": { + "type": ["integer", "null"] + } + }, + "type": "object" + }, + "type": ["array", "null"] + }, + "omni_channel": { + "type": ["boolean", "null"] + } + } + } + }, + "supported_sync_modes": ["full_refresh"], + "source_defined_primary_key": [["id"]], + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh" + }, + { + "stream": { + "name": "calls", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["integer", "null"] + }, + "direction": { + "type": ["string", "null"] + }, + "parent_call_id": { + "type": ["integer", "null"] + }, + "root_call_id": { + "type": ["integer", "null"] + }, + "phone_number_id": { + "type": ["integer", "null"] + }, + "phone_number": { + "type": ["string", "null"] + }, + "assigned_agent_id": { + "type": ["integer", "null"] + }, + "assigned_agent_name": { + "type": ["string", "null"] + }, + "assigned_team_id": { + "type": ["integer", "null"] + }, + "assigned_team_name": { + "type": ["string", "null"] + }, + "assigned_call_queue_id": { + "type": ["integer", "null"] + }, + "assigned_call_queue_name": { + "type": ["string", "null"] + }, + "assigned_ivr_id": { + "type": ["integer", "null"] + }, + "assigned_ivr_name": { + "type": ["string", "null"] + }, + "bill_duration": { + "type": ["number", "null"] + }, + "bill_duration_unit": { + "type": ["string", "null"] + }, + "created_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "updated_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "call_notes": { + "type": ["string", "null"] + }, + "integrated_resources": { + "items": { + "properties": { + "integration_name": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "id": { + "type": ["string", "null"] + } + }, + "type": "object" + }, + "type": ["array", "null"] + }, + "recording": { + "properties": { + "id": { + "type": ["integer", "null"] + }, + "url": { + "type": ["string", "null"] + }, + "transcription_url": { + "type": ["string", "null"] + }, + "duration": { + "type": ["number", "null"] + }, + "duration_unit": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + + "participants": { + "items": { + "properties": { + "id": { + "type": ["integer", "null"] + }, + "call_id": { + "type": ["integer", "null"] + }, + "caller_id": { + "type": ["integer", "null"] + }, + "caller_number": { + "type": ["string", "null"] + }, + "caller_name": { + "type": ["string", "null"] + }, + "participant_id": { + "type": ["integer", "null"] + }, + "participant_type": { + "type": ["string", "null"] + }, + "connection_type": { + "type": ["number", "null"] + }, + "call_status": { + "type": ["integer", "null"] + }, + "duration": { + "type": ["integer", "null"] + }, + "duration_unit": { + "type": ["string", "null"] + }, + "cost": { + "type": ["number", "null"] + }, + "cost_unit": { + "type": ["string", "null"] + }, + "enqueued_time": { + "type": ["string", "null"] + }, + "created_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "updated_time": { + "format": "date-time", + "type": ["string", "null"] + } + }, + "type": "object" + }, + "type": ["array", "null"] + } + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_time"], + "source_defined_primary_key": [["id"]], + "destination_sync_mode": "overwrite", + "sync_mode": "incremental" + }, + { + "stream": { + "name": "call_metrics", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["integer", "null"] + }, + "call_id": { + "type": ["integer", "null"] + }, + "ivr_time": { + "type": ["integer", "null"] + }, + "ivr_time_unit": { + "type": ["string", "null"] + }, + "hold_duration": { + "type": ["number", "null"] + }, + "hold_duration_unit": { + "type": ["string", "null"] + }, + "call_work_time": { + "type": ["number", "null"] + }, + "call_work_time_unit": { + "type": ["string", "null"] + }, + "total_ringing_time": { + "type": ["number", "null"] + }, + "total_ringing_time_unit": { + "type": ["string", "null"] + }, + "talk_time": { + "type": ["number", "null"] + }, + "talk_time_unit": { + "type": ["string", "null"] + }, + "answering_speed": { + "type": ["number", "null"] + }, + "answering_speed_unit": { + "type": ["string", "null"] + }, + "recording_duration": { + "type": ["number", "null"] + }, + "recording_duration_unit": { + "type": ["string", "null"] + }, + "bill_duration": { + "type": ["number", "null"] + }, + "bill_duration_unit": { + "type": ["string", "null"] + }, + "cost": { + "type": ["number", "null"] + }, + "cost_unit": { + "type": ["string", "null"] + }, + "csat": { + "properties": { + "transfer_made": { + "type": ["boolean", "null"] + }, + "outcome": { + "type": ["string", "null"] + }, + "time": { + "type": ["number", "null"] + }, + "time_unit": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + "created_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "updated_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "tags": { + "items": { + "properties": { + "id": { + "type": ["integer", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "default": { + "type": ["boolean", "null"] + } + }, + "type": "object" + }, + "type": ["array", "null"] + } + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_time"], + "source_defined_primary_key": [["id"]], + "destination_sync_mode": "overwrite", + "sync_mode": "incremental" + } + ] +} diff --git a/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_calls.json b/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_calls.json new file mode 100644 index 000000000000..743c16e299e6 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_calls.json @@ -0,0 +1,177 @@ +{ + "streams": [ + { + "stream": { + "name": "calls", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["integer", "null"] + }, + "direction": { + "type": ["string", "null"] + }, + "parent_call_id": { + "type": ["integer", "null"] + }, + "root_call_id": { + "type": ["integer", "null"] + }, + "phone_number_id": { + "type": ["integer", "null"] + }, + "phone_number": { + "type": ["string", "null"] + }, + "assigned_agent_id": { + "type": ["integer", "null"] + }, + "assigned_agent_name": { + "type": ["string", "null"] + }, + "assigned_team_id": { + "type": ["integer", "null"] + }, + "assigned_team_name": { + "type": ["string", "null"] + }, + "assigned_call_queue_id": { + "type": ["integer", "null"] + }, + "assigned_call_queue_name": { + "type": ["string", "null"] + }, + "assigned_ivr_id": { + "type": ["integer", "null"] + }, + "assigned_ivr_name": { + "type": ["string", "null"] + }, + "bill_duration": { + "type": ["number", "null"] + }, + "bill_duration_unit": { + "type": ["string", "null"] + }, + "created_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "updated_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "call_notes": { + "type": ["string", "null"] + }, + "integrated_resources": { + "items": { + "properties": { + "integration_name": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "id": { + "type": ["string", "null"] + } + }, + "type": "object" + }, + "type": ["array", "null"] + }, + "recording": { + "properties": { + "id": { + "type": ["integer", "null"] + }, + "url": { + "type": ["string", "null"] + }, + "transcription_url": { + "type": ["string", "null"] + }, + "duration": { + "type": ["number", "null"] + }, + "duration_unit": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + + "participants": { + "items": { + "properties": { + "id": { + "type": ["integer", "null"] + }, + "call_id": { + "type": ["integer", "null"] + }, + "caller_id": { + "type": ["integer", "null"] + }, + "caller_number": { + "type": ["string", "null"] + }, + "caller_name": { + "type": ["string", "null"] + }, + "participant_id": { + "type": ["integer", "null"] + }, + "participant_type": { + "type": ["string", "null"] + }, + "connection_type": { + "type": ["number", "null"] + }, + "call_status": { + "type": ["integer", "null"] + }, + "duration": { + "type": ["integer", "null"] + }, + "duration_unit": { + "type": ["string", "null"] + }, + "cost": { + "type": ["number", "null"] + }, + "cost_unit": { + "type": ["string", "null"] + }, + "enqueued_time": { + "type": ["string", "null"] + }, + "created_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "updated_time": { + "format": "date-time", + "type": ["string", "null"] + } + }, + "type": "object" + }, + "type": ["array", "null"] + } + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_time"], + "source_defined_primary_key": [["id"]], + "destination_sync_mode": "overwrite", + "sync_mode": "incremental" + } + ] +} diff --git a/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_full_refresh.json b/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_full_refresh.json new file mode 100644 index 000000000000..e634015fb51f --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_full_refresh.json @@ -0,0 +1,40 @@ +{ + "streams": [ + { + "stream": { + "name": "users", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "teams", + "json_schema": {}, + "supported_sync_modes": ["full_refresh"] + }, + "sync_mode": "full_refresh", + "destination_sync_mode": "overwrite" + }, + { + "stream": { + "name": "calls", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh" + }, + { + "stream": { + "name": "call_metrics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"] + }, + "destination_sync_mode": "overwrite", + "sync_mode": "full_refresh" + } + ] +} diff --git a/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_incremental.json b/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_incremental.json new file mode 100644 index 000000000000..6ba515640165 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_incremental.json @@ -0,0 +1,30 @@ +{ + "streams": [ + { + "stream": { + "name": "calls", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_time"], + "source_defined_primary_key": [["id"]] + }, + "destination_sync_mode": "append", + "cursor_field": ["updated_time"], + "sync_mode": "incremental" + }, + { + "stream": { + "name": "call_metrics", + "json_schema": {}, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_time"], + "source_defined_primary_key": [["id"]] + }, + "destination_sync_mode": "append", + "cursor_field": ["updated_time"], + "sync_mode": "incremental" + } + ] +} diff --git a/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_metrics.json b/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_metrics.json new file mode 100644 index 000000000000..2ac580a06e50 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/sample_files/configured_catalog_metrics.json @@ -0,0 +1,124 @@ +{ + "streams": [ + { + "stream": { + "name": "call_metrics", + "json_schema": { + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["integer", "null"] + }, + "call_id": { + "type": ["integer", "null"] + }, + "ivr_time": { + "type": ["integer", "null"] + }, + "ivr_time_unit": { + "type": ["string", "null"] + }, + "hold_duration": { + "type": ["number", "null"] + }, + "hold_duration_unit": { + "type": ["string", "null"] + }, + "call_work_time": { + "type": ["number", "null"] + }, + "call_work_time_unit": { + "type": ["string", "null"] + }, + "total_ringing_time": { + "type": ["number", "null"] + }, + "total_ringing_time_unit": { + "type": ["string", "null"] + }, + "talk_time": { + "type": ["number", "null"] + }, + "talk_time_unit": { + "type": ["string", "null"] + }, + "answering_speed": { + "type": ["number", "null"] + }, + "answering_speed_unit": { + "type": ["string", "null"] + }, + "recording_duration": { + "type": ["number", "null"] + }, + "recording_duration_unit": { + "type": ["string", "null"] + }, + "bill_duration": { + "type": ["number", "null"] + }, + "bill_duration_unit": { + "type": ["string", "null"] + }, + "cost": { + "type": ["number", "null"] + }, + "cost_unit": { + "type": ["string", "null"] + }, + "csat": { + "properties": { + "transfer_made": { + "type": ["boolean", "null"] + }, + "outcome": { + "type": ["string", "null"] + }, + "time": { + "type": ["number", "null"] + }, + "time_unit": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + "created_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "updated_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "tags": { + "items": { + "properties": { + "id": { + "type": ["integer", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "default": { + "type": ["boolean", "null"] + } + }, + "type": "object" + }, + "type": ["array", "null"] + } + } + } + }, + "supported_sync_modes": ["full_refresh", "incremental"], + "source_defined_cursor": true, + "default_cursor_field": ["updated_time"], + "source_defined_primary_key": [["id"]], + "destination_sync_mode": "overwrite", + "sync_mode": "incremental" + } + ] +} diff --git a/airbyte-integrations/connectors/source-freshcaller/setup.py b/airbyte-integrations/connectors/source-freshcaller/setup.py new file mode 100644 index 000000000000..862872947a06 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/setup.py @@ -0,0 +1,31 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + + +from setuptools import find_packages, setup + +MAIN_REQUIREMENTS = [ + "airbyte-cdk~=0.1", + "pendulum==1.2.0", +] + +TEST_REQUIREMENTS = [ + "pytest~=6.1", + "pytest-mock~=3.6.1", + "requests-mock~=1.9.3", + "source-acceptance-test", +] + +setup( + name="source_freshcaller", + description="Source implementation for Freshcaller.", + author="Jay Bujala (Snapcommerce)", + author_email="jay.bujala@snapcommerce.com", + packages=find_packages(), + install_requires=MAIN_REQUIREMENTS, + package_data={"": ["*.json", "schemas/*.json", "schemas/shared/*.json"]}, + extras_require={ + "tests": TEST_REQUIREMENTS, + }, +) diff --git a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/__init__.py b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/__init__.py new file mode 100644 index 000000000000..c4276956372e --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/__init__.py @@ -0,0 +1,3 @@ +from .source import SourceFreshcaller + +__all__ = ["SourceFreshcaller"] diff --git a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/schemas/call_metrics.json b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/schemas/call_metrics.json new file mode 100644 index 000000000000..51d07052b8a6 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/schemas/call_metrics.json @@ -0,0 +1,109 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["integer", "null"] + }, + "call_id": { + "type": ["integer", "null"] + }, + "ivr_time": { + "type": ["integer", "null"] + }, + "ivr_time_unit": { + "type": ["string", "null"] + }, + "hold_duration": { + "type": ["number", "null"] + }, + "hold_duration_unit": { + "type": ["string", "null"] + }, + "call_work_time": { + "type": ["number", "null"] + }, + "call_work_time_unit": { + "type": ["string", "null"] + }, + "total_ringing_time": { + "type": ["number", "null"] + }, + "total_ringing_time_unit": { + "type": ["string", "null"] + }, + "talk_time": { + "type": ["number", "null"] + }, + "talk_time_unit": { + "type": ["string", "null"] + }, + "answering_speed": { + "type": ["number", "null"] + }, + "answering_speed_unit": { + "type": ["string", "null"] + }, + "recording_duration": { + "type": ["number", "null"] + }, + "recording_duration_unit": { + "type": ["string", "null"] + }, + "bill_duration": { + "type": ["number", "null"] + }, + "bill_duration_unit": { + "type": ["string", "null"] + }, + "cost": { + "type": ["number", "null"] + }, + "cost_unit": { + "type": ["string", "null"] + }, + "csat": { + "properties": { + "transfer_made": { + "type": ["boolean", "null"] + }, + "outcome": { + "type": ["string", "null"] + }, + "time": { + "type": ["number", "null"] + }, + "time_unit": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + "created_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "updated_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "tags": { + "items": { + "properties": { + "id": { + "type": ["integer", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "default": { + "type": ["boolean", "null"] + } + }, + "type": "object" + }, + "type": ["array", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/schemas/calls.json b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/schemas/calls.json new file mode 100644 index 000000000000..11d2e5ab64c2 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/schemas/calls.json @@ -0,0 +1,162 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["integer", "null"] + }, + "direction": { + "type": ["string", "null"] + }, + "parent_call_id": { + "type": ["integer", "null"] + }, + "root_call_id": { + "type": ["integer", "null"] + }, + "phone_number_id": { + "type": ["integer", "null"] + }, + "phone_number": { + "type": ["string", "null"] + }, + "assigned_agent_id": { + "type": ["integer", "null"] + }, + "assigned_agent_name": { + "type": ["string", "null"] + }, + "assigned_team_id": { + "type": ["integer", "null"] + }, + "assigned_team_name": { + "type": ["string", "null"] + }, + "assigned_call_queue_id": { + "type": ["integer", "null"] + }, + "assigned_call_queue_name": { + "type": ["string", "null"] + }, + "assigned_ivr_id": { + "type": ["integer", "null"] + }, + "assigned_ivr_name": { + "type": ["string", "null"] + }, + "bill_duration": { + "type": ["number", "null"] + }, + "bill_duration_unit": { + "type": ["string", "null"] + }, + "created_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "updated_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "call_notes": { + "type": ["string", "null"] + }, + "integrated_resources": { + "items": { + "properties": { + "integration_name": { + "type": ["string", "null"] + }, + "type": { + "type": ["string", "null"] + }, + "id": { + "type": ["string", "null"] + } + }, + "type": "object" + }, + "type": ["array", "null"] + }, + "recording": { + "properties": { + "id": { + "type": ["integer", "null"] + }, + "url": { + "type": ["string", "null"] + }, + "transcription_url": { + "type": ["string", "null"] + }, + "duration": { + "type": ["number", "null"] + }, + "duration_unit": { + "type": ["string", "null"] + } + }, + "type": ["object", "null"] + }, + + "participants": { + "items": { + "properties": { + "id": { + "type": ["integer", "null"] + }, + "call_id": { + "type": ["integer", "null"] + }, + "caller_id": { + "type": ["integer", "null"] + }, + "caller_number": { + "type": ["string", "null"] + }, + "caller_name": { + "type": ["string", "null"] + }, + "participant_id": { + "type": ["integer", "null"] + }, + "participant_type": { + "type": ["string", "null"] + }, + "connection_type": { + "type": ["number", "null"] + }, + "call_status": { + "type": ["integer", "null"] + }, + "duration": { + "type": ["integer", "null"] + }, + "duration_unit": { + "type": ["string", "null"] + }, + "cost": { + "type": ["number", "null"] + }, + "cost_unit": { + "type": ["string", "null"] + }, + "enqueued_time": { + "type": ["string", "null"] + }, + "created_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "updated_time": { + "format": "date-time", + "type": ["string", "null"] + } + }, + "type": "object" + }, + "type": ["array", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/schemas/teams.json b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/schemas/teams.json new file mode 100644 index 000000000000..ac0a75fc902d --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/schemas/teams.json @@ -0,0 +1,30 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["integer", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "description": { + "type": ["string", "null"] + }, + "users": { + "items": { + "properties": { + "id": { + "type": ["integer", "null"] + } + }, + "type": "object" + }, + "type": ["array", "null"] + }, + "omni_channel": { + "type": ["boolean", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/schemas/users.json b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/schemas/users.json new file mode 100644 index 000000000000..2e257174594a --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/schemas/users.json @@ -0,0 +1,62 @@ +{ + "$schema": "http://json-schema.org/draft-04/schema#", + "type": "object", + "additionalProperties": true, + "properties": { + "id": { + "type": ["integer", "null"] + }, + "name": { + "type": ["string", "null"] + }, + "email": { + "type": ["string", "null"] + }, + "phone": { + "type": ["string", "null"] + }, + "status": { + "type": ["integer", "null"] + }, + "preference": { + "type": ["integer", "null"] + }, + "mobile_app_preference": { + "type": ["integer", "null"] + }, + "last_call_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "last_seen_time": { + "format": "date-time", + "type": ["string", "null"] + }, + "confirmed": { + "type": ["boolean", "null"] + }, + "language": { + "type": ["string", "null"] + }, + "time_zone": { + "type": ["string", "null"] + }, + "deleted": { + "type": ["boolean", "null"] + }, + "role": { + "type": ["string", "null"] + }, + "teams": { + "items": { + "properties": { + "id": { + "type": ["integer", "null"] + } + }, + "type": "object" + }, + "type": ["array", "null"] + } + } +} diff --git a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/source.py b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/source.py new file mode 100644 index 000000000000..6e34418d3122 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/source.py @@ -0,0 +1,46 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import logging +from typing import Any, List, Mapping, Tuple + +import requests +from airbyte_cdk import AirbyteLogger +from airbyte_cdk.sources import AbstractSource +from airbyte_cdk.sources.streams import Stream +from airbyte_cdk.sources.streams.http.auth import TokenAuthenticator +from source_freshcaller.streams import CallMetrics, Calls, Teams, Users + +logger = logging.getLogger("airbyte") + + +class FreshcallerTokenAuthenticator(TokenAuthenticator): + def get_auth_header(self) -> Mapping[str, Any]: + return {"X-Api-Auth": self._token} + + +class SourceFreshcaller(AbstractSource): + def check_connection(self, logger: AirbyteLogger, config: Mapping[str, Any]) -> Tuple[bool, Any]: + api_url = f"https://{config['domain']}.freshcaller.com/api/v1" + auth = FreshcallerTokenAuthenticator(token=config["api_key"]).get_auth_header() + url = "{api_url}/users".format(api_url=api_url) + auth.update({"Accept": "application/json"}) + auth.update({"Content-Type": "application/json"}) + + try: + session = requests.get(url, headers=auth) + session.raise_for_status() + return True, None + except Exception as e: + return False, e + + def streams(self, config: Mapping[str, Any]) -> List[Stream]: + authenticator = FreshcallerTokenAuthenticator(token=config["api_key"]) + args = {"authenticator": authenticator, "config": config} + return [ + Users(**args), + Teams(**args), + Calls(**args), + CallMetrics(**args), + ] diff --git a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/spec.json b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/spec.json new file mode 100644 index 000000000000..8c1e370bf3fd --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/spec.json @@ -0,0 +1,49 @@ +{ + "documentationUrl": "https://docs.airbyte.io/integrations/sources/freshcaller", + "connectionSpecification": { + "$schema": "http://json-schema.org/draft-07/schema#", + "title": "Freshcaller Spec", + "type": "object", + "required": [ + "domain", + "api_key", + "start_date" + ], + "additionalProperties": true, + "properties": { + "domain": { + "type": "string", + "title": "Domain for Freshcaller account", + "description": "Used to construct Base URL for the Freshcaller APIs", + "examples": [ + "snaptravel" + ] + }, + "api_key": { + "type": "string", + "title": "API Key", + "description": "Freshcaller API Key. See the docs for more information on how to obtain this key.", + "airbyte_secret": true + }, + "requests_per_minute": { + "title": "Requests per minute", + "type": "integer", + "description": "The number of requests per minute that this source allowed to use. There is a rate limit of 50 requests per minute per app per account." + }, + "start_date": { + "title": "Start Date", + "description": "UTC date and time. Any data created after this date will be replicated.", + "format": "date-time", + "pattern": "^[0-9]{4}-[0-9]{2}-[0-9]{2}T[0-9]{2}:[0-9]{2}:[0-9]{2}Z$", + "examples": [ + "2022-01-01T12:00:00Z" + ] + }, + "sync_lag_minutes": { + "title": "Lag in minutes for each sync", + "type": "integer", + "description": "Lag in minutes for each sync, i.e., at time T, data for the time range [prev_sync_time, T-30] will be fetched" + } + } + } +} \ No newline at end of file diff --git a/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/streams.py b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/streams.py new file mode 100644 index 000000000000..064e8f5de8cc --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/source_freshcaller/streams.py @@ -0,0 +1,202 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +from abc import ABC, abstractmethod +from typing import Any, Dict, Iterable, Mapping, MutableMapping, Optional, Union + +import pendulum +import requests +from airbyte_cdk.sources.streams.http import HttpStream + + +class FreshcallerStream(HttpStream, ABC): + """Abstract class curated for Freshcaller""" + + primary_key = "id" + data_field = "" + start = 1 + page_limit = 1000 + api_version = 2 + curr_page_param = "page" + + def __init__(self, config: Dict, **kwargs): + super().__init__(**kwargs) + self.config = config + + @property + def url_base(self) -> str: + return f"https://{self.config['domain']}.freshcaller.com/api/v1/" + + def request_params( + self, + stream_state: Mapping[str, Any], + stream_slice: Mapping[str, Any] = None, + next_page_token: Mapping[str, Any] = None, + ) -> MutableMapping[str, Any]: + + params = {"per_page": self.page_limit, self.curr_page_param: self.start} + + # Handle pagination by inserting the next page's token in the request parameters + if next_page_token: + self.logger.debug(f"The next page is: {next_page_token}") + params.update(next_page_token) + return params + + def request_headers(self, **kwargs) -> Mapping[str, Any]: + return {"Accept": "application/json", "User-Agent": "PostmanRuntime/7.28.0", "Content-Type": "application/json"} + + def next_page_token(self, response: requests.Response) -> Optional[Mapping[str, Any]]: + decoded_response = response.json() + meta_data = decoded_response.get("meta") + if meta_data: + total_pages = meta_data["total_pages"] + current_page = meta_data["current"] + if current_page < total_pages: + current_page += 1 + return {self.curr_page_param: current_page} + return None + + def parse_response(self, response: requests.Response, **kwargs) -> Iterable[Mapping]: + response_json = response.json() + yield from response_json.get(self.data_field, []) + + @property + def max_retries(self) -> Union[int, None]: + return 10 + + +class APIIncrementalFreshcallerStream(FreshcallerStream): + """ + Base abstract class for a "true" incremental stream, i.e., for an endpoint that supports + filtering by date or time + """ + + start_param = "by_time[from]" + end_param = "by_time[to]" + + def __init__(self, config: Dict, **kwargs): + super().__init__(config, **kwargs) + self.config = config + self.start_date = config["start_date"] + self.window_in_days = config.get("window_in_days", 5) + self.sync_lag_minutes = config.get("sync_lag_minutes", 30) + + @property + @abstractmethod + def cursor_field(self) -> str: + """ + Defining a cursor field indicates that a stream is incremental, so any incremental stream must extend this class + and define a cursor field. + """ + + def get_updated_state( + self, + current_stream_state: MutableMapping[str, Any], + latest_record: Mapping[str, Any], + ) -> Mapping[str, Any]: + """ + Override default get_updated_state CDK method to return the latest state by comparing the cursor value in the latest record with the stream's most recent state object + and returning an updated state object. + """ + last_synced_at = current_stream_state.get(self.cursor_field, self.start_date) + last_synced_at = pendulum.parse(last_synced_at) if isinstance(last_synced_at, str) else last_synced_at + return {self.cursor_field: max(pendulum.parse(latest_record.get(self.cursor_field)).in_tz("UTC"), last_synced_at)} + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + params = super().request_params(stream_state, stream_slice, next_page_token) + params[self.start_param] = stream_slice[self.start_param] + params[self.end_param] = stream_slice[self.end_param] + self.logger.info(f"Endpoint[{self.path()}] - Request params: {params}") + return params + + def stream_slices(self, stream_state: Mapping[str, Any] = None, **kwargs) -> Iterable[Optional[Mapping[str, Any]]]: + """ + Override default stream_slices CDK method to provide date_slices as page chunks for data fetch. + Returns list of dict, example: [{ + "by_time[from]": "2022-03-07 15:00:01", + "by_time[to]": "2022-03-07 18:00:00" + }, + { + "by_time[from]": "2022-03-07 18:00:01", + "by_time[to]": "2022-03-07 21:00:00" + }, + ...] + """ + start_date = pendulum.parse(self.start_date).in_timezone("UTC") + end_date = pendulum.utcnow().subtract(minutes=self.sync_lag_minutes) # have a safe lag + + # Determine stream_state, if no stream_state we use start_date + if stream_state: + start_date = stream_state.get(self.cursor_field) + start_date = pendulum.parse(start_date) if isinstance(start_date, str) else start_date + start_date = start_date.in_tz("UTC") + # use the lowest date between start_date and self.end_date, otherwise API fails if start_date is in future + start_date: pendulum.Pendulum = min(start_date, end_date) + date_slices = [] + + while start_date <= end_date: + end_date_slice = start_date.add(days=self.window_in_days) + # add 1 second for start next slice to not duplicate data from previous slice end date. + stream_slice = { + self.start_param: start_date.add(seconds=1).to_datetime_string(), + self.end_param: min(end_date_slice, end_date).to_datetime_string(), + } + date_slices.append(stream_slice) + start_date = end_date_slice + + return date_slices + + +class Users(FreshcallerStream): + """ + API docs: https://developers.freshcaller.com/api/#users + """ + + data_field = "users" + + def path(self, **kwargs) -> str: + return "users" + + +class Teams(FreshcallerStream): + """ + API docs: https://developers.freshcaller.com/api/#teams + """ + + data_field = "teams" + + def path(self, **kwargs) -> str: + return "teams" + + +class Calls(APIIncrementalFreshcallerStream): + """ + API docs: https://developers.freshcaller.com/api/#calls + """ + + data_field = "calls" + cursor_field = "created_time" + + def path(self, **kwargs) -> str: + return "calls" + + def request_params( + self, stream_state: Mapping[str, Any], stream_slice: Mapping[str, Any] = None, next_page_token: Mapping[str, Any] = None + ) -> MutableMapping[str, Any]: + params = {**super().request_params(stream_state, stream_slice, next_page_token), "has_ancestry": "true"} + return params + + +class CallMetrics(APIIncrementalFreshcallerStream): + """ + API docs: https://developers.freshcaller.com/api/#call-metrics + """ + + data_field = "call_metrics" + cursor_field = "created_time" + + def path(self, **kwargs) -> str: + return "call_metrics" diff --git a/airbyte-integrations/connectors/source-freshcaller/unit_tests/test_source.py b/airbyte-integrations/connectors/source-freshcaller/unit_tests/test_source.py new file mode 100644 index 000000000000..fd4055ebfaa9 --- /dev/null +++ b/airbyte-integrations/connectors/source-freshcaller/unit_tests/test_source.py @@ -0,0 +1,28 @@ +# +# Copyright (c) 2022 Airbyte, Inc., all rights reserved. +# + +import pendulum +from source_freshcaller.source import FreshcallerTokenAuthenticator, SourceFreshcaller + +now_dt = pendulum.now() + + +def test_authenticator(requests_mock): + URL = "https://example.com/" + TOKEN = "test_token" + config = { + "domain": "https://example.com", + "api_key": "test_token", + } + requests_mock.post(URL, json={"token": TOKEN}) + a = FreshcallerTokenAuthenticator(config["api_key"]) + auth_headers = a.get_auth_header() + assert auth_headers["X-Api-Auth"] == TOKEN + + +def test_count_streams(mocker): + source = SourceFreshcaller() + config_mock = mocker.MagicMock() + streams = source.streams(config_mock) + assert len(streams) == 4 diff --git a/docs/integrations/sources/freshcaller.md b/docs/integrations/sources/freshcaller.md new file mode 100644 index 000000000000..ff79efce6e6a --- /dev/null +++ b/docs/integrations/sources/freshcaller.md @@ -0,0 +1,43 @@ +# Freshcaller + +## Overview + +The Freshcaller source supports full refresh and incremental sync. Depending on your needs, one could choose appropriate sync mode - `full refresh` replicates all records every time a sync happens where as `incremental` replicates net-new records since the last successful sync. + +### Output schema + +The following endpoints are supported from this source: + +* [Users](https://developers.freshcaller.com/api/#users) +* [Teams](https://developers.freshcaller.com/api/#teams) +* [Calls](https://developers.freshcaller.com/api/#calls) +* [Call Metrics](https://developers.freshcaller.com/api/#call-metrics) + +If there are more endpoints you'd like Airbyte to support, please [create an issue.](https://github.com/airbytehq/airbyte/issues/new/choose) + +### Features + +| Feature | Supported? | +| :--- | :--- | +| Full Refresh Sync | Yes | +| Incremental Sync | Yes | +| SSL connection | Yes | +| Namespaces | No | + +### Performance considerations + +The Freshcaller connector should not run into Freshcaller API limitations under normal usage. Please [create an issue](https://github.com/airbytehq/airbyte/issues) if you see any rate limit issues that are not automatically retried successfully. + +## Getting started + +### Requirements + +* Freshcaller Account +* Freshcaller API Key + +### Setup guide + +Please read [How to find your API key](https://support.freshdesk.com/en/support/solutions/articles/225435-where-can-i-find-my-api-key-). + +## Changelog +| 0.1.0 | 2022-08-11 | [14759](https://github.com/airbytehq/airbyte/pull/14759) | 🎉 New Source: Freshcaller | \ No newline at end of file