Merge branch 'master' into ycherniaiev/issue-8929-update-title-descri…

…ption

Signed-off-by: Sergey Chvalyuk <[email protected]>

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.35.2-alpha
+current_version = 0.35.9-alpha
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.editorconfig

@@ -1,20 +1,5 @@
 root = true
 
-[*]
-charset = utf-8
-indent_size = 4
-indent_style = space
-insert_final_newline = true
-max_line_length = 140
-tab_width = 4
-ij_continuation_indent_size = 8
-ij_formatter_off_tag = @formatter:off
-ij_formatter_on_tag = @formatter:on
-ij_formatter_tags_enabled = false
-ij_smart_tabs = false
-ij_visual_guides = none
-ij_wrap_on_typing = false
-
 [{*.bash,*.sh,*.zsh}]
 indent_size = 2
 tab_width = 2
@@ -52,6 +37,20 @@ ij_markdown_min_lines_around_header = 1
 ij_markdown_min_lines_between_paragraphs = 1
 
 [{*.py,*.pyw,Tiltfile}]
+charset = utf-8
+indent_size = 4
+indent_style = space
+insert_final_newline = true
+max_line_length = 140
+tab_width = 4
+ij_continuation_indent_size = 8
+ij_formatter_off_tag = @formatter:off
+ij_formatter_on_tag = @formatter:on
+ij_formatter_tags_enabled = false
+ij_smart_tabs = false
+ij_visual_guides = none
+ij_wrap_on_typing = false
+
 ij_python_align_collections_and_comprehensions = true
 ij_python_align_multiline_imports = true
 ij_python_align_multiline_parameters = true

.env

@@ -1,32 +1,22 @@
-VERSION=0.35.2-alpha
+# This file only contains Docker relevant variables.
+#
+# Variables with defaults have been omitted to avoid duplication of defaults.
+# The only exception to the non-default rule are env vars related to scaling.
+#
+# See https://github.com/airbytehq/airbyte/blob/master/airbyte-config/models/src/main/java/io/airbyte/config/Configs.java
+# for the latest environment variables.
+#
+# # Contributors - please organise this env file according to the above linked file.
 
-# Airbyte Internal Job Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db
-DATABASE_USER=docker
-DATABASE_PASSWORD=docker
-DATABASE_HOST=db
-DATABASE_PORT=5432
-DATABASE_DB=airbyte
-# translate manually DATABASE_URL=jdbc:postgresql://${DATABASE_HOST}:${DATABASE_PORT/${DATABASE_DB} (do not include the username or password here)
-DATABASE_URL=jdbc:postgresql://db:5432/airbyte
-JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.29.15.001
-
-# Airbyte Internal Config Database, default to reuse the Job Database when they are empty
-# Usually you do not need to set them; they are explicitly left empty to mute docker compose warnings
-CONFIG_DATABASE_USER=
-CONFIG_DATABASE_PASSWORD=
-CONFIG_DATABASE_URL=
-CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.35.1.001
 
-RUN_DATABASE_MIGRATION_ON_STARTUP=true
+### SHARED ###
+VERSION=0.35.9-alpha
 
-# When using the airbyte-db via default docker image:
+# When using the airbyte-db via default docker image
 CONFIG_ROOT=/data
 DATA_DOCKER_MOUNT=airbyte_data
 DB_DOCKER_MOUNT=airbyte_db
 
-# Temporal.io worker configuration
-TEMPORAL_HOST=airbyte-temporal:7233
-
 # Workspace storage for running jobs (logs, etc)
 WORKSPACE_ROOT=/tmp/workspace
 WORKSPACE_DOCKER_MOUNT=airbyte_workspace
@@ -42,59 +32,62 @@ LOCAL_DOCKER_MOUNT=/tmp/airbyte_local
 # Issue: https://github.com/airbytehq/airbyte/issues/577
 HACK_LOCAL_ROOT_PARENT=/tmp
 
-# Maximum simultaneous jobs
-SUBMITTER_NUM_THREADS=10
 
-# Job container images
-# Usually you should not need to set these, they have defaults already set
-JOB_KUBE_SOCAT_IMAGE=
-JOB_KUBE_BUSYBOX_IMAGE=
-JOB_KUBE_CURL_IMAGE=
+### DATABASE ###
+# Airbyte Internal Job Database, see https://docs.airbyte.io/operator-guides/configuring-airbyte-db
+DATABASE_USER=docker
+DATABASE_PASSWORD=docker
+DATABASE_HOST=db
+DATABASE_PORT=5432
+DATABASE_DB=airbyte
+# translate manually DATABASE_URL=jdbc:postgresql://${DATABASE_HOST}:${DATABASE_PORT/${DATABASE_DB} (do not include the username or password here)
+DATABASE_URL=jdbc:postgresql://db:5432/airbyte
+JOBS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.29.15.001
 
-# Miscellaneous
-TRACKING_STRATEGY=segment
+# Airbyte Internal Config Database, defaults to Job Database if empty. Explicitly left empty to mute docker compose warnings.
+CONFIG_DATABASE_USER=
+CONFIG_DATABASE_PASSWORD=
+CONFIG_DATABASE_URL=
+CONFIGS_DATABASE_MINIMUM_FLYWAY_MIGRATION_VERSION=0.35.1.001
+
+### AIRBYTE SERVICES ###
+TEMPORAL_HOST=airbyte-temporal:7233
+INTERNAL_API_HOST=airbyte-server:8001
 WEBAPP_URL=http://localhost:8000/
+# Although not present as an env var, required for webapp configuration.
 API_URL=/api/v1/
-INTERNAL_API_HOST=airbyte-server:8001
-LOG_LEVEL=INFO
 
-WORKER_ENVIRONMENT=docker
 
-# Cloud log backups. Don't use this unless you know what you're doing. Mainly for Airbyte devs.
-# If you just want to capture Docker logs, you probably want to use something like this instead:
-# https://docs.docker.com/config/containers/logging/configure/
-S3_LOG_BUCKET=
-S3_LOG_BUCKET_REGION=
-AWS_ACCESS_KEY_ID=
-AWS_SECRET_ACCESS_KEY=
-S3_MINIO_ENDPOINT=
-S3_PATH_STYLE_ACCESS=
-
-GCS_LOG_BUCKET=
-
-# Docker Resource Limits
+### JOBS ###
+# Relevant to scaling.
+SYNC_JOB_MAX_ATTEMPTS=3
+SYNC_JOB_MAX_TIMEOUT_DAYS=3
 JOB_MAIN_CONTAINER_CPU_REQUEST=
 JOB_MAIN_CONTAINER_CPU_LIMIT=
 JOB_MAIN_CONTAINER_MEMORY_REQUEST=
 JOB_MAIN_CONTAINER_MEMORY_LIMIT=
 
-# Max attempts per sync and max retries per attempt
-SYNC_JOB_MAX_ATTEMPTS=3
 
-# Time in days to reach a timeout to cancel the synchronization
-SYNC_JOB_MAX_TIMEOUT_DAYS=3
+### LOGGING/MONITORING/TRACKING ###
+TRACKING_STRATEGY=segment
+# Although not present as an env var, expected by Log4J configuration.
+LOG_LEVEL=INFO
+# Although not present as an env var, helps Airbyte track job healthiness.
+SENTRY_DSN="https://[email protected]/6102835"
 
-# Set secret persistence store to use. Do not change this for existing installations!
-SECRET_PERSISTENCE=NONE
 
-# State Cloud Storage
-STATE_STORAGE_S3_BUCKET_NAME=
-STATE_STORAGE_S3_REGION=
+### APPLICATIONS ###
+# Scheduler #
+# Relevant to scaling.
+SUBMITTER_NUM_THREADS=10
 
-STATE_STORAGE_MINIO_BUCKET_NAME=
-STATE_STORAGE_MINIO_ENDPOINT=
+# Worker #
+# Relevant to scaling.
+MAX_SYNC_WORKERS=5
+MAX_SPEC_WORKERS=5
+MAX_CHECK_WORKERS=5
+MAX_DISCOVER_WORKERS=5
 
-STATE_STORAGE_GCS_BUCKET_NAME=
 
-# Sentry
-SENTRY_DSN="https://[email protected]/6102835"
+### FEATURE FLAGS ###
+NEW_SCHEDULER=false

.env.dev

@@ -1,3 +1,5 @@
+# For internal Airbyte dev use.
+
 VERSION=dev
 DATABASE_USER=docker
 DATABASE_PASSWORD=docker

.github/ISSUE_TEMPLATE/source-connector-type.md

@@ -0,0 +1,26 @@
+---
+
+name: Source Connector Type
+about: Add a new type or update an existing type in source connector
+title: '[EPIC] Add new type / update <type-name> in source connector <connector-name>'
+labels: area/connectors, needs-triage
+assignees: ''
+
+---
+
+## Summary
+(Choose one of the two below.)
+- [ ] Support new type <type-name>
+- [ ] Update existing type <type-name>
+
+## TODOs
+(Complete the TODOs based on the instruction, and convert each bullet point with the `[Issue]` tag into an issue.)
+- [ ] [Issue] Add a new destination acceptance test (DAT) test case for this type.
+- List every destination below, either update the destination to suppor this type, or override its DAT to bypass the new test case.
+  - [ ] Example: [Issue] support <type-name> in destination bigquery
+- [ ] [Issue] Make sure every destination can pass the new DAT test case.
+  - Even if a destination does not need to support this type, its DAT should not break.
+- List every source that should support this type below.
+  - [ ] Example: [Issue] support <type-name> in source github
+
+## Desired Timeline

.github/actions/ci-java-tests/action.yml

@@ -0,0 +1,28 @@
+name: "Runner CI Java Tests"
+description: "Runner CI Java Tests"
+inputs:
+  module-name:
+    required: true
+  module-folder:
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Install Java
+      uses: actions/setup-java@v1
+      with:
+        java-version: '17'
+
+    - name: "Build"
+      shell: bash
+      run: |
+        rm -rf ${{ inputs.module-folder }}/.venv ${{ inputs.module-folder }}/build
+        ROOT_DIR=$(git rev-parse --show-toplevel)
+        ARG=:$(python -c "import os; print(os.path.relpath('${{ inputs.module-folder }}', start='${ROOT_DIR}').replace('/', ':') )")
+        echo "./gradlew --no-daemon $ARG:build"
+        ./gradlew --no-daemon "$ARG:clean"
+        ./gradlew --no-daemon "$ARG:build"
+
+
+

.github/actions/ci-py-tests/action.yml

@@ -0,0 +1,85 @@
+name: "Runner CI Python Tests"
+description: "Runner CI Python Tests"
+inputs:
+  module-name:
+    required: true
+  module-folder:
+    required: true
+outputs:
+  coverage-paths:
+    description: "Coverage Paths"
+    value: ${{ steps.build-coverage-reports.outputs.coverage-paths }}
+  flake8-logs:
+    description: "Flake8 Logs"
+    value: ${{ steps.build-linter-reports.outputs.flake8-logs }}
+  mypy-logs:
+    description: "MyPy Logs"
+    value: ${{ steps.build-linter-reports.outputs.mypy-logs }}
+  black-diff:
+    description: "Black Diff"
+    value: ${{ steps.build-linter-reports.outputs.black-diff }}
+  isort-diff:
+    description: "Isort Diff"
+    value: ${{ steps.build-linter-reports.outputs.isort-diff }}
+runs:
+  using: "composite"
+  steps:
+    - name: Build Coverage Reports
+      id: build-coverage-reports
+      shell: bash
+      working-directory: ${{ inputs.module-folder }}
+      run: |
+        virtualenv .venv
+        source .venv/bin/activate
+        JSON_CONFIG='{"module": "${{ inputs.module-name }}", "folder": "${{ inputs.module-folder }}", "lang": "py"}'
+        pip install coverage[toml]~=6.2
+        mkdir -p .venv/source-acceptance-test
+        mkdir -p reports
+        SAT_DIR=$(git rev-parse --show-toplevel)/airbyte-integrations/bases/source-acceptance-test
+        PYPROJECT_CONFIG=$(git rev-parse --show-toplevel)/pyproject.toml
+        git ls-tree -r HEAD --name-only $SAT_DIR | while read src; do cp -f $src .venv/source-acceptance-test; done
+        pip install build
+        python -m build .venv/source-acceptance-test
+        pip install .venv/source-acceptance-test/dist/source_acceptance_test-*.whl
+        [ -f requirements.txt ] && pip install --quiet -r requirements.txt
+        pip install .[tests]
+        coverage run --rcfile=${PYPROJECT_CONFIG} -m pytest ./unit_tests || true
+        coverage xml --rcfile=${PYPROJECT_CONFIG} -o reports/coverage.xml || true
+
+        rm -rf .venv
+        echo "::set-output name=coverage-paths::reports/coverage.xml"
+
+    - name: Upload coverage to Codecov
+      if: ${{ always() }}
+      uses: codecov/codecov-action@v2
+
+    - name: Build Linter Reports
+      id: build-linter-reports
+      shell: bash
+      working-directory: ${{ inputs.module-folder }}
+      run: |
+        JSON_CONFIG='{"module": "${{ inputs.module-name }}", "folder": "${{ inputs.module-folder }}", "lang": "py"}'
+        REPORT_FOLDER=reports
+        PYPROJECT_CONFIG=$(git rev-parse --show-toplevel)/pyproject.toml
+
+        # run mypy
+        pip install lxml~=4.7 mypy~=0.910 .
+        mypy . --config-file=${PYPROJECT_CONFIG} | tee reports/mypy.log || true
+
+        # run black
+        pip install black~=21.12b0
+        XDG_CACHE_HOME=/dev/null black --config ${PYPROJECT_CONFIG} --diff . | tee reports/black.diff
+
+        # run isort
+        pip install isort~=5.10.1
+        cp ${PYPROJECT_CONFIG} ./pyproject.toml
+        isort --diff . | tee reports/isort.diff
+
+        # run flake8
+        pip install mccabe~=0.6.1 pyproject-flake8~=0.0.1a2
+        pflake8 --exit-zero . | grep ^. | tee reports/flake.txt
+
+        echo "::set-output name=mypy-logs::reports/mypy.log"
+        echo "::set-output name=black-diff::reports/black.diff"
+        echo "::set-output name=isort-diff::reports/isort.diff"
+        echo "::set-output name=flake8-logs::reports/flake.txt"