This repository has been archived by the owner on Dec 4, 2024. It is now read-only.
Update README.md #715
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: dbt-server CI/CD | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| jobs: | |
| hygiene: | |
| name: Run "code hygiene" checks (linter, formatter) | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 3 | |
| steps: | |
| - name: checkout repo | |
| uses: actions/checkout@v3 | |
| - name: setup python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.8" | |
| - name: install dependencies | |
| run: pip install -r requirements.txt -r dev-requirements.txt | |
| - name: Check formatting with black | |
| run: black --check . | |
| - name: Check lint errors with flake8 | |
| run: flake8 ./tests ./dbt_server | |
| test: | |
| env: | |
| # Smoke test required envs. | |
| # Postgres. | |
| SMOKE_TEST_POSTGRES_USER: root | |
| SMOKE_TEST_POSTGRES_PASSWORD: testpassword | |
| SMOKE_TEST_DBT_SERVER_START_UP_SECONDS: 15 | |
| # Snowflake. | |
| SMOKE_TEST_SNOWFLAKE_ACCOUNT: ${{ secrets.SMOKE_TEST_SNOWFLAKE_ACCOUNT }} | |
| SMOKE_TEST_SNOWFLAKE_DATABASE: ${{ secrets.SMOKE_TEST_SNOWFLAKE_DATABASE }} | |
| SMOKE_TEST_SNOWFLAKE_PASSWORD: ${{ secrets.SMOKE_TEST_SNOWFLAKE_PASSWORD }} | |
| SMOKE_TEST_SNOWFLAKE_ROLE: ${{ secrets.SMOKE_TEST_SNOWFLAKE_ROLE }} | |
| SMOKE_TEST_SNOWFLAKE_SCHEMA: ${{ secrets.SMOKE_TEST_SNOWFLAKE_SCHEMA }} | |
| SMOKE_TEST_SNOWFLAKE_USER: ${{ secrets.SMOKE_TEST_SNOWFLAKE_USER }} | |
| SMOKE_TEST_SNOWFLAKE_WAREHOUSE: ${{ secrets.SMOKE_TEST_SNOWFLAKE_WAREHOUSE }} | |
| name: test code | |
| runs-on: ubuntu-latest | |
| timeout-minutes: 8 | |
| strategy: | |
| matrix: | |
| dbt-core: | |
| - version: "1.5.0-latest" | |
| package: "dbt-core~=1.5.0" | |
| - version: "1.6.0-pre" | |
| package: "dbt-core~=1.6.0b1" | |
| prerelease: true | |
| steps: | |
| - name: checkout repo | |
| uses: actions/checkout@v3 | |
| - name: setup python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.8" | |
| - name: setup postgres | |
| run: sh bash/ubuntu-setup-postgres.sh | |
| - name: run tests - releases | |
| run: | | |
| pip install -r requirements.txt -r dev-requirements.txt | |
| pip install ${{ (matrix.dbt-core.prerelease && '--pre') || '' }} ${{ matrix.dbt-core.package }} dbt-postgres dbt-snowflake | |
| pytest | |
| - name: run tests - head | |
| run: | | |
| pip install -r requirements.txt -r dev-requirements.txt | |
| pip install "git+https://github.com/dbt-labs/dbt-core#egg=dbt-core&subdirectory=core" | |
| pip install "git+https://github.com/dbt-labs/dbt-core#egg=dbt-postgres&subdirectory=plugins/postgres" | |
| pip install "git+https://github.com/dbt-labs/dbt-snowflake.git#egg=dbt-snowflake" | |
| pytest | |
| build-push: | |
| name: build and push dbt server images | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| dbt-core: | |
| - version: "1.5.0-pre" | |
| package: "dbt-core~=1.5.0b1" | |
| prerelease: true | |
| - version: "1.5.0-latest" | |
| package: "dbt-core~=1.5.0" | |
| - version: "1.6.0-pre" | |
| package: "dbt-core~=1.6.0b1" | |
| prerelease: true | |
| dbt-database-adapter: | |
| - name: snowflake | |
| package: dbt-snowflake | |
| - name: bigquery | |
| package: dbt-bigquery | |
| - name: postgres | |
| package: dbt-postgres | |
| - name: redshift | |
| package: dbt-redshift | |
| - name: databricks | |
| package: dbt-databricks | |
| - name: spark | |
| package: dbt-spark | |
| #- name: trino Does not support dbt 1.6 yet! | |
| # package: dbt-trino | |
| include: | |
| # We do not build head images for trino or databricks | |
| - dbt-core: | |
| version: head | |
| package: git+https://github.com/dbt-labs/dbt-core#egg=dbt-core&subdirectory=core | |
| dbt-database-adapter: | |
| name: snowflake | |
| package: git+https://github.com/dbt-labs/dbt-snowflake.git#egg=dbt-snowflake | |
| - dbt-core: | |
| version: head | |
| package: git+https://github.com/dbt-labs/dbt-core#egg=dbt-core&subdirectory=core | |
| dbt-database-adapter: | |
| name: postgres | |
| package: git+https://github.com/dbt-labs/dbt-core#egg=dbt-postgres&subdirectory=plugins/postgres | |
| - dbt-core: | |
| version: head | |
| package: git+https://github.com/dbt-labs/dbt-core#egg=dbt-core&subdirectory=core | |
| dbt-database-adapter: | |
| name: redshift | |
| package: "git+https://github.com/dbt-labs/dbt-core#egg=dbt-postgres&subdirectory=plugins/postgres git+https://github.com/dbt-labs/dbt-redshift#egg=dbt-redshift" | |
| - dbt-core: | |
| version: head | |
| package: git+https://github.com/dbt-labs/dbt-core#egg=dbt-core&subdirectory=core | |
| dbt-database-adapter: | |
| name: bigquery | |
| package: git+https://github.com/dbt-labs/dbt-bigquery#egg=dbt-bigquery | |
| - dbt-core: | |
| version: head | |
| package: git+https://github.com/dbt-labs/dbt-core#egg=dbt-core&subdirectory=core | |
| dbt-database-adapter: | |
| name: spark | |
| package: git+https://github.com/dbt-labs/dbt-spark#egg=dbt-spark | |
| steps: | |
| - name: checkout code | |
| uses: actions/checkout@v3 | |
| - name: registry login | |
| uses: ./.github/actions/registry-login-action | |
| with: | |
| development-registry-endpoint: ${{ secrets.DEV_FROM_ACCOUNT_PUBLIC }}.dkr.ecr.us-east-1.amazonaws.com | |
| development-registry-username: ${{ secrets.DEV_AWS_ACCESS_KEY_ID_PUBLIC }} | |
| development-registry-password: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY_PUBLIC }} | |
| staging-registry-endpoint: ${{ secrets.STAGING_REGISTRY_ENDPOINT }} | |
| staging-registry-username: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }} | |
| staging-registry-password: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }} | |
| production-registry-endpoint: ${{ secrets.INFRA_ROOT_REGISTRY_ENDPOINT }} | |
| production-registry-username: ${{ secrets.INFRA_ROOT_AWS_ACCESS_KEY_ID }} | |
| production-registry-password: ${{ secrets.INFRA_ROOT_AWS_SECRET_ACCESS_KEY }} | |
| - name: build and push dbt server images | |
| uses: ./.github/actions/docker-build-publish | |
| with: | |
| image-name: dbt-server-${{ matrix.dbt-database-adapter.name }}-${{ matrix.dbt-core.version }} | |
| docker-file-path: Dockerfile | |
| docker-build-args: | | |
| # Use --pre flag when installing this version if it's a prerelease | |
| DBT_PIP_FLAGS=${{ (matrix.dbt-core.prerelease && '--pre') || '' }} | |
| DBT_CORE_PACKAGE=${{ matrix.dbt-core.package }} | |
| DBT_DATABASE_ADAPTER_PACKAGE=${{ matrix.dbt-database-adapter.package }} | |
| DATADOG_PACKAGE=ddtrace==1.4.4 | |
| development-registry-endpoint: ${{ secrets.DEV_FROM_ACCOUNT_PUBLIC }}.dkr.ecr.us-east-1.amazonaws.com | |
| staging-registry-endpoint: ${{ secrets.STAGING_REGISTRY_ENDPOINT }} | |
| production-registry-endpoint: ${{ secrets.INFRA_ROOT_REGISTRY_ENDPOINT }} | |
| push: ${{ github.event_name == 'push' }} | |
| - name: notify runtime of new images | |
| uses: ./.github/actions/notify-dbt-runtime | |
| if: ${{ github.event_name == 'push' }} | |
| with: | |
| image-name: dbt-server-${{ matrix.dbt-database-adapter.name }}-${{ matrix.dbt-core.version }} | |
| production-notifier-passphrase: ${{ secrets.INFRA_ROOT_NOTIFIER_PASSPHRASE }} | |
| production-registry-endpoint: ${{ secrets.INFRA_ROOT_REGISTRY_ENDPOINT }} | |
| production-runtime-endpoint: ${{ secrets.PRODUCTION_RUNTIME_ENDPOINT }} | |
| production-emea-notifier-passphrase: ${{ secrets.PRODUCTION_EMEA_NOTIFIER_PASSPHRASE }} | |
| production-emea-runtime-endpoint: ${{ secrets.PRODUCTION_EMEA_RUNTIME_ENDPOINT }} | |
| production-au-notifier-passphrase: ${{ secrets.PRODUCTION_AU_NOTIFIER_PASSPHRASE }} | |
| production-au-runtime-endpoint: ${{ secrets.PRODUCTION_AU_RUNTIME_ENDPOINT }} |