main.yml

name: dbt-server CI/CD

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main

jobs:
  hygiene:
    name: Run "code hygiene" checks (linter, formatter)
    runs-on: ubuntu-latest
    timeout-minutes: 3
    steps:
      - name: checkout repo
        uses: actions/checkout@v3

      - name: setup python
        uses: actions/setup-python@v4
        with:
          python-version: "3.8"

      - name: install dependencies
        run: pip install -r requirements.txt -r dev-requirements.txt

      - name: Check formatting with black
        run: black --check .

      - name: Check lint errors with flake8
        run: flake8 ./tests ./dbt_server

  test:
    env:
      # Smoke test required envs.
      # Postgres.
      SMOKE_TEST_POSTGRES_USER: root
      SMOKE_TEST_POSTGRES_PASSWORD: testpassword
      SMOKE_TEST_DBT_SERVER_START_UP_SECONDS: 15
      # Snowflake.
      SMOKE_TEST_SNOWFLAKE_ACCOUNT: ${{ secrets.SMOKE_TEST_SNOWFLAKE_ACCOUNT }}
      SMOKE_TEST_SNOWFLAKE_DATABASE: ${{ secrets.SMOKE_TEST_SNOWFLAKE_DATABASE }}
      SMOKE_TEST_SNOWFLAKE_PASSWORD: ${{ secrets.SMOKE_TEST_SNOWFLAKE_PASSWORD }}
      SMOKE_TEST_SNOWFLAKE_ROLE: ${{ secrets.SMOKE_TEST_SNOWFLAKE_ROLE }}
      SMOKE_TEST_SNOWFLAKE_SCHEMA: ${{ secrets.SMOKE_TEST_SNOWFLAKE_SCHEMA }}
      SMOKE_TEST_SNOWFLAKE_USER: ${{ secrets.SMOKE_TEST_SNOWFLAKE_USER }}
      SMOKE_TEST_SNOWFLAKE_WAREHOUSE: ${{ secrets.SMOKE_TEST_SNOWFLAKE_WAREHOUSE }}
    name: test code
    runs-on: ubuntu-latest
    timeout-minutes: 8
    strategy:
      matrix:
        dbt-core:
          - version: "1.5.0-latest"
            package: "dbt-core~=1.5.0"
          - version: "1.6.0-pre"
            package: "dbt-core~=1.6.0b1"
            prerelease: true

    steps:
      - name: checkout repo
        uses: actions/checkout@v3

      - name: setup python
        uses: actions/setup-python@v4
        with:
          python-version: "3.8"
      - name: setup postgres
        run: sh bash/ubuntu-setup-postgres.sh
      - name: run tests - releases
        run: |
          pip install -r requirements.txt -r dev-requirements.txt
          pip install ${{ (matrix.dbt-core.prerelease && '--pre') || '' }} ${{ matrix.dbt-core.package }} dbt-postgres dbt-snowflake
          pytest

      - name: run tests - head
        run: |
          pip install -r requirements.txt -r dev-requirements.txt
          pip install "git+https://github.com/dbt-labs/dbt-core#egg=dbt-core&subdirectory=core"
          pip install "git+https://github.com/dbt-labs/dbt-core#egg=dbt-postgres&subdirectory=plugins/postgres"
          pip install "git+https://github.com/dbt-labs/dbt-snowflake.git#egg=dbt-snowflake"
          pytest
  build-push:
    name: build and push dbt server images
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        dbt-core:
          - version: "1.5.0-pre"
            package: "dbt-core~=1.5.0b1"
            prerelease: true
          - version: "1.5.0-latest"
            package: "dbt-core~=1.5.0"
          - version: "1.6.0-pre"
            package: "dbt-core~=1.6.0b1"
            prerelease: true
        dbt-database-adapter:
          - name: snowflake
            package: dbt-snowflake
          - name: bigquery
            package: dbt-bigquery
          - name: postgres
            package: dbt-postgres
          - name: redshift
            package: dbt-redshift
          - name: databricks
            package: dbt-databricks
          - name: spark
            package: dbt-spark
          #- name: trino Does not support dbt 1.6 yet!
          #  package: dbt-trino
        include:
        # We do not build head images for trino or databricks
          - dbt-core:
              version: head
              package: git+https://github.com/dbt-labs/dbt-core#egg=dbt-core&subdirectory=core
            dbt-database-adapter:
              name: snowflake
              package: git+https://github.com/dbt-labs/dbt-snowflake.git#egg=dbt-snowflake
          - dbt-core:
              version: head
              package: git+https://github.com/dbt-labs/dbt-core#egg=dbt-core&subdirectory=core
            dbt-database-adapter:
              name: postgres
              package: git+https://github.com/dbt-labs/dbt-core#egg=dbt-postgres&subdirectory=plugins/postgres
          - dbt-core:
              version: head
              package: git+https://github.com/dbt-labs/dbt-core#egg=dbt-core&subdirectory=core
            dbt-database-adapter:
              name: redshift
              package: "git+https://github.com/dbt-labs/dbt-core#egg=dbt-postgres&subdirectory=plugins/postgres git+https://github.com/dbt-labs/dbt-redshift#egg=dbt-redshift"
          - dbt-core:
              version: head
              package: git+https://github.com/dbt-labs/dbt-core#egg=dbt-core&subdirectory=core
            dbt-database-adapter:
              name: bigquery
              package: git+https://github.com/dbt-labs/dbt-bigquery#egg=dbt-bigquery
          - dbt-core:
              version: head
              package: git+https://github.com/dbt-labs/dbt-core#egg=dbt-core&subdirectory=core
            dbt-database-adapter:
              name: spark
              package: git+https://github.com/dbt-labs/dbt-spark#egg=dbt-spark
    steps:
      - name: checkout code
        uses: actions/checkout@v3

      - name: registry login
        uses: ./.github/actions/registry-login-action
        with:
          development-registry-endpoint: ${{ secrets.DEV_FROM_ACCOUNT_PUBLIC }}.dkr.ecr.us-east-1.amazonaws.com
          development-registry-username: ${{ secrets.DEV_AWS_ACCESS_KEY_ID_PUBLIC }}
          development-registry-password: ${{ secrets.DEV_AWS_SECRET_ACCESS_KEY_PUBLIC }}
          staging-registry-endpoint: ${{ secrets.STAGING_REGISTRY_ENDPOINT }}
          staging-registry-username: ${{ secrets.STAGING_AWS_ACCESS_KEY_ID }}
          staging-registry-password: ${{ secrets.STAGING_AWS_SECRET_ACCESS_KEY }}
          production-registry-endpoint: ${{ secrets.INFRA_ROOT_REGISTRY_ENDPOINT }}
          production-registry-username: ${{ secrets.INFRA_ROOT_AWS_ACCESS_KEY_ID }}
          production-registry-password: ${{ secrets.INFRA_ROOT_AWS_SECRET_ACCESS_KEY }}

      - name: build and push dbt server images
        uses: ./.github/actions/docker-build-publish
        with:
          image-name: dbt-server-${{ matrix.dbt-database-adapter.name }}-${{ matrix.dbt-core.version }}
          docker-file-path: Dockerfile
          docker-build-args: |
            # Use --pre flag when installing this version if it's a prerelease
            DBT_PIP_FLAGS=${{ (matrix.dbt-core.prerelease && '--pre') || '' }}
            DBT_CORE_PACKAGE=${{ matrix.dbt-core.package }}
            DBT_DATABASE_ADAPTER_PACKAGE=${{ matrix.dbt-database-adapter.package }}
            DATADOG_PACKAGE=ddtrace==1.4.4
          development-registry-endpoint: ${{ secrets.DEV_FROM_ACCOUNT_PUBLIC }}.dkr.ecr.us-east-1.amazonaws.com
          staging-registry-endpoint: ${{ secrets.STAGING_REGISTRY_ENDPOINT }}
          production-registry-endpoint: ${{ secrets.INFRA_ROOT_REGISTRY_ENDPOINT }}
          push: ${{ github.event_name == 'push' }}

      - name: notify runtime of new images
        uses: ./.github/actions/notify-dbt-runtime
        if: ${{ github.event_name == 'push' }}
        with:
          image-name: dbt-server-${{ matrix.dbt-database-adapter.name }}-${{ matrix.dbt-core.version }}
          production-notifier-passphrase: ${{ secrets.INFRA_ROOT_NOTIFIER_PASSPHRASE }}
          production-registry-endpoint: ${{ secrets.INFRA_ROOT_REGISTRY_ENDPOINT }}
          production-runtime-endpoint: ${{ secrets.PRODUCTION_RUNTIME_ENDPOINT }}
          production-emea-notifier-passphrase: ${{ secrets.PRODUCTION_EMEA_NOTIFIER_PASSPHRASE }}
          production-emea-runtime-endpoint: ${{ secrets.PRODUCTION_EMEA_RUNTIME_ENDPOINT }}
          production-au-notifier-passphrase: ${{ secrets.PRODUCTION_AU_NOTIFIER_PASSPHRASE }}
          production-au-runtime-endpoint: ${{ secrets.PRODUCTION_AU_RUNTIME_ENDPOINT }}