gradle.yml

name: Airbyte CI

on:
  schedule:
    - cron: '0 */6 * * *'
  push:

jobs:
  launch_integration_tests:
    runs-on: ubuntu-latest
    if: github.ref == 'refs/heads/master'
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v2
      - name: Launch Integration Tests
        run: ./tools/bin/ci_integration_workflow_launcher.sh
        env:
          GITHUB_TOKEN: ${{ secrets.SLASH_COMMAND_PAT }}
  build:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v2

      - name: Check images exist
        run: ./tools/bin/check_images_exist.sh

      - name: Pip Caching
        uses: actions/cache@v2
        with:
          path: |
            ~/.cache/pip
          key: ${{ secrets.CACHE_VERSION }}-pip-${{ runner.os }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/requirements.txt') }}
          restore-keys: |
            ${{ secrets.CACHE_VERSION }}-pip-${{ runner.os }}-

      - name: Npm Caching
        uses: actions/cache@v2
        with:
          path: |
            ~/.npm
          key: ${{ secrets.CACHE_VERSION }}-npm-${{ runner.os }}-${{ hashFiles('**/package-lock.json') }}
          restore-keys: |
            ${{ secrets.CACHE_VERSION }}-npm-${{ runner.os }}-

      # this intentionally does not use restore-keys so we don't mess with gradle caching
      - name: Gradle and Python Caching
        uses: actions/cache@v2
        with:
          path: |
            ~/.gradle/caches
            ~/.gradle/wrapper
            **/.venv
          key: ${{ secrets.CACHE_VERSION }}-${{ runner.os }}-${{ hashFiles('**/*.gradle*') }}-${{ hashFiles('**/package-lock.json') }}-${{ hashFiles('**/setup.py') }}-${{ hashFiles('**/requirements.txt') }}

      - uses: actions/setup-java@v1
        with:
          java-version: '14'

      - uses: actions/setup-node@v1
        with:
          node-version: '14.7'

      - uses: actions/setup-python@v2
        with:
          python-version: '3.7'

      - name: Generate Template scaffold
        run: ./gradlew :airbyte-integrations:connector-templates:generator:testScaffoldTemplates --scan

      - name: Format
        run: ./gradlew --no-daemon format --scan

      - name: Ensure no file change
        run: git status --porcelain && test -z "$(git status --porcelain)"

      - name: Build
        run: CORE_ONLY=true ./gradlew --no-daemon build --scan

      - name: Ensure no file change
        run: git status --porcelain && test -z "$(git status --porcelain)"

      - name: Check documentation
        if: success() && github.ref == 'refs/heads/master'
        run: ./tools/site/link_checker.sh check_docs

      - name: Get Docker Space
        run: docker run --rm busybox df -h

      - name: Build Core Docker Images
        if: success() && github.ref == 'refs/heads/master'
        run: ./gradlew --no-daemon composeBuild --scan
        env:
          GIT_REVISION: ${{ github.sha }}

      - name: Image Cleanup
        run: ./tools/bin/clean_images.sh

      # make sure these always run before pushing core docker images
      - name: Run End-to-End Acceptance Tests
        if: success() && github.ref == 'refs/heads/master'
        run: ./tools/bin/acceptance_test.sh

      - name: Push Core Docker Images
        if: success() && github.ref == 'refs/heads/master'
        run: |
          docker login -u airbytebot -p ${DOCKER_PASSWORD}
          VERSION=dev docker-compose -f docker-compose.build.yaml push
        env:
          DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}

      - name: Slack Notification - Failure
        if: failure() && github.ref == 'refs/heads/master'
        uses: rtCamp/action-slack-notify@master
        env:
          SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
          SLACK_USERNAME: Buildozer
          SLACK_ICON: https://avatars.slack-edge.com/temp/2020-09-01/1342729352468_209b10acd6ff13a649a1.jpg
          SLACK_COLOR: DC143C
          SLACK_TITLE: "Build failure"
          SLACK_FOOTER: ""

      - name: Slack Notification - Success
        if: success() && github.ref == 'refs/heads/master'
        uses: rtCamp/action-slack-notify@master
        env:
          SLACK_WEBHOOK: ${{ secrets.BUILD_SLACK_WEBHOOK }}
          SLACK_USERNAME: Buildbot
          SLACK_TITLE: "Build Success"
          SLACK_FOOTER: ""
  test_docker:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v2

      - uses: actions/setup-java@v1
        with:
          java-version: '14'

      - uses: actions/setup-node@v1
        with:
          node-version: '14.7'

      - uses: actions/setup-python@v2
        with:
          python-version: '3.7'

      - name: Build Core Docker Images and Run Tests
        run: CORE_ONLY=true ./gradlew --no-daemon format composeBuild test -x :airbyte-webapp:test --scan
        env:
          GIT_REVISION: ${{ github.sha }}
          CORE_ONLY: true

      - name: Ensure no file change
        run: git status --porcelain && test -z "$(git status --porcelain)"

      - name: Run Docker End-to-End Acceptance Tests
        run: |
          ./tools/bin/acceptance_test.sh

  launch_e2e_frontend_tests:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Airbyte
        uses: actions/checkout@v2

      - uses: actions/setup-java@v1
        with:
          java-version: '14'

      - uses: actions/setup-node@v1
        with:
          node-version: '14.7'

      - uses: actions/setup-python@v2
        with:
          python-version: '3.7'

      - name: Build Core Docker Images and Run Tests
        run: CORE_ONLY=true ./gradlew --no-daemon build --scan

      - name: Run End-to-End Frontend Tests
        run: ./tools/bin/e2e_test.sh
# DISABLED UNTIL WE HAVE TEMPORAL ON KUBE
#  test_kube:
#    runs-on: ubuntu-latest
#    steps:
#      - name: Checkout Airbyte
#        uses: actions/checkout@v2
#
#      - uses: actions/setup-java@v1
#        with:
#          java-version: '14'
#
#      - uses: actions/setup-node@v1
#        with:
#          node-version: '14.7'
#
#      - uses: actions/setup-python@v2
#        with:
#          python-version: '3.7'
#
#      - name: Setup Minikube
#        uses: manusa/actions-setup-minikube@v2.3.0
#        with:
#          minikube version: 'v1.16.0'
#          kubernetes version: 'v1.19.2'
#
#      - name: Install socat
#        run: sudo apt-get install socat
#
#      - name: Build Core Docker Images and Run Tests
#        run: CORE_ONLY=true ./gradlew --no-daemon format composeBuild test -x :airbyte-webapp:test --scan
#        env:
#          GIT_REVISION: ${{ github.sha }}
#          CORE_ONLY: true
#
#      - name: Ensure no file change
#        run: git status --porcelain && test -z "$(git status --porcelain)"
#
#      - name: Run Kubernetes End-to-End Acceptance Tests
#        run: |
#          ./tools/bin/acceptance_test_kube.sh