kind-e2e.yaml

#
# Copyright 2021 The Dapr Authors
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#     http://www.apache.org/licenses/LICENSE-2.0
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
name: E2E tests on KinD

on:
  pull_request:
    branches:
      - master
      - 'release-*'
      - 'feature/*'

defaults:
  run:
    shell: bash
    working-directory: ./src/github.com/dapr/dapr

jobs:
  # This workflow runs our e2e tests in a local KinD cluster. Since it
  # does not required a paid cluster and special credentials, it does
  # not require /ok-to-test. That allows contributors to run E2E tests
  # in their Draft PRs without explicit approval from Dapr
  # maintainers.
  #
  # One other benefit of a fully-localized workflow is that tests are
  # running on a fresh cluster everytime, thus being decoupled from
  # shared resource issues.
  #
  # However, KinD currently does not support Windows nodes so this is
  # not intended to be a complete replacement for our AKS-based E2E
  # workflows.
  e2e:
    name: e2e
    runs-on: ubuntu-latest
    env:
      REGISTRY_PORT: 5000
      REGISTRY_NAME: kind-registry
      DAPR_REGISTRY: localhost:5000/dapr
      DAPR_TAG: dev
      DAPR_NAMESPACE: dapr-tests
      GOVER: 1.18
    strategy:
      fail-fast: false # Keep running if one leg fails.
      matrix:
        k8s-version:
        - v1.20.7
        - v1.21.1
        - v1.22.0
        - v1.23.0
        mode:
        - ha
        - non-ha
        # Map between K8s and KinD versions.
        # This is attempting to make it a bit clearer what's being tested.
        # See: https://github.com/kubernetes-sigs/kind/releases/tag/v0.11.1
        include:
        - k8s-version: v1.20.7
          kind-version: v0.11.1
          kind-image-sha: sha256:cbeaf907fc78ac97ce7b625e4bf0de16e3ea725daf6b04f930bd14c67c671ff9
        - k8s-version: v1.21.1
          kind-version: v0.11.1
          kind-image-sha: sha256:69860bda5563ac81e3c0057d654b5253219618a22ec3a346306239bba8cfa1a6
        - k8s-version: v1.22.0
          kind-version: v0.11.1
          kind-image-sha: sha256:b8bda84bb3a190e6e028b1760d277454a72267a5454b57db34437c34a588d047
        - k8s-version: v1.23.0
          kind-version: v0.11.1
          kind-image-sha: sha256:49824ab1727c04e56a21a5d8372a402fcd32ea51ac96a2706a12af38934f81ac
        exclude:
        - k8s-version: v1.20.7
          mode: non-ha
        - k8s-version: v1.21.1
          mode: non-ha
        - k8s-version: v1.22.0
          mode: non-ha
    steps:
    - name: Set up Go ${{ env.GOVER }}
      uses: actions/setup-go@v2
      with:
        go-version: ${{ env.GOVER }}
    - name: Check out code onto GOPATH
      uses: actions/checkout@v2
      with:
        path: ./src/github.com/dapr/dapr
    - name: Configure KinD
      # Generate a KinD configuration file that uses:
      # (a) a couple of worker nodes: this is needed to run both
      #     ZooKeeper + Kakfa, and
      # (b) a local registry: Due to `kind load` performance
      #     https://github.com/kubernetes-sigs/kind/issues/1165, using
      #     a local repository speeds up the image pushes into KinD
      #     significantly.
      working-directory: ./
      run: |
        cat > kind.yaml <<EOF
        apiVersion: kind.x-k8s.io/v1alpha4
        kind: Cluster
        nodes:
        - role: control-plane
          image: kindest/node:${{ matrix.k8s-version }}@${{ matrix.kind-image-sha }}
        - role: worker
          image: kindest/node:${{ matrix.k8s-version }}@${{ matrix.kind-image-sha }}
        - role: worker
          image: kindest/node:${{ matrix.k8s-version }}@${{ matrix.kind-image-sha }}
        - role: worker
          image: kindest/node:${{ matrix.k8s-version }}@${{ matrix.kind-image-sha }}
        containerdConfigPatches:
        - |-
          [plugins."io.containerd.grpc.v1.cri".registry.mirrors."localhost:$REGISTRY_PORT"]
            endpoint = ["http://$REGISTRY_NAME:$REGISTRY_PORT"]
        EOF

        # Log the generated kind.yaml for easy reference.
        cat kind.yaml

        # Set log target directory
        echo "DAPR_CONTAINER_LOG_PATH=$GITHUB_WORKSPACE/container_logs/${{ matrix.k8s-version }}_${{ matrix.mode }}" >> $GITHUB_ENV

    - name: Create KinD Cluster
      uses: helm/kind-action@v1.0.0
      with:
        config: kind.yaml
        cluster_name: kind
        version: ${{ matrix.kind-version }}

    - name: Get KinD info
      run: |
        kubectl cluster-info --context kind-kind
        NODE_IP=$(kubectl get nodes \
                  -lkubernetes.io/hostname!=kind-control-plane \
                  -ojsonpath='{.items[0].status.addresses[?(@.type=="InternalIP")].address}')
        echo "MINIKUBE_NODE_IP=$NODE_IP" >> $GITHUB_ENV

    - name: Setup test output
      shell: bash
      run: |
        export TEST_OUTPUT_FILE_PREFIX=$GITHUB_WORKSPACE/test_report
        echo "TEST_OUTPUT_FILE_PREFIX=$TEST_OUTPUT_FILE_PREFIX" >> $GITHUB_ENV

    - name: Setup local registry
      run: |
        # Run a registry.
        docker run -d --restart=always \
          -p $REGISTRY_PORT:$REGISTRY_PORT --name $REGISTRY_NAME registry:2
        # Connect the registry to the KinD network.
        docker network connect "kind" $REGISTRY_NAME

    - name: Setup Helm
      uses: azure/setup-helm@v1
      with:
        version: v3.3.4

    - name: Build and push Dapr
      run: |
        make build-linux
        make docker-build
        make docker-push

    - name: Build and push test apps
      run: |
        make build-e2e-app-all
        make push-e2e-app-all

    - name: Setup Dapr
      run: |
        make setup-helm-init
        make create-test-namespace
        if [[ "${{ matrix.mode }}" == "ha" ]]; then
          export HA_MODE=true
        else
          export HA_MODE=false
        fi
        make docker-deploy-k8s

    - name: Setup Redis
      run: |
        make setup-test-env-redis

    - name: Setup Kafka
      run: |
        make setup-test-env-kafka

    - name: Setup MongoDB
      run: |
        make setup-test-env-mongodb

    - name: Setup test components
      run: |
        make setup-test-components

    - name: Free up some diskspace
      run: |
        docker image prune -a -f

    - name: Run tests
      run: |
        make test-e2e-all

    - name: Save control plane logs
      if: always()
      run: |
        make save-dapr-control-plane-k8s-logs

    - name: Upload container logs
      if: always()
      uses: actions/upload-artifact@master
      with:
        name: ${{ matrix.k8s-version }}_${{ matrix.mode}}_container_logs
        path: ${{ env.DAPR_CONTAINER_LOG_PATH }}
    - name: Upload test results
      if: always()
      uses: actions/upload-artifact@master
      with:
        #TODO: .json suffix can be removed from artifact name after test analytics scripts are updated
        name: ${{ matrix.k8s-version }}_${{ matrix.mode }}_test_e2e.json
        path: ${{ env.TEST_OUTPUT_FILE_PREFIX }}_e2e.*