systest.yml

name: System tests
on:
  # Allow manually triggering this workflow
  workflow_dispatch:
    inputs:
      test_name:
        description: "Test to run"
        default: "."
      log_level:
        description: "Log level"
        default: "debug"
  push:
    branches:
      - staging
      - trying
env:
  GCLOUD_KEY: ${{ secrets.GCLOUD_KEY }}
  PROJECT_NAME: ${{ secrets.PROJECT_NAME }}
  CLUSTER_NAME: ${{ secrets.CLUSTER_NAME }}
  CLUSTER_ZONE: ${{ secrets.CLUSTER_ZONE }}
  DOCKER_USERNAME: ${{ secrets.DOCKER_USERNAME }}
  DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
  SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
  ES_USER: ${{ secrets.ES_USER }}
  ES_PASS: ${{ secrets.ES_PASS }}
  MAIN_ES_IP: ${{ secrets.MAIN_ES_IP }}
  TD_QUEUE_NAME: ${{ secrets.TD_QUEUE_NAME }}
  TD_QUEUE_ZONE: ${{ secrets.TD_QUEUE_ZONE }}
  DUMP_QUEUE_NAME: ${{ secrets.DUMP_QUEUE_NAME }}
  DUMP_QUEUE_ZONE: ${{ secrets.DUMP_QUEUE_ZONE }}
  CI_CLUSTER_NAME: ${{ secrets.CI_CLUSTER_NAME }}
  CI_GCP_CREDENTIALS: ${{ secrets.CI_GCP_CREDENTIALS }}
  CI_GCP_PROJECT_ID: ${{ secrets.CI_GCP_PROJECT_ID }}
  CI_REGION_NAME: ${{ secrets.CI_REGION_NAME }}
  USE_GKE_GCLOUD_AUTH_PLUGIN: True

concurrency:
  group: ${{ github.base_ref == 'staging' && 'smci-staging' || format('smci-{0}-{1}', github.workflow, github.ref) }}
  cancel-in-progress: ${{ github.base_ref == 'staging' && false || true }}

jobs:
  filter-changes:
    runs-on: ubuntu-22.04
    outputs:
      nondocchanges: ${{ steps.filter.outputs.nondoc }}
    steps:
      - uses: actions/checkout@v4
      - uses: dorny/paths-filter@v3
        id: filter
        with:
          predicate-quantifier: 'every'
          filters: |
            nondoc:
              - '!**/*.md'
              - '!.github/CODEOWNERS'

  build-docker-images:
    runs-on: ubuntu-22.04
    if: ${{ needs.filter-changes.outputs.nondocchanges == 'true' }}
    needs:
      - filter-changes
    timeout-minutes: 20
    permissions:
      contents: 'read'
      id-token: 'write'
    steps:
      - uses: actions/checkout@v4
        with:
          ssh-key: ${{ secrets.GH_ACTION_PRIVATE_KEY }}

      - name: Setup gcloud authentication
        uses: google-github-actions/auth@v2
        with:
          project_id: ${{ secrets.GCP_WI_PROJECT_ID }}
          workload_identity_provider: ${{ secrets.GCP_WI_PROVIDER_SA }}
          service_account: ${{ secrets.GCP_WI_SA }}
          token_format: access_token

      - name: Configure gcloud
        uses: google-github-actions/setup-gcloud@v2
        with:
          version: "469.0.0"

      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - uses: extractions/netrc@v2
        with:
          machine: github.com
          username: ${{ secrets.GH_ACTION_TOKEN_USER }}
          password: ${{ secrets.GH_ACTION_TOKEN }}
        if: vars.GOPRIVATE

      - name: Push go-spacemesh build to docker hub
        run: make dockerpush

      - name: Push go-bootstrapper build to docker hub
        run: make dockerpush-bs

      - name: Get commit hash
        id: vars
        shell: bash
        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT

      - uses: extractions/netrc@v2
        with:
          machine: github.com
          username: ${{ secrets.GH_ACTION_TOKEN_USER }}
          password: ${{ secrets.GH_ACTION_TOKEN }}
        if: vars.GOPRIVATE

      - name: Build tests docker image
        run: make -C systest docker

      - name: Push tests docker images
        run: make -C systest push

  provision-gke-cluster:
    runs-on: ubuntu-22.04
    if: ${{ needs.filter-changes.outputs.nondocchanges == 'true' }}
    needs:
      - filter-changes
    timeout-minutes: 20
    permissions:
      contents: 'read'
      id-token: 'write'
    steps:
      - uses: actions/checkout@v4
        with:
          ssh-key: ${{ secrets.GH_ACTION_PRIVATE_KEY }}

      - name: Setup gcloud authentication
        uses: google-github-actions/auth@v2
        with:
          project_id: ${{ secrets.GCP_WI_PROJECT_ID }}
          workload_identity_provider: ${{ secrets.GCP_WI_PROVIDER_SA }}
          service_account: ${{ secrets.GCP_WI_SA }}
          token_format: access_token

      - name: Configure gcloud
        uses: google-github-actions/setup-gcloud@v2
        with:
          version: "469.0.0"

      - name: Create Node Pool
        run: |
          NODE_POOL_NAME="systemtest-${{ github.run_id }}"
          echo "nodepool-name=${NODE_POOL_NAME}" >> $GITHUB_OUTPUT

          gcloud container node-pools create $NODE_POOL_NAME \
            --cluster ${{ secrets.CI_CLUSTER_NAME }} \
            --num-nodes 2 \
            --preemptible \
            --location ${{ secrets.CI_REGION_NAME }} \
            --machine-type ${{ secrets.CI_NODE_MACHINE_TYPE }} \
            --disk-type pd-ssd \
            --disk-size 300GB \
            --image-type COS_CONTAINERD \
            --enable-autorepair \
            --no-enable-autoupgrade \
            --node-labels env=dev,cluster=${{ secrets.CI_CLUSTER_NAME }},pipeline-id=${{ github.run_id }} \
            --metadata disable-legacy-endpoints=true \
            --service-account ${{ secrets.CI_GKE_NODEPOOL_SA }} \
            --project ${{ secrets.CI_GCP_PROJECT_ID }} \
            --quiet

          echo "Node pool created: $NODE_POOL_NAME"

  systest-gke:
    runs-on: ubuntu-22.04
    if: ${{ needs.filter-changes.outputs.nondocchanges == 'true' }}
    needs:
      - filter-changes
      - provision-gke-cluster
      - build-docker-images
    timeout-minutes: 70
    permissions:
      contents: 'read'
      id-token: 'write'
    steps:
      - uses: actions/checkout@v4
        with:
          ssh-key: ${{ secrets.GH_ACTION_PRIVATE_KEY }}

      - name: Setup kubectl
        uses: azure/setup-kubectl@v4
        with:
          version: "v1.27.16"

      - name: Setup gcloud authentication
        uses: google-github-actions/auth@v2
        with:
          project_id: ${{ secrets.GCP_WI_PROJECT_ID }}
          workload_identity_provider: ${{ secrets.GCP_WI_PROVIDER_SA }}
          service_account: ${{ secrets.GCP_WI_SA }}
          token_format: access_token

      - name: Configure gcloud
        uses: google-github-actions/setup-gcloud@v2
        with:
          version: "469.0.0"

      - name: Configure gke authentication plugin
        run: gcloud components install gke-gcloud-auth-plugin --quiet

      - name: Configure kubectl
        run: gcloud container clusters get-credentials ${{ secrets.CI_CLUSTER_NAME }} --region ${{ secrets.CI_REGION_NAME }} --project ${{ secrets.CI_GCP_PROJECT_ID }}

      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - uses: extractions/netrc@v2
        with:
          machine: github.com
          username: ${{ secrets.GH_ACTION_TOKEN_USER }}
          password: ${{ secrets.GH_ACTION_TOKEN }}
        if: vars.GOPRIVATE

      - name: Get commit hash
        id: vars
        shell: bash
        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT

      - uses: extractions/netrc@v2
        with:
          machine: github.com
          username: ${{ secrets.GH_ACTION_TOKEN_USER }}
          password: ${{ secrets.GH_ACTION_TOKEN }}
        if: vars.GOPRIVATE

      - name: set up go
        uses: actions/setup-go@v5
        with:
          check-latest: true
          go-version-file: "go.mod"

      - name: Run tests
        env:
          test_id: systest-${{ steps.vars.outputs.sha_short }}
          storage: premium-rwo=10Gi
          node_selector: pipeline-id=${{ github.run_id }}
          size: 20
          bootstrap: 4m
          level: ${{ inputs.log_level }}
          clusters: 4
          norbac: 1
        run: make -C systest run test_name=${{ inputs.test_name }}

      - name: Delete pod
        if: always()
        env:
          test_id: systest-${{ steps.vars.outputs.sha_short }}
        run: make -C systest clean

  delete-gke-cluster-pool:
    runs-on: ubuntu-22.04
    if: always()
    needs:
      - filter-changes
      - systest-gke
    timeout-minutes: 20
    permissions:
      contents: 'read'
      id-token: 'write'
    steps:
      - uses: actions/checkout@v4
        with:
          ssh-key: ${{ secrets.GH_ACTION_PRIVATE_KEY }}

      - name: Setup gcloud authentication
        uses: google-github-actions/auth@v2
        with:
          project_id: ${{ secrets.GCP_WI_PROJECT_ID }}
          workload_identity_provider: ${{ secrets.GCP_WI_PROVIDER_SA }}
          service_account: ${{ secrets.GCP_WI_SA }}
          token_format: access_token

      - name: Configure gcloud
        uses: google-github-actions/setup-gcloud@v2
        with:
          version: "469.0.0"

      - name: Delete Node Pool
        run: |
          NODE_POOL_NAME="systemtest-${{ github.run_id }}"
          gcloud container node-pools delete $NODE_POOL_NAME \
            --cluster ${{ secrets.CI_CLUSTER_NAME }} \
            --location ${{ secrets.CI_REGION_NAME }} \
            --project ${{ secrets.CI_GCP_PROJECT_ID }} \
            --quiet
          echo "Node pool deleted: $NODE_POOL_NAME"

  systest-dedicated-cluster:
    runs-on: ubuntu-22.04
    if: ${{ needs.filter-changes.outputs.nondocchanges == 'true' }}
    needs:
      - filter-changes
      - build-docker-images
    timeout-minutes: 70
    permissions:
      contents: 'read'
      id-token: 'write'
    steps:
      - uses: actions/checkout@v4
        with:
          ssh-key: ${{ secrets.GH_ACTION_PRIVATE_KEY }}

      - name: Setup kubectl
        uses: azure/setup-kubectl@v4
        with:
          version: "v1.27.16"

      - name: Save cluster kubeconfig
        run: |
          mkdir -p ${{ github.workspace }}/.kube
          echo "${{ secrets.CI_KUBECONFIG }}" | base64 -d > ${{ github.workspace }}/.kube/config
          echo "KUBECONFIG=${{ github.workspace }}/.kube/config" >> $GITHUB_ENV

      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - uses: extractions/netrc@v2
        with:
          machine: github.com
          username: ${{ secrets.GH_ACTION_TOKEN_USER }}
          password: ${{ secrets.GH_ACTION_TOKEN }}
        if: vars.GOPRIVATE

      - name: Get commit hash
        id: vars
        shell: bash
        run: echo "sha_short=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT

      - uses: extractions/netrc@v2
        with:
          machine: github.com
          username: ${{ secrets.GH_ACTION_TOKEN_USER }}
          password: ${{ secrets.GH_ACTION_TOKEN }}
        if: vars.GOPRIVATE

      - name: set up go
        uses: actions/setup-go@v5
        with:
          check-latest: true
          go-version-file: "go.mod"

      - name: Run tests
        env:
          test_id: systest-${{ steps.vars.outputs.sha_short }}
          storage: default=10Gi
          node_selector: cluster-type=baremetal
          size: 20
          bootstrap: 4m
          level: ${{ inputs.log_level }}
          clusters: 4
          norbac: 1
        run: make -C systest run test_name=${{ inputs.test_name }}

      - name: Delete pod
        if: always()
        env:
          test_id: systest-${{ steps.vars.outputs.sha_short }}
        run: make -C systest clean

  systest-status:
    if: always()
    needs:
      - filter-changes
      - systest-gke
      - systest-dedicated-cluster
    runs-on: ubuntu-22.04
    steps:
      - name: Check test status
        id: check
        run: |
          if [[ "${{ needs.filter-changes.outputs.nondocchanges }}" == "false" ]]; then
            echo "status=success" >> $GITHUB_OUTPUT
            echo "reason=only documentation changes" >> $GITHUB_OUTPUT
            exit 0
          fi

          if [[ "${{ needs.systest-gke.result }}" == "success" ]]; then
            echo "status=success" >> $GITHUB_OUTPUT
            echo "reason=GKE tests passed" >> $GITHUB_OUTPUT
            exit 0
          fi

          if [[ "${{ needs.systest-dedicated-cluster.result }}" == "success" ]]; then
            echo "status=success" >> $GITHUB_OUTPUT
            echo "reason=dedicated cluster tests passed" >> $GITHUB_OUTPUT
            exit 0
          fi

          echo "status=failure" >> $GITHUB_OUTPUT
          echo "reason=all tests failed" >> $GITHUB_OUTPUT

      - name: Set final status
        env:
          STATUS: ${{ steps.check.outputs.status }}
          REASON: ${{ steps.check.outputs.reason }}
        run: |
          echo "Final status: ${{ steps.check.outputs.status }}"
          echo "Reason: ${{ steps.check.outputs.reason }}"
          if [[ "${{ steps.check.outputs.status }}" == "failure" ]]; then
            exit 1
          fi