ci: add basic e2e smoke test

.gitignore

@@ -1,3 +1,5 @@
 mocks
 
 /virter
+
+/tests/vms.toml

.gitlab-ci.yml

@@ -10,6 +10,38 @@ stages:
   - test
   - deploy
 
+variables:
+  VMSHED_VERSION: v0.14.4
+  BUILD_HELPERS_VERSION: a87d4f95c35fa24eebe1a157408ef59cebd1f3f4
+
+prepare-bins:
+  stage: build
+  image: ${LINBIT_DOCKER_REGISTRY}/build-helpers:${BUILD_HELPERS_VERSION}
+  script:
+    - . /usr/local/bin/gitlab-utils.sh
+    - ci_prepare_tools
+    - ci_fetch_binary vmshed vmshed-$VMSHED_VERSION https://github.com/LINBIT/vmshed/releases/download/$VMSHED_VERSION/vmshed-linux-amd64
+  cache:
+    paths:
+      - download
+  artifacts:
+    paths:
+      - download
+      - bin
+
+prepare-docker:
+  stage: build
+  image:
+    # Pinned to 1.6.0 until https://github.com/GoogleContainerTools/kaniko/issues/1790 is resolved
+    name: gcr.io/kaniko-project/executor:v1.6.0-debug
+    entrypoint: [ "" ]
+  script:
+    - set -ex
+    - mkdir -p /kaniko/.docker
+    - ln -snf "$DOCKER_AUTH_CONFIG_FILE" /kaniko/.docker/config.json
+    - /kaniko/executor --registry-mirror $DOCKER_REGISTRY_MIRROR --dockerfile $CI_PROJECT_DIR/tests/Dockerfile --context $CI_PROJECT_DIR/tests --cache --destination ${LINBIT_DOCKER_REGISTRY}/virter/ssh:$CI_COMMIT_REF_SLUG
+
+
 build:
   stage: build
   script:
@@ -53,6 +85,31 @@ lint:
       junit:
         - lint.xml
 
+e2e-test:
+  stage: test
+  variables:
+    EXCLUDED: amazonlinux-2
+  tags:
+    - libvirt
+  script:
+    - ln -snf $DOCKER_AUTH_CONFIG_FILE $HOME/.docker/config.json
+    - docker pull ${LINBIT_DOCKER_REGISTRY}/virter/ssh:$CI_COMMIT_REF_SLUG
+    - cp virter bin/virter
+    - export "PATH=$PWD/bin:$PATH"
+    - virter registry update
+    - cd tests/
+    - EXCLUDED_BASE_IMAGES=$EXCLUDED PULL_LOCATION=${LINBIT_DOCKER_REGISTRY}/vm ./all-base-images.sh > vms.toml
+    - vmshed --set values.DockerImage=${LINBIT_DOCKER_REGISTRY}/virter/ssh:$CI_COMMIT_REF_SLUG --nvms ${LINBIT_CI_MAX_CPUS:-20}
+  artifacts:
+    when: always
+    paths:
+      - tests/tests-out/
+    reports:
+      junit:
+        # These reports are created by vmshed, reporting if the test command itself succeeded.
+        - tests/tests-out/test-results/*.xml
+
+
 sync-images:
   stage: deploy
   rules:

tests/Dockerfile

@@ -0,0 +1,3 @@
+FROM alpine:latest
+
+RUN apk add openssh-client

tests/all-base-images.sh

@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+# Generate a vms.toml for vmshed, using known base images.
+set -e
+
+cat <<EOF
+name = "smoke"
+provision_file = "provision.toml"
+provision_timeout = "2m"
+provision_memory = "1G"
+provision_cpus = 1
+EOF
+
+while read -r name url; do
+	if echo "$name" | grep -q -E "^($EXCLUDED_BASE_IMAGES)\$" ; then
+		echo "$name was excluded" >&2
+		continue
+	fi
+
+	if [ -n "$PULL_LOCATION" ]; then
+		virter image pull "$name" "$PULL_LOCATION/$name" >&2
+	else
+		virter image pull "$name" >&2
+	fi
+
+	cat <<EOF
+
+[[vms]]
+vcpus = 1
+memory = "1G"
+base_image = "$name"
+EOF
+
+done < <(virter image ls --available | tail -n +2)

tests/data/example.txt

@@ -0,0 +1,3 @@
+ABCDEFGHIJKLMNOPQRSTUVWXYZ
+abcdefghijklmnopqrstuvwxyz
+0123456789

tests/data/example.txt.sha256sum

@@ -0,0 +1 @@
+134d9234f01411b06bd56ebb4b7ae6208444bc26a91f5755cc9a07131ed78c39  example.txt

tests/provision.toml

@@ -0,0 +1,16 @@
+[[steps]]
+[steps.shell]
+script = """
+set -ex
+
+if ! command -v rsync ; then
+    if command -v apt-get ; then
+        apt-get update
+        apt-get install -y rsync
+    elif command -v dnf ; then
+        dnf install -y rsync
+    elif command -v yum ; then
+        yum install -y rsync
+    fi
+fi
+"""

tests/test-run.toml

@@ -0,0 +1,59 @@
+[values]
+DockerImage = "ssh"
+
+# Check that rsync steps work
+[[steps]]
+[steps.rsync]
+source = "data/"
+dest = "/virter"
+
+# Check that docker steps work
+[[steps]]
+[steps.docker]
+image = "{{ .DockerImage }}"
+command = ["sh", "-exc", """
+mkdir -p /virter/out
+test -n "$TARGETS"
+IFS=,; for t in $TARGETS; do
+    test "$t" = "$(ssh $t hostname -s)"
+    scp -r $t:/virter/example.txt example.txt
+    sha256sum -c /virter/workspace/data/example.txt.sha256sum
+    mv example.txt /virter/out/example-$t.txt
+done
+"""]
+[steps.docker.copy]
+source = "/virter/out"
+dest = "{{ .OutDir }}"
+
+# Checks that the previous docker step copied files as expected
+[[steps]]
+[steps.docker]
+image = "{{ .DockerImage }}"
+command = ["sh", "-exc", """
+IFS=,; for t in $TARGETS; do
+    test -f "/virter/workspace/{{ .OutDir }}/out/example-$t.txt"
+done
+"""]
+
+# Checks that shell steps work as expected
+[[steps]]
+[steps.shell]
+script = """
+set -ex
+
+test -f /virter/example.txt
+cd /virter
+sha256sum -c example.txt.sha256sum
+
+# CentOS 6 doesn't use systemd, Upstart is a mess, so don't even bother...
+[ command -v systemctl ] || exit 0
+
+while true; do
+	running=$(systemctl is-system-running || true)
+	[ "$running" = initializing -o "$running" = starting ] && { sleep 1; continue; }
+	[ "$running" = running ] && break
+	echo "System in unexpected state '$running'; failed units:" 1>&2
+	systemctl list-units --failed 1>&2
+	exit 1
+done
+"""

tests/tests.toml

@@ -0,0 +1,12 @@
+test_suite_file = "test-run.toml"
+test_timeout = "1m"
+artifacts = [
+    "/var/log/journal",
+    "/var/log/messages",
+    "/run/log/journal",
+]
+
+[tests]
+[tests.smoke]
+vms = [1]
+needallplatforms = true