Add e2e tests and fix page build CI (#197)

* incomplete * feat(tests): new e2e test framework with jq * ci: run e2e in build workflow
kubewharf · Oct 17, 2023 · 2b87c4e · 2b87c4e
1 parent 71ffea4
commit 2b87c4e
Show file tree

Hide file tree

Showing 14 changed files with 271 additions and 145 deletions.
diff --git a/.editorconfig b/.editorconfig
@@ -9,6 +9,10 @@ charset = utf-8
 indent_style = tab
 indent_size = 4
 
+[*.js]
+indent_style = space
+indent_size = 4
+
 [*.{yaml,tpl}]
 indent_style = space
 indent_size = 2
@@ -32,3 +36,7 @@ indent_size = 2
 [Makefile]
 indent_style = tab
 indent_size = 2
+
+[*.jq]
+indent_style = space
+indent_size = 2
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -69,119 +69,96 @@ jobs:
           name: ci-docker-image
           path: output/kelemetry-ci.tar
 
-  demo-traces:
+  e2e:
     name: Generate demo traces
     runs-on: [ubuntu-20.04]
     needs: [build-docker]
-    strategy:
-      matrix:
-        scripts:
-          - |
-            cat <<EOF >>"$GITHUB_OUTPUT"
-            TRACE_NAME=deployment
-            CURL_PARAMS=--data-urlencode cluster=tracetest --data-urlencode resource=deployments --data-urlencode namespace=default --data-urlencode name=demo
-            EOF
-
-            kubectl create deployment demo --image=alpine:3.16 --replicas=2 -- sleep infinity
-            sleep 5
-            kubectl scale deployment demo --replicas=4
-            sleep 5
-            kubectl set image deployments demo alpine=alpine:3.17
-            sleep 5
-            kubectl scale deployment demo --replicas=2
-            sleep 5
-            kubectl delete deployment demo
-            sleep 30
     steps:
       - uses: actions/checkout@v3
 
       - name: Install kind
-        run: wget -O kind https://kind.sigs.k8s.io/dl/v0.17.0/kind-linux-amd64 && chmod +x kind && sudo mv kind /usr/local/bin/kind
+        run: wget --no-verbose -O kind https://kind.sigs.k8s.io/dl/v0.17.0/kind-linux-amd64 && chmod +x kind && sudo mv kind /usr/local/bin/kind
       - name: Start test cluster
         run: make kind
 
+      - name: Install kubectl
+        run: |
+          wget --no-verbose -O kubectl https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl && \
+            chmod +x kubectl && \
+            sudo mv kubectl /usr/local/bin/kubectl
+
       - name: Download docker image
         uses: actions/download-artifact@v3
         with:
           name: ci-docker-image
           path: local
       - name: Load docker image
         run: docker load -i local/kelemetry-ci.tar
-      - name: Start quickstart setup
+      - name: Start quickstartup setup
         run: make quickstart KELEMETRY_IMAGE=kelemetry:ci
 
-      - name: Install kubectl
-        run: |
-          wget -O kubectl https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl && \
-            chmod +x kubectl && \
-            sudo mv kubectl /usr/local/bin/kubectl
-
-      - name: Trigger test trace
-        id: script
-        run: echo "${{matrix.scripts}}" | bash
-
-      - name: Kelemetry process logs
-        run: make quickstart KELEMETRY_IMAGE=kelemetry:ci COMPOSE_COMMAND="logs kelemetry"
-
-      - name: Invoke jaeger-query
-        id: curl-jaeger-query
+      - name: Run e2e tests
         run: |
-          set -x
-          set -o pipefail
-
-          curl -i -G \
-              --data-urlencode ts="$(date --iso-8601=seconds)" \
-              ${{steps.script.outputs.CURL_PARAMS}} \
-              localhost:8080/redirect >curl-output.http
-          TRACE_ID=$(grep "Location: /trace/" curl-output.http | cut -d/ -f3 | tr -d '\r')
-
-          mkdir -p output/api/traces
-          for mode in ff{0,1,2,3}{0,1,5,6}000000; do
-            mode_trace=${mode}${TRACE_ID:10}
-            curl -o output/api/traces/$mode_trace http://localhost:16686/api/traces/$mode_trace
-          done
-
-          echo ${{steps.script.outputs.TRACE_NAME}} >output/trace_name
-          echo ${TRACE_ID:10} >output/trace_id
+          mkdir output
+          OUTPUT_TRACE=$(realpath output) bash e2e/run-all.sh
 
       - name: Upload trace output
         uses: actions/upload-artifact@v3
         with:
-          name: trace-${{steps.script.outputs.TRACE_NAME}}
+          name: trace-output
           path: output
 
-  build-github-pages:
-    name: Build GitHub Pages
+  build-site:
+    name: Build site
     runs-on: [ubuntu-20.04]
-    needs: [compile-binary, dot-usage, demo-traces]
+    needs: [compile-binary, dot-usage, e2e]
     steps:
       - uses: actions/download-artifact@v3
         with:
+          name: trace-output
           path: output
       - name: Download Jaeger UI
         run: |
-          rm -r output/ci-docker-image # too large and not relevant
           mkdir -p local/jaeger-ui
           wget -O - https://github.com/jaegertracing/jaeger-ui/releases/download/v1.28.1/assets.tar.gz | tar xzv -C local/jaeger-ui
           mv local/jaeger-ui/packages/jaeger-ui/build/static output/static
           sed 's#<base href="/"#<base href="/kelemetry/"#g' local/jaeger-ui/packages/jaeger-ui/build/index.html >spa.html
+      - name: Generate static SPA links
+        run: |
           mkdir -p output/api/traces
-          for trace_dir in output/trace-*; do
-            basename=$(basename $trace_dir)
-            cat <<EOF >$trace_dir/index.html
+          for trace_id_file in output/trace-*/trace_id; do
+            trace_name=$(basename $(dirname $trace_id_file))
+            base_trace_id=$(cat $trace_id_file)
+            default_trace_id=ff21000000${base_trace_id}
+
+            cat <<EOF >output/${trace_name}/index.html
               <html>
                 <head>
-                  <title>Redirecting to ${basename}</title>
-                  <meta http-equiv="refresh" content="0;URL='/kelemetry/trace/ff26000000$(cat $trace_dir/trace_id)'" />
+                  <title>Redirecting to ${trace_name}</title>
+                  <meta http-equiv="refresh" content="0;URL='/kelemetry/trace/${default_trace_id}'" />
                 </head>
               </html>
           EOF
-            mkdir -p output/trace/ff26000000$(cat $trace_dir/trace_id)
-            cp spa.html output/trace/ff26000000$(cat $trace_dir/trace_id)/index.html
-            cp $trace_dir/api/traces/* output/api/traces/
+
+            mkdir -p output/trace/${default_trace_id}
+            cp spa.html output/trace/${default_trace_id}/index.html
           done
-          mv output/dot-usage/* output/
-      - run: |
+
+      - name: Download dot-usage artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: dot-usage
+          path: dot-usage
+      - name: Download binary artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: kelemetry-linux-amd64
+          path: output/kelemetry-linux-amd64
+
+      - name: Generate index page
+        run: |
+          mv dot-usage/* output/
+
           cat <<EOF >output/index.html
           <html>
             <head>
@@ -190,7 +167,7 @@ jobs:
             </head>
             <body>
               <h1>Kelemetry</h1>
-              <p>Check repo on <a href="https://github.com/kubewharf/kelemetry">GitHub</a>.
+              <p>Check repo on <a href="https://github.com/kubewharf/kelemetry">GitHub</a>.</p>
 
               <h2>Artifacts</h2>
               <p>
@@ -214,7 +191,7 @@ jobs:
           for trace_dir in output/trace-*; do
             basename=$(basename $trace_dir)
             cat <<EOF >>output/index.html
-                <li><a href="$basename/">$(cat $trace_dir/trace_name)</a></li>
+                <li><a href="$basename/">$(cat $trace_dir/trace_display_name)</a></li>
           EOF
           done
           cat <<EOF >>output/index.html
@@ -226,83 +203,11 @@ jobs:
         uses: actions/upload-pages-artifact@v1
         with:
           path: output
-  take-screenshot:
-    name: Take screenshot
-    runs-on: [ubuntu-20.04]
-    needs: [build-github-pages]
-    steps:
-      - uses: actions/download-artifact@v3
-        with:
-          name: github-pages
-          path: pages-artifact
-      - name: Extract pages artifact
-        run: mkdir pages && tar xf pages-artifact/artifact.tar -C pages
-      - name: Start static HTTP server in background
-        run: docker run -d -v $PWD/pages:/var/www/html/kelemetry -p 5000:5000 busybox httpd -f -p 0.0.0.0:5000 -h /var/www/html
-      - run: npm install puppeteer
-      - name: Take screenshots
-        run: |
-          node <<'EOF'
-            const pt = require("puppeteer")
-            const fs = require("fs/promises")
-
-            ;(async () => {
-              const browser = await pt.launch({headless: "new"})
-
-              const traces = await fs.readdir("pages")
-              console.log("traces:", traces)
-
-              await fs.mkdir("pages/screenshots")
-
-              const promises = []
-              for(const trace of traces) {
-                if(!trace.startsWith("trace-")) continue
-
-                promises.push((async () => {
-                  const traceName = await fs.readFile(`pages/${trace}/trace_name`, {encoding: "utf8"})
-                  const partialTraceId = await fs.readFile(`pages/${trace}/trace_id`, {encoding: "utf8"})
-                  const traceId = "ff26000000" + partialTraceId
-
-                  console.log(`Loading trace ${traceName}`)
-
-                  const page = await browser.newPage()
-                  await page.goto(`http://localhost:5000/kelemetry/trace/${traceId}/`)
-                  await page.setViewport({width: 1920, height: 1080})
-
-                  console.log("waiting for service element")
-                  await page.waitForSelector(".span-svc-name")
-                  const services = await page.$$(".span-svc-name")
-
-                  await services[5].click()
-
-                  console.log("waiting for log header element")
-                  const logHeader = await page.waitForSelector(".AccordianLogs--header")
-                  await logHeader.click()
-
-                  await new Promise(resolve => setTimeout(resolve, 1000))
-
-                  console.log("dump screenshot")
-                  await page.screenshot({path: `pages/screenshots/${traceName.trim()}.png`})
-                })())
-              }
-
-              console.log("waiting", promises)
-              await Promise.all(promises)
-
-              await browser.close()
-              console.log("ok")
-            })()
-          EOF
-      - name: Upload screenshots as artifacts
-        uses: actions/upload-artifact@v3
-        with:
-          name: screenshots
-          path: pages/screenshots
 
   deploy-github-pages:
     name: Deploy GitHub Pages
     runs-on: [ubuntu-20.04]
-    needs: [build-github-pages]
+    needs: [build-site]
     if: github.event_name == 'push' && (github.ref_name == 'main' || github.ref_name == 'ci-test')
     concurrency:
       group: gh-pages

diff --git a/.gitignore b/.gitignore
@@ -8,3 +8,4 @@ dump*.json
 /depgraph.*
 /local
 /USAGE.txt
+node_modules
diff --git a/Makefile b/Makefile
@@ -68,7 +68,7 @@ else
 	TAG := $(shell git describe --always)-$(shell git diff --exit-code >/dev/null && echo clean || (echo dirty- && git ls-files | xargs cat --show-all | crc32 /dev/stdin))
 endif
 
-.PHONY: run dump-rotate test usage dot kind stack pre-commit
+.PHONY: run dump-rotate test usage dot kind stack pre-commit fmt local-docker-build e2e
 run: output/kelemetry $(DUMP_ROTATE_DEP)
 	GIN_MODE=debug \
 		$(RUN_PREFIX) ./output/kelemetry $(RUN_SUFFIX) \
@@ -162,6 +162,7 @@ endef
 
 export QUICKSTART_JQ_PATCH
 quickstart:
+	echo $(COMPOSE_COMMAND)
 	docker compose -f quickstart.docker-compose.yaml \
 		-f <(jq -n --arg KELEMETRY_IMAGE "$(KELEMETRY_IMAGE)" "$$QUICKSTART_JQ_PATCH") \
 		up --no-recreate --no-start
@@ -181,3 +182,13 @@ fmt:
 	golines -m140 --base-formatter=gofumpt -w .
 	goimports -l -w .
 	gci write -s standard -s default -s 'prefix(github.com/kubewharf/kelemetry)' .
+
+local-docker-build:
+	make output/kelemetry
+	cp hack/tfconfig.yaml output
+	docker build --build-arg BIN_FILE=kelemetry --build-arg TFCONFIG=tfconfig.yaml -f ./hack/local.Dockerfile -t kelemetry:local output
+
+e2e: local-docker-build
+	make quickstart COMPOSE_COMMAND='down --remove-orphans --volumes' KELEMETRY_IMAGE=kelemetry:local
+	make quickstart COMPOSE_COMMAND='up --build -d --remove-orphans' KELEMETRY_IMAGE=kelemetry:local
+	bash e2e/run-all.sh
diff --git a/docs/DEV.md b/docs/DEV.md
@@ -312,3 +312,21 @@ graph LR
 Kelemetrix is a separate component that consumes audit logs into metric points.
 Tag providers provide string tags to use in kelemetrix.toml,
 and quantifiers provide scalar quantities.
+
+## Testing
+
+Kelemetry runs end-to-end tests on GitHub CI.
+Each test suite is a directory under the [`e2e`](../e2e) directory
+with the following structure:
+
+- `client.sh`: Executes a series of operations on a running cluster at `$KUBECONFIG`
+  to trigger generation of a test trace.
+  It can be assumed that the default `make quickstart` configuration is running for the cluster.
+- `config.sh`: Defines the following `local` Bash variables.
+  It is directly `source`d by the e2e test script.
+  - `TRACE_DISPLAY_NAME`, the display name of the trace shown on the [demo index](https://kubewharf.io/kelemetry/)
+  - `TRACE_SEARCH_TAGS`, an associative array for the search tags to uniquely identify the test trace generated by `client.sh`.
+- `validate.jq`: A `jq` script that takes the test trace JSON as input
+  and [fails](https://jqlang.github.io/jq/manual/#error) if errors are encountered.
+  The script may use library functions defined under [`e2e/lib`](../e2e/lib).
+  The current framework uses [gojq](https://github.com/itchyny/gojq/) as the `jq` implementation.
diff --git a/e2e/deployment/client.sh b/e2e/deployment/client.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+kubectl create deployment demo --image=alpine:3.16 --replicas=2 -- sleep infinity
+sleep 5
+kubectl scale deployment demo --replicas=4
+sleep 5
+kubectl set image deployments demo alpine=alpine:3.17
+sleep 5
+kubectl scale deployment demo --replicas=2
+sleep 5
+kubectl delete deployment demo
+sleep 30
diff --git a/e2e/deployment/config.sh b/e2e/deployment/config.sh
@@ -0,0 +1,8 @@
+local TRACE_DISPLAY_NAME="Deployment"
+
+declare -A TRACE_SEARCH_TAGS
+TRACE_SEARCH_TAGS[cluster]=tracetest
+TRACE_SEARCH_TAGS[group]=apps
+TRACE_SEARCH_TAGS[resource]=deployments
+TRACE_SEARCH_TAGS[namespace]=default
+TRACE_SEARCH_TAGS[name]=demo