Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Run e2e test with both kind and k3s for all runtimes #323

Merged
merged 2 commits into from
Sep 22, 2023
Merged

Run e2e test with both kind and k3s for all runtimes #323

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
c6eaabc
use both kind and k3s for all runtimes
jprendes Sep 18, 2023
3f774fe
wait for k3s system pods to be ready
jprendes Sep 21, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .github/workflows/action-test-kind.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,7 +36,7 @@ jobs:
path: dist
- name: run
timeout-minutes: 5
run: make test/k8s
run: make test/k8s-${{ inputs.runtime }}
# only runs when the previous step fails
- name: inspect failed pods
if: failure()
Expand Down
4 changes: 2 additions & 2 deletions .github/workflows/ci.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ jobs:
matrix:
# 20.04 uses cgroupv1, 22.04 uses cgroupv2
os: ["ubuntu-20.04", "ubuntu-22.04"]
runtime: ["wasmtime"]
runtime: ["wasmtime", "wasmedge", "wasmer"]
uses: ./.github/workflows/action-test-kind.yml
with:
os: ${{ matrix.os }}
Expand All @@ -82,7 +82,7 @@ jobs:
strategy:
matrix:
os: ["ubuntu-20.04", "ubuntu-22.04"]
runtime: ["wasmedge", "wasmer"]
runtime: ["wasmtime", "wasmedge", "wasmer"]
uses: ./.github/workflows/action-test-k3s.yml
with:
os: ${{ matrix.os }}
Expand Down
69 changes: 28 additions & 41 deletions Makefile
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -82,19 +82,17 @@ test-%:
.PHONY: install install-%
install: $(RUNTIMES:%=install-%);

install-%:
install-%: build-%
mkdir -p $(PREFIX)/bin
$(INSTALL) target/$(TARGET)/containerd-shim-$*-v1 $(PREFIX)/bin/
$(LN) ./containerd-shim-$*-v1 $(PREFIX)/bin/containerd-shim-$*d-v1
$(LN) ./containerd-shim-$*-v1 $(PREFIX)/bin/containerd-$*d

# dist is not phony, so that if the folder exist we don't try to do work
dist:
$(MAKE) $(RUNTIMES:%=dist-%) TARGET="$(TARGET)"
.PHONY: dist dist-%
dist: $(RUNTIMES:%=dist-%);

.PHONY: dist-%
dist-%: build-%
$(MAKE) install-$* PREFIX="$(PWD)/dist" TARGET="$(TARGET)"
dist-%:
[ -f $(PWD)/dist/bin/containerd-shim-$*-v1 ] || $(MAKE) install-$* PREFIX="$(PWD)/dist" TARGET="$(TARGET)"

.PHONY: test-image
test-image: dist/img.tar
Expand All @@ -121,21 +119,21 @@ load: dist/img.tar
bin/kind: test/k8s/Dockerfile
$(DOCKER_BUILD) --output=bin/ -f test/k8s/Dockerfile --target=kind .

test/k8s/_out/img: test/k8s/Dockerfile dist
mkdir -p $(@D) && $(DOCKER_BUILD) -f test/k8s/Dockerfile --iidfile=$(@) --load .
test/k8s/_out/img-%: test/k8s/Dockerfile dist-%
mkdir -p $(@D) && $(DOCKER_BUILD) -f test/k8s/Dockerfile --build-arg="RUNTIME=$*" --iidfile=$(@) --load .

.PHONY: test/nginx
test/nginx:
docker pull docker.io/nginx:latest
mkdir -p $@/out && docker save -o $@/out/img.tar docker.io/nginx:latest

.PHONY: test/k8s/cluster
test/k8s/cluster: dist/img.tar bin/kind test/k8s/_out/img
bin/kind create cluster --name $(KIND_CLUSTER_NAME) --image="$(shell cat test/k8s/_out/img)" && \
.PHONY: test/k8s/cluster-%
test/k8s/cluster-%: dist/img.tar bin/kind test/k8s/_out/img-%
bin/kind create cluster --name $(KIND_CLUSTER_NAME) --image="$(shell cat test/k8s/_out/img-$*)" && \
bin/kind load image-archive --name $(KIND_CLUSTER_NAME) $(<)

.PHONY: test/k8s
test/k8s: test/k8s/cluster
.PHONY: test/k8s-%
test/k8s-%: test/k8s/cluster-%
kubectl --context=kind-$(KIND_CLUSTER_NAME) apply -f test/k8s/deploy.yaml
kubectl --context=kind-$(KIND_CLUSTER_NAME) wait deployment wasi-demo --for condition=Available=True --timeout=90s

Expand All @@ -152,34 +150,23 @@ bin/k3s:
bin/k3s/clean:
bin/k3s-runwasi-uninstall.sh

.PHONY: test/k3s-wasmedge
test/k3s-wasmedge: dist/img.tar bin/k3s dist
sudo cp /var/lib/rancher/k3s/agent/etc/containerd/config.toml /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \
echo '[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.wasm]' | sudo tee -a /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \
echo ' runtime_type = "$(PWD)/dist/bin/containerd-shim-wasmedge-v1"' | sudo tee -a /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \
echo "CONTAINERD_NAMESPACE='default'" | sudo tee /etc/systemd/system/k3s-runwasi.service.env && \
echo "NO_PROXY=192.168.0.0/16" | sudo tee -a /etc/systemd/system/k3s-runwasi.service.env && \
sudo systemctl daemon-reload && \
sudo systemctl restart k3s-runwasi && \
timeout 60 bash -c -- 'while true; do sudo bin/k3s ctr version && break; sleep 1; done' && \
sudo bin/k3s ctr image import --all-platforms dist/img.tar && \
.PHONY: test/k3s-%
test/k3s-%: dist/img.tar bin/k3s dist-%
sudo cp /var/lib/rancher/k3s/agent/etc/containerd/config.toml /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl
echo '[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.wasm]' | sudo tee -a /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl
echo ' runtime_type = "$(PWD)/dist/bin/containerd-shim-$*-v1"' | sudo tee -a /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl
echo "CONTAINERD_NAMESPACE='$(CONTAINERD_NAMESPACE)'" | sudo tee /etc/systemd/system/k3s-runwasi.service.env
echo "NO_PROXY=192.168.0.0/16" | sudo tee -a /etc/systemd/system/k3s-runwasi.service.env
sudo systemctl daemon-reload
sudo systemctl restart k3s-runwasi
timeout 60 bash -c -- 'while ! sudo bin/k3s ctr version; do sleep 1; done'
sudo bin/k3s ctr image import --all-platforms dist/img.tar
timeout 60 bash -c -- 'while [ "$$(sudo bin/k3s kubectl get pods --all-namespaces --no-headers | wc -l)" == "0" ]; do sleep 1; done'
timeout 60 bash -c -- 'while [ "$$(sudo bin/k3s kubectl get pods --all-namespaces --no-headers | grep -vE "Completed|Running" | wc -l)" != "0" ]; do sleep 1; done'
sudo bin/k3s kubectl get pods --all-namespaces
sudo bin/k3s kubectl apply -f test/k8s/deploy.yaml
sudo bin/k3s kubectl wait deployment wasi-demo --for condition=Available=True --timeout=120s && \
sudo bin/k3s kubectl get pods -o wide

.PHONY: test/k3s-wasmer
test/k3s-wasmer: dist/img.tar bin/k3s dist
sudo cp /var/lib/rancher/k3s/agent/etc/containerd/config.toml /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \
echo '[plugins."io.containerd.grpc.v1.cri".containerd.runtimes.wasm]' | sudo tee -a /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \
echo ' runtime_type = "$(PWD)/dist/bin/containerd-shim-wasmer-v1"' | sudo tee -a /var/lib/rancher/k3s/agent/etc/containerd/config.toml.tmpl && \
echo "CONTAINERD_NAMESPACE='default'" | sudo tee /etc/systemd/system/k3s-runwasi.service.env && \
echo "NO_PROXY=192.168.0.0/16" | sudo tee -a /etc/systemd/system/k3s-runwasi.service.env && \
sudo systemctl daemon-reload && \
sudo systemctl restart k3s-runwasi && \
timeout 60 bash -c -- 'while true; do sudo bin/k3s ctr version && break; sleep 1; done' && \
sudo bin/k3s ctr image import --all-platforms dist/img.tar && \
sudo bin/k3s kubectl apply -f test/k8s/deploy.yaml
sudo bin/k3s kubectl wait deployment wasi-demo --for condition=Available=True --timeout=120s && \
sudo bin/k3s kubectl get pods --all-namespaces
sudo bin/k3s kubectl wait deployment wasi-demo --for condition=Available=True --timeout=120s
sudo bin/k3s kubectl get pods -o wide

.PHONY: test/k3s/clean
Expand Down