bpftrace-compiler: add cache, loading kernel modules, fixes

eve-tools/bpftrace-compiler/.gitignore

@@ -1 +1,2 @@
 bpftrace-compiler
+kernel-commits.mk

eve-tools/bpftrace-compiler/Dockerfile

@@ -0,0 +1,17 @@
+# Copyright (c) 2024 Zededa, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+FROM alpine:3.20
+
+# hadolint ignore=DL3018
+RUN apk add --no-cache qemu-system-x86_64 qemu-system-aarch64 go make git docker
+
+COPY init_docker_tests.sh /init_docker_tests.sh
+RUN chmod u+x /init_docker_tests.sh
+
+RUN mkdir /src
+COPY kernel-commits.mk /
+
+WORKDIR /src
+
+ENTRYPOINT ["/init_docker_tests.sh"]

eve-tools/bpftrace-compiler/Makefile

@@ -4,7 +4,15 @@
 bpftrace-compiler: *.go
 	go build
 
-.PHONY: test
+.PHONY: test bare-test docker-test
 
-test:
-	go test -test.timeout 30m -v -race .
+bare-test:
+	go test -test.timeout 1h -v -race .
+
+test: bare-test
+
+kernel-commits.mk: ../../kernel-commits.mk
+	cp ../../kernel-commits.mk .
+
+docker-test: kernel-commits.mk
+	docker run --cap-add=SYS_PTRACE --cap-add=SYS_ADMIN --privileged --security-opt seccomp=unconfined -v $(shell pwd):/src $(shell docker build -q .) make bare-test

eve-tools/bpftrace-compiler/cmdRun.go

@@ -8,6 +8,8 @@ import (
 	"fmt"
 	"log"
 	"os"
+	"path/filepath"
+	"strings"
 	"time"
 )
 
@@ -22,7 +24,7 @@ type remoteRun interface {
 	end()
 }
 
-func (r *run) run(bpfFile string, uc userspaceContainer, timeout time.Duration) {
+func (r *run) run(bpfFile string, uc userspaceContainer, kernelModules []string, timeout time.Duration) {
 	fh, err := os.CreateTemp("/var/tmp", "bpftrace-aot")
 	if err != nil {
 		log.Fatalf("could not create temp file: %v", err)
@@ -33,7 +35,7 @@ func (r *run) run(bpfFile string, uc userspaceContainer, timeout time.Duration)
 
 	arch := cleanArch(r.arch())
 	lkConf := r.lkConf()
-	err = compile(arch, lkConf, uc, bpfFile, outputFile)
+	err = compileWithCache(arch, lkConf, uc, kernelModules, bpfFile, outputFile)
 	if err != nil {
 		log.Fatalf("compiling for %s/%s failed: %v", arch, lkConf, err)
 	}
@@ -46,21 +48,65 @@ func (r *run) run(bpfFile string, uc userspaceContainer, timeout time.Duration)
 
 }
 
-func compile(arch string, lkConf lkConf, uc userspaceContainer, bpfFile string, outputFile string) error {
+func compileWithCache(arch string, lkConf lkConf, uc userspaceContainer, kernelModules []string, bpfFile string, outputFile string) error {
+	var err error
+	if bpftraceCompilerDir == "" {
+		return compile(arch, lkConf, uc, kernelModules, bpfFile, outputFile)
+	}
+
+	ucString := ""
+	if uc != nil {
+		ucString = uc.String()
+	}
+
+	bpfFileContent, err := os.ReadFile(bpfFile)
+	if err != nil {
+		return fmt.Errorf("Could not read '%s': %v", bpfFile, err)
+	}
+	hash := hashDir([]string{"root"}, arch, lkConf.String(), ucString, strings.Join(kernelModules, ","), string(bpfFileContent))
+
+	hashPath := filepath.Join(bpftraceCompilerDir, "cache", hash)
+
+	compileAndStoreInCache := func() error {
+		err := compile(arch, lkConf, uc, kernelModules, bpfFile, outputFile)
+		if err != nil {
+			return err
+		}
+		err = copyFile(outputFile, hashPath)
+		if err != nil {
+			log.Printf("could not store in cache (%s): %v", hashPath, err)
+		}
+
+		return nil
+
+	}
+	_, err = os.Stat(hashPath)
+	if err != nil {
+		log.Printf("could not find compiled script in cache, compiling now ...")
+		return compileAndStoreInCache()
+	}
+	err = copyFile(hashPath, outputFile)
+	if err != nil {
+		log.Printf("copying %s to %s failed: %err, compiling now ...", hashPath, outputFile, err)
+		return compileAndStoreInCache()
+	}
+	log.Printf("found compiled script in cache ...")
+
+	return nil
+}
+
+func compile(arch string, lkConf lkConf, uc userspaceContainer, kernelModules []string, bpfFile string, outputFile string) error {
 	arch = cleanArch(arch)
 	imageDir, err := os.MkdirTemp("/var/tmp", "bpftrace-image")
 	if err != nil {
-		log.Fatal(err)
+		log.Fatalf("creating image dir %s failed: %v", imageDir, err)
 	}
 	defer os.RemoveAll(imageDir)
 	createImage(arch, lkConf, uc, imageDir)
 
-	var qr *qemuRunner
-	if arch == "arm64" {
-		qr = newQemuArm64Runner(imageDir, bpfFile, outputFile)
-	} else if arch == "amd64" {
-		qr = newQemuAmd64Runner(imageDir, bpfFile, outputFile)
-	}
+	qr := newQemuRunner(arch, imageDir, bpfFile, outputFile)
+
+	qr.withLoadKernelModule(kernelModules)
 
 	qemuOutput, err := qr.run()
 	if err != nil {

eve-tools/bpftrace-compiler/compile_test.go

@@ -36,7 +36,7 @@ func testCompile(t *testing.T, arch, eveKernel string) {
 	defer aotFh.Close()
 	defer os.Remove(aotFilename)
 
-	err = compile(arch, lkConf{kernel: eveKernel}, nil, btFilename, aotFilename)
+	err = compile(arch, lkConf{kernel: eveKernel}, nil, []string{}, btFilename, aotFilename)
 	if err != nil {
 		t.Fatalf("compiling failed: %v", err)
 	}

eve-tools/bpftrace-compiler/edgeview.go

@@ -132,8 +132,9 @@ func (er *edgeviewRun) stopHTTPDebugPort() {
 func (er *edgeviewRun) forwardHTTPDebugPort() error {
 	var err error
 
+	hostForward := "localhost:6543"
 	er.ctx, er.cancel = context.WithTimeout(context.Background(), 5*time.Minute)
-	er.cmd = exec.CommandContext(er.ctx, "bash", er.path, "tcp/127.0.0.1:6543")
+	er.cmd = exec.CommandContext(er.ctx, "bash", er.path, fmt.Sprintf("tcp/%s", hostForward))
 
 	er.tty, err = pty.Start(er.cmd)
 	if err != nil {
@@ -150,7 +151,7 @@ func (er *edgeviewRun) forwardHTTPDebugPort() error {
 			line := scanner.Text()
 			output += line + "\n"
 
-			if strings.Contains(line, "127.0.0.1:6543") {
+			if strings.Contains(line, hostForward) {
 				forwardLine = strings.TrimFunc(line, func(r rune) bool {
 					return !unicode.IsGraphic(r)
 				})

eve-tools/bpftrace-compiler/go.mod

@@ -1,6 +1,6 @@
 module bpftrace-compiler
 
-go 1.22.5
+go 1.21
 
 require (
 	github.com/creack/pty v1.1.18
@@ -11,6 +11,7 @@ require (
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/cobra v1.8.0
 	golang.org/x/crypto v0.21.0
+	golang.org/x/mod v0.14.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 
@@ -85,7 +86,6 @@ require (
 	go.opentelemetry.io/otel/sdk v1.21.0 // indirect
 	go.opentelemetry.io/otel/trace v1.21.0 // indirect
 	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
-	golang.org/x/mod v0.14.0 // indirect
 	golang.org/x/net v0.23.0 // indirect
 	golang.org/x/sync v0.6.0 // indirect
 	golang.org/x/sys v0.18.0 // indirect

eve-tools/bpftrace-compiler/httpRun.go

@@ -139,11 +139,11 @@ func (hr *httpRun) runBpftrace(aotFile string, timeout time.Duration) error {
 
 	err = <-errChan
 	if err != nil {
-		log.Fatalf("received err: %v", err)
+		log.Fatalf("received error from http: %v", err)
 	}
 	err = multipartWriter.Close()
 	if err != nil && !strings.Contains(err.Error(), "io: read/write on closed pipe") {
-		log.Fatalf("closing multipart writer failed: %v %T", err, err)
+		log.Fatalf("closing multipart writer failed: %v", err)
 	}
 
 	return nil

eve-tools/bpftrace-compiler/init_docker_tests.sh

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+# Copyright (c) 2024 Zededa, Inc.
+# SPDX-License-Identifier: Apache-2.0
+
+mkdir -p /sys/fs/cgroup/init
+echo 1 > /sys/fs/cgroup/init/cgroup.procs
+echo +cpu > /sys/fs/cgroup/cgroup.subtree_control
+
+/usr/bin/dockerd -l fatal &
+exec "$@"