Merge branch 'main' into fix/api-spec

cmd/vela-worker/client.go

@@ -11,7 +11,7 @@ import (
 )
 
 // helper function to setup the queue from the CLI arguments.
-func setupClient(s *Server) (*vela.Client, error) {
+func setupClient(s *Server, token string) (*vela.Client, error) {
 	logrus.Debug("creating vela client from worker configuration")
 
 	// create a new Vela client from the server configuration
@@ -21,11 +21,10 @@ func setupClient(s *Server) (*vela.Client, error) {
 	if err != nil {
 		return nil, err
 	}
-
 	// set token for authentication with the server
 	//
 	// https://pkg.go.dev/github.com/go-vela/sdk-go/vela?tab=doc#AuthenticationService.SetTokenAuth
-	vela.Authentication.SetTokenAuth(s.Secret)
+	vela.Authentication.SetTokenAuth(token)
 
 	return vela, nil
 }

cmd/vela-worker/exec.go

@@ -36,6 +36,18 @@ func (w *Worker) exec(index int) error {
 		return nil
 	}
 
+	// GET build token from server to setup execBuildClient
+	bt, _, err := w.VelaClient.Build.GetBuildToken(item.Repo.GetOrg(), item.Repo.GetName(), item.Build.GetNumber())
+	if err != nil {
+		logrus.Errorf("Unable to GetBuildToken: %s", err)
+		return err
+	}
+	// set up build client with build token as auth
+	execBuildClient, err := setupClient(w.Config.Server, bt.GetToken())
+	if err != nil {
+		return err
+	}
+
 	// create logger with extra metadata
 	//
 	// https://pkg.go.dev/github.com/sirupsen/logrus?tab=doc#WithFields
@@ -79,7 +91,7 @@ func (w *Worker) exec(index int) error {
 		LogStreamingTimeout: w.Config.Executor.LogStreamingTimeout,
 		EnforceTrustedRepos: w.Config.Executor.EnforceTrustedRepos,
 		PrivilegedImages:    w.Config.Runtime.PrivilegedImages,
-		Client:              w.VelaClient,
+		Client:              execBuildClient,
 		Hostname:            w.Config.API.Address.Hostname(),
 		Runtime:             w.Runtime,
 		Build:               item.Build,

cmd/vela-worker/operate.go

@@ -22,8 +22,8 @@ import (
 func (w *Worker) operate(ctx context.Context) error {
 	var err error
 
-	// setup the client
-	w.VelaClient, err = setupClient(w.Config.Server)
+	// setup the vela client with the server
+	w.VelaClient, err = setupClient(w.Config.Server, w.Config.Server.Secret)
 	if err != nil {
 		return err
 	}

cmd/vela-worker/register.go

@@ -12,7 +12,7 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-// checkIn is a helper function to to phone home to the server.
+// checkIn is a helper function to phone home to the server.
 func (w *Worker) checkIn(config *library.Worker) error {
 	// check to see if the worker already exists in the database
 	logrus.Infof("retrieving worker %s from the server", config.GetHostname())

executor/linux/build_test.go

@@ -32,7 +32,9 @@ import (
 
 func TestLinux_CreateBuild(t *testing.T) {
 	// setup types
-	compiler, _ := native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil))
+	set := flag.NewFlagSet("test", 0)
+	set.String("clone-image", "target/vela-git:latest", "doc")
+	compiler, _ := native.New(cli.NewContext(nil, set, nil))
 
 	_build := testBuild()
 	_repo := testRepo()
@@ -184,7 +186,9 @@ func TestLinux_CreateBuild(t *testing.T) {
 
 func TestLinux_AssembleBuild_EnforceTrustedRepos(t *testing.T) {
 	// setup types
-	compiler, _ := native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil))
+	set := flag.NewFlagSet("test", 0)
+	set.String("clone-image", "target/vela-git:latest", "doc")
+	compiler, _ := native.New(cli.NewContext(nil, set, nil))
 
 	_build := testBuild()
 
@@ -1013,7 +1017,9 @@ func TestLinux_AssembleBuild_EnforceTrustedRepos(t *testing.T) {
 
 func TestLinux_PlanBuild(t *testing.T) {
 	// setup types
-	compiler, _ := native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil))
+	set := flag.NewFlagSet("test", 0)
+	set.String("clone-image", "target/vela-git:latest", "doc")
+	compiler, _ := native.New(cli.NewContext(nil, set, nil))
 
 	_build := testBuild()
 	_repo := testRepo()
@@ -1158,7 +1164,9 @@ func TestLinux_PlanBuild(t *testing.T) {
 
 func TestLinux_AssembleBuild(t *testing.T) {
 	// setup types
-	compiler, _ := native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil))
+	set := flag.NewFlagSet("test", 0)
+	set.String("clone-image", "target/vela-git:latest", "doc")
+	compiler, _ := native.New(cli.NewContext(nil, set, nil))
 
 	_build := testBuild()
 	_repo := testRepo()
@@ -1297,8 +1305,7 @@ func TestLinux_AssembleBuild(t *testing.T) {
 
 			switch test.runtime {
 			case constants.DriverKubernetes:
-				_pod := testPodFor(_pipeline)
-				_runtime, err = kubernetes.NewMock(_pod)
+				_runtime, err = kubernetes.NewMock(&v1.Pod{}) // do not use _pod here! AssembleBuild will load it.
 				if err != nil {
 					t.Errorf("unable to create kubernetes runtime engine: %v", err)
 				}
@@ -1329,6 +1336,30 @@ func TestLinux_AssembleBuild(t *testing.T) {
 				t.Errorf("unable to create build: %v", err)
 			}
 
+			// Kubernetes runtime needs to set up the Mock after CreateBuild is called
+			if test.runtime == constants.DriverKubernetes {
+				go func() {
+					_mockRuntime := _runtime.(kubernetes.MockKubernetesRuntime)
+					// This handles waiting until runtime.AssembleBuild has prepared the PodTracker.
+					_mockRuntime.WaitForPodTrackerReady()
+					// Normally, runtime.StreamBuild (which runs in a goroutine) calls PodTracker.Start.
+					_mockRuntime.StartPodTracker(context.Background())
+
+					_pod := testPodFor(_pipeline)
+
+					// Now wait until the pod is created at the end of runtime.AssembleBuild.
+					_mockRuntime.WaitForPodCreate(_pod.GetNamespace(), _pod.GetName())
+
+					// Mark services running and secrets as completed, but no steps have started.
+					err := _mockRuntime.SimulateStatusUpdate(_pod,
+						testContainerStatuses(_pipeline, true, 0, 0),
+					)
+					if err != nil {
+						t.Errorf("%s - failed to simulate pod update: %s", test.name, err)
+					}
+				}()
+			}
+
 			err = _engine.AssembleBuild(context.Background())
 
 			if test.failure {
@@ -1363,7 +1394,9 @@ func TestLinux_AssembleBuild(t *testing.T) {
 
 func TestLinux_ExecBuild(t *testing.T) {
 	// setup types
-	compiler, _ := native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil))
+	set := flag.NewFlagSet("test", 0)
+	set.String("clone-image", "target/vela-git:latest", "doc")
+	compiler, _ := native.New(cli.NewContext(nil, set, nil))
 
 	_build := testBuild()
 	_repo := testRepo()
@@ -1529,6 +1562,44 @@ func TestLinux_ExecBuild(t *testing.T) {
 			// go-vela/server has logic to set it to an expected state.
 			_engine.build.SetStatus("running")
 
+			// Kubernetes runtime needs to set up the Mock after CreateBuild is called
+			if test.runtime == constants.DriverKubernetes {
+				err = _runtime.(kubernetes.MockKubernetesRuntime).SetupMock()
+				if err != nil {
+					t.Errorf("Kubernetes runtime SetupMock returned err: %v", err)
+				}
+
+				_runtime.(kubernetes.MockKubernetesRuntime).StartPodTracker(context.Background())
+
+				go func() {
+					_runtime.(kubernetes.MockKubernetesRuntime).SimulateResync(nil)
+
+					var stepsRunningCount int
+
+					percents := []int{0, 0, 50, 100}
+					lastIndex := len(percents) - 1
+					for index, stepsCompletedPercent := range percents {
+						if index == 0 || index == lastIndex {
+							stepsRunningCount = 0
+						} else {
+							stepsRunningCount = 1
+						}
+
+						err := _runtime.(kubernetes.MockKubernetesRuntime).SimulateStatusUpdate(_pod,
+							testContainerStatuses(
+								_pipeline, true, stepsRunningCount, stepsCompletedPercent,
+							),
+						)
+						if err != nil {
+							t.Errorf("%s - failed to simulate pod update: %s", test.name, err)
+						}
+
+						// simulate exec build duration
+						time.Sleep(100 * time.Microsecond)
+					}
+				}()
+			}
+
 			err = _engine.ExecBuild(context.Background())
 
 			if test.failure {
@@ -1563,7 +1634,9 @@ func TestLinux_ExecBuild(t *testing.T) {
 
 func TestLinux_StreamBuild(t *testing.T) {
 	// setup types
-	compiler, _ := native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil))
+	set := flag.NewFlagSet("test", 0)
+	set.String("clone-image", "target/vela-git:latest", "doc")
+	compiler, _ := native.New(cli.NewContext(nil, set, nil))
 
 	_build := testBuild()
 	_repo := testRepo()
@@ -1882,8 +1955,19 @@ func TestLinux_StreamBuild(t *testing.T) {
 				t.Errorf("%s unable to create build: %v", test.name, err)
 			}
 
-			// simulate ExecBuild() which runs concurrently with StreamBuild()
+			// simulate AssembleBuild()/ExecBuild() which run concurrently with StreamBuild()
 			go func() {
+				// This Kubernetes setup would normally be called within AssembleBuild()
+				if test.runtime == constants.DriverKubernetes {
+					err = _runtime.(kubernetes.MockKubernetesRuntime).SetupMock()
+					if err != nil {
+						t.Errorf("Kubernetes runtime SetupMock returned err: %v", err)
+					}
+
+					// Runtime.StreamBuild calls PodTracker.Start after the PodTracker is marked Ready
+					_runtime.(kubernetes.MockKubernetesRuntime).MarkPodTrackerReady()
+				}
+
 				if test.earlyBuildDone {
 					// imitate build getting canceled or otherwise finishing before ExecBuild gets called.
 					done()
@@ -1953,7 +2037,9 @@ func TestLinux_StreamBuild(t *testing.T) {
 
 func TestLinux_DestroyBuild(t *testing.T) {
 	// setup types
-	compiler, _ := native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil))
+	set := flag.NewFlagSet("test", 0)
+	set.String("clone-image", "target/vela-git:latest", "doc")
+	compiler, _ := native.New(cli.NewContext(nil, set, nil))
 
 	_build := testBuild()
 	_repo := testRepo()
@@ -2092,6 +2178,14 @@ func TestLinux_DestroyBuild(t *testing.T) {
 				t.Errorf("%s unable to create build: %v", test.name, err)
 			}
 
+			// Kubernetes runtime needs to set up the Mock after CreateBuild is called
+			if test.runtime == constants.DriverKubernetes {
+				err = _runtime.(kubernetes.MockKubernetesRuntime).SetupMock()
+				if err != nil {
+					t.Errorf("Kubernetes runtime SetupMock returned err: %v", err)
+				}
+			}
+
 			err = _engine.DestroyBuild(context.Background())
 
 			if test.failure {

executor/linux/secret_test.go

@@ -235,6 +235,14 @@ func TestLinux_Secret_delete(t *testing.T) {
 			// add init container info to client
 			_ = _engine.CreateBuild(context.Background())
 
+			// Kubernetes runtime needs to set up the Mock after CreateBuild is called
+			if test.runtime.Driver() == constants.DriverKubernetes {
+				err = _engine.Runtime.(kubernetes.MockKubernetesRuntime).SetupMock()
+				if err != nil {
+					t.Errorf("Kubernetes runtime SetupMock returned err: %v", err)
+				}
+			}
+
 			_engine.steps.Store(test.container.ID, test.step)
 
 			err = _engine.secret.destroy(context.Background(), test.container)
@@ -256,7 +264,9 @@ func TestLinux_Secret_delete(t *testing.T) {
 
 func TestLinux_Secret_exec(t *testing.T) {
 	// setup types
-	compiler, _ := native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil))
+	set := flag.NewFlagSet("test", 0)
+	set.String("clone-image", "target/vela-git:latest", "doc")
+	compiler, _ := native.New(cli.NewContext(nil, set, nil))
 
 	_build := testBuild()
 	_repo := testRepo()
@@ -349,6 +359,16 @@ func TestLinux_Secret_exec(t *testing.T) {
 			// add init container info to client
 			_ = _engine.CreateBuild(context.Background())
 
+			// Kubernetes runtime needs to set up the Mock after CreateBuild is called
+			if test.runtime == constants.DriverKubernetes {
+				err = _runtime.(kubernetes.MockKubernetesRuntime).SetupMock()
+				if err != nil {
+					t.Errorf("Kubernetes runtime SetupMock returned err: %v", err)
+				}
+
+				go _runtime.(kubernetes.MockKubernetesRuntime).SimulateResync(nil)
+			}
+
 			err = _engine.secret.exec(context.Background(), &p.Secrets)
 
 			if test.failure {
@@ -623,6 +643,14 @@ func TestLinux_Secret_stream(t *testing.T) {
 			// add init container info to client
 			_ = _engine.CreateBuild(context.Background())
 
+			// Kubernetes runtime needs to set up the Mock after CreateBuild is called
+			if test.runtime.Driver() == constants.DriverKubernetes {
+				err = _engine.Runtime.(kubernetes.MockKubernetesRuntime).SetupMock()
+				if err != nil {
+					t.Errorf("Kubernetes runtime SetupMock returned err: %v", err)
+				}
+			}
+
 			err = _engine.secret.stream(context.Background(), test.container)
 
 			if test.failure {

executor/linux/stage_test.go

@@ -31,7 +31,9 @@ func TestLinux_CreateStage(t *testing.T) {
 	_user := testUser()
 	_metadata := testMetadata()
 
-	compiler, _ := native.New(cli.NewContext(nil, flag.NewFlagSet("test", 0), nil))
+	set := flag.NewFlagSet("test", 0)
+	set.String("clone-image", "target/vela-git:latest", "doc")
+	compiler, _ := native.New(cli.NewContext(nil, set, nil))
 
 	_pipeline, _, err := compiler.
 		Duplicate().

go.mod

@@ -8,9 +8,9 @@ require (
 	github.com/docker/docker v20.10.22+incompatible
 	github.com/docker/go-units v0.5.0
 	github.com/gin-gonic/gin v1.8.1
-	github.com/go-vela/sdk-go v0.17.0
-	github.com/go-vela/server v0.17.0
-	github.com/go-vela/types v0.17.0
+	github.com/go-vela/sdk-go v0.17.1-0.20230224164256-73b956037d5b
+	github.com/go-vela/server v0.17.1-0.20230224163114-c4283eb079e8
+	github.com/go-vela/types v0.17.1-0.20230223155025-1c8a34f71425
 	github.com/google/go-cmp v0.5.9
 	github.com/joho/godotenv v1.4.0
 	github.com/opencontainers/image-spec v1.0.2

go.sum

@@ -152,12 +152,12 @@ github.com/go-playground/validator/v10 v10.10.0/go.mod h1:74x4gJWsvQexRdW8Pn3dXS
 github.com/go-redis/redis/v8 v8.11.5 h1:AcZZR7igkdvfVmQTPnu9WE37LRrO/YrBH5zWyjDC0oI=
 github.com/go-redis/redis/v8 v8.11.5/go.mod h1:gREzHqY1hg6oD9ngVRbLStwAWKhA0FEgq8Jd4h5lpwo=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/go-vela/sdk-go v0.17.0 h1:vVck8Xm5/co6zrDntG1mdjenT5N0vgDJ30ImFZTUEjY=
-github.com/go-vela/sdk-go v0.17.0/go.mod h1:R7oHeYj1ThBREPzqsvevUEBxp8knmOviFbspvuGqZsU=
-github.com/go-vela/server v0.17.0 h1:sEJ4a9mus43Qw4E431OYwN8qvYugoA0ZmaU0sFGgESk=
-github.com/go-vela/server v0.17.0/go.mod h1:Z8YT/IFJTQ80a63GINRHCaFBTkkkHAL7mDohw4xLbEE=
-github.com/go-vela/types v0.17.0 h1:nvKBbNO8BSiLtYPMScT0XWosGqEWX85UKSkkclb2DVA=
-github.com/go-vela/types v0.17.0/go.mod h1:6KoRkvXMw9DkAcLdtI7PxPqMlT2Bl0DiigQamLGGjwo=
+github.com/go-vela/sdk-go v0.17.1-0.20230224164256-73b956037d5b h1:N/g4kgUFlb36JZBlhuq3ifJE+DTIJTqO21nGXsf+76A=
+github.com/go-vela/sdk-go v0.17.1-0.20230224164256-73b956037d5b/go.mod h1:RurQS+lXlmzv4I/QS/qMI1Rqb36tfA+NubekWVRIwXc=
+github.com/go-vela/server v0.17.1-0.20230224163114-c4283eb079e8 h1:CUXtUYOu8Jz9IYuTMXivu4GXr8BdQXLAYjFYnDnkndE=
+github.com/go-vela/server v0.17.1-0.20230224163114-c4283eb079e8/go.mod h1:WJZQJ2nwSSDuB56LskZ3gEoSWI0IwKRtxaFE8GdTLIc=
+github.com/go-vela/types v0.17.1-0.20230223155025-1c8a34f71425 h1:tdjas7NJLWlU2vmETaU36wjbd+zvWPLtznE4uwtnFlw=
+github.com/go-vela/types v0.17.1-0.20230223155025-1c8a34f71425/go.mod h1:6KoRkvXMw9DkAcLdtI7PxPqMlT2Bl0DiigQamLGGjwo=
 github.com/goccy/go-json v0.9.7 h1:IcB+Aqpx/iMHu5Yooh7jEzJk1JZ7Pjtmys2ukPr7EeM=
 github.com/goccy/go-json v0.9.7/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=

runtime/kubernetes/build.go

@@ -212,7 +212,7 @@ func (c *client) AssembleBuild(ctx context.Context, b *pipeline.Build) error {
 		}
 	}
 
-	// setup containerTeachers now that all containers are defined.
+	// setup containerTrackers now that all containers are defined.
 	c.PodTracker.TrackContainers(c.Pod.Spec.Containers)
 
 	// send signal to StreamBuild now that PodTracker is ready to be started.

runtime/kubernetes/kubernetes.go

@@ -5,14 +5,14 @@
 package kubernetes
 
 import (
-	"github.com/sirupsen/logrus"
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 
 	velav1alpha1 "github.com/go-vela/worker/runtime/kubernetes/apis/vela/v1alpha1"
 	velaK8sClient "github.com/go-vela/worker/runtime/kubernetes/generated/clientset/versioned"
+	"github.com/sirupsen/logrus"
 )
 
 type config struct {