From 33e5aa25645c0f49b281f4c9bab2fa926e7d1098 Mon Sep 17 00:00:00 2001 From: Josh Deprez Date: Thu, 6 Jun 2024 11:16:50 +1000 Subject: [PATCH 1/2] kubernetes-exec is now a flag --- EXPERIMENTS.md | 10 -- agent/agent_configuration.go | 1 + agent/agent_worker.go | 1 + agent/job_runner.go | 38 ++++--- agent/tags.go | 3 +- clicommand/agent_start.go | 3 + clicommand/bootstrap.go | 3 + clicommand/global.go | 162 +++++++++++++++------------- internal/experiments/experiments.go | 4 +- internal/job/config.go | 3 + internal/job/executor.go | 2 +- 11 files changed, 125 insertions(+), 105 deletions(-) diff --git a/EXPERIMENTS.md b/EXPERIMENTS.md index ee6e2a3960..7940a6464f 100644 --- a/EXPERIMENTS.md +++ b/EXPERIMENTS.md @@ -37,16 +37,6 @@ After repository checkout, resolve `BUILDKITE_COMMIT` to a commit hash. This mak **Status**: broadly useful, we'd like this to be the standard behaviour in 4.0. 👍👍 -### `kubernetes-exec` - -Modifies `start` and `bootstrap` in such a way that they can run in separate Kubernetes containers in the same pod. - -Currently, this experiment is being used by [agent-stack-k8s](https://github.com/buildkite/agent-stack-k8s). - -This will result in errors unless orchestrated in a similar manner to that project. Please see the [README](https://github.com/buildkite/agent-stack-k8s/blob/main/README.md) of that repository for more details. - -**Status**: Being used in a preview release of agent-stack-k8s. As it has little applicability outside of Kubernetes, this will not be the default behaviour. - ### `polyglot-hooks` Allows the agent to run hooks written in languages other than bash. This enables the agent to run hooks written in any language, as long as the language has a runtime available on the agent. Polyglot hooks can be in interpreted languages, so long as they have a valid shebang, and the interpreter specified in the shebang is installed on the agent. diff --git a/agent/agent_configuration.go b/agent/agent_configuration.go index 690789e4b3..11a7f41e60 100644 --- a/agent/agent_configuration.go +++ b/agent/agent_configuration.go @@ -35,6 +35,7 @@ type AgentConfiguration struct { LocalHooksEnabled bool StrictSingleHooks bool RunInPty bool + KubernetesExec bool SigningJWKSFile string // Where to find the key to sign pipeline uploads with (passed through to jobs, they might be uploading pipelines) SigningJWKSKeyID string // The key ID to sign pipeline uploads with diff --git a/agent/agent_worker.go b/agent/agent_worker.go index 17f47a20e9..6dac6a2f2e 100644 --- a/agent/agent_worker.go +++ b/agent/agent_worker.go @@ -687,6 +687,7 @@ func (a *AgentWorker) RunJob(ctx context.Context, acceptResponse *api.Job) error JobStatusInterval: time.Duration(a.agent.JobStatusInterval) * time.Second, AgentConfiguration: a.agentConfiguration, AgentStdout: a.agentStdout, + KubernetesExec: a.agentConfiguration.KubernetesExec, }) if err != nil { return fmt.Errorf("Failed to initialize job: %w", err) diff --git a/agent/job_runner.go b/agent/job_runner.go index 9ca488cb1e..b69ddb4f4e 100644 --- a/agent/job_runner.go +++ b/agent/job_runner.go @@ -50,28 +50,30 @@ const ( // Certain env can only be set by agent configuration. // We show the user a warning in the bootstrap if they use any of these at a job level. var ProtectedEnv = map[string]struct{}{ - "BUILDKITE_AGENT_ENDPOINT": {}, "BUILDKITE_AGENT_ACCESS_TOKEN": {}, "BUILDKITE_AGENT_DEBUG": {}, + "BUILDKITE_AGENT_ENDPOINT": {}, "BUILDKITE_AGENT_PID": {}, "BUILDKITE_BIN_PATH": {}, - "BUILDKITE_CONFIG_PATH": {}, "BUILDKITE_BUILD_PATH": {}, - "BUILDKITE_GIT_MIRRORS_PATH": {}, - "BUILDKITE_GIT_MIRRORS_SKIP_UPDATE": {}, - "BUILDKITE_HOOKS_PATH": {}, - "BUILDKITE_PLUGINS_PATH": {}, - "BUILDKITE_SSH_KEYSCAN": {}, - "BUILDKITE_GIT_SUBMODULES": {}, "BUILDKITE_COMMAND_EVAL": {}, - "BUILDKITE_PLUGINS_ENABLED": {}, - "BUILDKITE_LOCAL_HOOKS_ENABLED": {}, + "BUILDKITE_CONFIG_PATH": {}, + "BUILDKITE_CONTAINER_COUNT": {}, + "BUILDKITE_GIT_CLEAN_FLAGS": {}, "BUILDKITE_GIT_CLONE_FLAGS": {}, - "BUILDKITE_GIT_FETCH_FLAGS": {}, "BUILDKITE_GIT_CLONE_MIRROR_FLAGS": {}, + "BUILDKITE_GIT_FETCH_FLAGS": {}, "BUILDKITE_GIT_MIRRORS_LOCK_TIMEOUT": {}, - "BUILDKITE_GIT_CLEAN_FLAGS": {}, + "BUILDKITE_GIT_MIRRORS_PATH": {}, + "BUILDKITE_GIT_MIRRORS_SKIP_UPDATE": {}, + "BUILDKITE_GIT_SUBMODULES": {}, + "BUILDKITE_HOOKS_PATH": {}, + "BUILDKITE_KUBERNETES_EXEC": {}, + "BUILDKITE_LOCAL_HOOKS_ENABLED": {}, + "BUILDKITE_PLUGINS_ENABLED": {}, + "BUILDKITE_PLUGINS_PATH": {}, "BUILDKITE_SHELL": {}, + "BUILDKITE_SSH_KEYSCAN": {}, } type JobRunnerConfig struct { @@ -99,6 +101,9 @@ type JobRunnerConfig struct { // Whether to set debug HTTP Requests in the job DebugHTTP bool + // Whether the job is executing as a k8s pod + KubernetesExec bool + // Stdout of the parent agent process. Used for job log stdout writing arg, for simpler containerized log collection. AgentStdout io.Writer } @@ -335,7 +340,8 @@ func NewJobRunner(ctx context.Context, l logger.Logger, apiClient APIClient, con processEnv := append(os.Environ(), env...) // The process that will run the bootstrap script - if experiments.IsEnabled(ctx, experiments.KubernetesExec) { + if conf.KubernetesExec { + // Thank you Mario, but our bootstrap is in another container containerCount, err := strconv.Atoi(os.Getenv("BUILDKITE_CONTAINER_COUNT")) if err != nil { return nil, fmt.Errorf("failed to parse BUILDKITE_CONTAINER_COUNT: %w", err) @@ -346,7 +352,7 @@ func NewJobRunner(ctx context.Context, l logger.Logger, apiClient APIClient, con Stderr: r.jobLogs, ClientCount: containerCount, }) - } else { + } else { // not Kubernetes // The bootstrap-script gets parsed based on the operating system cmd, err := shellwords.Split(conf.AgentConfiguration.BootstrapScript) if err != nil { @@ -488,6 +494,10 @@ func (r *JobRunner) createEnvironment(ctx context.Context) ([]string, error) { env["BUILDKITE_STRICT_SINGLE_HOOKS"] = fmt.Sprintf("%t", r.conf.AgentConfiguration.StrictSingleHooks) env["BUILDKITE_SIGNAL_GRACE_PERIOD_SECONDS"] = fmt.Sprintf("%d", int(r.conf.AgentConfiguration.SignalGracePeriod/time.Second)) + if r.conf.KubernetesExec { + env["BUILDKITE_KUBERNETES_EXEC"] = "true" + } + // propagate CancelSignal to bootstrap, unless it's the default SIGTERM if r.conf.CancelSignal != process.SIGTERM { env["BUILDKITE_CANCEL_SIGNAL"] = r.conf.CancelSignal.String() diff --git a/agent/tags.go b/agent/tags.go index 9ba29f3ce6..9925f37ce6 100644 --- a/agent/tags.go +++ b/agent/tags.go @@ -10,7 +10,6 @@ import ( "strings" "time" - "github.com/buildkite/agent/v3/internal/experiments" "github.com/buildkite/agent/v3/logger" "github.com/buildkite/roko" "github.com/denisbrodbeck/machineid" @@ -78,7 +77,7 @@ type tagFetcher struct { func (t *tagFetcher) Fetch(ctx context.Context, l logger.Logger, conf FetchTagsConfig) []string { tags := conf.Tags - if experiments.IsEnabled(ctx, experiments.KubernetesExec) { + if t.k8s != nil { k8sTags, err := t.k8s() if err != nil { l.Warn("Could not fetch tags from k8s: %s", err) diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go index 05c4bb7c0b..8d48e76d1e 100644 --- a/clicommand/agent_start.go +++ b/clicommand/agent_start.go @@ -165,6 +165,7 @@ type AgentStartConfig struct { Experiments []string `cli:"experiment" normalize:"list"` Profile string `cli:"profile"` StrictSingleHooks bool `cli:"strict-single-hooks"` + KuberentesExec bool `cli:"kubernetes-exec"` // API config DebugHTTP bool `cli:"debug-http"` @@ -680,6 +681,7 @@ var AgentStartCommand = cli.Command{ ProfileFlag, RedactedVars, StrictSingleHooksFlag, + KubernetesExecFlag, // Deprecated flags which will be removed in v4 cli.StringSliceFlag{ @@ -955,6 +957,7 @@ var AgentStartCommand = cli.Command{ TracingBackend: cfg.TracingBackend, TracingServiceName: cfg.TracingServiceName, VerificationFailureBehaviour: cfg.VerificationFailureBehavior, + KubernetesExec: cfg.KuberentesExec, SigningJWKSFile: cfg.SigningJWKSFile, SigningJWKSKeyID: cfg.SigningJWKSKeyID, diff --git a/clicommand/bootstrap.go b/clicommand/bootstrap.go index 3245081db6..160e6af051 100644 --- a/clicommand/bootstrap.go +++ b/clicommand/bootstrap.go @@ -97,6 +97,7 @@ type BootstrapConfig struct { TracingServiceName string `cli:"tracing-service-name"` NoJobAPI bool `cli:"no-job-api"` DisableWarningsFor []string `cli:"disable-warnings-for" normalize:"list"` + KubernetesExec bool `cli:"kubernetes-exec"` } var BootstrapCommand = cli.Command{ @@ -371,6 +372,7 @@ var BootstrapCommand = cli.Command{ ProfileFlag, RedactedVars, StrictSingleHooksFlag, + KubernetesExecFlag, }, Action: func(c *cli.Context) error { ctx := context.Background() @@ -452,6 +454,7 @@ var BootstrapCommand = cli.Command{ TracingServiceName: cfg.TracingServiceName, JobAPI: !cfg.NoJobAPI, DisabledWarnings: cfg.DisableWarningsFor, + KubernetesExec: cfg.KubernetesExec, }) cctx, cancel := context.WithCancel(ctx) diff --git a/clicommand/global.go b/clicommand/global.go index 7e3c2a66a8..6de66f5baf 100644 --- a/clicommand/global.go +++ b/clicommand/global.go @@ -21,93 +21,103 @@ const ( DefaultEndpoint = "https://agent.buildkite.com/v3" ) -var AgentAccessTokenFlag = cli.StringFlag{ - Name: "agent-access-token", - Value: "", - Usage: "The access token used to identify the agent", - EnvVar: "BUILDKITE_AGENT_ACCESS_TOKEN", -} +var ( + AgentAccessTokenFlag = cli.StringFlag{ + Name: "agent-access-token", + Value: "", + Usage: "The access token used to identify the agent", + EnvVar: "BUILDKITE_AGENT_ACCESS_TOKEN", + } -var AgentRegisterTokenFlag = cli.StringFlag{ - Name: "token", - Value: "", - Usage: "Your account agent token", - EnvVar: "BUILDKITE_AGENT_TOKEN", -} + AgentRegisterTokenFlag = cli.StringFlag{ + Name: "token", + Value: "", + Usage: "Your account agent token", + EnvVar: "BUILDKITE_AGENT_TOKEN", + } -var EndpointFlag = cli.StringFlag{ - Name: "endpoint", - Value: DefaultEndpoint, - Usage: "The Agent API endpoint", - EnvVar: "BUILDKITE_AGENT_ENDPOINT", -} + EndpointFlag = cli.StringFlag{ + Name: "endpoint", + Value: DefaultEndpoint, + Usage: "The Agent API endpoint", + EnvVar: "BUILDKITE_AGENT_ENDPOINT", + } -var NoHTTP2Flag = cli.BoolFlag{ - Name: "no-http2", - Usage: "Disable HTTP2 when communicating with the Agent API.", - EnvVar: "BUILDKITE_NO_HTTP2", -} + NoHTTP2Flag = cli.BoolFlag{ + Name: "no-http2", + Usage: "Disable HTTP2 when communicating with the Agent API.", + EnvVar: "BUILDKITE_NO_HTTP2", + } -var DebugFlag = cli.BoolFlag{ - Name: "debug", - Usage: "Enable debug mode. Synonym for ′--log-level debug′. Takes precedence over ′--log-level′", - EnvVar: "BUILDKITE_AGENT_DEBUG", -} + DebugFlag = cli.BoolFlag{ + Name: "debug", + Usage: "Enable debug mode. Synonym for ′--log-level debug′. Takes precedence over ′--log-level′", + EnvVar: "BUILDKITE_AGENT_DEBUG", + } -var LogLevelFlag = cli.StringFlag{ - Name: "log-level", - Value: "notice", - Usage: "Set the log level for the agent, making logging more or less verbose. Defaults to notice. Allowed values are: debug, info, error, warn, fatal", - EnvVar: "BUILDKITE_AGENT_LOG_LEVEL", -} + LogLevelFlag = cli.StringFlag{ + Name: "log-level", + Value: "notice", + Usage: "Set the log level for the agent, making logging more or less verbose. Defaults to notice. Allowed values are: debug, info, error, warn, fatal", + EnvVar: "BUILDKITE_AGENT_LOG_LEVEL", + } -var ProfileFlag = cli.StringFlag{ - Name: "profile", - Usage: "Enable a profiling mode, either cpu, memory, mutex or block", - EnvVar: "BUILDKITE_AGENT_PROFILE", -} + ProfileFlag = cli.StringFlag{ + Name: "profile", + Usage: "Enable a profiling mode, either cpu, memory, mutex or block", + EnvVar: "BUILDKITE_AGENT_PROFILE", + } -var DebugHTTPFlag = cli.BoolFlag{ - Name: "debug-http", - Usage: "Enable HTTP debug mode, which dumps all request and response bodies to the log", - EnvVar: "BUILDKITE_AGENT_DEBUG_HTTP", -} + DebugHTTPFlag = cli.BoolFlag{ + Name: "debug-http", + Usage: "Enable HTTP debug mode, which dumps all request and response bodies to the log", + EnvVar: "BUILDKITE_AGENT_DEBUG_HTTP", + } -var NoColorFlag = cli.BoolFlag{ - Name: "no-color", - Usage: "Don't show colors in logging", - EnvVar: "BUILDKITE_AGENT_NO_COLOR", -} + NoColorFlag = cli.BoolFlag{ + Name: "no-color", + Usage: "Don't show colors in logging", + EnvVar: "BUILDKITE_AGENT_NO_COLOR", + } -var StrictSingleHooksFlag = cli.BoolFlag{ - Name: "strict-single-hooks", - Usage: "Enforces that only one checkout hook, and only one command hook, can be run", - EnvVar: "BUILDKITE_STRICT_SINGLE_HOOKS", -} + StrictSingleHooksFlag = cli.BoolFlag{ + Name: "strict-single-hooks", + Usage: "Enforces that only one checkout hook, and only one command hook, can be run", + EnvVar: "BUILDKITE_STRICT_SINGLE_HOOKS", + } -var ExperimentsFlag = cli.StringSliceFlag{ - Name: "experiment", - Value: &cli.StringSlice{}, - Usage: "Enable experimental features within the buildkite-agent", - EnvVar: "BUILDKITE_AGENT_EXPERIMENT", -} + KubernetesExecFlag = cli.BoolFlag{ + Name: "kubernetes-exec", + Usage: "This is intended to be used only by the Buildkite k8s stack " + + "(github.com/buildkite/agent-stack-k8s); it enables a Unix socket for transporting " + + "logs and exit statuses between containers in a pod", + EnvVar: "BUILDKITE_KUBERNETES_EXEC", + } -var RedactedVars = cli.StringSliceFlag{ - Name: "redacted-vars", - Usage: "Pattern of environment variable names containing sensitive values", - EnvVar: "BUILDKITE_REDACTED_VARS", - Value: &cli.StringSlice{ - "*_PASSWORD", - "*_SECRET", - "*_TOKEN", - "*_PRIVATE_KEY", - "*_ACCESS_KEY", - "*_SECRET_KEY", - // Connection strings frequently contain passwords, e.g. - // https://user:pass@host/ or Server=foo;Database=my-db;User Id=user;Password=pass; - "*_CONNECTION_STRING", - }, -} + ExperimentsFlag = cli.StringSliceFlag{ + Name: "experiment", + Value: &cli.StringSlice{}, + Usage: "Enable experimental features within the buildkite-agent", + EnvVar: "BUILDKITE_AGENT_EXPERIMENT", + } + + RedactedVars = cli.StringSliceFlag{ + Name: "redacted-vars", + Usage: "Pattern of environment variable names containing sensitive values", + EnvVar: "BUILDKITE_REDACTED_VARS", + Value: &cli.StringSlice{ + "*_PASSWORD", + "*_SECRET", + "*_TOKEN", + "*_PRIVATE_KEY", + "*_ACCESS_KEY", + "*_SECRET_KEY", + // Connection strings frequently contain passwords, e.g. + // https://user:pass@host/ or Server=foo;Database=my-db;User Id=user;Password=pass; + "*_CONNECTION_STRING", + }, + } +) func globalFlags() []cli.Flag { return []cli.Flag{ diff --git a/internal/experiments/experiments.go b/internal/experiments/experiments.go index 39c6f6f850..234b0c7deb 100644 --- a/internal/experiments/experiments.go +++ b/internal/experiments/experiments.go @@ -26,7 +26,6 @@ const ( AgentAPI = "agent-api" DescendingSpawnPriority = "descending-spawn-priority" InterpolationPrefersRuntimeEnv = "interpolation-prefers-runtime-env" - KubernetesExec = "kubernetes-exec" NormalisedUploadPaths = "normalised-upload-paths" OverrideZeroExitOnCancel = "override-zero-exit-on-cancel" PTYRaw = "pty-raw" @@ -42,6 +41,7 @@ const ( InbuiltStatusPage = "inbuilt-status-page" IsolatedPluginCheckout = "isolated-plugin-checkout" JobAPI = "job-api" + KubernetesExec = "kubernetes-exec" ) var ( @@ -49,7 +49,6 @@ var ( AgentAPI: {}, DescendingSpawnPriority: {}, InterpolationPrefersRuntimeEnv: {}, - KubernetesExec: {}, NormalisedUploadPaths: {}, OverrideZeroExitOnCancel: {}, PolyglotHooks: {}, @@ -65,6 +64,7 @@ var ( InbuiltStatusPage: standardPromotionMsg(InbuiltStatusPage, "v3.48.0"), IsolatedPluginCheckout: standardPromotionMsg(IsolatedPluginCheckout, "v3.67.0"), JobAPI: standardPromotionMsg(JobAPI, "v3.64.0"), + KubernetesExec: "The kubernetes-exec experiment has been replaced with the --kubernetes-exec flag as of agent v3.74.0", } // Used to track experiments possibly in use. diff --git a/internal/job/config.go b/internal/job/config.go index a19fae9c3d..b962b8bb44 100644 --- a/internal/job/config.go +++ b/internal/job/config.go @@ -168,6 +168,9 @@ type ExecutorConfig struct { // Whether to start the JobAPI JobAPI bool + // Whether to connect to the Kubernetes socket + KubernetesExec bool + // The warnings that have been disabled by the user DisabledWarnings []string } diff --git a/internal/job/executor.go b/internal/job/executor.go index 4e7db32e03..6e99d1503a 100644 --- a/internal/job/executor.go +++ b/internal/job/executor.go @@ -100,7 +100,7 @@ func (e *Executor) Run(ctx context.Context) (exitCode int) { e.shell.InterruptSignal = e.ExecutorConfig.CancelSignal e.shell.SignalGracePeriod = e.ExecutorConfig.SignalGracePeriod } - if experiments.IsEnabled(ctx, experiments.KubernetesExec) { + if e.KubernetesExec { kubernetesClient := &kubernetes.Client{} if err := e.startKubernetesClient(ctx, kubernetesClient); err != nil { e.shell.Errorf("Failed to start kubernetes client: %v", err) From fc0bad12dd9eb4725db114503867762d085020e0 Mon Sep 17 00:00:00 2001 From: Josh Deprez Date: Thu, 6 Jun 2024 12:56:17 +1000 Subject: [PATCH 2/2] Enable k8s tags only for k8s exec --- agent/tags.go | 3 ++- clicommand/agent_start.go | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/agent/tags.go b/agent/tags.go index 9925f37ce6..f7dd535292 100644 --- a/agent/tags.go +++ b/agent/tags.go @@ -18,6 +18,7 @@ import ( type FetchTagsConfig struct { Tags []string + TagsFromK8s bool TagsFromEC2MetaData bool TagsFromEC2MetaDataPaths []string TagsFromEC2Tags bool @@ -77,7 +78,7 @@ type tagFetcher struct { func (t *tagFetcher) Fetch(ctx context.Context, l logger.Logger, conf FetchTagsConfig) []string { tags := conf.Tags - if t.k8s != nil { + if conf.TagsFromK8s { k8sTags, err := t.k8s() if err != nil { l.Warn("Could not fetch tags from k8s: %s", err) diff --git a/clicommand/agent_start.go b/clicommand/agent_start.go index 8d48e76d1e..0702196823 100644 --- a/clicommand/agent_start.go +++ b/clicommand/agent_start.go @@ -1065,6 +1065,7 @@ var AgentStartCommand = cli.Command{ tags := agent.FetchTags(ctx, l, agent.FetchTagsConfig{ Tags: cfg.Tags, + TagsFromK8s: cfg.KuberentesExec, TagsFromEC2MetaData: (cfg.TagsFromEC2MetaData || cfg.TagsFromEC2), TagsFromEC2MetaDataPaths: cfg.TagsFromEC2MetaDataPaths, TagsFromEC2Tags: cfg.TagsFromEC2Tags,