Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

kubernetes-exec is now a flag #2814

Merged
merged 2 commits into from
Jun 6, 2024
Merged

kubernetes-exec is now a flag #2814

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
33e5aa2
kubernetes-exec is now a flag
DrJosh9000 Jun 6, 2024
fc0bad1
Enable k8s tags only for k8s exec
DrJosh9000 Jun 6, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 0 additions & 10 deletions EXPERIMENTS.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -37,16 +37,6 @@ After repository checkout, resolve `BUILDKITE_COMMIT` to a commit hash. This mak

**Status**: broadly useful, we'd like this to be the standard behaviour in 4.0. 👍👍

### `kubernetes-exec`

Modifies `start` and `bootstrap` in such a way that they can run in separate Kubernetes containers in the same pod.

Currently, this experiment is being used by [agent-stack-k8s](https://github.com/buildkite/agent-stack-k8s).

This will result in errors unless orchestrated in a similar manner to that project. Please see the [README](https://github.com/buildkite/agent-stack-k8s/blob/main/README.md) of that repository for more details.

**Status**: Being used in a preview release of agent-stack-k8s. As it has little applicability outside of Kubernetes, this will not be the default behaviour.

### `polyglot-hooks`

Allows the agent to run hooks written in languages other than bash. This enables the agent to run hooks written in any language, as long as the language has a runtime available on the agent. Polyglot hooks can be in interpreted languages, so long as they have a valid shebang, and the interpreter specified in the shebang is installed on the agent.
Expand Down
1 change: 1 addition & 0 deletions agent/agent_configuration.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,6 +35,7 @@ type AgentConfiguration struct {
LocalHooksEnabled bool
StrictSingleHooks bool
RunInPty bool
KubernetesExec bool

SigningJWKSFile string // Where to find the key to sign pipeline uploads with (passed through to jobs, they might be uploading pipelines)
SigningJWKSKeyID string // The key ID to sign pipeline uploads with
Expand Down
1 change: 1 addition & 0 deletions agent/agent_worker.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -687,6 +687,7 @@ func (a *AgentWorker) RunJob(ctx context.Context, acceptResponse *api.Job) error
JobStatusInterval: time.Duration(a.agent.JobStatusInterval) * time.Second,
AgentConfiguration: a.agentConfiguration,
AgentStdout: a.agentStdout,
KubernetesExec: a.agentConfiguration.KubernetesExec,
})
if err != nil {
return fmt.Errorf("Failed to initialize job: %w", err)
Expand Down
38 changes: 24 additions & 14 deletions agent/job_runner.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -50,28 +50,30 @@ const (
// Certain env can only be set by agent configuration.
// We show the user a warning in the bootstrap if they use any of these at a job level.
var ProtectedEnv = map[string]struct{}{
"BUILDKITE_AGENT_ENDPOINT": {},
"BUILDKITE_AGENT_ACCESS_TOKEN": {},
"BUILDKITE_AGENT_DEBUG": {},
"BUILDKITE_AGENT_ENDPOINT": {},
"BUILDKITE_AGENT_PID": {},
"BUILDKITE_BIN_PATH": {},
"BUILDKITE_CONFIG_PATH": {},
"BUILDKITE_BUILD_PATH": {},
"BUILDKITE_GIT_MIRRORS_PATH": {},
"BUILDKITE_GIT_MIRRORS_SKIP_UPDATE": {},
"BUILDKITE_HOOKS_PATH": {},
"BUILDKITE_PLUGINS_PATH": {},
"BUILDKITE_SSH_KEYSCAN": {},
"BUILDKITE_GIT_SUBMODULES": {},
"BUILDKITE_COMMAND_EVAL": {},
"BUILDKITE_PLUGINS_ENABLED": {},
"BUILDKITE_LOCAL_HOOKS_ENABLED": {},
"BUILDKITE_CONFIG_PATH": {},
"BUILDKITE_CONTAINER_COUNT": {},
"BUILDKITE_GIT_CLEAN_FLAGS": {},
"BUILDKITE_GIT_CLONE_FLAGS": {},
"BUILDKITE_GIT_FETCH_FLAGS": {},
"BUILDKITE_GIT_CLONE_MIRROR_FLAGS": {},
"BUILDKITE_GIT_FETCH_FLAGS": {},
"BUILDKITE_GIT_MIRRORS_LOCK_TIMEOUT": {},
"BUILDKITE_GIT_CLEAN_FLAGS": {},
"BUILDKITE_GIT_MIRRORS_PATH": {},
"BUILDKITE_GIT_MIRRORS_SKIP_UPDATE": {},
"BUILDKITE_GIT_SUBMODULES": {},
"BUILDKITE_HOOKS_PATH": {},
"BUILDKITE_KUBERNETES_EXEC": {},
"BUILDKITE_LOCAL_HOOKS_ENABLED": {},
"BUILDKITE_PLUGINS_ENABLED": {},
"BUILDKITE_PLUGINS_PATH": {},
"BUILDKITE_SHELL": {},
"BUILDKITE_SSH_KEYSCAN": {},
}

type JobRunnerConfig struct {
Expand Down Expand Up @@ -99,6 +101,9 @@ type JobRunnerConfig struct {
// Whether to set debug HTTP Requests in the job
DebugHTTP bool

// Whether the job is executing as a k8s pod
KubernetesExec bool

// Stdout of the parent agent process. Used for job log stdout writing arg, for simpler containerized log collection.
AgentStdout io.Writer
}
Expand Down Expand Up @@ -335,7 +340,8 @@ func NewJobRunner(ctx context.Context, l logger.Logger, apiClient APIClient, con
processEnv := append(os.Environ(), env...)

// The process that will run the bootstrap script
if experiments.IsEnabled(ctx, experiments.KubernetesExec) {
if conf.KubernetesExec {
// Thank you Mario, but our bootstrap is in another container
containerCount, err := strconv.Atoi(os.Getenv("BUILDKITE_CONTAINER_COUNT"))
if err != nil {
return nil, fmt.Errorf("failed to parse BUILDKITE_CONTAINER_COUNT: %w", err)
Expand All @@ -346,7 +352,7 @@ func NewJobRunner(ctx context.Context, l logger.Logger, apiClient APIClient, con
Stderr: r.jobLogs,
ClientCount: containerCount,
})
} else {
} else { // not Kubernetes
// The bootstrap-script gets parsed based on the operating system
cmd, err := shellwords.Split(conf.AgentConfiguration.BootstrapScript)
if err != nil {
Expand Down Expand Up @@ -488,6 +494,10 @@ func (r *JobRunner) createEnvironment(ctx context.Context) ([]string, error) {
env["BUILDKITE_STRICT_SINGLE_HOOKS"] = fmt.Sprintf("%t", r.conf.AgentConfiguration.StrictSingleHooks)
env["BUILDKITE_SIGNAL_GRACE_PERIOD_SECONDS"] = fmt.Sprintf("%d", int(r.conf.AgentConfiguration.SignalGracePeriod/time.Second))

if r.conf.KubernetesExec {
env["BUILDKITE_KUBERNETES_EXEC"] = "true"
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This seems to be new. Just for my curiosity, what is this impacting?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Previously, if kubernetes-exec were enabled (as an experiment), it would be passed down to the subprocess via the BUILDKITE_AGENT_EXPERIMENTS variable. Now it could be set in the parent process via a flag, we should ensure that if enabled it should be passed to the subprocess.

But we'll make the k8s controller set this anyway, so it's probably unnecessary.

}

// propagate CancelSignal to bootstrap, unless it's the default SIGTERM
if r.conf.CancelSignal != process.SIGTERM {
env["BUILDKITE_CANCEL_SIGNAL"] = r.conf.CancelSignal.String()
Expand Down
3 changes: 1 addition & 2 deletions agent/tags.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,6 @@ import (
"strings"
"time"

"github.com/buildkite/agent/v3/internal/experiments"
"github.com/buildkite/agent/v3/logger"
"github.com/buildkite/roko"
"github.com/denisbrodbeck/machineid"
Expand Down Expand Up @@ -78,7 +77,7 @@ type tagFetcher struct {
func (t *tagFetcher) Fetch(ctx context.Context, l logger.Logger, conf FetchTagsConfig) []string {
tags := conf.Tags

if experiments.IsEnabled(ctx, experiments.KubernetesExec) {
if t.k8s != nil {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is this line right? the t.k8s seems to be set always?

Copy link
Contributor Author

@DrJosh9000 DrJosh9000 Jun 6, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Good point, on one hand this is defensive in case we set t.k8s optionally, on the other YAGNI.

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I changed it so that it used the same logic as before (only run t.k8s to create tags if kubernetes-exec is enabled).

k8sTags, err := t.k8s()
if err != nil {
l.Warn("Could not fetch tags from k8s: %s", err)
Expand Down
3 changes: 3 additions & 0 deletions clicommand/agent_start.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -165,6 +165,7 @@ type AgentStartConfig struct {
Experiments []string `cli:"experiment" normalize:"list"`
Profile string `cli:"profile"`
StrictSingleHooks bool `cli:"strict-single-hooks"`
KuberentesExec bool `cli:"kubernetes-exec"`
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Heads up there's a typo here KuberentesExec

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

lol, whoops. i've just put a PR in to fix it, but there should be no impact


// API config
DebugHTTP bool `cli:"debug-http"`
Expand Down Expand Up @@ -680,6 +681,7 @@ var AgentStartCommand = cli.Command{
ProfileFlag,
RedactedVars,
StrictSingleHooksFlag,
KubernetesExecFlag,

// Deprecated flags which will be removed in v4
cli.StringSliceFlag{
Expand Down Expand Up @@ -955,6 +957,7 @@ var AgentStartCommand = cli.Command{
TracingBackend: cfg.TracingBackend,
TracingServiceName: cfg.TracingServiceName,
VerificationFailureBehaviour: cfg.VerificationFailureBehavior,
KubernetesExec: cfg.KuberentesExec,

SigningJWKSFile: cfg.SigningJWKSFile,
SigningJWKSKeyID: cfg.SigningJWKSKeyID,
Expand Down
3 changes: 3 additions & 0 deletions clicommand/bootstrap.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -97,6 +97,7 @@ type BootstrapConfig struct {
TracingServiceName string `cli:"tracing-service-name"`
NoJobAPI bool `cli:"no-job-api"`
DisableWarningsFor []string `cli:"disable-warnings-for" normalize:"list"`
KubernetesExec bool `cli:"kubernetes-exec"`
}

var BootstrapCommand = cli.Command{
Expand Down Expand Up @@ -371,6 +372,7 @@ var BootstrapCommand = cli.Command{
ProfileFlag,
RedactedVars,
StrictSingleHooksFlag,
KubernetesExecFlag,
},
Action: func(c *cli.Context) error {
ctx := context.Background()
Expand Down Expand Up @@ -452,6 +454,7 @@ var BootstrapCommand = cli.Command{
TracingServiceName: cfg.TracingServiceName,
JobAPI: !cfg.NoJobAPI,
DisabledWarnings: cfg.DisableWarningsFor,
KubernetesExec: cfg.KubernetesExec,
})

cctx, cancel := context.WithCancel(ctx)
Expand Down
162 changes: 86 additions & 76 deletions clicommand/global.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,93 +21,103 @@ const (
DefaultEndpoint = "https://agent.buildkite.com/v3"
)

var AgentAccessTokenFlag = cli.StringFlag{
Name: "agent-access-token",
Value: "",
Usage: "The access token used to identify the agent",
EnvVar: "BUILDKITE_AGENT_ACCESS_TOKEN",
}
var (
AgentAccessTokenFlag = cli.StringFlag{
Name: "agent-access-token",
Value: "",
Usage: "The access token used to identify the agent",
EnvVar: "BUILDKITE_AGENT_ACCESS_TOKEN",
}

var AgentRegisterTokenFlag = cli.StringFlag{
Name: "token",
Value: "",
Usage: "Your account agent token",
EnvVar: "BUILDKITE_AGENT_TOKEN",
}
AgentRegisterTokenFlag = cli.StringFlag{
Name: "token",
Value: "",
Usage: "Your account agent token",
EnvVar: "BUILDKITE_AGENT_TOKEN",
}

var EndpointFlag = cli.StringFlag{
Name: "endpoint",
Value: DefaultEndpoint,
Usage: "The Agent API endpoint",
EnvVar: "BUILDKITE_AGENT_ENDPOINT",
}
EndpointFlag = cli.StringFlag{
Name: "endpoint",
Value: DefaultEndpoint,
Usage: "The Agent API endpoint",
EnvVar: "BUILDKITE_AGENT_ENDPOINT",
}

var NoHTTP2Flag = cli.BoolFlag{
Name: "no-http2",
Usage: "Disable HTTP2 when communicating with the Agent API.",
EnvVar: "BUILDKITE_NO_HTTP2",
}
NoHTTP2Flag = cli.BoolFlag{
Name: "no-http2",
Usage: "Disable HTTP2 when communicating with the Agent API.",
EnvVar: "BUILDKITE_NO_HTTP2",
}

var DebugFlag = cli.BoolFlag{
Name: "debug",
Usage: "Enable debug mode. Synonym for ′--log-level debug′. Takes precedence over ′--log-level′",
EnvVar: "BUILDKITE_AGENT_DEBUG",
}
DebugFlag = cli.BoolFlag{
Name: "debug",
Usage: "Enable debug mode. Synonym for ′--log-level debug′. Takes precedence over ′--log-level′",
EnvVar: "BUILDKITE_AGENT_DEBUG",
}

var LogLevelFlag = cli.StringFlag{
Name: "log-level",
Value: "notice",
Usage: "Set the log level for the agent, making logging more or less verbose. Defaults to notice. Allowed values are: debug, info, error, warn, fatal",
EnvVar: "BUILDKITE_AGENT_LOG_LEVEL",
}
LogLevelFlag = cli.StringFlag{
Name: "log-level",
Value: "notice",
Usage: "Set the log level for the agent, making logging more or less verbose. Defaults to notice. Allowed values are: debug, info, error, warn, fatal",
EnvVar: "BUILDKITE_AGENT_LOG_LEVEL",
}

var ProfileFlag = cli.StringFlag{
Name: "profile",
Usage: "Enable a profiling mode, either cpu, memory, mutex or block",
EnvVar: "BUILDKITE_AGENT_PROFILE",
}
ProfileFlag = cli.StringFlag{
Name: "profile",
Usage: "Enable a profiling mode, either cpu, memory, mutex or block",
EnvVar: "BUILDKITE_AGENT_PROFILE",
}

var DebugHTTPFlag = cli.BoolFlag{
Name: "debug-http",
Usage: "Enable HTTP debug mode, which dumps all request and response bodies to the log",
EnvVar: "BUILDKITE_AGENT_DEBUG_HTTP",
}
DebugHTTPFlag = cli.BoolFlag{
Name: "debug-http",
Usage: "Enable HTTP debug mode, which dumps all request and response bodies to the log",
EnvVar: "BUILDKITE_AGENT_DEBUG_HTTP",
}

var NoColorFlag = cli.BoolFlag{
Name: "no-color",
Usage: "Don't show colors in logging",
EnvVar: "BUILDKITE_AGENT_NO_COLOR",
}
NoColorFlag = cli.BoolFlag{
Name: "no-color",
Usage: "Don't show colors in logging",
EnvVar: "BUILDKITE_AGENT_NO_COLOR",
}

var StrictSingleHooksFlag = cli.BoolFlag{
Name: "strict-single-hooks",
Usage: "Enforces that only one checkout hook, and only one command hook, can be run",
EnvVar: "BUILDKITE_STRICT_SINGLE_HOOKS",
}
StrictSingleHooksFlag = cli.BoolFlag{
Name: "strict-single-hooks",
Usage: "Enforces that only one checkout hook, and only one command hook, can be run",
EnvVar: "BUILDKITE_STRICT_SINGLE_HOOKS",
}

var ExperimentsFlag = cli.StringSliceFlag{
Name: "experiment",
Value: &cli.StringSlice{},
Usage: "Enable experimental features within the buildkite-agent",
EnvVar: "BUILDKITE_AGENT_EXPERIMENT",
}
KubernetesExecFlag = cli.BoolFlag{
Name: "kubernetes-exec",
Usage: "This is intended to be used only by the Buildkite k8s stack " +
"(github.com/buildkite/agent-stack-k8s); it enables a Unix socket for transporting " +
"logs and exit statuses between containers in a pod",
EnvVar: "BUILDKITE_KUBERNETES_EXEC",
}

var RedactedVars = cli.StringSliceFlag{
Name: "redacted-vars",
Usage: "Pattern of environment variable names containing sensitive values",
EnvVar: "BUILDKITE_REDACTED_VARS",
Value: &cli.StringSlice{
"*_PASSWORD",
"*_SECRET",
"*_TOKEN",
"*_PRIVATE_KEY",
"*_ACCESS_KEY",
"*_SECRET_KEY",
// Connection strings frequently contain passwords, e.g.
// https://user:pass@host/ or Server=foo;Database=my-db;User Id=user;Password=pass;
"*_CONNECTION_STRING",
},
}
ExperimentsFlag = cli.StringSliceFlag{
Name: "experiment",
Value: &cli.StringSlice{},
Usage: "Enable experimental features within the buildkite-agent",
EnvVar: "BUILDKITE_AGENT_EXPERIMENT",
}

RedactedVars = cli.StringSliceFlag{
Name: "redacted-vars",
Usage: "Pattern of environment variable names containing sensitive values",
EnvVar: "BUILDKITE_REDACTED_VARS",
Value: &cli.StringSlice{
"*_PASSWORD",
"*_SECRET",
"*_TOKEN",
"*_PRIVATE_KEY",
"*_ACCESS_KEY",
"*_SECRET_KEY",
// Connection strings frequently contain passwords, e.g.
// https://user:pass@host/ or Server=foo;Database=my-db;User Id=user;Password=pass;
"*_CONNECTION_STRING",
},
}
)

func globalFlags() []cli.Flag {
return []cli.Flag{
Expand Down
4 changes: 2 additions & 2 deletions internal/experiments/experiments.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,6 @@ const (
AgentAPI = "agent-api"
DescendingSpawnPriority = "descending-spawn-priority"
InterpolationPrefersRuntimeEnv = "interpolation-prefers-runtime-env"
KubernetesExec = "kubernetes-exec"
NormalisedUploadPaths = "normalised-upload-paths"
OverrideZeroExitOnCancel = "override-zero-exit-on-cancel"
PTYRaw = "pty-raw"
Expand All @@ -42,14 +41,14 @@ const (
InbuiltStatusPage = "inbuilt-status-page"
IsolatedPluginCheckout = "isolated-plugin-checkout"
JobAPI = "job-api"
KubernetesExec = "kubernetes-exec"
)

var (
Available = map[string]struct{}{
AgentAPI: {},
DescendingSpawnPriority: {},
InterpolationPrefersRuntimeEnv: {},
KubernetesExec: {},
NormalisedUploadPaths: {},
OverrideZeroExitOnCancel: {},
PolyglotHooks: {},
Expand All @@ -65,6 +64,7 @@ var (
InbuiltStatusPage: standardPromotionMsg(InbuiltStatusPage, "v3.48.0"),
IsolatedPluginCheckout: standardPromotionMsg(IsolatedPluginCheckout, "v3.67.0"),
JobAPI: standardPromotionMsg(JobAPI, "v3.64.0"),
KubernetesExec: "The kubernetes-exec experiment has been replaced with the --kubernetes-exec flag as of agent v3.74.0",
}

// Used to track experiments possibly in use.
Expand Down
Loading