From a2d45ed75c3c81a4f8e50be7ccce3003d56422b8 Mon Sep 17 00:00:00 2001 From: Marin Dzhigarov Date: Mon, 13 Jan 2025 14:47:28 +0200 Subject: [PATCH] Bumps gexe to 0.4.0 (#270) * Bumps gexe to 0.4.0 Additional fixes: - Fixes a bug in ssh.go where the retry logic did not work due to using StartProc instead of RunProc in the backoff callback - Fixes the ssh key setup on Mac where id_rsa private key was getting created with wrong permissions * Bumps checkout action version * Fix how we pass env vars to the ssh agent. See also https://github.com/vladimirvivien/gexe/issues/64 --------- Co-authored-by: Marin Dzhigarov --- .ci/build/build.go | 1 - .github/workflows/compile-test.yaml | 6 +++--- go.mod | 5 +++-- go.sum | 16 +++++----------- k8s/client.go | 3 ++- k8s/kube_config.go | 2 +- ssh/agent.go | 18 +++++++++++------- ssh/agent_test.go | 12 ++++++++---- ssh/scp.go | 4 ++-- ssh/ssh.go | 4 ++-- starlark/capture_local.go | 2 +- testing/key.go | 12 ++++++++++-- testing/kindcluster.go | 2 +- 13 files changed, 49 insertions(+), 38 deletions(-) diff --git a/.ci/build/build.go b/.ci/build/build.go index 8e525dfd..78414c31 100644 --- a/.ci/build/build.go +++ b/.ci/build/build.go @@ -30,7 +30,6 @@ func main() { func gobuild(arch, os, ldflags, binary string) { b := gexe.New() - b.Conf.SetPanicOnErr(true) b.SetVar("arch", arch) b.SetVar("os", os) b.SetVar("ldflags", ldflags) diff --git a/.github/workflows/compile-test.yaml b/.github/workflows/compile-test.yaml index a7eca0a6..6f36c851 100644 --- a/.github/workflows/compile-test.yaml +++ b/.github/workflows/compile-test.yaml @@ -7,13 +7,13 @@ jobs: steps: - name: Set up Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: - go-version: 1.22.0 + go-version: 1.22.1 id: go - name: Check out code into the Go module directory - uses: actions/checkout@v1 + uses: actions/checkout@v4 - name: test run: | diff --git a/go.mod b/go.mod index 1a49965a..924d56dc 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.1 - github.com/vladimirvivien/gexe v0.1.1 + github.com/vladimirvivien/gexe v0.4.0 go.starlark.net v0.0.0-20241226192728-8dfa5b98479f k8s.io/api v0.30.1 k8s.io/apimachinery v0.30.1 @@ -30,7 +30,6 @@ require ( github.com/go-openapi/swag v0.22.3 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.4 // indirect - github.com/golangci/golangci-lint v1.63.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect github.com/google/go-cmp v0.6.0 // indirect github.com/google/gofuzz v1.2.0 // indirect @@ -46,7 +45,9 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/nxadm/tail v1.4.8 // indirect + github.com/rogpeppe/go-internal v1.13.1 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/testify v1.10.0 // indirect golang.org/x/net v0.33.0 // indirect golang.org/x/oauth2 v0.10.0 // indirect golang.org/x/sys v0.28.0 // indirect diff --git a/go.sum b/go.sum index 06dec7ba..f758734a 100644 --- a/go.sum +++ b/go.sum @@ -11,8 +11,6 @@ github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxER github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= -github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= -github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= @@ -39,8 +37,6 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= -github.com/golangci/golangci-lint v1.63.4 h1:bJQFQ3hSfUto597dkL7ipDzOxsGEpiWdLiZ359OWOBI= -github.com/golangci/golangci-lint v1.63.4/go.mod h1:Hx0B7Lg5/NXbaOHem8+KU+ZUIzMI6zNj/7tFwdnn10I= github.com/google/gnostic-models v0.6.8 h1:yo/ABAfM5IMRsS1VnXjTBvUb61tFIHozhlYvRgGre9I= github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYuF5OpfBSENuI8U= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -104,9 +100,8 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= +github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -123,10 +118,10 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/vladimirvivien/gexe v0.1.1 h1:2A0SBaOSKH+cwLVdt6H+KkHZotZWRNLlWygANGw5DxE= -github.com/vladimirvivien/gexe v0.1.1/go.mod h1:LHQL00w/7gDUKIak24n801ABp8C+ni6eBht9vGVst8w= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= +github.com/vladimirvivien/gexe v0.4.0 h1:yk51bQu4HRlkt+MzXGQbSucvg7VIyOU4U0fR+awNN6c= +github.com/vladimirvivien/gexe v0.4.0/go.mod h1:fp7cy60ON1xjhtEI/+bfSEIXX35qgmI+iRYlGOqbBFM= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.starlark.net v0.0.0-20241226192728-8dfa5b98479f h1:Zs/py28HDFATSDzPcfIzrBFjVsV7HzDEGNNVZIGsjm0= @@ -161,7 +156,6 @@ golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= diff --git a/k8s/client.go b/k8s/client.go index 057083f9..04ecc240 100644 --- a/k8s/client.go +++ b/k8s/client.go @@ -6,9 +6,10 @@ package k8s import ( "context" "fmt" - "k8s.io/apimachinery/pkg/api/errors" "strings" + "k8s.io/apimachinery/pkg/api/errors" + "github.com/sirupsen/logrus" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" diff --git a/k8s/kube_config.go b/k8s/kube_config.go index 19a3dc49..707e8b24 100644 --- a/k8s/kube_config.go +++ b/k8s/kube_config.go @@ -17,7 +17,7 @@ import ( func FetchWorkloadConfig(clusterName, clusterNamespace, mgmtKubeConfigPath string) (string, error) { var filePath string cmdStr := fmt.Sprintf(`kubectl get secrets/%s-kubeconfig --template '{{.data.value}}' --namespace=%s --kubeconfig %s`, clusterName, clusterNamespace, mgmtKubeConfigPath) - p := gexe.StartProc(cmdStr) + p := gexe.RunProc(cmdStr) if p.Err() != nil { return filePath, fmt.Errorf("kubectl get secrets failed: %s: %s", p.Err(), p.Result()) } diff --git a/ssh/agent.go b/ssh/agent.go index d7908964..a53e2868 100644 --- a/ssh/agent.go +++ b/ssh/agent.go @@ -24,7 +24,7 @@ type Agent interface { AddKey(keyPath string) error RemoveKey(keyPath string) error Stop() error - GetEnvVariables() string + GetEnvVariables() []string } // agentInfo captures the connection information of the ssh-agent @@ -50,7 +50,7 @@ func (agent *agent) AddKey(keyPath string) error { return errors.New("ssh-add not found") } - p := e.Envs(agent.GetEnvVariables()). + p := e.Envs(agent.GetEnvVariables()...). RunProc(fmt.Sprintf("%s %s", sshAddCmd, keyPath)) if err := p.Err(); err != nil { return errors.Wrapf(err, "could not add key %s to ssh-agent", keyPath) @@ -67,7 +67,7 @@ func (agent *agent) RemoveKey(keyPath string) error { return errors.New("ssh-add not found") } - p := e.Envs(agent.GetEnvVariables()). + p := e.Envs(agent.GetEnvVariables()...). RunProc(fmt.Sprintf("%s -d %s", sshAddCmd, keyPath)) if err := p.Err(); err != nil { return errors.Wrapf(err, "could not add key %s to ssh-agent", keyPath) @@ -88,15 +88,19 @@ func (agent *agent) Stop() error { } logrus.Debugf("stopping the ssh-agent with Pid: %s", agent.Pid) - p := gexe.Envs(agent.GetEnvVariables()).RunProc("ssh-agent -k") + e := gexe.New() + p := e.Envs(agent.GetEnvVariables()...).RunProc("ssh-agent -k") logrus.Debugf("ssh-agent stopped: %s", p.Result()) return p.Err() } // GetEnvVariables returns the space separated key=value information used to communicate with the ssh-agent -func (agent *agent) GetEnvVariables() string { - return fmt.Sprintf("%s=%s %s=%s", AgentPidIdentifier, agent.Pid, AuthSockIdentifier, agent.AuthSockPath) +func (agent *agent) GetEnvVariables() []string { + return []string{ + fmt.Sprintf("%s=%s", AgentPidIdentifier, agent.Pid), + fmt.Sprintf("%s=%s", AuthSockIdentifier, agent.AuthSockPath), + } } // StartAgent starts the ssh-agent process and returns the SSH authentication parameters. @@ -108,7 +112,7 @@ func StartAgent() (Agent, error) { } logrus.Debugf("starting %s", sshAgentCmd) - p := e.StartProc(fmt.Sprintf("%s -s", sshAgentCmd)) + p := e.RunProc(fmt.Sprintf("%s -s", sshAgentCmd)) if p.Err() != nil { return nil, errors.Wrap(p.Err(), "failed to start ssh agent") } diff --git a/ssh/agent_test.go b/ssh/agent_test.go index 92302f65..da8f714b 100644 --- a/ssh/agent_test.go +++ b/ssh/agent_test.go @@ -132,13 +132,17 @@ func TestAgent(t *testing.T) { name: "GetEnvVariables", assert: func(t *testing.T, agent Agent) { vars := agent.GetEnvVariables() - if len(strings.Split(vars, " ")) != 2 { + if len(vars) != 2 { t.Fatalf("not enough variables") } - match, err := regexp.MatchString(`SSH_AGENT_PID=[0-9]+ SSH_AUTH_SOCK=\S*`, vars) - if err != nil || !match { - t.Fatalf("format does not match") + matchPID, err := regexp.MatchString(`SSH_AGENT_PID=[0-9]+`, vars[0]) + if err != nil || !matchPID { + t.Fatalf("SSH_AGENT_PID format does not match") + } + matchSock, err := regexp.MatchString(`SSH_AUTH_SOCK=\S*`, vars[1]) + if err != nil || !matchSock { + t.Fatalf("SSH_AUTH_SOCK format does not match") } }, }, diff --git a/ssh/scp.go b/ssh/scp.go index 9fb67103..e89d8536 100644 --- a/ssh/scp.go +++ b/ssh/scp.go @@ -47,7 +47,7 @@ func CopyFrom(args SSHArgs, agent Agent, rootDir string, sourcePath string) erro if agent != nil { logrus.Debugf("scp: copyFrom: adding agent info: %s", agent.GetEnvVariables()) - e = e.Envs(agent.GetEnvVariables()) + e = e.Envs(agent.GetEnvVariables()...) } maxRetries := args.MaxRetries @@ -97,7 +97,7 @@ func CopyTo(args SSHArgs, agent Agent, sourcePath, targetPath string) error { if agent != nil { logrus.Debugf("scp: adding agent info: %s", agent.GetEnvVariables()) - e = e.Envs(agent.GetEnvVariables()) + e = e.Envs(agent.GetEnvVariables()...) } maxRetries := args.MaxRetries diff --git a/ssh/ssh.go b/ssh/ssh.go index 769c4fc1..02d405bd 100644 --- a/ssh/ssh.go +++ b/ssh/ssh.go @@ -64,7 +64,7 @@ func sshRunProc(args SSHArgs, agent Agent, cmd string) (io.Reader, error) { if agent != nil { logrus.Debugf("Adding agent info: %s", agent.GetEnvVariables()) - e = e.Envs(agent.GetEnvVariables()) + e = e.Envs(agent.GetEnvVariables()...) } var proc *exec.Proc @@ -74,7 +74,7 @@ func sshRunProc(args SSHArgs, agent Agent, cmd string) (io.Reader, error) { } retries := wait.Backoff{Steps: maxRetries, Duration: time.Millisecond * 80, Jitter: 0.1} if err := wait.ExponentialBackoff(retries, func() (bool, error) { - p := e.StartProc(effectiveCmd) + p := e.RunProc(effectiveCmd) if p.Err() != nil { logrus.Warn(fmt.Sprintf("ssh: failed to connect to %s: error '%s %s': retrying connection", args.Host, p.Err(), p.Result())) return false, p.Err() diff --git a/starlark/capture_local.go b/starlark/capture_local.go index 52b7a0a0..835c6c52 100644 --- a/starlark/capture_local.go +++ b/starlark/capture_local.go @@ -47,7 +47,7 @@ func captureLocalFunc(thread *starlark.Thread, b *starlark.Builtin, args starlar return starlark.String(msg), nil } - p := gexe.StartProc(cmdStr) + p := gexe.RunProc(cmdStr) // upon error, write error in file, return filepath if p.Err() != nil { msg := fmt.Sprintf("%s error: %s: %s", identifiers.captureLocal, p.Err(), p.Result()) diff --git a/testing/key.go b/testing/key.go index b545434f..493d30fe 100644 --- a/testing/key.go +++ b/testing/key.go @@ -13,8 +13,16 @@ import ( // WriteKeys copies the static private key in variable privateKey // to a local file. func WriteKeys(rootPath string) error { + // Use 0444 on github and 0600 on local dev because for some reason the docker volume mount in the action runner requires 0444 to work with the file + isGitHubActions := os.Getenv("GITHUB_ACTIONS") == "true" + var keyPerm os.FileMode + if isGitHubActions { + keyPerm = 0444 + } else { + keyPerm = 0600 + } pkPath := filepath.Join(rootPath, "id_rsa") - pkFile, err := os.OpenFile(pkPath, os.O_RDWR|os.O_CREATE, 0444) + pkFile, err := os.OpenFile(pkPath, os.O_RDWR|os.O_CREATE, keyPerm) if err != nil { return err } @@ -25,7 +33,7 @@ func WriteKeys(rootPath string) error { } pubPath := filepath.Join(rootPath, "id_rsa.pub") - pubFile, err := os.OpenFile(pubPath, os.O_RDWR|os.O_CREATE, 0444) + pubFile, err := os.OpenFile(pubPath, os.O_RDWR|os.O_CREATE, 0644) if err != nil { return err } diff --git a/testing/kindcluster.go b/testing/kindcluster.go index 1b3827f3..934a78bd 100644 --- a/testing/kindcluster.go +++ b/testing/kindcluster.go @@ -63,7 +63,7 @@ func (k *KindCluster) MakeKubeConfigFile(path string) error { defer f.Close() logrus.Infof("Retrieving kind kubeconfig for cluster: kind get kubeconfig --name %s", k.name) - p := k.e.StartProc(fmt.Sprintf(`kind get kubeconfig --name %s`, k.name)) + p := k.e.RunProc(fmt.Sprintf(`kind get kubeconfig --name %s`, k.name)) if p.Err() != nil { return fmt.Errorf("failed to generate kind kubeconfig: %s: %s", p.Result(), p.Err()) }