Skip to content

Commit

Permalink
feat(git): log parsed gitURL and warn if local (#345)
Browse files Browse the repository at this point in the history
  • Loading branch information
@johnstcn
johnstcn authored Sep 12, 2024
1 parent df8ea67 commit e14b952
Show file tree
Hide file tree
Showing 3 changed files with 118 additions and 89 deletions.
56 changes: 30 additions & 26 deletions envbuilder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -112,22 +112,25 @@ func Run(ctx context.Context, opts options.Options) error {
var fallbackErr error
var cloned bool
if opts.GitURL != "" {
cloneOpts, err := git.CloneOptionsFromOptions(opts)
if err != nil {
return fmt.Errorf("git clone options: %w", err)
}

endStage := startStage("📦 Cloning %s to %s...",
newColor(color.FgCyan).Sprintf(opts.GitURL),
newColor(color.FgCyan).Sprintf(cloneOpts.Path),
newColor(color.FgCyan).Sprintf(opts.WorkspaceFolder),
)

stageNum := stageNumber
w := git.ProgressWriter(func(line string) { opts.Logger(log.LevelInfo, "#%d: %s", stageNum, line) })
logStage := func(format string, args ...any) {
opts.Logger(log.LevelInfo, "#%d: %s", stageNum, fmt.Sprintf(format, args...))
}

cloneOpts, err := git.CloneOptionsFromOptions(logStage, opts)
if err != nil {
return fmt.Errorf("git clone options: %w", err)
}

w := git.ProgressWriter(logStage)
defer w.Close()
cloneOpts.Progress = w

cloned, fallbackErr = git.CloneRepo(ctx, cloneOpts)
cloned, fallbackErr = git.CloneRepo(ctx, logStage, cloneOpts)
if fallbackErr == nil {
if cloned {
endStage("📦 Cloned repository!")
Expand All @@ -144,7 +147,7 @@ func Run(ctx context.Context, opts options.Options) error {
// Always clone the repo in remote repo build mode into a location that
// we control that isn't affected by the users changes.
if opts.RemoteRepoBuildMode {
cloneOpts, err := git.CloneOptionsFromOptions(opts)
cloneOpts, err := git.CloneOptionsFromOptions(logStage, opts)
if err != nil {
return fmt.Errorf("git clone options: %w", err)
}
Expand All @@ -155,12 +158,11 @@ func Run(ctx context.Context, opts options.Options) error {
newColor(color.FgCyan).Sprintf(cloneOpts.Path),
)

stageNum := stageNumber
w := git.ProgressWriter(func(line string) { opts.Logger(log.LevelInfo, "#%d: %s", stageNum, line) })
w := git.ProgressWriter(logStage)
defer w.Close()
cloneOpts.Progress = w

fallbackErr = git.ShallowCloneRepo(ctx, cloneOpts)
fallbackErr = git.ShallowCloneRepo(ctx, logStage, cloneOpts)
if fallbackErr == nil {
endStage("📦 Cloned repository!")
buildTimeWorkspaceFolder = cloneOpts.Path
Expand Down Expand Up @@ -891,25 +893,28 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error)
var fallbackErr error
var cloned bool
if opts.GitURL != "" {
endStage := startStage("📦 Cloning %s to %s...",
newColor(color.FgCyan).Sprintf(opts.GitURL),
newColor(color.FgCyan).Sprintf(opts.WorkspaceFolder),
)
stageNum := stageNumber
logStage := func(format string, args ...any) {
opts.Logger(log.LevelInfo, "#%d: %s", stageNum, fmt.Sprintf(format, args...))
}

// In cache probe mode we should only attempt to clone the full
// repository if remote repo build mode isn't enabled.
if !opts.RemoteRepoBuildMode {
cloneOpts, err := git.CloneOptionsFromOptions(opts)
cloneOpts, err := git.CloneOptionsFromOptions(logStage, opts)
if err != nil {
return nil, fmt.Errorf("git clone options: %w", err)
}

endStage := startStage("📦 Cloning %s to %s...",
newColor(color.FgCyan).Sprintf(opts.GitURL),
newColor(color.FgCyan).Sprintf(cloneOpts.Path),
)

stageNum := stageNumber
w := git.ProgressWriter(func(line string) { opts.Logger(log.LevelInfo, "#%d: %s", stageNum, line) })
w := git.ProgressWriter(logStage)
defer w.Close()
cloneOpts.Progress = w

cloned, fallbackErr = git.CloneRepo(ctx, cloneOpts)
cloned, fallbackErr = git.CloneRepo(ctx, logStage, cloneOpts)
if fallbackErr == nil {
if cloned {
endStage("📦 Cloned repository!")
Expand All @@ -923,7 +928,7 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error)

_ = w.Close()
} else {
cloneOpts, err := git.CloneOptionsFromOptions(opts)
cloneOpts, err := git.CloneOptionsFromOptions(logStage, opts)
if err != nil {
return nil, fmt.Errorf("git clone options: %w", err)
}
Expand All @@ -934,12 +939,11 @@ func RunCacheProbe(ctx context.Context, opts options.Options) (v1.Image, error)
newColor(color.FgCyan).Sprintf(cloneOpts.Path),
)

stageNum := stageNumber
w := git.ProgressWriter(func(line string) { opts.Logger(log.LevelInfo, "#%d: %s", stageNum, line) })
w := git.ProgressWriter(logStage)
defer w.Close()
cloneOpts.Progress = w

fallbackErr = git.ShallowCloneRepo(ctx, cloneOpts)
fallbackErr = git.ShallowCloneRepo(ctx, logStage, cloneOpts)
if fallbackErr == nil {
endStage("📦 Cloned repository!")
buildTimeWorkspaceFolder = cloneOpts.Path
Expand Down
68 changes: 44 additions & 24 deletions git/git.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,6 @@ import (
"github.com/coder/envbuilder/options"

giturls "github.com/chainguard-dev/git-urls"
"github.com/coder/envbuilder/log"
"github.com/go-git/go-billy/v5"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing"
Expand Down Expand Up @@ -47,11 +46,12 @@ type CloneRepoOptions struct {
// be cloned again.
//
// The bool returned states whether the repository was cloned or not.
func CloneRepo(ctx context.Context, opts CloneRepoOptions) (bool, error) {
func CloneRepo(ctx context.Context, logf func(string, ...any), opts CloneRepoOptions) (bool, error) {
parsed, err := giturls.Parse(opts.RepoURL)
if err != nil {
return false, fmt.Errorf("parse url %q: %w", opts.RepoURL, err)
}
logf("Parsed Git URL as %q", parsed.Redacted())
if parsed.Hostname() == "dev.azure.com" {
// Azure DevOps requires capabilities multi_ack / multi_ack_detailed,
// which are not fully implemented and by default are included in
Expand All @@ -73,6 +73,7 @@ func CloneRepo(ctx context.Context, opts CloneRepoOptions) (bool, error) {
transport.UnsupportedCapabilities = []capability.Capability{
capability.ThinPack,
}
logf("Workaround for Azure DevOps: marking thin-pack as unsupported")
}

err = opts.Storage.MkdirAll(opts.Path, 0o755)
Expand Down Expand Up @@ -131,7 +132,7 @@ func CloneRepo(ctx context.Context, opts CloneRepoOptions) (bool, error) {
// clone will not be performed.
//
// The bool returned states whether the repository was cloned or not.
func ShallowCloneRepo(ctx context.Context, opts CloneRepoOptions) error {
func ShallowCloneRepo(ctx context.Context, logf func(string, ...any), opts CloneRepoOptions) error {
opts.Depth = 1
opts.SingleBranch = true

Expand All @@ -150,7 +151,7 @@ func ShallowCloneRepo(ctx context.Context, opts CloneRepoOptions) error {
}
}

cloned, err := CloneRepo(ctx, opts)
cloned, err := CloneRepo(ctx, logf, opts)
if err != nil {
return err
}
Expand Down Expand Up @@ -182,14 +183,14 @@ func ReadPrivateKey(path string) (gossh.Signer, error) {

// LogHostKeyCallback is a HostKeyCallback that just logs host keys
// and does nothing else.
func LogHostKeyCallback(logger log.Func) gossh.HostKeyCallback {
func LogHostKeyCallback(logger func(string, ...any)) gossh.HostKeyCallback {
return func(hostname string, remote net.Addr, key gossh.PublicKey) error {
var sb strings.Builder
_ = knownhosts.WriteKnownHost(&sb, hostname, remote, key)
// skeema/knownhosts uses a fake public key to determine the host key
// algorithms. Ignore this one.
if s := sb.String(); !strings.Contains(s, "fake-public-key ZmFrZSBwdWJsaWMga2V5") {
logger(log.LevelInfo, "#1: 🔑 Got host key: %s", strings.TrimSpace(s))
logger("🔑 Got host key: %s", strings.TrimSpace(s))
}
return nil
}
Expand All @@ -203,6 +204,8 @@ func LogHostKeyCallback(logger log.Func) gossh.HostKeyCallback {
// | https?://host.tld/repo | Not Set | Set | HTTP Basic |
// | https?://host.tld/repo | Set | Not Set | HTTP Basic |
// | https?://host.tld/repo | Set | Set | HTTP Basic |
// | file://path/to/repo | - | - | None |
// | path/to/repo | - | - | None |
// | All other formats | - | - | SSH |
//
// For SSH authentication, the default username is "git" but will honour
Expand All @@ -214,58 +217,73 @@ func LogHostKeyCallback(logger log.Func) gossh.HostKeyCallback {
// If SSH_KNOWN_HOSTS is not set, the SSH auth method will be configured
// to accept and log all host keys. Otherwise, host key checking will be
// performed as usual.
func SetupRepoAuth(options *options.Options) transport.AuthMethod {
func SetupRepoAuth(logf func(string, ...any), options *options.Options) transport.AuthMethod {
if options.GitURL == "" {
options.Logger(log.LevelInfo, "#1: ❔ No Git URL supplied!")
logf("❔ No Git URL supplied!")
return nil
}
if strings.HasPrefix(options.GitURL, "http://") || strings.HasPrefix(options.GitURL, "https://") {
parsedURL, err := giturls.Parse(options.GitURL)
if err != nil {
logf("❌ Failed to parse Git URL: %s", err.Error())
return nil
}

if parsedURL.Scheme == "http" || parsedURL.Scheme == "https" {
// Special case: no auth
if options.GitUsername == "" && options.GitPassword == "" {
options.Logger(log.LevelInfo, "#1: 👤 Using no authentication!")
logf("👤 Using no authentication!")
return nil
}
// Basic Auth
// NOTE: we previously inserted the credentials into the repo URL.
// This was removed in https://github.com/coder/envbuilder/pull/141
options.Logger(log.LevelInfo, "#1: 🔒 Using HTTP basic authentication!")
logf("🔒 Using HTTP basic authentication!")
return &githttp.BasicAuth{
Username: options.GitUsername,
Password: options.GitPassword,
}
}

if parsedURL.Scheme == "file" {
// go-git will try to fallback to using the `git` command for local
// filesystem clones. However, it's more likely than not that the
// `git` command is not present in the container image. Log a warning
// but continue. Also, no auth.
logf("🚧 Using local filesystem clone! This requires the git executable to be present!")
return nil
}

// Generally git clones over SSH use the 'git' user, but respect
// GIT_USERNAME if set.
if options.GitUsername == "" {
options.GitUsername = "git"
}

// Assume SSH auth for all other formats.
options.Logger(log.LevelInfo, "#1: 🔑 Using SSH authentication!")
logf("🔑 Using SSH authentication!")

var signer ssh.Signer
if options.GitSSHPrivateKeyPath != "" {
s, err := ReadPrivateKey(options.GitSSHPrivateKeyPath)
if err != nil {
options.Logger(log.LevelError, "#1: ❌ Failed to read private key from %s: %s", options.GitSSHPrivateKeyPath, err.Error())
logf("❌ Failed to read private key from %s: %s", options.GitSSHPrivateKeyPath, err.Error())
} else {
options.Logger(log.LevelInfo, "#1: 🔑 Using %s key!", s.PublicKey().Type())
logf("🔑 Using %s key!", s.PublicKey().Type())
signer = s
}
}

// If no SSH key set, fall back to agent auth.
if signer == nil {
options.Logger(log.LevelError, "#1: 🔑 No SSH key found, falling back to agent!")
logf("🔑 No SSH key found, falling back to agent!")
auth, err := gitssh.NewSSHAgentAuth(options.GitUsername)
if err != nil {
options.Logger(log.LevelError, "#1: ❌ Failed to connect to SSH agent: %s", err.Error())
logf("❌ Failed to connect to SSH agent: " + err.Error())
return nil // nothing else we can do
}
if os.Getenv("SSH_KNOWN_HOSTS") == "" {
options.Logger(log.LevelWarn, "#1: 🔓 SSH_KNOWN_HOSTS not set, accepting all host keys!")
auth.HostKeyCallback = LogHostKeyCallback(options.Logger)
logf("🔓 SSH_KNOWN_HOSTS not set, accepting all host keys!")
auth.HostKeyCallback = LogHostKeyCallback(logf)
}
return auth
}
Expand All @@ -283,19 +301,20 @@ func SetupRepoAuth(options *options.Options) transport.AuthMethod {

// Duplicated code due to Go's type system.
if os.Getenv("SSH_KNOWN_HOSTS") == "" {
options.Logger(log.LevelWarn, "#1: 🔓 SSH_KNOWN_HOSTS not set, accepting all host keys!")
auth.HostKeyCallback = LogHostKeyCallback(options.Logger)
logf("🔓 SSH_KNOWN_HOSTS not set, accepting all host keys!")
auth.HostKeyCallback = LogHostKeyCallback(logf)
}
return auth
}

func CloneOptionsFromOptions(options options.Options) (CloneRepoOptions, error) {
func CloneOptionsFromOptions(logf func(string, ...any), options options.Options) (CloneRepoOptions, error) {
caBundle, err := options.CABundle()
if err != nil {
return CloneRepoOptions{}, err
}

cloneOpts := CloneRepoOptions{
RepoURL: options.GitURL,
Path: options.WorkspaceFolder,
Storage: options.Filesystem,
Insecure: options.Insecure,
Expand All @@ -304,13 +323,12 @@ func CloneOptionsFromOptions(options options.Options) (CloneRepoOptions, error)
CABundle: caBundle,
}

cloneOpts.RepoAuth = SetupRepoAuth(&options)
cloneOpts.RepoAuth = SetupRepoAuth(logf, &options)
if options.GitHTTPProxyURL != "" {
cloneOpts.ProxyOptions = transport.ProxyOptions{
URL: options.GitHTTPProxyURL,
}
}
cloneOpts.RepoURL = options.GitURL

return cloneOpts, nil
}
Expand All @@ -331,7 +349,7 @@ func (w *progressWriter) Close() error {
return err2
}

func ProgressWriter(write func(line string)) io.WriteCloser {
func ProgressWriter(write func(line string, args ...any)) io.WriteCloser {
reader, writer := io.Pipe()
done := make(chan struct{})
go func() {
Expand All @@ -347,6 +365,8 @@ func ProgressWriter(write func(line string)) io.WriteCloser {
if line == "" {
continue
}
// Escape % signs so that they don't get interpreted as format specifiers
line = strings.Replace(line, "%", "%%", -1)
write(strings.TrimSpace(line))
}
}
Expand Down
Loading

0 comments on commit e14b952

Please sign in to comment.