diff --git a/go.sum b/go.sum index 427230ee..f30a1b61 100644 --- a/go.sum +++ b/go.sum @@ -487,7 +487,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.11.0/go.mod h1:XjsvQN+RJGWI2TWy1/kqaE16HrR2J/FWgkYjdZQsX9M= -github.com/hashicorp/consul/api v1.12.0/go.mod h1:6pVBMo0ebnYdt2S3H87XhekM/HHrUoTD2XXb/VrZVy0= github.com/hashicorp/consul/sdk v0.8.0/go.mod h1:GBvyrGALthsZObzUGsfgHZQDXjg4lOjagTIwIR1vPms= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= @@ -749,7 +748,6 @@ github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= github.com/sagikazarmark/crypt v0.3.0/go.mod h1:uD/D+6UF4SrIR1uGEv7bBNkNqLGqUr43MRiaGWX1Nig= -github.com/sagikazarmark/crypt v0.4.0/go.mod h1:ALv2SRj7GxYV4HO9elxH9nS6M9gW+xDNxqmyJ6RfDFM= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= @@ -988,7 +986,6 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211015210444-4f30a5c0130f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= -golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211209124913-491a49abca63/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20211216030914-fe4d6282115f h1:hEYJvxw1lSnWIl8X9ofsYMklzaDs90JI2az5YMd4fPM= golang.org/x/net v0.0.0-20211216030914-fe4d6282115f/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= @@ -1128,7 +1125,6 @@ golang.org/x/sys v0.0.0-20211007075335-d3039528d8ac/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211205182925-97ca703d548d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -1254,7 +1250,6 @@ google.golang.org/api v0.57.0/go.mod h1:dVPlbZyBo2/OjBpmvNdpn2GRm6rPy75jyU7bmhdr google.golang.org/api v0.59.0/go.mod h1:sT2boj7M9YJxZzgeZqXogmhfmRWDtPzT31xkieUbuZU= google.golang.org/api v0.61.0/go.mod h1:xQRti5UdCmoCEqFxcz93fTl338AVqDgyaDRuOZ3hg9I= google.golang.org/api v0.62.0/go.mod h1:dKmwPCydfsad4qCH08MSdgWjfHOyfpd4VtDGgRFdavw= -google.golang.org/api v0.63.0/go.mod h1:gs4ij2ffTRXwuzzgJl/56BdwJaA194ijkfn++9tDuPo= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= diff --git a/internal/daemon/daemon.go b/internal/daemon/daemon.go index 545c8306..88b649c0 100644 --- a/internal/daemon/daemon.go +++ b/internal/daemon/daemon.go @@ -1,32 +1,120 @@ package daemon import ( + "fmt" + "os" + "path/filepath" + "time" + + "github.com/ckotzbauer/sbom-git-operator/internal" "github.com/ckotzbauer/sbom-git-operator/internal/git" "github.com/ckotzbauer/sbom-git-operator/internal/kubernetes" "github.com/ckotzbauer/sbom-git-operator/internal/syft" + "github.com/robfig/cron" "github.com/sirupsen/logrus" "github.com/spf13/viper" ) -func RunBackgroundService() { +type CronService struct { + cron string +} + +func Start(cronTime string) { + cr := internal.Unescape(cronTime) + logrus.Debugf("Cron set to: %v", cr) + + cs := CronService{cron: cr} + cs.printNextExecution() + + c := cron.New() + c.AddFunc(cr, func() { cs.runBackgroundService() }) + c.Start() +} + +func (c *CronService) printNextExecution() { + s, err := cron.Parse(c.cron) + if err != nil { + logrus.WithError(err).Fatal("Cron cannot be parsed") + } + + nextRun := s.Next(time.Now()) + logrus.Debugf("Next background-service run at: %v", nextRun) +} + +func (c *CronService) runBackgroundService() { + logrus.Info("Execute background-service") workingTree := viper.GetString("git-workingtree") gitAccount := git.New(viper.GetString("git-access-token"), viper.GetString("git-author-name"), viper.GetString("git-author-email")) - gitAccount.Clone(viper.GetString("git-repository"), workingTree, viper.GetString("git-branch")) + gitAccount.PrepareRepository(viper.GetString("git-repository"), workingTree, viper.GetString("git-branch")) client := kubernetes.NewClient() namespaces := client.ListNamespaces(viper.GetString("namespace-label-selector")) logrus.Debugf("Discovered %v namespaces", len(namespaces)) + processedSbomFiles := []string{} + for _, ns := range namespaces { pods := client.ListPods(ns.Name, viper.GetString("pod-label-selector")) logrus.Debugf("Discovered %v pods in namespace %v", len(pods), ns.Name) digests := client.GetContainerDigests(pods) for _, d := range digests { - syft.ExecuteSyft(d, workingTree) + sbomPath := syft.ExecuteSyft(d, workingTree) + processedSbomFiles = append(processedSbomFiles, sbomPath) } + + gitAccount.CommitAll(workingTree, fmt.Sprintf("Created new SBOMs for pods in namespace %s", ns.Name)) } - gitAccount.CommitAll(workingTree, "Created new SBOMs") + logrus.Debug("Start to remove old SBOMs") + ignoreDirs := []string{".git"} + err := filepath.Walk(workingTree, deleteObsoleteFiles(workingTree, ignoreDirs, processedSbomFiles, gitAccount)) + if err != nil { + logrus.WithError(err).Error("Could not cleanup old SBOMs") + } else { + gitAccount.CommitAndPush(workingTree, "Deleted old SBOMs") + } + + c.printNextExecution() +} + +func deleteObsoleteFiles(workingTree string, ignoreDirs, processedSbomFiles []string, gitAccount git.GitAccount) filepath.WalkFunc { + return func(path string, info os.FileInfo, err error) error { + if err != nil { + logrus.WithError(err).Errorf("An error occurred while processing %s", path) + return nil + } + + if info.IsDir() { + dir := filepath.Base(path) + for _, d := range ignoreDirs { + if d == dir { + return filepath.SkipDir + } + } + } + + if info.Name() == "sbom.json" { + found := false + for _, f := range processedSbomFiles { + if f == path { + found = true + break + } + } + + if !found { + rel, _ := filepath.Rel(workingTree, path) + gitAccount.RemoveFile(workingTree, rel) + if err != nil { + logrus.WithError(err).Errorf("File could not be deleted %s", path) + } else { + logrus.Debugf("Deleted old SBOM: %s", path) + } + } + } + + return nil + } } diff --git a/internal/git/git.go b/internal/git/git.go index e6bd074d..6a37988d 100644 --- a/internal/git/git.go +++ b/internal/git/git.go @@ -22,18 +22,32 @@ func New(token, name, email string) GitAccount { return GitAccount{Token: token, Name: name, Email: email} } -func (g *GitAccount) Clone(repo, path, branch string) { - // TODO: Detect if repo is already cloned and skip it in that case. - - r, err := git.PlainClone(path, false, &git.CloneOptions{ - URL: repo, - Depth: 1, - Progress: os.Stdout, - Auth: g.tokenAuth(), - }) +func (g *GitAccount) alreadyCloned(path string) (*git.Repository, error) { + r, err := git.PlainOpen(path) + + if err == git.ErrRepositoryNotExists { + return nil, nil + } + + return r, nil +} + +func (g *GitAccount) PrepareRepository(repo, path, branch string) { + r, err := g.alreadyCloned(path) + cloned := false + + if r == nil && err == nil { + cloned = true + r, err = git.PlainClone(path, false, &git.CloneOptions{ + URL: repo, + Depth: 1, + Progress: os.Stdout, + Auth: g.tokenAuth(), + }) + } if err != nil { - logrus.WithError(err).Error("Clone failed") + logrus.WithError(err).Error("Open or clone failed") return } @@ -53,30 +67,42 @@ func (g *GitAccount) Clone(repo, path, branch string) { return } - // TODO: msg="Pull failed" error="empty git-upload-pack given" - err = w.Pull(&git.PullOptions{ - Auth: g.tokenAuth(), - }) + if !cloned { + // TODO: msg="Pull failed" error="empty git-upload-pack given" + err = w.Pull(&git.PullOptions{ + Auth: g.tokenAuth(), + }) - if err != nil { - logrus.WithError(err).Error("Pull failed") + if err != nil { + logrus.WithError(err).Error("Pull failed") + } } - logrus.Info("Git-Repository is prepared!") + logrus.Debug("Git-Repository is prepared!") } -func (g *GitAccount) CommitAll(path, message string) { +func (g *GitAccount) openExistingRepo(path string) (*git.Repository, *git.Worktree) { r, err := git.PlainOpen(path) if err != nil { logrus.WithError(err).Error("Open failed") - return + return nil, nil } w, err := r.Worktree() if err != nil { logrus.WithError(err).Error("Worktree failed") + return nil, nil + } + + return r, w +} + +func (g *GitAccount) CommitAll(path, message string) { + r, w := g.openExistingRepo(path) + + if r == nil && w == nil { return } @@ -88,7 +114,7 @@ func (g *GitAccount) CommitAll(path, message string) { } if status.IsClean() { - logrus.Info("Git-Worktree is clean") + logrus.Debug("Git-Worktree is clean, skip commit") return } @@ -99,6 +125,47 @@ func (g *GitAccount) CommitAll(path, message string) { return } + g.commitAndPush(w, r, message) +} + +func (g *GitAccount) RemoveFile(workTree, path string) { + r, w := g.openExistingRepo(workTree) + + if r == nil && w == nil { + return + } + + _, err := w.Remove(path) + + if err != nil { + logrus.WithError(err).Error("Remove failed") + return + } +} + +func (g *GitAccount) CommitAndPush(path, message string) { + r, w := g.openExistingRepo(path) + + if r == nil && w == nil { + return + } + + status, err := w.Status() + + if err != nil { + logrus.WithError(err).Error("Status failed") + return + } + + if status.IsClean() { + logrus.Debug("Git-Worktree is clean, skip commit") + return + } + + g.commitAndPush(w, r, message) +} + +func (g *GitAccount) commitAndPush(w *git.Worktree, r *git.Repository, message string) { commit, err := w.Commit(message, &git.CommitOptions{ Author: &object.Signature{ Name: g.Name, diff --git a/internal/kubernetes/kubernetes.go b/internal/kubernetes/kubernetes.go index 88be538b..4e127cfe 100644 --- a/internal/kubernetes/kubernetes.go +++ b/internal/kubernetes/kubernetes.go @@ -43,7 +43,7 @@ func prepareLabelSelector(selector string) meta.ListOptions { if len(selector) > 0 { listOptions.LabelSelector = internal.Unescape(selector) - logrus.Debugf("Applied labelSelector %v", listOptions.LabelSelector) + logrus.Tracef("Applied labelSelector %v", listOptions.LabelSelector) } return listOptions diff --git a/internal/syft/syft.go b/internal/syft/syft.go index dfc886a5..215b6835 100644 --- a/internal/syft/syft.go +++ b/internal/syft/syft.go @@ -13,13 +13,13 @@ import ( "github.com/sirupsen/logrus" ) -func ExecuteSyft(img kubernetes.ImageDigest, gitWorkingTree string) { +func ExecuteSyft(img kubernetes.ImageDigest, gitWorkingTree string) string { name := strings.ReplaceAll(img.Digest, "@", "/") name = strings.ReplaceAll(gitWorkingTree+"/"+name+"/sbom.json", ":", "_") if pathExists(name) { logrus.Debugf("Skip image %s", img.Digest) - return + return name } logrus.Debugf("Processing image %s", img.Digest) @@ -32,7 +32,7 @@ func ExecuteSyft(img kubernetes.ImageDigest, gitWorkingTree string) { if err != nil { logrus.WithError(err).Error("Image-Pull failed") - return + return name } cmd := exec.Command("syft", imagePath, "-o", "json") @@ -44,7 +44,7 @@ func ExecuteSyft(img kubernetes.ImageDigest, gitWorkingTree string) { if err != nil { logrus.WithError(err).WithField("stderr", errb.String()).Error("Syft stopped with error") - return + return name } dir := filepath.Dir(name) @@ -52,7 +52,7 @@ func ExecuteSyft(img kubernetes.ImageDigest, gitWorkingTree string) { if err != nil { logrus.WithError(err).Error("Directory could not be created") - return + return name } data := []byte(stdout) @@ -61,6 +61,8 @@ func ExecuteSyft(img kubernetes.ImageDigest, gitWorkingTree string) { if err != nil { logrus.WithError(err).Error("SBOM could not be saved") } + + return name } func pathExists(path string) bool { diff --git a/main.go b/main.go index 6dd62e7c..f948df31 100644 --- a/main.go +++ b/main.go @@ -5,11 +5,9 @@ import ( "net/http" "os" "runtime" - "time" "github.com/ckotzbauer/sbom-git-operator/internal" "github.com/ckotzbauer/sbom-git-operator/internal/daemon" - "github.com/robfig/cron" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "github.com/spf13/viper" @@ -33,19 +31,7 @@ var ( internal.SetUpLogs(os.Stdout, verbosity) printVersion() - cr := internal.Unescape(viper.GetString("cron")) - logrus.Debugf("Cron set to: %v", cr) - s, err := cron.Parse(cr) - if err != nil { - logrus.WithError(err).Fatal("Cron cannot be parsed") - } - - nextRun := s.Next(time.Now()) - logrus.Debugf("Next background-service run at: %v", nextRun) - - c := cron.New() - c.AddFunc(cr, func() { daemon.RunBackgroundService() }) - c.Start() + daemon.Start(viper.GetString("cron")) logrus.Info("Webserver is running at port 8080") http.HandleFunc("/health", health) @@ -65,7 +51,7 @@ func init() { rootCmd.PersistentFlags().String("git-access-token", "", "Git-Access-Token.") rootCmd.PersistentFlags().String("git-author-name", "", "Author name to use for Git-Commits.") rootCmd.PersistentFlags().String("git-author-email", "", "Author email to use for Git-Commits.") - rootCmd.PersistentFlags().String("pod-label-selector", "l", "Kubernetes Label-Selector for pods.") + rootCmd.PersistentFlags().String("pod-label-selector", "", "Kubernetes Label-Selector for pods.") rootCmd.PersistentFlags().String("namespace-label-selector", "", "Kubernetes Label-Selector for namespaces.") }