From c3475277e7026cf00dcac609fa05acb9d66ea663 Mon Sep 17 00:00:00 2001 From: Radoslav Dimitrov Date: Wed, 15 May 2024 10:59:21 +0300 Subject: [PATCH 01/16] Refactor Frizbee Signed-off-by: Radoslav Dimitrov --- .gitignore | 4 + README.md | 2 +- cmd/action/action.go | 85 +++++ cmd/{ghactions => action}/list.go | 47 ++- cmd/{ghactions => action}/one.go | 38 +- cmd/containerimage/containerimage.go | 35 -- cmd/containerimage/yaml.go | 67 ---- cmd/dockercompose/dockercompose.go | 62 --- cmd/ghactions/common.go | 26 -- cmd/ghactions/ghactions.go | 100 ----- cmd/ghactions/replacer.go | 98 ----- cmd/image/image.go | 80 ++++ cmd/image/list.go | 94 +++++ cmd/{containerimage => image}/one.go | 47 ++- cmd/kubernetes/kubernetes.go | 68 ---- cmd/root.go | 12 +- cmd/version/version.go | 5 +- internal/cli/cli.go | 212 ++++++++++ internal/cmd/doc.go | 17 - internal/cmd/yamlreplacer.go | 160 -------- internal/ghrest/ghrest.go | 14 +- {pkg/utils => internal/store}/cache.go | 2 +- internal/traverse/traverse.go | 138 +++++++ pkg/config/config.go | 12 +- pkg/constants/constants.go | 97 ----- pkg/containers/containers.go | 50 --- pkg/containers/containers_test.go | 87 ----- pkg/containers/replace.go | 100 ----- pkg/containers/replace_test.go | 75 ---- pkg/errors/errors.go | 24 -- pkg/ghactions/errors.go | 26 -- pkg/ghactions/ghactions.go | 272 ------------- pkg/ghactions/ghactions_test.go | 361 ------------------ pkg/ghactions/utils.go | 61 --- pkg/interfaces/interfaces.go | 30 ++ pkg/interfaces/rest.go | 33 -- pkg/replacer/action/action.go | 163 ++++++++ .../action/utils.go} | 88 ++++- pkg/replacer/image/image.go | 101 +++++ pkg/replacer/image/utils.go | 76 ++++ pkg/replacer/replacer.go | 317 +++++++++++++++ pkg/replacer/replacer_test.go | 1 + pkg/utils/cli/billy.go | 29 -- pkg/utils/cli/replacer.go | 89 ----- pkg/utils/utils.go | 71 ---- 45 files changed, 1481 insertions(+), 2095 deletions(-) create mode 100644 cmd/action/action.go rename cmd/{ghactions => action}/list.go (62%) rename cmd/{ghactions => action}/one.go (64%) delete mode 100644 cmd/containerimage/containerimage.go delete mode 100644 cmd/containerimage/yaml.go delete mode 100644 cmd/dockercompose/dockercompose.go delete mode 100644 cmd/ghactions/common.go delete mode 100644 cmd/ghactions/ghactions.go delete mode 100644 cmd/ghactions/replacer.go create mode 100644 cmd/image/image.go create mode 100644 cmd/image/list.go rename cmd/{containerimage => image}/one.go (50%) delete mode 100644 cmd/kubernetes/kubernetes.go create mode 100644 internal/cli/cli.go delete mode 100644 internal/cmd/doc.go delete mode 100644 internal/cmd/yamlreplacer.go rename {pkg/utils => internal/store}/cache.go (99%) create mode 100644 internal/traverse/traverse.go delete mode 100644 pkg/constants/constants.go delete mode 100644 pkg/containers/containers.go delete mode 100644 pkg/containers/containers_test.go delete mode 100644 pkg/containers/replace.go delete mode 100644 pkg/containers/replace_test.go delete mode 100644 pkg/errors/errors.go delete mode 100644 pkg/ghactions/errors.go delete mode 100644 pkg/ghactions/ghactions.go delete mode 100644 pkg/ghactions/ghactions_test.go delete mode 100644 pkg/ghactions/utils.go create mode 100644 pkg/interfaces/interfaces.go delete mode 100644 pkg/interfaces/rest.go create mode 100644 pkg/replacer/action/action.go rename pkg/{ghactions/ghactions_helpers.go => replacer/action/utils.go} (52%) create mode 100644 pkg/replacer/image/image.go create mode 100644 pkg/replacer/image/utils.go create mode 100644 pkg/replacer/replacer.go create mode 100644 pkg/replacer/replacer_test.go delete mode 100644 pkg/utils/cli/billy.go delete mode 100644 pkg/utils/cli/replacer.go delete mode 100644 pkg/utils/utils.go diff --git a/.gitignore b/.gitignore index fe5ae38..a49b579 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,7 @@ dist/ # Go workspace file go.work + +.idea/ + +frizbee \ No newline at end of file diff --git a/README.md b/README.md index b4d9352..06f8c43 100644 --- a/README.md +++ b/README.md @@ -84,7 +84,7 @@ To get the digest for a single image tag, you can use the `containerimage one` c frizbee containerimage one quay.io/stacklok/frizbee:latest ``` -This will print the image refrence with the digest for the image tag provided. +This will print the image reference with the digest for the image tag provided. ## Contributing diff --git a/cmd/action/action.go b/cmd/action/action.go new file mode 100644 index 0000000..18e90ca --- /dev/null +++ b/cmd/action/action.go @@ -0,0 +1,85 @@ +// +// Copyright 2023 Stacklok, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package action provides command-line utilities to work with GitHub Actions. +package action + +import ( + "github.com/stacklok/frizbee/internal/cli" + "github.com/stacklok/frizbee/pkg/config" + "github.com/stacklok/frizbee/pkg/replacer" + "os" + + "github.com/spf13/cobra" +) + +// CmdGHActions represents the actions command +func CmdGHActions() *cobra.Command { + cmd := &cobra.Command{ + Use: "action", + Short: "Replace tags in GitHub Actions workflows", + Long: `This utility replaces tag or branch references in GitHub Actions workflows +with the latest commit hash of the referenced tag or branch. + +Example: + + $ frizbee action -d .github/workflows + +This will replace all tag or branch references in all GitHub Actions workflows +for the given directory. + +` + cli.TokenHelpText + "\n", + Aliases: []string{"ghactions"}, // backwards compatibility + RunE: replaceCmd, + SilenceUsage: true, + } + + // flags + cli.DeclareFrizbeeFlags(cmd, ".github/workflows") + + // sub-commands + cmd.AddCommand(CmdOne()) + cmd.AddCommand(CmdList()) + + return cmd +} + +func replaceCmd(cmd *cobra.Command, _ []string) error { + // Extract the CLI flags from the cobra command + cliFlags, err := cli.NewHelper(cmd) + if err != nil { + return err + } + + // Set up the config + cfg, err := config.FromCommand(cmd) + if err != nil { + return err + } + + // Create a new replacer + r := replacer.New(cfg). + WithUserRegex(cliFlags.Regex). + WithGitHubClient(os.Getenv(cli.GitHubTokenEnvKey)) + + // Replace the tags in the given directory + res, err := r.ParseGitHubActions(cmd.Context(), cliFlags.Dir) + if err != nil { + return err + } + + // Process the output files + return cliFlags.ProcessOutput(res.Processed, res.Modified) +} diff --git a/cmd/ghactions/list.go b/cmd/action/list.go similarity index 62% rename from cmd/ghactions/list.go rename to cmd/action/list.go index acd75d7..05786cb 100644 --- a/cmd/ghactions/list.go +++ b/cmd/action/list.go @@ -13,16 +13,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -package ghactions +package action import ( "encoding/json" "fmt" - "github.com/olekukonko/tablewriter" "github.com/spf13/cobra" - - "github.com/stacklok/frizbee/pkg/ghactions" + "github.com/stacklok/frizbee/internal/cli" + "github.com/stacklok/frizbee/pkg/config" + "github.com/stacklok/frizbee/pkg/replacer" + "os" + "strconv" ) // CmdList represents the one sub-command @@ -33,32 +35,49 @@ func CmdList() *cobra.Command { Long: `This utility lists all the github actions used in the workflows Example: - frizbee ghactions list + frizbee action list -d .github/workflows `, Aliases: []string{"ls"}, RunE: list, SilenceUsage: true, } - cmd.Flags().StringP("dir", "d", ".github/workflows", "workflows directory") + cli.DeclareFrizbeeFlags(cmd, ".github/workflows") cmd.Flags().StringP("output", "o", "table", "output format. Can be 'json' or 'table'") return cmd } func list(cmd *cobra.Command, _ []string) error { - dir := cmd.Flag("dir").Value.String() - actions, err := ghactions.ListActionsInDirectory(dir) + // Extract the CLI flags from the cobra command + cliFlags, err := cli.NewHelper(cmd) + if err != nil { + return err + } + + // Set up the config + cfg, err := config.FromCommand(cmd) + if err != nil { + return err + } + + // Create a new replacer + r := replacer.New(cfg). + WithUserRegex(cliFlags.Regex). + WithGitHubClient(os.Getenv(cli.GitHubTokenEnvKey)) + + // List the references in the directory + res, err := r.ListGitHibActions(cliFlags.Dir) if err != nil { - return fmt.Errorf("failed to list actions: %w", err) + return err } output := cmd.Flag("output").Value.String() switch output { case "json": - jsonBytes, err := json.MarshalIndent(actions, "", " ") + jsonBytes, err := json.MarshalIndent(res.Entities, "", " ") if err != nil { - return fmt.Errorf("failed to marshal actions: %w", err) + return err } jsonString := string(jsonBytes) @@ -66,9 +85,9 @@ func list(cmd *cobra.Command, _ []string) error { return nil case "table": table := tablewriter.NewWriter(cmd.OutOrStdout()) - table.SetHeader([]string{"Owner", "Repo", "Action", "Ref"}) - for _, a := range actions { - table.Append([]string{a.Owner, a.Repo, a.Action, a.Ref}) + table.SetHeader([]string{"No", "Type", "Name", "Ref"}) + for i, a := range res.Entities { + table.Append([]string{strconv.Itoa(i + 1), a.Type, a.Name, a.Ref}) } table.Render() return nil diff --git a/cmd/ghactions/one.go b/cmd/action/one.go similarity index 64% rename from cmd/ghactions/one.go rename to cmd/action/one.go index 1148c65..513fcf9 100644 --- a/cmd/ghactions/one.go +++ b/cmd/action/one.go @@ -13,16 +13,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -package ghactions +package action import ( "fmt" + "github.com/stacklok/frizbee/internal/cli" + "github.com/stacklok/frizbee/pkg/config" + "github.com/stacklok/frizbee/pkg/replacer" "os" "github.com/spf13/cobra" - - "github.com/stacklok/frizbee/internal/ghrest" - "github.com/stacklok/frizbee/pkg/ghactions" ) // CmdOne represents the one sub-command @@ -35,39 +35,47 @@ with the latest commit hash of the referenced tag or branch. Example: - $ frizbee ghactions one actions/checkout@v4.1.1 + $ frizbee action one actions/checkout@v4.1.1 This will replace the tag or branch reference for the commit hash of the referenced tag or branch. -` + TokenHelpText + "\n", +` + cli.TokenHelpText + "\n", Args: cobra.ExactArgs(1), RunE: replaceOne, SilenceUsage: true, } + cli.DeclareFrizbeeFlags(cmd, "") return cmd } func replaceOne(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() ref := args[0] - ghcli := ghrest.NewGhRest(os.Getenv(GitHubTokenEnvKey)) - - act, ref, err := ghactions.ParseActionReference(ref) + // Extract the CLI flags from the cobra command + cliFlags, err := cli.NewHelper(cmd) if err != nil { - return fmt.Errorf("failed to parse action reference '%s': %w", ref, err) + return err } - sum, err := ghactions.GetChecksum(ctx, ghcli, act, ref) + // Set up the config + cfg, err := config.FromCommand(cmd) if err != nil { - return fmt.Errorf("failed to get checksum for action '%s': %w", ref, err) + return err } - if ref != sum { - fmt.Fprintf(cmd.OutOrStdout(), "%s@%s\n", act, sum) + // Create a new replacer + r := replacer.New(cfg). + WithUserRegex(cliFlags.Regex). + WithGitHubClient(os.Getenv(cli.GitHubTokenEnvKey)) + + // Replace the passed reference + res, err := r.ParseSingleGitHubAction(cmd.Context(), ref) + if err != nil { + return err } + fmt.Fprintln(cmd.OutOrStdout(), res) return nil } diff --git a/cmd/containerimage/containerimage.go b/cmd/containerimage/containerimage.go deleted file mode 100644 index d542cf1..0000000 --- a/cmd/containerimage/containerimage.go +++ /dev/null @@ -1,35 +0,0 @@ -// Copyright 2023 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package containerimage provides command-line utilities to work with container images. -package containerimage - -import ( - "github.com/spf13/cobra" -) - -// CmdContainerImage represents the containerimage command -func CmdContainerImage() *cobra.Command { - cmd := &cobra.Command{ - Use: "containerimage", - Short: "Replace container image references with checksums", - RunE: replaceYAML, - SilenceUsage: true, - } - - cmd.AddCommand(CmdOne()) - cmd.AddCommand(CmdYAML()) - - return cmd -} diff --git a/cmd/containerimage/yaml.go b/cmd/containerimage/yaml.go deleted file mode 100644 index e4ee8a8..0000000 --- a/cmd/containerimage/yaml.go +++ /dev/null @@ -1,67 +0,0 @@ -// Copyright 2023 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package containerimage provides command-line utilities to work with container images. -package containerimage - -import ( - "fmt" - - "github.com/spf13/cobra" - - intcmd "github.com/stacklok/frizbee/internal/cmd" - "github.com/stacklok/frizbee/pkg/config" -) - -// CmdYAML represents the yaml sub-command -func CmdYAML() *cobra.Command { - cmd := &cobra.Command{ - Use: "yaml", - Short: "Replace container image references with checksums in YAML files", - Long: `This utility replaces a tag or branch reference in a container image references -with the digest hash of the referenced tag in YAML files. - -Example: - - $ frizbee containerimage yaml --dir . --dry-run --quiet --error -`, - RunE: replaceYAML, - SilenceUsage: true, - } - - // flags - cmd.Flags().StringP("image-regex", "i", "image", "regex to match container image references") - - intcmd.DeclareYAMLReplacerFlags(cmd) - - return cmd -} - -func replaceYAML(cmd *cobra.Command, _ []string) error { - cfg, err := config.FromContext(cmd.Context()) - if err != nil { - return fmt.Errorf("failed to get config from context: %w", err) - } - ir, err := cmd.Flags().GetString("image-regex") - if err != nil { - return fmt.Errorf("failed to get image-regex flag: %w", err) - } - - replacer, err := intcmd.NewYAMLReplacer(cmd, intcmd.WithImageRegex(ir)) - if err != nil { - return err - } - - return replacer.Do(cmd.Context(), cfg) -} diff --git a/cmd/dockercompose/dockercompose.go b/cmd/dockercompose/dockercompose.go deleted file mode 100644 index fd1c0e8..0000000 --- a/cmd/dockercompose/dockercompose.go +++ /dev/null @@ -1,62 +0,0 @@ -// Copyright 2023 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package dockercompose provides command-line utilities to work with container images. -package dockercompose - -import ( - "fmt" - - "github.com/spf13/cobra" - - intcmd "github.com/stacklok/frizbee/internal/cmd" - "github.com/stacklok/frizbee/pkg/config" -) - -// CmdCompose represents the compose yaml sub-command -func CmdCompose() *cobra.Command { - cmd := &cobra.Command{ - Use: "docker-compose", - Aliases: []string{"dockercompose", "compose"}, - Short: "Replace container image references with checksums in docker-compose YAML files", - Long: `This utility replaces a tag or branch reference in a container image references -with the digest hash of the referenced tag in docker-compose YAML files. - -Example: - - $ frizbee docker-compose --dir . --dry-run --quiet --error -`, - RunE: replaceYAML, - SilenceUsage: true, - } - - // flags - intcmd.DeclareYAMLReplacerFlags(cmd) - - return cmd -} - -func replaceYAML(cmd *cobra.Command, _ []string) error { - cfg, err := config.FromContext(cmd.Context()) - if err != nil { - return fmt.Errorf("failed to get config from context: %w", err) - } - - replacer, err := intcmd.NewYAMLReplacer(cmd) - if err != nil { - return err - } - - return replacer.Do(cmd.Context(), cfg) -} diff --git a/cmd/ghactions/common.go b/cmd/ghactions/common.go deleted file mode 100644 index 71c2987..0000000 --- a/cmd/ghactions/common.go +++ /dev/null @@ -1,26 +0,0 @@ -// -// Copyright 2023 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ghactions - -const ( - // GitHubTokenEnvKey is the environment variable key for the GitHub token - //nolint:gosec // This is not a hardcoded credential - GitHubTokenEnvKey = "GITHUB_TOKEN" - - // TokenHelpText is the help text for the GitHub token - TokenHelpText = "NOTE: It's recommended to set the " + GitHubTokenEnvKey + - " environment variable given that GitHub has tighter rate limits on anonymous calls." -) diff --git a/cmd/ghactions/ghactions.go b/cmd/ghactions/ghactions.go deleted file mode 100644 index 8345463..0000000 --- a/cmd/ghactions/ghactions.go +++ /dev/null @@ -1,100 +0,0 @@ -// -// Copyright 2023 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package ghactions provides command-line utilities to work with GitHub Actions. -package ghactions - -import ( - "fmt" - "os" - - "github.com/spf13/cobra" - - "github.com/stacklok/frizbee/internal/ghrest" - "github.com/stacklok/frizbee/pkg/config" - cliutils "github.com/stacklok/frizbee/pkg/utils/cli" -) - -// CmdGHActions represents the ghactions command -func CmdGHActions() *cobra.Command { - cmd := &cobra.Command{ - Use: "ghactions", - Short: "Replace tags in GitHub Actions workflows", - Long: `This utility replaces tag or branch references in GitHub Actions workflows -with the latest commit hash of the referenced tag or branch. - -Example: - - $ frizbee ghactions -d .github/workflows - -This will replace all tag or branch references in all GitHub Actions workflows -for the given directory. - -` + TokenHelpText + "\n", - Aliases: []string{"actions", "action"}, - RunE: replace, - SilenceUsage: true, - } - - // flags - cmd.Flags().StringP("dir", "d", ".github/workflows", "workflows directory") - - cliutils.DeclareReplacerFlags(cmd) - - // sub-commands - cmd.AddCommand(CmdOne()) - cmd.AddCommand(CmdList()) - - return cmd -} - -func replace(cmd *cobra.Command, _ []string) error { - dir := cmd.Flag("dir").Value.String() - dryRun, err := cmd.Flags().GetBool("dry-run") - if err != nil { - return fmt.Errorf("failed to get dry-run flag: %w", err) - } - errOnModified, err := cmd.Flags().GetBool("error") - if err != nil { - return fmt.Errorf("failed to get error flag: %w", err) - } - quiet, err := cmd.Flags().GetBool("quiet") - if err != nil { - return fmt.Errorf("failed to get quiet flag: %w", err) - } - cfg, err := config.FromContext(cmd.Context()) - if err != nil { - return fmt.Errorf("failed to get config from context: %w", err) - } - - dir = cliutils.ProcessDirNameForBillyFS(dir) - - ctx := cmd.Context() - - ghcli := ghrest.NewGhRest(os.Getenv(GitHubTokenEnvKey)) - - replacer := &replacer{ - Replacer: cliutils.Replacer{ - Dir: dir, - DryRun: dryRun, - Quiet: quiet, - ErrOnModified: errOnModified, - Cmd: cmd, - }, - restIf: ghcli, - } - - return replacer.do(ctx, cfg) -} diff --git a/cmd/ghactions/replacer.go b/cmd/ghactions/replacer.go deleted file mode 100644 index a6a1678..0000000 --- a/cmd/ghactions/replacer.go +++ /dev/null @@ -1,98 +0,0 @@ -// -// Copyright 2023 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package ghactions provides command-line utilities to work with GitHub Actions. -package ghactions - -import ( - "context" - "fmt" - "path/filepath" - "sync/atomic" - - "github.com/go-git/go-billy/v5/osfs" - "golang.org/x/sync/errgroup" - "gopkg.in/yaml.v3" - - "github.com/stacklok/frizbee/pkg/config" - ferrors "github.com/stacklok/frizbee/pkg/errors" - "github.com/stacklok/frizbee/pkg/ghactions" - "github.com/stacklok/frizbee/pkg/interfaces" - "github.com/stacklok/frizbee/pkg/utils" - cliutils "github.com/stacklok/frizbee/pkg/utils/cli" -) - -type replacer struct { - cliutils.Replacer - restIf interfaces.REST -} - -func (r *replacer) do(ctx context.Context, cfg *config.Config) error { - basedir := filepath.Dir(r.Dir) - base := filepath.Base(r.Dir) - bfs := osfs.New(basedir, osfs.WithBoundOS()) - - outfiles := map[string]string{} - - var modified atomic.Bool - modified.Store(false) - - // error group - var eg errgroup.Group - cache := utils.NewRefCacher() - - err := ghactions.TraverseGitHubActionWorkflows(bfs, base, func(path string, wflow *yaml.Node) error { - eg.Go(func() error { - r.Logf("Processing %s\n", path) - m, err := ghactions.ModifyReferencesInYAMLWithCache(ctx, r.restIf, wflow, &cfg.GHActions, cache) - if err != nil { - return fmt.Errorf("failed to process YAML file %s: %w", path, err) - } - - modified.Store(modified.Load() || m) - - buf, err := utils.YAMLToBuffer(wflow) - if err != nil { - return fmt.Errorf("failed to convert YAML to buffer: %w", err) - } - - if m { - r.Logf("Modified %s\n", path) - outfiles[path] = buf.String() - } - - return nil - }) - - return nil - }) - if err != nil { - return err - } - - if err := eg.Wait(); err != nil { - return err - } - - if err := r.ProcessOutput(bfs, outfiles); err != nil { - return err - } - - if r.ErrOnModified && modified.Load() { - return ferrors.ErrModifiedFiles - } - - return nil -} diff --git a/cmd/image/image.go b/cmd/image/image.go new file mode 100644 index 0000000..b04a538 --- /dev/null +++ b/cmd/image/image.go @@ -0,0 +1,80 @@ +// +// Copyright 2023 Stacklok, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package image provides command-line utilities to work with container images. +package image + +import ( + "github.com/spf13/cobra" + "github.com/stacklok/frizbee/internal/cli" + "github.com/stacklok/frizbee/pkg/config" + "github.com/stacklok/frizbee/pkg/replacer" +) + +// CmdContainerImage represents the containers command +func CmdContainerImage() *cobra.Command { + cmd := &cobra.Command{ + Use: "image", + Short: "Replace container image references with checksums", + Long: `This utility replaces tag or branch references in yaml/yml files +with the latest commit hash of the referenced tag or branch. + +Example: + + $ frizbee image -d + +This will replace all tag or branch references in all yaml files for the given directory. +`, + RunE: replaceCmd, + SilenceUsage: true, + Aliases: []string{"containerimage", "dockercompose", "compose"}, // backwards compatibility + } + + // flags + cli.DeclareFrizbeeFlags(cmd, ".") + + // sub-commands + cmd.AddCommand(CmdOne()) + cmd.AddCommand(CmdList()) + + return cmd +} + +func replaceCmd(cmd *cobra.Command, _ []string) error { + // Extract the CLI flags from the cobra command + cliFlags, err := cli.NewHelper(cmd) + if err != nil { + return err + } + + // Set up the config + cfg, err := config.FromCommand(cmd) + if err != nil { + return err + } + + // Create a new replacer + r := replacer.New(cfg). + WithUserRegex(cliFlags.Regex) + + // Replace the tags in the directory + res, err := r.ParseContainerImages(cmd.Context(), cliFlags.Dir) + if err != nil { + return err + } + + // Process the output files + return cliFlags.ProcessOutput(res.Processed, res.Modified) +} diff --git a/cmd/image/list.go b/cmd/image/list.go new file mode 100644 index 0000000..7b9039b --- /dev/null +++ b/cmd/image/list.go @@ -0,0 +1,94 @@ +// +// Copyright 2023 Stacklok, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package image + +import ( + "encoding/json" + "fmt" + "github.com/olekukonko/tablewriter" + "github.com/spf13/cobra" + "github.com/stacklok/frizbee/internal/cli" + "github.com/stacklok/frizbee/pkg/config" + "github.com/stacklok/frizbee/pkg/replacer" + "strconv" +) + +// CmdList represents the one sub-command +func CmdList() *cobra.Command { + cmd := &cobra.Command{ + Use: "list", + Short: "Lists the used container images", + Long: `This utility lists all container images used in the files in the directory + +Example: + frizbee image list -d +`, + Aliases: []string{"ls"}, + RunE: list, + SilenceUsage: true, + } + + cli.DeclareFrizbeeFlags(cmd, ".") + cmd.Flags().StringP("output", "o", "table", "output format. Can be 'json' or 'table'") + + return cmd +} + +func list(cmd *cobra.Command, _ []string) error { + // Extract the CLI flags from the cobra command + cliFlags, err := cli.NewHelper(cmd) + if err != nil { + return err + } + + // Set up the config + cfg, err := config.FromCommand(cmd) + if err != nil { + return err + } + + // Create a new replacer + r := replacer.New(cfg). + WithUserRegex(cliFlags.Regex) + + // List the references in the directory + res, err := r.ListContainerImages(cliFlags.Dir) + if err != nil { + return err + } + + output := cmd.Flag("output").Value.String() + switch output { + case "json": + jsonBytes, err := json.MarshalIndent(res.Entities, "", " ") + if err != nil { + return err + } + jsonString := string(jsonBytes) + fmt.Fprintf(cmd.OutOrStdout(), "%s\n", jsonString) + return nil + case "table": + table := tablewriter.NewWriter(cmd.OutOrStdout()) + table.SetHeader([]string{"No", "Type", "Name", "Ref"}) + for i, a := range res.Entities { + table.Append([]string{strconv.Itoa(i + 1), a.Type, a.Name, a.Ref}) + } + table.Render() + return nil + default: + return fmt.Errorf("unknown output format: %s", output) + } +} diff --git a/cmd/containerimage/one.go b/cmd/image/one.go similarity index 50% rename from cmd/containerimage/one.go rename to cmd/image/one.go index 03c6781..195867a 100644 --- a/cmd/containerimage/one.go +++ b/cmd/image/one.go @@ -1,10 +1,11 @@ +// // Copyright 2023 Stacklok, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -12,53 +13,65 @@ // See the License for the specific language governing permissions and // limitations under the License. -package containerimage +package image import ( "fmt" - - "github.com/google/go-containerregistry/pkg/name" "github.com/spf13/cobra" - - "github.com/stacklok/frizbee/pkg/containers" + "github.com/stacklok/frizbee/internal/cli" + "github.com/stacklok/frizbee/pkg/config" + "github.com/stacklok/frizbee/pkg/replacer" ) // CmdOne represents the one sub-command func CmdOne() *cobra.Command { cmd := &cobra.Command{ Use: "one", - Short: "Replace the tag in container image reference", - Long: `This utility replaces a tag or branch reference in a container image reference -with the digest hash of the referenced tag. + Short: "Replace the tag with a digest reference", + Long: `This utility replaces a tag of a container reference +with the corresponding digest. Example: - $ frizbee containerimage one ghcr.io/stacklok/minder/helm/minder:0.20231123.829_ref.26ca90b + $ frizbee image one ghcr.io/stacklok/minder/server:latest + +This will replace a tag of the container reference with the corresponding digest. + `, Args: cobra.ExactArgs(1), RunE: replaceOne, SilenceUsage: true, } + cli.DeclareFrizbeeFlags(cmd, "") return cmd } func replaceOne(cmd *cobra.Command, args []string) error { - ctx := cmd.Context() ref := args[0] - r, err := name.ParseReference(ref) + // Extract the CLI flags from the cobra command + cliFlags, err := cli.NewHelper(cmd) + if err != nil { + return err + } + + // Set up the config + cfg, err := config.FromCommand(cmd) if err != nil { - return fmt.Errorf("failed to parse reference: %w", err) + return err } - img := r.Context().String() + // Create a new replacer + r := replacer.New(cfg). + WithUserRegex(cliFlags.Regex) - sum, err := containers.GetDigest(ctx, ref) + // Replace the passed reference + res, err := r.ParseSingleContainerImage(cmd.Context(), ref) if err != nil { - return fmt.Errorf("failed to get checksum for action '%s': %w", ref, err) + return err } - fmt.Fprintf(cmd.OutOrStdout(), "%s@%s\n", img, sum) + fmt.Fprintf(cmd.OutOrStdout(), res) return nil } diff --git a/cmd/kubernetes/kubernetes.go b/cmd/kubernetes/kubernetes.go deleted file mode 100644 index 542b5df..0000000 --- a/cmd/kubernetes/kubernetes.go +++ /dev/null @@ -1,68 +0,0 @@ -// Copyright 2023 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package kubernetes provides command-line utilities to work with kubernetes manifests. -package kubernetes - -import ( - "fmt" - - "github.com/spf13/cobra" - - intcmd "github.com/stacklok/frizbee/internal/cmd" - "github.com/stacklok/frizbee/pkg/config" -) - -// CmdK8s represents the k8s yaml sub-command -func CmdK8s() *cobra.Command { - cmd := &cobra.Command{ - Use: "kubernetes", - Aliases: []string{"k8s"}, - Short: "Replace container image references with checksums in kubernetes YAML files", - Long: `This utility replaces a tag or branch reference in a container image references -with the digest hash of the referenced tag in YAML files. - -Example: - - $ frizbee kubernetes --dir . --dry-run --quiet --error -`, - RunE: replaceYAML, - SilenceUsage: true, - } - - // flags - cmd.Flags().StringP("image-regex", "i", "image", "regex to match container image references") - - intcmd.DeclareYAMLReplacerFlags(cmd) - - return cmd -} - -func replaceYAML(cmd *cobra.Command, _ []string) error { - cfg, err := config.FromContext(cmd.Context()) - if err != nil { - return fmt.Errorf("failed to get config from context: %w", err) - } - ir, err := cmd.Flags().GetString("image-regex") - if err != nil { - return fmt.Errorf("failed to get image-regex flag: %w", err) - } - - replacer, err := intcmd.NewYAMLReplacer(cmd, intcmd.WithImageRegex(ir)) - if err != nil { - return err - } - - return replacer.Do(cmd.Context(), cfg) -} diff --git a/cmd/root.go b/cmd/root.go index 3085fe3..4e3c3b1 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -19,14 +19,12 @@ package cmd import ( "context" "fmt" + "github.com/stacklok/frizbee/cmd/action" + "github.com/stacklok/frizbee/cmd/image" "os" "github.com/spf13/cobra" - "github.com/stacklok/frizbee/cmd/containerimage" - "github.com/stacklok/frizbee/cmd/dockercompose" - "github.com/stacklok/frizbee/cmd/ghactions" - "github.com/stacklok/frizbee/cmd/kubernetes" "github.com/stacklok/frizbee/cmd/version" "github.com/stacklok/frizbee/pkg/config" ) @@ -41,10 +39,8 @@ func Execute() { rootCmd.PersistentFlags().StringP("config", "c", ".frizbee.yml", "config file (default is .frizbee.yml)") - rootCmd.AddCommand(ghactions.CmdGHActions()) - rootCmd.AddCommand(containerimage.CmdContainerImage()) - rootCmd.AddCommand(dockercompose.CmdCompose()) - rootCmd.AddCommand(kubernetes.CmdK8s()) + rootCmd.AddCommand(action.CmdGHActions()) + rootCmd.AddCommand(image.CmdContainerImage()) rootCmd.AddCommand(version.CmdVersion()) if err := rootCmd.ExecuteContext(context.Background()); err != nil { diff --git a/cmd/version/version.go b/cmd/version/version.go index b056a1b..177dced 100644 --- a/cmd/version/version.go +++ b/cmd/version/version.go @@ -18,8 +18,7 @@ package version import ( "github.com/spf13/cobra" - - "github.com/stacklok/frizbee/pkg/constants" + "github.com/stacklok/frizbee/internal/cli" ) // CmdVersion is the Cobra command for the version command. @@ -29,7 +28,7 @@ func CmdVersion() *cobra.Command { Short: "Print frizbee CLI version", Long: "The frizbee version command prints the version of the frizbee CLI.", Run: func(cmd *cobra.Command, _ []string) { - cmd.Println(constants.VerboseCLIVersion) + cmd.Println(cli.VerboseCLIVersion) }, } } diff --git a/internal/cli/cli.go b/internal/cli/cli.go new file mode 100644 index 0000000..5a20033 --- /dev/null +++ b/internal/cli/cli.go @@ -0,0 +1,212 @@ +package cli + +import ( + "fmt" + "github.com/go-git/go-billy/v5/osfs" + "github.com/spf13/cobra" + "io" + "os" + "path/filepath" + "runtime/debug" + "strings" + "text/template" +) + +const ( + // UserAgent is the user agent string used by frizbee. + // + // TODO (jaosorior): Add version information to this. + UserAgent = "frizbee" + // GitHubTokenEnvKey is the environment variable key for the GitHub token + //nolint:gosec // This is not a hardcoded credential + GitHubTokenEnvKey = "GITHUB_TOKEN" + + // TokenHelpText is the help text for the GitHub token + TokenHelpText = "NOTE: It's recommended to set the " + GitHubTokenEnvKey + + " environment variable given that GitHub has tighter rate limits on anonymous calls." + verboseTemplate = `Version: {{ .Version }} +Go Version: {{.GoVersion}} +Git Commit: {{.Commit}} +Commit Date: {{.Time}} +OS/Arch: {{.OS}}/{{.Arch}} +Dirty: {{.Modified}} +` +) + +// Helper is a common struct for implementing a CLI command that replaces +// files. +type Helper struct { + Dir string + DryRun bool + Quiet bool + ErrOnModified bool + Regex string + Cmd *cobra.Command +} + +type versionInfo struct { + Version string + GoVersion string + Time string + Commit string + OS string + Arch string + Modified bool +} + +var ( + // CLIVersion is the version of the frizbee CLI. + // nolint: gochecknoglobals + CLIVersion = "dev" + // VerboseCLIVersion is the verbose version of the frizbee CLI. + // nolint: gochecknoglobals + VerboseCLIVersion = "" +) + +// nolint:init +func init() { + buildinfo, ok := debug.ReadBuildInfo() + if !ok { + return + } + + var vinfo versionInfo + vinfo.Version = CLIVersion + vinfo.GoVersion = buildinfo.GoVersion + + for _, kv := range buildinfo.Settings { + switch kv.Key { + case "vcs.time": + vinfo.Time = kv.Value + case "vcs.revision": + vinfo.Commit = kv.Value + case "vcs.modified": + vinfo.Modified = kv.Value == "true" + case "GOOS": + vinfo.OS = kv.Value + case "GOARCH": + vinfo.Arch = kv.Value + } + } + VerboseCLIVersion = vinfo.String() +} + +func (vvs *versionInfo) String() string { + stringBuilder := &strings.Builder{} + tmpl := template.Must(template.New("version").Parse(verboseTemplate)) + err := tmpl.Execute(stringBuilder, vvs) + if err != nil { + panic(err) + } + return stringBuilder.String() +} + +func NewHelper(cmd *cobra.Command) (*Helper, error) { + dir := "." + if cmd.Flags().Lookup("dir") != nil { + dir = removeTrailingSlash(cmd.Flag("dir").Value.String()) + } + dryRun, err := cmd.Flags().GetBool("dry-run") + if err != nil { + return nil, fmt.Errorf("failed to get dry-run flag: %w", err) + } + errOnModified, err := cmd.Flags().GetBool("error") + if err != nil { + return nil, fmt.Errorf("failed to get error flag: %w", err) + } + quiet, err := cmd.Flags().GetBool("quiet") + if err != nil { + return nil, fmt.Errorf("failed to get quiet flag: %w", err) + } + regex, err := cmd.Flags().GetString("regex") + if err != nil { + return nil, fmt.Errorf("failed to get regex flag: %w", err) + } + + return &Helper{ + Cmd: cmd, + Dir: dir, + DryRun: dryRun, + ErrOnModified: errOnModified, + Quiet: quiet, + Regex: regex, + }, nil +} + +// DeclareFrizbeeFlags declares the flags common to all replacer commands. +func DeclareFrizbeeFlags(cmd *cobra.Command, defaultDir string) { + cmd.Flags().BoolP("dry-run", "n", false, "don't modify files") + cmd.Flags().BoolP("quiet", "q", false, "don't print anything") + cmd.Flags().BoolP("error", "e", false, "exit with error code if any file is modified") + cmd.Flags().StringP("regex", "r", "", "regex to match artifact references") + cmd.Flags().StringP("platform", "p", "", "platform to match artifact references, e.g. linux/amd64") + + if defaultDir != "" { + cmd.Flags().StringP("dir", "d", defaultDir, "directory path to parse") + } + +} + +// Logf logs the given message to the given command's stderr if the command is +// not quiet. +func (r *Helper) Logf(format string, args ...interface{}) { + if !r.Quiet { + fmt.Fprintf(r.Cmd.ErrOrStderr(), format, args...) + } +} + +// ProcessOutput processes the given output files. +// If the command is quiet, the output is discarded. +// If the command is a dry run, the output is written to the command's stdout. +// Otherwise, the output is written to the given filesystem. +func (r *Helper) ProcessOutput(processed []string, modified map[string]string) error { + basedir := filepath.Dir(r.Dir) + bfs := osfs.New(basedir, osfs.WithBoundOS()) + var out io.Writer + for _, path := range processed { + if !r.Quiet { + r.Logf("Processed: %s\n", path) + } + } + for path, content := range modified { + if r.Quiet { + out = io.Discard + } else if r.DryRun { + out = r.Cmd.OutOrStdout() + } else { + f, err := bfs.OpenFile(path, os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + return fmt.Errorf("failed to open file %s: %w", path, err) + } + + defer func() { + if err := f.Close(); err != nil { + fmt.Fprintf(r.Cmd.ErrOrStderr(), "failed to close file %s: %v", path, err) + } + }() + + out = f + } + if !r.Quiet { + r.Logf("Modified: %s\n", path) + } + _, err := fmt.Fprintf(out, "%s", content) + if err != nil { + return fmt.Errorf("failed to write to file %s: %w", path, err) + } + } + + return nil +} + +// removeTrailingSlash processes the given directory name for use with +// go-billy filesystems. +func removeTrailingSlash(dir string) string { + // remove trailing / from dir. This doesn't play well with + // the go-billy filesystem and walker we use. + if dir[len(dir)-1] == '/' { + return dir[:len(dir)-1] + } + + return dir +} diff --git a/internal/cmd/doc.go b/internal/cmd/doc.go deleted file mode 100644 index ec8b8ce..0000000 --- a/internal/cmd/doc.go +++ /dev/null @@ -1,17 +0,0 @@ -// -// Copyright 2023 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package cmd provide common implementations for commands -package cmd diff --git a/internal/cmd/yamlreplacer.go b/internal/cmd/yamlreplacer.go deleted file mode 100644 index e13c82f..0000000 --- a/internal/cmd/yamlreplacer.go +++ /dev/null @@ -1,160 +0,0 @@ -// -// Copyright 2023 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "bytes" - "context" - "fmt" - "io/fs" - "path/filepath" - "sync/atomic" - - "github.com/go-git/go-billy/v5/osfs" - "github.com/spf13/cobra" - "golang.org/x/sync/errgroup" - - "github.com/stacklok/frizbee/pkg/config" - "github.com/stacklok/frizbee/pkg/containers" - ferrors "github.com/stacklok/frizbee/pkg/errors" - "github.com/stacklok/frizbee/pkg/utils" - cliutils "github.com/stacklok/frizbee/pkg/utils/cli" -) - -// DeclareYAMLReplacerFlags declares the flags for the YAML replacer -func DeclareYAMLReplacerFlags(cli *cobra.Command) { - cli.Flags().StringP("dir", "d", ".", "manifests file or directory") - - cliutils.DeclareReplacerFlags(cli) -} - -// YAMLReplacer replaces container image references in YAML files -type YAMLReplacer struct { - cliutils.Replacer - ImageRegex string -} - -// WithImageRegex sets the image regex -func WithImageRegex(regex string) func(*YAMLReplacer) { - return func(r *YAMLReplacer) { - r.ImageRegex = regex - } -} - -// NewYAMLReplacer creates a new YAMLReplacer from the given -// command-line arguments and options -func NewYAMLReplacer(cli *cobra.Command, opts ...func(*YAMLReplacer)) (*YAMLReplacer, error) { - dir := cli.Flag("dir").Value.String() - dryRun, err := cli.Flags().GetBool("dry-run") - if err != nil { - return nil, fmt.Errorf("failed to get dry-run flag: %w", err) - } - errOnModified, err := cli.Flags().GetBool("error") - if err != nil { - return nil, fmt.Errorf("failed to get error flag: %w", err) - } - quiet, err := cli.Flags().GetBool("quiet") - if err != nil { - return nil, fmt.Errorf("failed to get quiet flag: %w", err) - } - - dir = cliutils.ProcessDirNameForBillyFS(dir) - - r := &YAMLReplacer{ - Replacer: cliutils.Replacer{ - Dir: dir, - DryRun: dryRun, - Quiet: quiet, - ErrOnModified: errOnModified, - Cmd: cli, - }, - ImageRegex: "image", - } - for _, opt := range opts { - opt(r) - } - - return r, nil -} - -// Do runs the YAMLReplacer -func (r *YAMLReplacer) Do(ctx context.Context, _ *config.Config) error { - basedir := filepath.Dir(r.Dir) - base := filepath.Base(r.Dir) - // NOTE: For some reason using boundfs causes a panic when trying to open a file. - // I instead falled back to chroot which is the default. - bfs := osfs.New(basedir) - - outfiles := map[string]string{} - - var modified atomic.Bool - modified.Store(false) - - var eg errgroup.Group - cache := utils.NewRefCacher() - - err := utils.Traverse(bfs, base, func(path string, info fs.FileInfo) error { - eg.Go(func() error { - if !utils.IsYAMLFile(info) { - return nil - } - - f, err := bfs.Open(path) - if err != nil { - return fmt.Errorf("failed to open file %s: %w", path, err) - } - - // nolint:errcheck // ignore error - defer f.Close() - - r.Logf("Processing %s\n", path) - - buf := bytes.Buffer{} - m, err := containers.ReplaceReferenceFromYAMLWithCache(ctx, r.ImageRegex, f, &buf, cache) - if err != nil { - return fmt.Errorf("failed to process YAML file %s: %w", path, err) - } - - modified.Store(modified.Load() || m) - - if m { - r.Logf("Modified %s\n", path) - outfiles[path] = buf.String() - } - - return nil - }) - - return nil - }) - if err != nil { - return err - } - - if err := eg.Wait(); err != nil { - return err - } - - if err := r.ProcessOutput(bfs, outfiles); err != nil { - return err - } - - if r.ErrOnModified && modified.Load() { - return ferrors.ErrModifiedFiles - } - - return nil -} diff --git a/internal/ghrest/ghrest.go b/internal/ghrest/ghrest.go index 158174d..57d5b2c 100644 --- a/internal/ghrest/ghrest.go +++ b/internal/ghrest/ghrest.go @@ -25,20 +25,20 @@ import ( "github.com/google/go-github/v61/github" ) -// GhRest is the struct that contains the GitHub REST API client +// Client is the struct that contains the GitHub REST API client // this struct implements the REST API -type GhRest struct { +type Client struct { client *github.Client } -// NewGhRest creates a new instance of GhRest -func NewGhRest(token string) *GhRest { +// NewClient creates a new instance of GhRest +func NewClient(token string) *Client { ghcli := github.NewClient(nil) if token != "" { ghcli = ghcli.WithAuthToken(token) } - return &GhRest{ + return &Client{ client: ghcli, } } @@ -47,12 +47,12 @@ func NewGhRest(token string) *GhRest { // which will be resolved to the BaseURL of the Client. Relative URLS should // always be specified without a preceding slash. If specified, the value // pointed to by body is JSON encoded and included as the request body. -func (c *GhRest) NewRequest(method, requestUrl string, body any) (*http.Request, error) { +func (c *Client) NewRequest(method, requestUrl string, body any) (*http.Request, error) { return c.client.NewRequest(method, requestUrl, body) } // Do sends an API request and returns the API response. -func (c *GhRest) Do(ctx context.Context, req *http.Request) (*http.Response, error) { +func (c *Client) Do(ctx context.Context, req *http.Request) (*http.Response, error) { var buf bytes.Buffer // The GitHub client closes the response body, so we need to capture it diff --git a/pkg/utils/cache.go b/internal/store/cache.go similarity index 99% rename from pkg/utils/cache.go rename to internal/store/cache.go index 02333e4..96cf7e0 100644 --- a/pkg/utils/cache.go +++ b/internal/store/cache.go @@ -13,7 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package utils +package store import ( "github.com/puzpuzpuz/xsync" diff --git a/internal/traverse/traverse.go b/internal/traverse/traverse.go new file mode 100644 index 0000000..18556e7 --- /dev/null +++ b/internal/traverse/traverse.go @@ -0,0 +1,138 @@ +package traverse + +import ( + "fmt" + "github.com/go-git/go-billy/v5" + "io/fs" + "os" + "path/filepath" + "strings" +) + +// TraverseGHWFunc is a function that gets called with each file in a GitHub Actions workflow +// directory. It receives the path to the file. +type TraverseGHWFunc func(path string) error + +// TraverseFunc is a function that gets called with each file in a directory. +type TraverseFunc func(path string, info fs.FileInfo) error + +// TraverseYAMLDockerfiles traverses all yaml/yml in the given directory +// and calls the given function with each workflow. +func TraverseYAMLDockerfiles(bfs billy.Filesystem, base string, fun TraverseGHWFunc) error { + return Traverse(bfs, base, func(path string, info fs.FileInfo) error { + if !isYAMLOrDockerfile(info) { + return nil + } + + if err := fun(path); err != nil { + return fmt.Errorf("failed to process file %s: %w", path, err) + } + + return nil + }) +} + +// Traverse traverses the given directory and calls the given function with each file. +func Traverse(bfs billy.Filesystem, base string, fun TraverseFunc) error { + return Walk(bfs, base, func(path string, info fs.FileInfo, err error) error { + if err != nil { + return nil + } + + return fun(path, info) + }) +} + +// isYAMLOrDockerfile returns true if the given file is a YAML or Dockerfile. +func isYAMLOrDockerfile(info fs.FileInfo) bool { + // Skip if not a file + if info.IsDir() { + return false + } + + // Filter out files that are not yml, yaml or dockerfiles + if strings.HasSuffix(info.Name(), ".yml") || strings.HasSuffix(info.Name(), ".yaml") || + strings.Contains(strings.ToLower(info.Name()), "dockerfile") { + return true + } + + return false +} + +// walk recursively descends path, calling walkFn +// adapted from https://golang.org/src/path/filepath/path.go +func walk(fs billy.Filesystem, path string, info os.FileInfo, walkFn filepath.WalkFunc) error { + if !info.IsDir() { + return walkFn(path, info, nil) + } + + names, err := readDirNames(fs, path) + err1 := walkFn(path, info, err) + // If err != nil, walk can't walk into this directory. + // err1 != nil means walkFn want walk to skip this directory or stop walking. + // Therefore, if one of err and err1 isn't nil, walk will return. + if err != nil || err1 != nil { + // The caller's behavior is controlled by the return value, which is decided + // by walkFn. walkFn may ignore err and return nil. + // If walkFn returns SkipDir, it will be handled by the caller. + // So walk should return whatever walkFn returns. + return err1 + } + + for _, name := range names { + filename := filepath.Join(path, name) + fileInfo, err := fs.Lstat(filename) + if err != nil { + if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir { + return err + } + } else { + err = walk(fs, filename, fileInfo, walkFn) + if err != nil { + if !fileInfo.IsDir() || err != filepath.SkipDir { + return err + } + } + } + } + return nil +} + +// Walk walks the file tree rooted at root, calling fn for each file or +// directory in the tree, including root. All errors that arise visiting files +// and directories are filtered by fn: see the WalkFunc documentation for +// details. +// +// The files are walked in lexical order, which makes the output deterministic +// but requires Walk to read an entire directory into memory before proceeding +// to walk that directory. Walk does not follow symbolic links. +// +// Function adapted from https://github.com/golang/go/blob/3b770f2ccb1fa6fecc22ea822a19447b10b70c5c/src/path/filepath/path.go#L500 +func Walk(fs billy.Filesystem, root string, walkFn filepath.WalkFunc) error { + info, err := fs.Lstat(root) + if err != nil { + err = walkFn(root, nil, err) + } else { + err = walk(fs, root, info, walkFn) + } + + if err == filepath.SkipDir { + return nil + } + + return err +} + +func readDirNames(fs billy.Filesystem, dir string) ([]string, error) { + files, err := fs.ReadDir(dir) + if err != nil { + return nil, err + } + + var names []string + for _, file := range files { + names = append(names, file.Name()) + } + + return names, nil +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 3e91b01..0743b5c 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -17,9 +17,9 @@ package config import ( - "context" "errors" "fmt" + "github.com/spf13/cobra" "os" "path/filepath" @@ -39,18 +39,24 @@ var ( ErrNoConfigInContext = errors.New("no configuration found in context") ) -// FromContext returns the configuration from the context. -func FromContext(ctx context.Context) (*Config, error) { +// FromCommand returns the configuration from the cobra command. +func FromCommand(cmd *cobra.Command) (*Config, error) { + ctx := cmd.Context() cfg, ok := ctx.Value(ContextConfigKey).(*Config) if !ok { return nil, ErrNoConfigInContext } + // If the platform flag is set, override the platform in the configuration. + if cmd.Flags().Lookup("platform") != nil { + cfg.Platform = cmd.Flag("platform").Value.String() + } return cfg, nil } // Config is the frizbee configuration. type Config struct { + Platform string `yaml:"platform" mapstructure:"platform"` GHActions GHActions `yaml:"ghactions" mapstructure:"ghactions"` } diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go deleted file mode 100644 index a87cef6..0000000 --- a/pkg/constants/constants.go +++ /dev/null @@ -1,97 +0,0 @@ -// -// Copyright 2023 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package constants provides constants for the frizbee utilities. -package constants - -import ( - "runtime/debug" - "strings" - "text/template" -) - -const ( - // UserAgent is the user agent string used by frizbee. - // - // TODO (jaosorior): Add version information to this. - UserAgent = "frizbee" -) - -var ( - // CLIVersion is the version of the frizbee CLI. - // nolint: gochecknoglobals - CLIVersion = "dev" - // VerboseCLIVersion is the verbose version of the frizbee CLI. - // nolint: gochecknoglobals - VerboseCLIVersion = "" -) - -type versionInfo struct { - Version string - GoVersion string - Time string - Commit string - OS string - Arch string - Modified bool -} - -const ( - verboseTemplate = `Version: {{ .Version }} -Go Version: {{.GoVersion}} -Git Commit: {{.Commit}} -Commit Date: {{.Time}} -OS/Arch: {{.OS}}/{{.Arch}} -Dirty: {{.Modified}} -` -) - -// nolint:init -func init() { - buildinfo, ok := debug.ReadBuildInfo() - if !ok { - return - } - - var vinfo versionInfo - vinfo.Version = CLIVersion - vinfo.GoVersion = buildinfo.GoVersion - - for _, kv := range buildinfo.Settings { - switch kv.Key { - case "vcs.time": - vinfo.Time = kv.Value - case "vcs.revision": - vinfo.Commit = kv.Value - case "vcs.modified": - vinfo.Modified = kv.Value == "true" - case "GOOS": - vinfo.OS = kv.Value - case "GOARCH": - vinfo.Arch = kv.Value - } - } - VerboseCLIVersion = vinfo.String() -} - -func (vvs *versionInfo) String() string { - stringBuilder := &strings.Builder{} - tmpl := template.Must(template.New("version").Parse(verboseTemplate)) - err := tmpl.Execute(stringBuilder, vvs) - if err != nil { - panic(err) - } - return stringBuilder.String() -} diff --git a/pkg/containers/containers.go b/pkg/containers/containers.go deleted file mode 100644 index f69834a..0000000 --- a/pkg/containers/containers.go +++ /dev/null @@ -1,50 +0,0 @@ -// -// Copyright 2023 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package containers provides functions to replace tags for checksums -package containers - -import ( - "context" - "fmt" - - "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/v1/remote" - - "github.com/stacklok/frizbee/pkg/constants" -) - -// GetDigest returns the digest of a container image reference. -func GetDigest(ctx context.Context, refstr string) (string, error) { - ref, err := name.ParseReference(refstr) - if err != nil { - return "", fmt.Errorf("failed to parse reference: %w", err) - } - - return GetDigestFromRef(ctx, ref) -} - -// GetDigestFromRef returns the digest of a container image reference -// from a name.Reference. -func GetDigestFromRef(ctx context.Context, ref name.Reference) (string, error) { - desc, err := remote.Get(ref, - remote.WithContext(ctx), - remote.WithUserAgent(constants.UserAgent)) - if err != nil { - return "", fmt.Errorf("failed to get remote reference: %w", err) - } - - return desc.Digest.String(), nil -} diff --git a/pkg/containers/containers_test.go b/pkg/containers/containers_test.go deleted file mode 100644 index 137552d..0000000 --- a/pkg/containers/containers_test.go +++ /dev/null @@ -1,87 +0,0 @@ -// -// Copyright 2023 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package containers provides functions to replace tags for checksums -package containers - -import ( - "context" - "testing" - "time" - - "github.com/stretchr/testify/assert" -) - -func TestGetDigest(t *testing.T) { - t.Parallel() - - type args struct { - refstr string - } - tests := []struct { - name string - args args - want string - wantErr bool - }{ - { - name: "valid 1", - args: args{ - refstr: "ghcr.io/stacklok/minder/helm/minder:0.20231123.829_ref.26ca90b", - }, - want: "sha256:a29f8a8d28f0af7f70a4b3dd3e33c8c8cc5cf9e88e802e2700cf272a0b6140ec", - }, - { - name: "valid 2", - args: args{ - refstr: "devopsfaith/krakend:2.5.0", - }, - want: "sha256:6a3c8e5e1a4948042bfb364ed6471e16b4a26d0afb6c3c01ebcb88b3fa551036", - }, - { - name: "invalid ref string", - args: args{ - refstr: "ghcr.io/stacklok/minder/helm/minder!", - }, - wantErr: true, - }, - { - name: "unexistent container in unexistent registry", - args: args{ - refstr: "beeeeer.io/ipa/toppling-goliath/king-sue:1.0.0", - }, - wantErr: true, - }, - } - for _, tt := range tests { - tt := tt - - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - got, err := GetDigest(ctx, tt.args.refstr) - if tt.wantErr { - assert.Error(t, err) - assert.Empty(t, got) - return - } - - assert.NoError(t, err) - assert.Equal(t, tt.want, got) - }) - } -} diff --git a/pkg/containers/replace.go b/pkg/containers/replace.go deleted file mode 100644 index 9f5d4a9..0000000 --- a/pkg/containers/replace.go +++ /dev/null @@ -1,100 +0,0 @@ -// -// Copyright 2023 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package containers - -import ( - "bufio" - "context" - "fmt" - "io" - "regexp" - - "github.com/google/go-containerregistry/pkg/name" - - "github.com/stacklok/frizbee/pkg/utils" -) - -// ReplaceImageReferenceFromYAML replaces the image reference in the input text with the digest -func ReplaceImageReferenceFromYAML(ctx context.Context, input io.Reader, output io.Writer) (bool, error) { - return ReplaceReferenceFromYAML(ctx, "image", input, output) -} - -// ReplaceReferenceFromYAML replaces the image reference in the input text with the digest -func ReplaceReferenceFromYAML(ctx context.Context, keyRegex string, input io.Reader, output io.Writer) (bool, error) { - cache := utils.NewUnsafeCacher() - return ReplaceReferenceFromYAMLWithCache(ctx, keyRegex, input, output, cache) -} - -// ReplaceReferenceFromYAMLWithCache replaces the image reference in the input text with the digest -// and uses the provided cache to store the digests. -func ReplaceReferenceFromYAMLWithCache( - ctx context.Context, keyRegex string, input io.Reader, output io.Writer, cache utils.RefCacher) (bool, error) { - scanner := bufio.NewScanner(input) - re, err := regexp.Compile(fmt.Sprintf(`(\s*%s):\s*([^\s]+)`, keyRegex)) - if err != nil { - return false, fmt.Errorf("failed to compile regex: %w", err) - } - - modified := false - - for scanner.Scan() { - line := scanner.Text() - updatedLine := re.ReplaceAllStringFunc(line, func(match string) string { - submatches := re.FindStringSubmatch(match) - if len(submatches) != 3 { - return match - } - - imageReferenceWithTag := submatches[2] - ref, err := name.ParseReference(imageReferenceWithTag) - if err != nil { - return match - } - - var digest string - if d, ok := cache.Load(ref.Identifier()); ok { - digest = d - } else { - digest, err = GetDigestFromRef(ctx, ref) - if err != nil { - return match - } - - cache.Store(ref.Identifier(), digest) - } - - imgWithoutTag := ref.Context().Name() - outstr := imgWithoutTag + "@" + digest - - if imageReferenceWithTag != outstr { - modified = true - } - - replacement := fmt.Sprintf("${1}: %s # %s", outstr, ref.Identifier()) - return re.ReplaceAllString(match, replacement) - }) - - if _, err := io.WriteString(output, updatedLine+"\n"); err != nil { - return false, err - } - } - - if err := scanner.Err(); err != nil { - return false, err - } - - return modified, nil -} diff --git a/pkg/containers/replace_test.go b/pkg/containers/replace_test.go deleted file mode 100644 index 2b0496b..0000000 --- a/pkg/containers/replace_test.go +++ /dev/null @@ -1,75 +0,0 @@ -// -// Copyright 2023 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package containers - -import ( - "context" - "strings" - "testing" - - "github.com/stretchr/testify/assert" -) - -func TestReplaceImageReference(t *testing.T) { - t.Parallel() - - ctx := context.Background() - - testCases := []struct { - name string - input string - expectedOutput string - modified bool - }{ - { - name: "Replace image reference", - input: ` -version: v1 -services: - - name: kube-apiserver - image: registry.k8s.io/kube-apiserver:v1.20.0 - - name: kube-controller-manager - image: registry.k8s.io/kube-controller-manager:v1.15.0 -`, - expectedOutput: ` -version: v1 -services: - - name: kube-apiserver - image: registry.k8s.io/kube-apiserver@sha256:8b8125d7a6e4225b08f04f65ca947b27d0cc86380bf09fab890cc80408230114 # v1.20.0 - - name: kube-controller-manager - image: registry.k8s.io/kube-controller-manager@sha256:835f32a5cdb30e86f35675dd91f9c7df01d48359ab8b51c1df866a2c7ea2e870 # v1.15.0 -`, - modified: true, - }, - // Add more test cases as needed - } - - // Define a regular expression to match YAML tags containing "image" - for _, tc := range testCases { - tc := tc - - t.Run(tc.name, func(t *testing.T) { - t.Parallel() - - var output strings.Builder - m, err := ReplaceImageReferenceFromYAML(ctx, strings.NewReader(tc.input), &output) - assert.NoError(t, err) - - assert.Equal(t, tc.expectedOutput, output.String()) - assert.Equal(t, tc.modified, m, "modified") - }) - } -} diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go deleted file mode 100644 index affea4a..0000000 --- a/pkg/errors/errors.go +++ /dev/null @@ -1,24 +0,0 @@ -// -// Copyright 2024 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package errors provides error values, constants and functions. -package errors - -import "errors" - -var ( - // ErrModifiedFiles is returned when modified files are found - ErrModifiedFiles = errors.New("modified files") -) diff --git a/pkg/ghactions/errors.go b/pkg/ghactions/errors.go deleted file mode 100644 index e407307..0000000 --- a/pkg/ghactions/errors.go +++ /dev/null @@ -1,26 +0,0 @@ -// -// Copyright 2023 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ghactions - -import "errors" - -var ( - // ErrInvalidAction is returned when parsing the action fails. - ErrInvalidAction = errors.New("invalid action") - - // ErrInvalidActionReference is returned when parsing the action reference fails. - ErrInvalidActionReference = errors.New("action reference is not a tag nor branch") -) diff --git a/pkg/ghactions/ghactions.go b/pkg/ghactions/ghactions.go deleted file mode 100644 index 22be0c5..0000000 --- a/pkg/ghactions/ghactions.go +++ /dev/null @@ -1,272 +0,0 @@ -// -// Copyright 2023 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package ghactions provides functions to locate action references and -// replace tags for checksums in GitHub Actions workflows. -package ghactions - -import ( - "context" - "fmt" - "path/filepath" - "strings" - - mapset "github.com/deckarep/golang-set/v2" - "github.com/go-git/go-billy/v5/osfs" - "gopkg.in/yaml.v3" - - "github.com/stacklok/frizbee/pkg/config" - "github.com/stacklok/frizbee/pkg/interfaces" - "github.com/stacklok/frizbee/pkg/utils" -) - -// IsLocal returns true if the input is a local path. -func IsLocal(input string) bool { - return strings.HasPrefix(input, "./") || strings.HasPrefix(input, "../") -} - -// ParseActionReference parses an action reference into action and reference. -func ParseActionReference(input string) (action string, reference string, err error) { - frags := strings.Split(input, "@") - if len(frags) != 2 { - return "", "", fmt.Errorf("invalid action reference: %s", input) - } - - return frags[0], frags[1], nil -} - -// GetChecksum returns the checksum for a given action and tag. -func GetChecksum(ctx context.Context, restIf interfaces.REST, action, ref string) (string, error) { - owner, repo, err := parseActionFragments(action) - if err != nil { - return "", err - } - - // Check if we're using a checksum - if isChecksum(ref) { - return ref, nil - } - - res, err := getCheckSumForTag(ctx, restIf, owner, repo, ref) - if err != nil { - return "", fmt.Errorf("failed to get checksum for tag: %w", err) - } else if res != "" { - return res, nil - } - - // check branch - res, err = getCheckSumForBranch(ctx, restIf, owner, repo, ref) - if err != nil { - return "", fmt.Errorf("failed to get checksum for branch: %w", err) - } else if res != "" { - return res, nil - } - - return "", ErrInvalidActionReference -} - -// ModifyReferencesInYAML takes the given YAML structure and replaces -// all references to tags with the checksum of the tag. -// Note that the given YAML structure is modified in-place. -// The function returns true if any references were modified. -func ModifyReferencesInYAML(ctx context.Context, restIf interfaces.REST, node *yaml.Node, cfg *config.GHActions) (bool, error) { - cache := utils.NewUnsafeCacher() - return ModifyReferencesInYAMLWithCache(ctx, restIf, node, cfg, cache) -} - -// ModifyReferencesInYAMLWithCache takes the given YAML structure and replaces -// all references to tags with the checksum of the tag. -// Note that the given YAML structure is modified in-place. -// The function returns true if any references were modified. -// The function uses the provided cache to store the checksums. -func ModifyReferencesInYAMLWithCache( - ctx context.Context, restIf interfaces.REST, node *yaml.Node, cfg *config.GHActions, cache utils.RefCacher) (bool, error) { - // `uses` will be immediately before the action - // name in the YAML `Content` array. We use a toggle - // to track if we've found `uses` and then look for - // the next node. - foundUses := false - modified := false - - for _, v := range node.Content { - if v.Value == "uses" { - foundUses = true - continue - } - - if foundUses { - foundUses = false - - // If the value is a local path, skip it - if IsLocal(v.Value) { - continue - } - - if shouldExclude(cfg, v.Value) { - continue - } - - act, ref, err := ParseActionReference(v.Value) - if err != nil { - return modified, fmt.Errorf("failed to parse action reference '%s': %w", v.Value, err) - } - - if shouldExclude(cfg, act) { - continue - } - - var sum string - - // Check if we have a cached value - if val, ok := cache.Load(v.Value); ok { - sum = val - } else { - sum, err = GetChecksum(ctx, restIf, act, ref) - if err != nil { - return modified, fmt.Errorf("failed to get checksum for action '%s': %w", v.Value, err) - } - - cache.Store(v.Value, sum) - } - - if ref != sum { - v.SetString(fmt.Sprintf("%s@%s", act, sum)) - v.LineComment = ref - modified = true - } - continue - } - - // Otherwise recursively look more - m, err := ModifyReferencesInYAMLWithCache(ctx, restIf, v, cfg, cache) - if err != nil { - return m, err - } - modified = modified || m - } - return modified, nil -} - -// Action represents an action reference. -type Action struct { - Action string `json:"action"` - Owner string `json:"owner"` - Repo string `json:"repo"` - Ref string `json:"ref"` -} - -// ListActionsInYAML returns a list of actions referenced in the given YAML structure. -func setOfActions(node *yaml.Node) (mapset.Set[Action], error) { - actions := mapset.NewThreadUnsafeSet[Action]() - foundUses := false - - for _, v := range node.Content { - if v.Value == "uses" { - foundUses = true - continue - } - - if foundUses { - foundUses = false - - // If the value is a local path, skip it - if IsLocal(v.Value) { - continue - } - - a, err := parseValue(v.Value) - if err != nil { - return nil, fmt.Errorf("failed to parse action reference '%s': %w", v.Value, err) - } - - actions.Add(*a) - continue - } - - // Otherwise recursively look more - childUses, err := setOfActions(v) - if err != nil { - return nil, err - } - actions = actions.Union(childUses) - } - - return actions, nil -} - -// ListActionsInYAML returns a list of actions referenced in the given YAML structure. -func ListActionsInYAML(node *yaml.Node) ([]Action, error) { - actions, err := setOfActions(node) - if err != nil { - return nil, err - } - - return actions.ToSlice(), nil -} - -// ListActionsInDirectory returns a list of actions referenced in the given directory. -func ListActionsInDirectory(dir string) ([]Action, error) { - base := filepath.Base(dir) - bfs := osfs.New(filepath.Dir(dir), osfs.WithBoundOS()) - actions := mapset.NewThreadUnsafeSet[Action]() - - err := TraverseGitHubActionWorkflows(bfs, base, func(path string, wflow *yaml.Node) error { - wfActions, err := setOfActions(wflow) - if err != nil { - return fmt.Errorf("failed to get actions from YAML file %s: %w", path, err) - } - - actions = actions.Union(wfActions) - return nil - }) - if err != nil { - return nil, err - } - - return actions.ToSlice(), nil -} - -func parseValue(val string) (*Action, error) { - action, ref, err := ParseActionReference(val) - if err != nil { - return nil, fmt.Errorf("failed to parse action reference '%s': %w", val, err) - } - - owner, repo, err := parseActionFragments(action) - if err != nil { - return nil, fmt.Errorf("failed to parse action fragments '%s': %w", action, err) - } - - return &Action{ - Action: action, - Owner: owner, - Repo: repo, - Ref: ref, - }, nil -} - -func shouldExclude(cfg *config.GHActions, input string) bool { - for _, e := range cfg.Exclude { - if e == input { - return true - } - } - return false -} - -// isChecksum returns true if the input is a checksum. -func isChecksum(ref string) bool { - return len(ref) == 40 -} diff --git a/pkg/ghactions/ghactions_test.go b/pkg/ghactions/ghactions_test.go deleted file mode 100644 index 839ed84..0000000 --- a/pkg/ghactions/ghactions_test.go +++ /dev/null @@ -1,361 +0,0 @@ -// -// Copyright 2023 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ghactions_test - -import ( - "context" - "os" - "strings" - "testing" - - "github.com/stretchr/testify/require" - "gopkg.in/yaml.v3" - - "github.com/stacklok/frizbee/internal/ghrest" - "github.com/stacklok/frizbee/pkg/config" - "github.com/stacklok/frizbee/pkg/ghactions" -) - -func TestParseActionReference(t *testing.T) { - t.Parallel() - - type args struct { - input string - } - type returns struct { - action string - reference string - } - tests := []struct { - name string - args args - returns returns - wantErr bool - }{ - { - name: "actions/checkout@v4.1.1", - args: args{ - input: "actions/checkout@v4.1.1", - }, - returns: returns{ - action: "actions/checkout", - reference: "v4.1.1", - }, - wantErr: false, - }, - { - name: "actions/checkout@v3.6.0", - args: args{ - input: "actions/checkout@v3.6.0", - }, - returns: returns{ - action: "actions/checkout", - reference: "v3.6.0", - }, - wantErr: false, - }, - { - name: "actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f", - args: args{ - input: "actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f", - }, - returns: returns{ - action: "actions/checkout", - reference: "1d96c772d19495a3b5c517cd2bc0cb401ea0529f", - }, - wantErr: false, - }, - { - name: "actions/checkout-invalid", - args: args{ - input: "actions/checkout-invalid", - }, - returns: returns{ - action: "", - reference: "", - }, - wantErr: true, - }, - } - for _, tt := range tests { - tt := tt - - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - gotact, gotref, err := ghactions.ParseActionReference(tt.args.input) - if tt.wantErr { - require.Error(t, err, "Wanted error, got none") - return - } - require.NoError(t, err, "Wanted no error, got %v", err) - require.Equal(t, tt.returns.action, gotact, "Wanted %v, got %v", tt.returns.action, gotact) - require.Equal(t, tt.returns.reference, gotref, "Wanted %v, got %v", tt.returns.reference, gotref) - }) - } -} - -func TestGetChecksum(t *testing.T) { - t.Parallel() - - tok := os.Getenv("GITHUB_TOKEN") - - type args struct { - action string - ref string - } - tests := []struct { - name string - args args - want string - wantErr bool - }{ - { - name: "actions/checkout with v4.1.1", - args: args{ - action: "actions/checkout", - ref: "v4.1.1", - }, - want: "b4ffde65f46336ab88eb53be808477a3936bae11", - wantErr: false, - }, - { - name: "actions/checkout with v3.6.0", - args: args{ - action: "actions/checkout", - ref: "v3.6.0", - }, - want: "f43a0e5ff2bd294095638e18286ca9a3d1956744", - wantErr: false, - }, - { - name: "actions/checkout with checksum returns checksum", - args: args{ - action: "actions/checkout", - ref: "1d96c772d19495a3b5c517cd2bc0cb401ea0529f", - }, - want: "1d96c772d19495a3b5c517cd2bc0cb401ea0529f", - wantErr: false, - }, - { - name: "aquasecurity/trivy-action with 0.14.0", - args: args{ - action: "aquasecurity/trivy-action", - ref: "0.14.0", - }, - want: "2b6a709cf9c4025c5438138008beaddbb02086f0", - wantErr: false, - }, - { - name: "aquasecurity/trivy-action with branch returns checksum", - args: args{ - action: "aquasecurity/trivy-action", - ref: "bump-trivy", - }, - want: "fb5e1b36be448e92ca98648c661bd7e9da1f1317", - wantErr: false, - }, - { - name: "actions/checkout with invalid tag returns error", - args: args{ - action: "actions/checkout", - ref: "v4.1.1.1", - }, - want: "", - wantErr: true, - }, - { - name: "actions/checkout with invalid action returns error", - args: args{ - action: "invalid-action", - ref: "v4.1.1", - }, - want: "", - wantErr: true, - }, - { - name: "actions/checkout with empty action returns error", - args: args{ - action: "", - ref: "v4.1.1", - }, - want: "", - wantErr: true, - }, - { - name: "actions/checkout with empty tag returns error", - args: args{ - action: "actions/checkout", - ref: "", - }, - want: "", - wantErr: true, - }, - { - name: "bufbuild/buf-setup-action with v1 is an array", - args: args{ - action: "bufbuild/buf-setup-action", - ref: "v1", - }, - want: "480a0ee8a588045b52a847b48138c6f377a89519", - }, - { - name: "anchore/sbom-action/download-syft with a sub-action works", - args: args{ - action: "anchore/sbom-action/download-syft", - ref: "v0.14.3", - }, - want: "78fc58e266e87a38d4194b2137a3d4e9bcaf7ca1", - }, - } - for _, tt := range tests { - tt := tt - - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - ghcli := ghrest.NewGhRest(tok) - got, err := ghactions.GetChecksum(context.Background(), ghcli, tt.args.action, tt.args.ref) - if tt.wantErr { - require.Error(t, err, "Wanted error, got none") - require.Empty(t, got, "Wanted empty string, got %v", got) - return - } - require.NoError(t, err, "Wanted no error, got %v", err) - require.Equal(t, tt.want, got, "Wanted %v, got %v", tt.want, got) - }) - } -} - -const ( - workflowYAML = ` -name: CI -on: - push: - branches: - - main - pull_request: - branches: - - main -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: checkout - uses: actions/checkout@v4 - - name: setup go - uses: actions/setup-go@v4 -` -) - -func TestModifyReferencesInYAML(t *testing.T) { - t.Parallel() - - tok := os.Getenv("GITHUB_TOKEN") - - tests := []struct { - name string - mustContain []string - mustNotContain []string - wantErr bool - cfg *config.GHActions - }{ - { - name: "modify all", - wantErr: false, - mustContain: []string{ - " uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4", - " uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4", - }, - mustNotContain: []string{ - " uses: actions/checkout@v4", - " uses: actions/setup-go@v4", - }, - cfg: &config.GHActions{ - Filter: config.Filter{ - Exclude: []string{}, - }, - }, - }, - { - name: "exclude full uses", - wantErr: false, - mustContain: []string{ - " uses: actions/checkout@v4", - " uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4", - }, - mustNotContain: []string{ - " uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4", - " uses: actions/setup-go@v4", - }, - cfg: &config.GHActions{ - Filter: config.Filter{ - Exclude: []string{ - "actions/checkout@v4", - }, - }, - }, - }, - { - name: "exclude just the action name", - wantErr: false, - mustContain: []string{ - " uses: actions/checkout@v4", - " uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4", - }, - mustNotContain: []string{ - " uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4", - " uses: actions/setup-go@v4", - }, - cfg: &config.GHActions{ - Filter: config.Filter{ - Exclude: []string{ - "actions/checkout", - }, - }, - }, - }, - } - - for _, tt := range tests { - tt := tt - - t.Run(tt.name, func(t *testing.T) { - t.Parallel() - - ghcli := ghrest.NewGhRest(tok) - - var root yaml.Node - err := yaml.Unmarshal([]byte(workflowYAML), &root) - require.NoError(t, err, "Error unmarshalling YAML, got %v", err) - - got, err := ghactions.ModifyReferencesInYAML(context.Background(), ghcli, &root, tt.cfg) - if tt.wantErr { - require.Error(t, err, "Wanted error, got none") - require.Empty(t, got, "Wanted empty string, got %v", got) - return - } - require.NoError(t, err, "Wanted no error, got %v", err) - - out, err := yaml.Marshal(&root) - require.NoError(t, err, "Error marhsalling YAML, got %v", err) - stringSlice := strings.Split(string(out), "\n") - - require.Subset(t, stringSlice, tt.mustContain, "Expected %v to not appear in %v", tt.mustContain, stringSlice) - require.NotSubset(t, stringSlice, tt.mustNotContain, "Expected %v to not appear in %v", tt.mustNotContain, stringSlice) - }) - } -} diff --git a/pkg/ghactions/utils.go b/pkg/ghactions/utils.go deleted file mode 100644 index 8ee3575..0000000 --- a/pkg/ghactions/utils.go +++ /dev/null @@ -1,61 +0,0 @@ -// -// Copyright 2023 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ghactions - -import ( - "fmt" - "io/fs" - - "github.com/go-git/go-billy/v5" - "gopkg.in/yaml.v3" - - "github.com/stacklok/frizbee/pkg/utils" -) - -// TraverseGHWFunc is a function that gets called with each file in a GitHub Actions workflow -// directory. It receives the path to the file and the parsed workflow. -type TraverseGHWFunc func(path string, wflow *yaml.Node) error - -// TraverseGitHubActionWorkflows traverses the GitHub Actions workflows in the given directory -// and calls the given function with each workflow. -func TraverseGitHubActionWorkflows(bfs billy.Filesystem, base string, fun TraverseGHWFunc) error { - return utils.Traverse(bfs, base, func(path string, info fs.FileInfo) error { - if !utils.IsYAMLFile(info) { - return nil - } - - f, err := bfs.Open(path) - if err != nil { - return fmt.Errorf("failed to open file %s: %w", path, err) - } - - // nolint:errcheck // ignore error - defer f.Close() - - dec := yaml.NewDecoder(f) - - var wflow yaml.Node - if err := dec.Decode(&wflow); err != nil { - return fmt.Errorf("failed to decode file %s: %w", path, err) - } - - if err := fun(path, &wflow); err != nil { - return fmt.Errorf("failed to process file %s: %w", path, err) - } - - return nil - }) -} diff --git a/pkg/interfaces/interfaces.go b/pkg/interfaces/interfaces.go new file mode 100644 index 0000000..3dc3e69 --- /dev/null +++ b/pkg/interfaces/interfaces.go @@ -0,0 +1,30 @@ +package interfaces + +import ( + "context" + "github.com/stacklok/frizbee/internal/store" + "github.com/stacklok/frizbee/pkg/config" + "net/http" +) + +// EntityRef represents an action reference. +type EntityRef struct { + Name string `json:"name"` + Ref string `json:"ref"` + Type string `json:"type"` +} + +type Parser interface { + GetRegex() string + Replace(ctx context.Context, matchedLine string, restIf REST, cfg config.Config, cache store.RefCacher, keepPrefix bool) (string, error) + ConvertToEntityRef(reference string) (*EntityRef, error) +} + +// The REST interface allows to wrap clients to talk to remotes +// When talking to GitHub, wrap a github client to provide this interface +type REST interface { + // NewRequest creates an HTTP request. + NewRequest(method, url string, body any) (*http.Request, error) + // Do executes an HTTP request. + Do(ctx context.Context, req *http.Request) (*http.Response, error) +} diff --git a/pkg/interfaces/rest.go b/pkg/interfaces/rest.go deleted file mode 100644 index 33c7f28..0000000 --- a/pkg/interfaces/rest.go +++ /dev/null @@ -1,33 +0,0 @@ -// -// Copyright 2023 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package interfaces provides generic interfaces for communicating with remotes -// like github -package interfaces - -import ( - "context" - "net/http" -) - -// The REST interface allows to wrap clients to talk to remotes -// When talking to GitHub, wrap a github client to provide this interface -type REST interface { - // NewRequest creates an HTTP request. - NewRequest(method, url string, body any) (*http.Request, error) - - // Do executes an HTTP request. - Do(ctx context.Context, req *http.Request) (*http.Response, error) -} diff --git a/pkg/replacer/action/action.go b/pkg/replacer/action/action.go new file mode 100644 index 0000000..5e3be8e --- /dev/null +++ b/pkg/replacer/action/action.go @@ -0,0 +1,163 @@ +package action + +import ( + "context" + "fmt" + "github.com/stacklok/frizbee/internal/store" + "github.com/stacklok/frizbee/pkg/config" + "github.com/stacklok/frizbee/pkg/interfaces" + "github.com/stacklok/frizbee/pkg/replacer/image" + "strings" +) + +const ( + // GitHubActionsRegex is regular expression pattern to match GitHub Actions usage + GitHubActionsRegex = `uses:\s*[^\s]+/[^\s]+@[^\s]+|uses:\s*docker://[^\s]+:[^\s]+` + prefixUses = "uses: " + prefixDocker = "docker://" + ReferenceType = "action" +) + +type Parser struct { + regex string +} + +func New(regex string) *Parser { + if regex == "" { + regex = GitHubActionsRegex + } + return &Parser{ + regex: regex, + } +} + +func (p *Parser) GetRegex() string { + return p.regex +} + +func (p *Parser) Replace(ctx context.Context, matchedLine string, restIf interfaces.REST, cfg config.Config, cache store.RefCacher, keepPrefix bool) (string, error) { + var err error + + // Trim the uses prefix + actionRef := strings.TrimPrefix(matchedLine, prefixUses) + + // Determine if the action reference has a docker prefix + if strings.Contains(actionRef, prefixDocker) { + actionRef, err = p.replaceDocker(ctx, actionRef, restIf, cfg, cache, keepPrefix) + } else { + actionRef, err = p.replaceAction(ctx, actionRef, restIf, cfg, cache, keepPrefix) + } + if err != nil { + return "", err + } + + // Add back the uses prefix, if needed + if keepPrefix { + actionRef = fmt.Sprintf("%s%s", prefixUses, actionRef) + } + + // Return the new action reference + return actionRef, nil +} + +func (p *Parser) replaceAction(ctx context.Context, matchedLine string, restIf interfaces.REST, cfg config.Config, cache store.RefCacher, keepPrefix bool) (string, error) { + actionRef := matchedLine + + // If the value is a local path or should be excluded, skip it + if isLocal(actionRef) || shouldExclude(&cfg.GHActions, actionRef) { + return matchedLine, nil + } + + // Parse the action reference + act, ref, err := ParseActionReference(actionRef) + if err != nil { + return matchedLine, nil + } + + // Check if the parsed reference should be excluded + if shouldExclude(&cfg.GHActions, act) { + return matchedLine, nil + } + var sum string + + // Check if we have a cache + if cache != nil { + // Check if we have a cached value + if val, ok := cache.Load(actionRef); ok { + sum = val + } else { + // Get the checksum for the action reference + sum, err = GetChecksum(ctx, restIf, act, ref) + if err != nil { + return matchedLine, nil + } + // Store the checksum in the cache + cache.Store(actionRef, sum) + } + } else { + // Get the checksum for the action reference + sum, err = GetChecksum(ctx, restIf, act, ref) + if err != nil { + return matchedLine, nil + } + } + // If the checksum is different from the reference, update the reference + // Otherwise, return the original line + if ref == sum { + return matchedLine, nil + } + + return fmt.Sprintf("%s@%s # %s", act, sum, ref), nil +} + +func (p *Parser) replaceDocker(ctx context.Context, matchedLine string, _ interfaces.REST, cfg config.Config, cache store.RefCacher, keepPrefix bool) (string, error) { + var err error + // Trim the docker prefix + actionRef := strings.TrimPrefix(matchedLine, prefixDocker) + + // If the value is a local path or should be excluded, skip it + if isLocal(actionRef) || shouldExclude(&cfg.GHActions, actionRef) { + return matchedLine, nil + } + + // Get the digest of the docker:// image reference + actionRef, err = image.GetImageDigestFromRef(ctx, actionRef, cfg.Platform, cache, false) + if err != nil { + return "", err + } + + // Check if the parsed reference should be excluded + if shouldExclude(&cfg.GHActions, actionRef) { + return matchedLine, nil + } + + // Add back the docker prefix, if needed + if keepPrefix { + actionRef = fmt.Sprintf("%s%s", prefixDocker, actionRef) + } + return actionRef, nil +} + +func (p *Parser) ConvertToEntityRef(reference string) (*interfaces.EntityRef, error) { + reference = strings.TrimPrefix(reference, prefixUses) + refType := ReferenceType + separator := "@" + // Update the separator in case this is a docker reference with a digest + if strings.Contains(reference, prefixDocker) { + reference = strings.TrimPrefix(reference, prefixDocker) + if !strings.Contains(reference, separator) && strings.Contains(reference, ":") { + separator = ":" + } + refType = image.ReferenceType + } + frags := strings.Split(reference, separator) + if len(frags) != 2 { + return nil, fmt.Errorf("invalid action reference: %s", reference) + } + + return &interfaces.EntityRef{ + Name: frags[0], + Ref: frags[1], + Type: refType, + }, nil +} diff --git a/pkg/ghactions/ghactions_helpers.go b/pkg/replacer/action/utils.go similarity index 52% rename from pkg/ghactions/ghactions_helpers.go rename to pkg/replacer/action/utils.go index ac77125..a6d2f8f 100644 --- a/pkg/ghactions/ghactions_helpers.go +++ b/pkg/replacer/action/utils.go @@ -1,33 +1,80 @@ -// -// Copyright 2023 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package ghactions +package action import ( "context" "encoding/json" + "errors" "fmt" + "github.com/google/go-github/v61/github" + "github.com/stacklok/frizbee/pkg/config" + "github.com/stacklok/frizbee/pkg/interfaces" "net/http" "net/url" "strings" +) - "github.com/google/go-github/v61/github" +var ( + // ErrInvalidAction is returned when parsing the action fails. + ErrInvalidAction = errors.New("invalid action") - "github.com/stacklok/frizbee/pkg/interfaces" + // ErrInvalidActionReference is returned when parsing the action reference fails. + ErrInvalidActionReference = errors.New("action reference is not a tag nor branch") ) +// isLocal returns true if the input is a local path. +func isLocal(input string) bool { + return strings.HasPrefix(input, "./") || strings.HasPrefix(input, "../") +} + +func shouldExclude(cfg *config.GHActions, input string) bool { + for _, e := range cfg.Exclude { + if e == input { + return true + } + } + return false +} + +// ParseActionReference parses an action reference into action and reference. +func ParseActionReference(input string) (action string, reference string, err error) { + frags := strings.Split(input, "@") + if len(frags) != 2 { + return "", "", fmt.Errorf("invalid action reference: %s", input) + } + + return frags[0], frags[1], nil +} + +// GetChecksum returns the checksum for a given action and tag. +func GetChecksum(ctx context.Context, restIf interfaces.REST, action, ref string) (string, error) { + owner, repo, err := parseActionFragments(action) + if err != nil { + return "", err + } + + // Check if we're using a checksum + if isChecksum(ref) { + return ref, nil + } + + res, err := getCheckSumForTag(ctx, restIf, owner, repo, ref) + if err != nil { + return "", fmt.Errorf("failed to get checksum for tag: %w", err) + } else if res != "" { + return res, nil + } + + // check branch + res, err = getCheckSumForBranch(ctx, restIf, owner, repo, ref) + if err != nil { + return "", fmt.Errorf("failed to get checksum for branch: %w", err) + } else if res != "" { + return res, nil + } + + return "", ErrInvalidActionReference +} + func parseActionFragments(action string) (owner string, repo string, err error) { frags := strings.Split(action, "/") @@ -40,6 +87,11 @@ func parseActionFragments(action string) (owner string, repo string, err error) return frags[0], frags[1], nil } +// isChecksum returns true if the input is a checksum. +func isChecksum(ref string) bool { + return len(ref) == 40 +} + func getCheckSumForTag(ctx context.Context, restIf interfaces.REST, owner, repo, tag string) (string, error) { path, err := url.JoinPath("repos", owner, repo, "git", "refs", "tags", tag) if err != nil { diff --git a/pkg/replacer/image/image.go b/pkg/replacer/image/image.go new file mode 100644 index 0000000..a87b91d --- /dev/null +++ b/pkg/replacer/image/image.go @@ -0,0 +1,101 @@ +package image + +import ( + "context" + "fmt" + "github.com/stacklok/frizbee/internal/store" + "github.com/stacklok/frizbee/pkg/config" + "github.com/stacklok/frizbee/pkg/interfaces" + "strings" +) + +const ( + // ContainerImageRegex is regular expression pattern to match container image usage in YAML + ContainerImageRegex = `image\s*:\s*["']?([^\s"']+/[^\s"']+(:[^\s"']+)?(@[^\s"']+)?)["']?|FROM\s+([^\s]+(/[^\s]+)?(:[^\s]+)?(@[^\s]+)?)` // `\b(image|FROM)\s*:?(\s*([^\s]+))?` + prefixFROM = "FROM " + prefixImage = "image: " + ReferenceType = "container" +) + +type Parser struct { + regex string +} + +func New(regex string) *Parser { + if regex == "" { + regex = ContainerImageRegex + } + return &Parser{ + regex: regex, + } +} + +func (p *Parser) GetRegex() string { + return p.regex +} + +func (p *Parser) Replace(ctx context.Context, matchedLine string, _ interfaces.REST, cfg config.Config, cache store.RefCacher, keepPrefix bool) (string, error) { + // Trim the prefix + hasFROMPrefix := false + imageRef := matchedLine + + // Check if the image reference has the FROM prefix, i.e. Dockerfile + if strings.HasPrefix(imageRef, prefixFROM) { + imageRef = strings.TrimPrefix(imageRef, prefixFROM) + // Check if the image reference should be excluded, i.e. scratch + if shouldExclude(imageRef) { + return matchedLine, nil + } + hasFROMPrefix = true + } else if strings.HasPrefix(imageRef, prefixImage) { + // Check if the image reference has the image prefix, i.e. Kubernetes or Docker Compose YAML + imageRef = strings.TrimPrefix(imageRef, prefixImage) + } + + // Get the digest of the image reference + imageRefWithDigest, err := GetImageDigestFromRef(ctx, imageRef, cfg.Platform, cache, hasFROMPrefix) + if err != nil { + return "", err + } + + // Add the prefix back, if needed + if keepPrefix { + if hasFROMPrefix { + imageRefWithDigest = prefixFROM + imageRefWithDigest + } else { + imageRefWithDigest = prefixImage + imageRefWithDigest + } + // Return the modified line with the prefix + return imageRefWithDigest, nil + + } + // Return the modified line without the prefix + return imageRefWithDigest, nil +} + +func (p *Parser) ConvertToEntityRef(reference string) (*interfaces.EntityRef, error) { + reference = strings.TrimPrefix(reference, prefixImage) + reference = strings.TrimPrefix(reference, prefixFROM) + var sep string + var frags []string + if strings.Contains(reference, "@") { + sep = "@" + } else if strings.Contains(reference, ":") { + sep = ":" + } + + if sep != "" { + frags = strings.Split(reference, sep) + if len(frags) != 2 { + return nil, fmt.Errorf("invalid container reference: %s", reference) + } + } else { + frags = []string{reference, "latest"} + } + + return &interfaces.EntityRef{ + Name: frags[0], + Ref: frags[1], + Type: ReferenceType, + }, nil +} diff --git a/pkg/replacer/image/utils.go b/pkg/replacer/image/utils.go new file mode 100644 index 0000000..8c74bb3 --- /dev/null +++ b/pkg/replacer/image/utils.go @@ -0,0 +1,76 @@ +package image + +import ( + "context" + "fmt" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/stacklok/frizbee/internal/cli" + "github.com/stacklok/frizbee/internal/store" + "strings" +) + +// GetImageDigestFromRef returns the digest of a container image reference +// from a name.Reference. +func GetImageDigestFromRef(ctx context.Context, imageRef, platform string, cache store.RefCacher, isDockerfileRef bool) (string, error) { + // Parse the image reference + ref, err := name.ParseReference(imageRef) + if err != nil { + return "", err + } + opts := []remote.Option{ + remote.WithContext(ctx), + remote.WithUserAgent(cli.UserAgent), + remote.WithAuthFromKeychain(authn.DefaultKeychain), + } + + // Set the platform if provided + if platform != "" { + platformSplit := strings.Split(platform, "/") + if len(platformSplit) != 2 { + return "", fmt.Errorf("platform must be in the format os/arch") + } + opts = append(opts, remote.WithPlatform(v1.Platform{ + OS: platformSplit[0], + Architecture: platformSplit[1], + })) + } + + // Get the digest of the image reference + var digest string + + if cache != nil { + if d, ok := cache.Load(imageRef); ok { + digest = d + } + desc, err := remote.Get(ref, opts...) + if err != nil { + return "", err + } + digest = desc.Digest.String() + cache.Store(imageRef, digest) + } else { + desc, err := remote.Get(ref, opts...) + if err != nil { + return "", err + } + digest = desc.Digest.String() + } + + // Compare the digest with the reference and return the original reference if they already match + if digest == ref.Identifier() { + return imageRef, nil + } + + // Return the image reference with the digest differently if it is a Dockerfile reference + if isDockerfileRef { + return fmt.Sprintf("%s:%s@%s", ref.Context().Name(), ref.Identifier(), digest), nil + } + return fmt.Sprintf("%s@%s # %s", ref.Context().Name(), digest, ref.Identifier()), nil +} + +func shouldExclude(ref string) bool { + return ref == "scratch" +} diff --git a/pkg/replacer/replacer.go b/pkg/replacer/replacer.go new file mode 100644 index 0000000..d2f32d6 --- /dev/null +++ b/pkg/replacer/replacer.go @@ -0,0 +1,317 @@ +// +// Copyright 2023 Stacklok, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package replacer provide common replacer implementation +package replacer + +import ( + "bufio" + "context" + "fmt" + mapset "github.com/deckarep/golang-set/v2" + "github.com/go-git/go-billy/v5/osfs" + "github.com/stacklok/frizbee/internal/ghrest" + "github.com/stacklok/frizbee/internal/store" + "github.com/stacklok/frizbee/internal/traverse" + "github.com/stacklok/frizbee/pkg/config" + "github.com/stacklok/frizbee/pkg/interfaces" + "github.com/stacklok/frizbee/pkg/replacer/action" + "github.com/stacklok/frizbee/pkg/replacer/image" + "golang.org/x/sync/errgroup" + "io" + "path/filepath" + "regexp" + "sort" + "strings" + "sync" +) + +type ParserType string + +// Replacer replaces container image references in YAML files +type Replacer struct { + parser interfaces.Parser + interfaces.REST + config.Config + regex string +} + +type ReplaceResult struct { + Processed []string + Modified map[string]string +} + +type ListResult struct { + Processed []string + Entities []interfaces.EntityRef +} + +func (r *Replacer) WithGitHubClient(token string) *Replacer { + client := ghrest.NewClient(token) + r.REST = client + return r +} + +func (r *Replacer) WithUserRegex(regex string) *Replacer { + r.regex = regex + return r +} + +// New creates a new FileReplacer from the given +// command-line arguments and options +func New(cfg *config.Config) *Replacer { + // Return the replacer + return &Replacer{ + Config: *cfg, + } +} + +func (r *Replacer) ParseSingleGitHubAction(ctx context.Context, entityRef string) (string, error) { + r.parser = action.New(r.regex) + return r.parser.Replace(ctx, entityRef, r.REST, r.Config, nil, false) +} + +func (r *Replacer) ParseGitHubActions(ctx context.Context, dir string) (*ReplaceResult, error) { + r.parser = action.New(r.regex) + return r.parsePath(ctx, dir) +} + +func (r *Replacer) ListGitHibActions(dir string) (*ListResult, error) { + r.parser = action.New(r.regex) + return r.listReferences(dir) +} + +func (r *Replacer) ParseSingleContainerImage(ctx context.Context, entityRef string) (string, error) { + r.parser = image.New(r.regex) + return r.parser.Replace(ctx, entityRef, r.REST, r.Config, nil, false) +} + +func (r *Replacer) ParseContainerImages(ctx context.Context, dir string) (*ReplaceResult, error) { + r.parser = image.New(r.regex) + return r.parsePath(ctx, dir) +} + +func (r *Replacer) ListContainerImages(dir string) (*ListResult, error) { + r.parser = image.New(r.regex) + return r.listReferences(dir) +} + +func (r *Replacer) parsePath(ctx context.Context, dir string) (*ReplaceResult, error) { + var eg errgroup.Group + var mu sync.Mutex + + basedir := filepath.Dir(dir) + base := filepath.Base(dir) + bfs := osfs.New(basedir, osfs.WithBoundOS()) + + cache := store.NewRefCacher() + + res := ReplaceResult{ + Processed: make([]string, 0), + Modified: make(map[string]string), + } + + // Traverse all YAML/YML files in dir + err := traverse.TraverseYAMLDockerfiles(bfs, base, func(path string) error { + eg.Go(func() error { + file, err := bfs.Open(path) + if err != nil { + return fmt.Errorf("failed to open file %s: %w", path, err) + } + // nolint:errcheck // ignore error + defer file.Close() + + // Store the file name to the processed batch + res.Processed = append(res.Processed, path) + + // Parse the content of the file and update the matching references + modified, updatedFile, err := r.parseAndReplaceReferencesInFile(ctx, file, cache) + if err != nil { + return fmt.Errorf("failed to modify references in %s: %w", path, err) + } + + // Store the updated file content if it was modified + if modified { + mu.Lock() + res.Modified[path] = updatedFile + mu.Unlock() + } + + // All good + return nil + }) + return nil + }) + if err != nil { + return nil, err + } + + if err := eg.Wait(); err != nil { + return nil, err + } + + // All good + return &res, nil +} + +func (r *Replacer) listReferences(dir string) (*ListResult, error) { + var eg errgroup.Group + var mu sync.Mutex + + basedir := filepath.Dir(dir) + base := filepath.Base(dir) + bfs := osfs.New(basedir, osfs.WithBoundOS()) + + res := ListResult{ + Processed: make([]string, 0), + Entities: make([]interfaces.EntityRef, 0), + } + + found := mapset.NewSet[interfaces.EntityRef]() + + // Traverse all YAML/YML files in dir + err := traverse.TraverseYAMLDockerfiles(bfs, base, func(path string) error { + eg.Go(func() error { + file, err := bfs.Open(path) + if err != nil { + return fmt.Errorf("failed to open file %s: %w", path, err) + } + // nolint:errcheck // ignore error + defer file.Close() + + // Store the file name to the processed batch + res.Processed = append(res.Processed, path) + + // Parse the content of the file and listReferences the matching references + foundRefs, err := r.listReferencesInFile(file) + if err != nil { + return fmt.Errorf("failed to listReferences references in %s: %w", path, err) + } + mu.Lock() + found = found.Union(foundRefs) + mu.Unlock() + // All good + return nil + }) + return nil + }) + if err != nil { + return nil, err + } + + if err := eg.Wait(); err != nil { + return nil, err + } + res.Entities = found.ToSlice() + + // Sort the slice by the Name field using sort.Slice + sort.Slice(res.Entities, func(i, j int) bool { + return res.Entities[i].Name < res.Entities[j].Name + }) + + // All good + return &res, nil +} + +// parseAndReplaceReferencesInFile takes the given file reader and returns its content +// after replacing all references to tags with the checksum of the tag. +// The function returns an empty string and an error if it fails to process the file +// It also uses the provided cache to store the checksums. +func (r *Replacer) parseAndReplaceReferencesInFile(ctx context.Context, f io.Reader, cache store.RefCacher) (bool, string, error) { + var contentBuilder strings.Builder + modified := false + + // Compile the regular expression + re, err := regexp.Compile(r.parser.GetRegex()) + if err != nil { + return false, "", err + } + + // Read the file line by line + scanner := bufio.NewScanner(f) + for scanner.Scan() { + var err error + var resultingLine string + line := scanner.Text() + + // See if we can match an entity reference in the line + newLine := re.ReplaceAllStringFunc(line, func(entityRef string) string { + // Modify the reference in the line + resultingLine, err = r.parser.Replace(ctx, entityRef, r.REST, r.Config, cache, true) + return resultingLine + }) + + // Handle the case where the reference could not be modified, i.e. failed to parse it correctly + if err != nil { + return false, "", fmt.Errorf("failed to modify reference in line: %s: %w", line, err) + } + + // Check if the line was modified and set the modified flag to true if it was + if newLine != line { + modified = true + } + + // Write the line to the content builder buffer + contentBuilder.WriteString(newLine + "\n") + } + + // Check for errors during the scan + if err := scanner.Err(); err != nil { + return false, "", err + } + + // Return the workflow content + return modified, contentBuilder.String(), nil +} + +// listReferencesInFile takes the given file reader and returns its content +// after replacing all references to tags with the checksum of the tag. +// The function returns an empty string and an error if it fails to process the file +// It also uses the provided cache to store the checksums. +func (r *Replacer) listReferencesInFile(f io.Reader) (mapset.Set[interfaces.EntityRef], error) { + found := mapset.NewSet[interfaces.EntityRef]() + + // Compile the regular expression + re, err := regexp.Compile(r.parser.GetRegex()) + if err != nil { + return nil, err + } + + // Read the file line by line + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + + // See if we can match an entity reference in the line + foundEntries := re.FindAllString(line, -1) + if foundEntries != nil { + for _, entry := range foundEntries { + e, err := r.parser.ConvertToEntityRef(entry) + if err != nil { + continue + } + found.Add(*e) + } + } + } + + // Check for errors during the scan + if err := scanner.Err(); err != nil { + return nil, err + } + + // Return the found references + return found, nil +} diff --git a/pkg/replacer/replacer_test.go b/pkg/replacer/replacer_test.go new file mode 100644 index 0000000..51c530c --- /dev/null +++ b/pkg/replacer/replacer_test.go @@ -0,0 +1 @@ +package replacer diff --git a/pkg/utils/cli/billy.go b/pkg/utils/cli/billy.go deleted file mode 100644 index 88d12d7..0000000 --- a/pkg/utils/cli/billy.go +++ /dev/null @@ -1,29 +0,0 @@ -// -// Copyright 2023 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package cli provides utilities for frizbee's CLI. -package cli - -// ProcessDirNameForBillyFS processes the given directory name for use with -// go-billy filesystems. -func ProcessDirNameForBillyFS(dir string) string { - // remove trailing / from dir. This doesn't play well with - // the go-billy filesystem and walker we use. - if dir[len(dir)-1] == '/' { - return dir[:len(dir)-1] - } - - return dir -} diff --git a/pkg/utils/cli/replacer.go b/pkg/utils/cli/replacer.go deleted file mode 100644 index c2fb34d..0000000 --- a/pkg/utils/cli/replacer.go +++ /dev/null @@ -1,89 +0,0 @@ -// -// Copyright 2023 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package cli provides utilities for frizbee's CLI. -package cli - -import ( - "fmt" - "io" - "os" - - "github.com/go-git/go-billy/v5" - "github.com/spf13/cobra" -) - -// DeclareReplacerFlags declares the flags common to all replacer commands. -// Note that `dir` is not declared here because it is command-specific. -func DeclareReplacerFlags(cmd *cobra.Command) { - cmd.Flags().BoolP("dry-run", "n", false, "don't modify files") - cmd.Flags().BoolP("quiet", "q", false, "don't print anything") - cmd.Flags().BoolP("error", "e", false, "exit with error code if any file is modified") -} - -// Replacer is a common struct for implementing a CLI command that replaces -// files. -type Replacer struct { - Dir string - DryRun bool - Quiet bool - ErrOnModified bool - Cmd *cobra.Command -} - -// Logf logs the given message to the given command's stderr if the command is -// not quiet. -func (r *Replacer) Logf(format string, args ...interface{}) { - if !r.Quiet { - fmt.Fprintf(r.Cmd.ErrOrStderr(), format, args...) - } -} - -// ProcessOutput processes the given output files. -// If the command is quiet, the output is discarded. -// If the command is a dry run, the output is written to the command's stdout. -// Otherwise, the output is written to the given filesystem. -func (r *Replacer) ProcessOutput(bfs billy.Filesystem, outfiles map[string]string) error { - - var out io.Writer - - for path, content := range outfiles { - if r.Quiet { - out = io.Discard - } else if r.DryRun { - out = r.Cmd.OutOrStdout() - } else { - f, err := bfs.OpenFile(path, os.O_WRONLY|os.O_TRUNC, 0644) - if err != nil { - return fmt.Errorf("failed to open file %s: %w", path, err) - } - - defer func() { - if err := f.Close(); err != nil { - fmt.Fprintf(r.Cmd.ErrOrStderr(), "failed to close file %s: %v", path, err) - } - }() - - out = f - } - - _, err := fmt.Fprintf(out, "%s", content) - if err != nil { - return fmt.Errorf("failed to write to file %s: %w", path, err) - } - } - - return nil -} diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go deleted file mode 100644 index c6e91f3..0000000 --- a/pkg/utils/utils.go +++ /dev/null @@ -1,71 +0,0 @@ -// -// Copyright 2023 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package utils provides utilities for frizbee -package utils - -import ( - "fmt" - "io/fs" - "strings" - - "github.com/go-git/go-billy/v5" - billyutil "github.com/go-git/go-billy/v5/util" - "gopkg.in/yaml.v3" -) - -// YAMLToBuffer converts a YAML node to a string buffer -func YAMLToBuffer(wflow *yaml.Node) (fmt.Stringer, error) { - buf := strings.Builder{} - enc := yaml.NewEncoder(&buf) - enc.SetIndent(2) - if err := enc.Encode(wflow); err != nil { - return nil, fmt.Errorf("failed to encode YAML: %w", err) - } - - // nolint:errcheck // ignore error - defer enc.Close() - - return &buf, nil -} - -// TraverseFunc is a function that gets called with each file in a directory. -type TraverseFunc func(path string, info fs.FileInfo) error - -// Traverse traverses the given directory and calls the given function with each file. -func Traverse(bfs billy.Filesystem, base string, fun TraverseFunc) error { - return billyutil.Walk(bfs, base, func(path string, info fs.FileInfo, err error) error { - if err != nil { - return nil - } - - return fun(path, info) - }) -} - -// IsYAMLFile returns true if the given file is a YAML file. -func IsYAMLFile(info fs.FileInfo) bool { - // skip if not a file - if info.IsDir() { - return false - } - - // skip if not a .yml or .yaml file - if strings.HasSuffix(info.Name(), ".yml") || strings.HasSuffix(info.Name(), ".yaml") { - return true - } - - return false -} From 6fd9044a1ff78a30b4cf4ca5bea6407a1df8286c Mon Sep 17 00:00:00 2001 From: Radoslav Dimitrov Date: Mon, 27 May 2024 17:21:41 +0300 Subject: [PATCH 02/16] Add/update the stacklok license header Signed-off-by: Radoslav Dimitrov --- README.md | 22 +++++++++++++++------- cmd/action/action.go | 2 +- cmd/action/list.go | 2 +- cmd/action/one.go | 2 +- cmd/image/image.go | 2 +- cmd/image/list.go | 2 +- cmd/image/one.go | 2 +- cmd/version/version.go | 2 +- internal/cli/cli.go | 15 +++++++++++++++ internal/ghrest/ghrest.go | 2 +- internal/store/cache.go | 2 +- internal/traverse/traverse.go | 15 +++++++++++++++ pkg/config/config.go | 2 +- pkg/interfaces/interfaces.go | 15 +++++++++++++++ pkg/replacer/action/action.go | 15 +++++++++++++++ pkg/replacer/action/utils.go | 15 +++++++++++++++ pkg/replacer/image/image.go | 15 +++++++++++++++ pkg/replacer/image/utils.go | 15 +++++++++++++++ pkg/replacer/replacer.go | 2 +- pkg/replacer/replacer_test.go | 15 +++++++++++++++ 20 files changed, 146 insertions(+), 18 deletions(-) diff --git a/README.md b/README.md index 06f8c43..3fc82e5 100644 --- a/README.md +++ b/README.md @@ -47,10 +47,10 @@ Frizbee can be used to generate checksums for GitHub Actions. This is useful for verifying that the contents of a GitHub Action have not changed. To quickly replace the GitHub Action references for your project, you can use -the `ghactions` command: +the `action` command: ```bash -frizbee ghactions -d path/to/your/repo/.github/workflows/ +frizbee action -d path/to/your/repo/.github/workflows/ ``` This will write all the replacements to the files in the directory provided. @@ -65,10 +65,10 @@ It also supports exiting with a non-zero exit code if any replacements are found This is handy for CI/CD pipelines. If you want to generate the replacement for a single GitHub Action, you can use -the `ghactions one` command: +the `action one` command: ```bash -frizbee ghactions one metal-toolbox/container-push/.github/workflows/container-push.yml@main +frizbee action one metal-toolbox/container-push/.github/workflows/container-push.yml@main ``` This is useful if you're developing and want to quickly test the replacement. @@ -76,12 +76,20 @@ This is useful if you're developing and want to quickly test the replacement. ### Container Images Frizbee can be used to generate checksums for container images. This is useful -for verifying that the contents of a container image have not changed. +for verifying that the contents of a container image have not changed. This works +for all yaml/yml and Dockerfile fies in the directory provided by the `-d` flag. -To get the digest for a single image tag, you can use the `containerimage one` command: +To quickly replace the container image references for your project, you can use +the `image` command: ```bash -frizbee containerimage one quay.io/stacklok/frizbee:latest +frizbee image -d path/to/your/yaml/files/ +``` + +To get the digest for a single image tag, you can use the `image one` command: + +```bash +frizbee image one quay.io/stacklok/frizbee:latest ``` This will print the image reference with the digest for the image tag provided. diff --git a/cmd/action/action.go b/cmd/action/action.go index 18e90ca..8a8300a 100644 --- a/cmd/action/action.go +++ b/cmd/action/action.go @@ -1,5 +1,5 @@ // -// Copyright 2023 Stacklok, Inc. +// Copyright 2024 Stacklok, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cmd/action/list.go b/cmd/action/list.go index 05786cb..8afee25 100644 --- a/cmd/action/list.go +++ b/cmd/action/list.go @@ -1,5 +1,5 @@ // -// Copyright 2023 Stacklok, Inc. +// Copyright 2024 Stacklok, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cmd/action/one.go b/cmd/action/one.go index 513fcf9..62d30f4 100644 --- a/cmd/action/one.go +++ b/cmd/action/one.go @@ -1,5 +1,5 @@ // -// Copyright 2023 Stacklok, Inc. +// Copyright 2024 Stacklok, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cmd/image/image.go b/cmd/image/image.go index b04a538..292b233 100644 --- a/cmd/image/image.go +++ b/cmd/image/image.go @@ -1,5 +1,5 @@ // -// Copyright 2023 Stacklok, Inc. +// Copyright 2024 Stacklok, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cmd/image/list.go b/cmd/image/list.go index 7b9039b..dd1983f 100644 --- a/cmd/image/list.go +++ b/cmd/image/list.go @@ -1,5 +1,5 @@ // -// Copyright 2023 Stacklok, Inc. +// Copyright 2024 Stacklok, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cmd/image/one.go b/cmd/image/one.go index 195867a..2b6870f 100644 --- a/cmd/image/one.go +++ b/cmd/image/one.go @@ -1,5 +1,5 @@ // -// Copyright 2023 Stacklok, Inc. +// Copyright 2024 Stacklok, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/cmd/version/version.go b/cmd/version/version.go index 177dced..836c5ed 100644 --- a/cmd/version/version.go +++ b/cmd/version/version.go @@ -1,5 +1,5 @@ // -// Copyright 2023 Stacklok, Inc. +// Copyright 2024 Stacklok, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 5a20033..e1676a8 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -1,3 +1,18 @@ +// +// Copyright 2024 Stacklok, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cli import ( diff --git a/internal/ghrest/ghrest.go b/internal/ghrest/ghrest.go index 57d5b2c..77051e3 100644 --- a/internal/ghrest/ghrest.go +++ b/internal/ghrest/ghrest.go @@ -1,5 +1,5 @@ // -// Copyright 2023 Stacklok, Inc. +// Copyright 2024 Stacklok, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/internal/store/cache.go b/internal/store/cache.go index 96cf7e0..b6993f3 100644 --- a/internal/store/cache.go +++ b/internal/store/cache.go @@ -1,5 +1,5 @@ // -// Copyright 2023 Stacklok, Inc. +// Copyright 2024 Stacklok, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/internal/traverse/traverse.go b/internal/traverse/traverse.go index 18556e7..7ab8c78 100644 --- a/internal/traverse/traverse.go +++ b/internal/traverse/traverse.go @@ -1,3 +1,18 @@ +// +// Copyright 2024 Stacklok, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package traverse import ( diff --git a/pkg/config/config.go b/pkg/config/config.go index 0743b5c..f34dc45 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -1,5 +1,5 @@ // -// Copyright 2023 Stacklok, Inc. +// Copyright 2024 Stacklok, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/interfaces/interfaces.go b/pkg/interfaces/interfaces.go index 3dc3e69..8b17a2c 100644 --- a/pkg/interfaces/interfaces.go +++ b/pkg/interfaces/interfaces.go @@ -1,3 +1,18 @@ +// +// Copyright 2024 Stacklok, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package interfaces import ( diff --git a/pkg/replacer/action/action.go b/pkg/replacer/action/action.go index 5e3be8e..a8a3355 100644 --- a/pkg/replacer/action/action.go +++ b/pkg/replacer/action/action.go @@ -1,3 +1,18 @@ +// +// Copyright 2024 Stacklok, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package action import ( diff --git a/pkg/replacer/action/utils.go b/pkg/replacer/action/utils.go index a6d2f8f..02e9826 100644 --- a/pkg/replacer/action/utils.go +++ b/pkg/replacer/action/utils.go @@ -1,3 +1,18 @@ +// +// Copyright 2024 Stacklok, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package action import ( diff --git a/pkg/replacer/image/image.go b/pkg/replacer/image/image.go index a87b91d..24be3f5 100644 --- a/pkg/replacer/image/image.go +++ b/pkg/replacer/image/image.go @@ -1,3 +1,18 @@ +// +// Copyright 2024 Stacklok, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package image import ( diff --git a/pkg/replacer/image/utils.go b/pkg/replacer/image/utils.go index 8c74bb3..154ef26 100644 --- a/pkg/replacer/image/utils.go +++ b/pkg/replacer/image/utils.go @@ -1,3 +1,18 @@ +// +// Copyright 2024 Stacklok, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package image import ( diff --git a/pkg/replacer/replacer.go b/pkg/replacer/replacer.go index d2f32d6..488fb30 100644 --- a/pkg/replacer/replacer.go +++ b/pkg/replacer/replacer.go @@ -1,5 +1,5 @@ // -// Copyright 2023 Stacklok, Inc. +// Copyright 2024 Stacklok, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. diff --git a/pkg/replacer/replacer_test.go b/pkg/replacer/replacer_test.go index 51c530c..42c1760 100644 --- a/pkg/replacer/replacer_test.go +++ b/pkg/replacer/replacer_test.go @@ -1 +1,16 @@ +// +// Copyright 2024 Stacklok, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package replacer From 4d7ab811b8f4f98327edc026ddf83543d0dc2cd1 Mon Sep 17 00:00:00 2001 From: Radoslav Dimitrov Date: Mon, 27 May 2024 18:05:57 +0300 Subject: [PATCH 03/16] Streamline path and one commands to one action or image command Signed-off-by: Radoslav Dimitrov --- .tests/actions/.github/dependabot.yml | 45 ++++ .../.github/workflows/argo-helm-diff.yaml | 179 ++++++++++++++++ .tests/actions/.github/workflows/docker.yaml | 12 ++ .../workflows/gh-runner-image-publish.yaml | 67 ++++++ .../.github/workflows/gh-runner-image.yaml | 24 +++ .../.github/workflows/krakend-check.yaml | 44 ++++ .../actions/.github/workflows/security.yaml | 24 +++ .../.github/workflows/terraform-apply.yaml | 87 ++++++++ .../.github/workflows/terraform-preview.yaml | 141 +++++++++++++ .tests/containers/Dockerfile | 62 ++++++ .tests/containers/docker-compose.yaml | 192 ++++++++++++++++++ .tests/containers/k8s.yaml | 19 ++ cmd/action/action.go | 36 ++-- cmd/action/list.go | 13 +- cmd/action/one.go | 81 -------- cmd/image/image.go | 34 +++- cmd/image/list.go | 12 +- cmd/image/one.go | 77 ------- internal/cli/cli.go | 32 +-- pkg/errors/errors.go | 24 +++ pkg/interfaces/interfaces.go | 10 +- pkg/replacer/action/action.go | 77 ++++--- pkg/replacer/image/image.go | 40 ++-- pkg/replacer/image/utils.go | 25 ++- pkg/replacer/replacer.go | 64 +++--- pkg/replacer/replacer_test.go | 191 +++++++++++++++++ 26 files changed, 1299 insertions(+), 313 deletions(-) create mode 100644 .tests/actions/.github/dependabot.yml create mode 100644 .tests/actions/.github/workflows/argo-helm-diff.yaml create mode 100644 .tests/actions/.github/workflows/docker.yaml create mode 100644 .tests/actions/.github/workflows/gh-runner-image-publish.yaml create mode 100644 .tests/actions/.github/workflows/gh-runner-image.yaml create mode 100644 .tests/actions/.github/workflows/krakend-check.yaml create mode 100644 .tests/actions/.github/workflows/security.yaml create mode 100644 .tests/actions/.github/workflows/terraform-apply.yaml create mode 100644 .tests/actions/.github/workflows/terraform-preview.yaml create mode 100644 .tests/containers/Dockerfile create mode 100644 .tests/containers/docker-compose.yaml create mode 100644 .tests/containers/k8s.yaml delete mode 100644 cmd/action/one.go delete mode 100644 cmd/image/one.go create mode 100644 pkg/errors/errors.go diff --git a/.tests/actions/.github/dependabot.yml b/.tests/actions/.github/dependabot.yml new file mode 100644 index 0000000..22a54de --- /dev/null +++ b/.tests/actions/.github/dependabot.yml @@ -0,0 +1,45 @@ +# +# Copyright 2023 Stacklok, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "daily" + - package-ecosystem: "terraform" + directory: "/dashboards" + schedule: + interval: "daily" + - package-ecosystem: "terraform" + directory: "/dns" + schedule: + interval: "daily" + - package-ecosystem: "terraform" + directory: "/iam" + schedule: + interval: "daily" + - package-ecosystem: "terraform" + directory: "/production" + schedule: + interval: "daily" + - package-ecosystem: "terraform" + directory: "/sandbox" + schedule: + interval: "daily" + - package-ecosystem: "terraform" + directory: "/staging" + schedule: + interval: "daily" diff --git a/.tests/actions/.github/workflows/argo-helm-diff.yaml b/.tests/actions/.github/workflows/argo-helm-diff.yaml new file mode 100644 index 0000000..574be8c --- /dev/null +++ b/.tests/actions/.github/workflows/argo-helm-diff.yaml @@ -0,0 +1,179 @@ +name: Argo Helm Diff + +on: + pull_request: + branches: + - main + paths: + - "**/*.yaml" + types: + - synchronize + - opened + - reopened + +jobs: + diff-helm-template: + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + # charts published to private repos + # Note you also need to grant the infra repo access to those packages: + # https://docs.github.com/en/packages/learn-github-packages/configuring-a-packages-access-control-and-visibility#about-setting-visibility-and-access-permissions-for-packages + packages: read + + strategy: + matrix: + # TODO: can we get this from a directory listing? + environment: ["staging2", "production", "sandbox"] + + env: + # This finds "Application" kinds, then extracts the chart coordinates (repo, version, etc) + # and the valuesObject, and outputs them into a merged YAML document. This assumes no collisions + # on the value keys `installName`, `chartName`, `chartRepo`, and `chartVersion`. + YQ_QUERY: 'select(.kind == "Application" and .spec.source.chart) | + .spec.source.targetRevision as $version | .spec.source.chart as $chart | .spec.source.repoURL as $repo | + .metadata.name as $name | .spec.destination.namespace as $namespace | + .spec.source.helm.valuesObject | + .installName = $name | .installNamespace = $namespace | .chartName = $chart | .chartRepo = $repo | .chartVersion = $version' + + steps: + - name: Checkout base branch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.base.sha }} + path: base + + - name: Checkout pull request branch + uses: actions/checkout@v4 + with: + ref: ${{ github.event.pull_request.head.sha }} + path: pr + + - name: Setup Helm + uses: azure/setup-helm@v4 + with: + version: "3.14.0" + + - name: Log in to read private GitHub packages + run: helm registry login $BASE_REPO --username ${{ github.repository_owner }} --password ${{ secrets.GITHUB_TOKEN }} + env: + BASE_REPO: ghcr.io/stacklok + + - name: create directories + run: | + mkdir -p base-tmp + mkdir -p pr-tmp + + - name: Extract base helm values with yq + # TODO: would be nice to be able to install instead of using a container + uses: mikefarah/yq@master + with: + cmd: | + if [[ -e "base/argocd/${{ matrix.environment }}/cluster-config/" ]]; then + yq -s '"base-tmp/" + (.chartName | sub("/", "_")) + "." + (.chartRepo | sub("/", "_")) + ".yaml"' "$YQ_QUERY" base/argocd/${{ matrix.environment }}/cluster-config/*.yaml + fi + + - name: Compute helm templates for base + run: | + for f in base-tmp/*.yaml; do + INSTALL_NAME=$(grep installName: $f | cut -d: -f2 | tr -d ' ' | sed 's/#.*//') + INSTALL_NS=$(grep installNamespace: $f | cut -d: -f2 | tr -d ' ' | sed 's/#.*//') + CHART_NAME=$(grep chartName: $f | cut -d: -f2 | tr -d ' ' | sed 's/#.*//') + CHART_REPO=$(grep chartRepo: $f | cut -d: -f2- | tr -d " '\"" | sed 's/#.*//') + CHART_VERSION=$(grep chartVersion: $f | cut -d: -f2- | tr -d " '\"" | sed 's/#.*//') + # Bare (OCI format) repos have a different format in helm template + if [[ "$CHART_REPO" != *":"* ]]; then + CHART_NAME="oci://$CHART_REPO/$CHART_NAME" + CHART_REPO="" + else + CHART_REPO="--repo $CHART_REPO" + fi + helm template --version "$CHART_VERSION" --namespace "$INSTALL_NS" $CHART_REPO $INSTALL_NAME $CHART_NAME -f $f > ${f%.yaml}.out || true # OCI fetch not yet working + echo "Expanding $f: $?" + done + + - name: Extract PR helm values with yq + # TODO: would be nice to be able to install instead of using a container + uses: mikefarah/yq@master + with: + cmd: yq -s '"pr-tmp/" + (.chartName | sub("/", "_")) + "." + (.chartRepo | sub("/", "_")) + ".yaml"' "$YQ_QUERY" pr/argocd/${{ matrix.environment }}/cluster-config/*.yaml + + - name: Compute helm templates for PR + run: | + for f in pr-tmp/*.yaml; do + INSTALL_NAME=$(grep installName: $f | cut -d: -f2 | tr -d ' ' | sed 's/#.*//') + INSTALL_NS=$(grep installNamespace: $f | cut -d: -f2 | tr -d ' ' | sed 's/#.*//') + CHART_NAME=$(grep chartName: $f | cut -d: -f2 | tr -d ' ' | sed 's/#.*//') + CHART_REPO=$(grep chartRepo: $f | cut -d: -f2- | tr -d " '\"" | sed 's/#.*//') + CHART_VERSION=$(grep chartVersion: $f | cut -d: -f2- | tr -d " '\"" | sed 's/#.*//') + # Bare (OCI format) repos have a different format in helm template + if [[ "$CHART_REPO" != *":"* ]]; then + CHART_NAME="oci://$CHART_REPO/$CHART_NAME" + CHART_REPO="" + else + CHART_REPO="--repo $CHART_REPO" + fi + helm template --version "$CHART_VERSION" --namespace "$INSTALL_NS" $CHART_REPO $INSTALL_NAME $CHART_NAME -f $f > ${f%.yaml}.out || true # OCI fetch not yet working + echo "Expanding $f: $?" + done + + - name: Diff template output + id: diff + run: | + # Remove *.yaml to only leave *.out + rm -f base-tmp/*.yaml pr-tmp/*.yaml + + # We use a file rather than a GitHub output, because sometimes the diff is too big. + # See https://github.com/actions/github-script/issues/266#issuecomment-1158990264 for + # an example from a different project. + (diff -Nu base-tmp pr-tmp || echo "no diff" ) > yaml.diff + + echo "Diff:" + cat yaml.diff + + - name: Present diff + uses: actions/github-script@v7 + id: diff-comment + if: github.event_name == 'pull_request' + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { readFile } = require("fs/promises"); + let planComment = await readFile("yaml.diff", "utf8"); + + if (planComment.length > 65535) { + planComment = "TRUNCATED DIFF, see 'Diff template output' for full output:\n\n" + planComment.substring(0, 60000); + } + + // 1. Retrieve existing bot comments for the PR + + const output = ` + ## Helm diffs \`${{ matrix.environment }}\` + + \`\`\` + ` + planComment + ` + \`\`\` + `; + + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + const botComment = comments.find(comment => { + return comment.user.type === 'Bot' && comment.body.includes('') + }); + if (botComment) { + github.rest.issues.deleteComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + }); + } + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: output + }); diff --git a/.tests/actions/.github/workflows/docker.yaml b/.tests/actions/.github/workflows/docker.yaml new file mode 100644 index 0000000..d995899 --- /dev/null +++ b/.tests/actions/.github/workflows/docker.yaml @@ -0,0 +1,12 @@ +name: Linter +on: pull_request +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - uses: xt0rted/markdownlint-problem-matcher@v1 + - name: "Run Markdown linter" + uses: docker://avtodev/markdown-lint:v1 + with: + args: src/*.md \ No newline at end of file diff --git a/.tests/actions/.github/workflows/gh-runner-image-publish.yaml b/.tests/actions/.github/workflows/gh-runner-image-publish.yaml new file mode 100644 index 0000000..bf9400b --- /dev/null +++ b/.tests/actions/.github/workflows/gh-runner-image-publish.yaml @@ -0,0 +1,67 @@ +--- +name: GH Runner image publish + +on: + push: + branches: + - main + workflow_dispatch: + +jobs: + publish: + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + env: + GH_RUNNER_IMAGE: ghcr.io/stacklok/gh-runner + steps: + - name: Checkout + uses: actions/checkout@v4.1.4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.3.0 + + - name: Login to ghcr.io + uses: docker/login-action@v3.1.0 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + - name: Compute version number + id: version-string + run: | + DATE="$(date +%Y%m%d)" + COMMIT="$(git rev-parse --short HEAD)" + echo "tag=0.$DATE.$GITHUB_RUN_NUMBER+ref.$COMMIT" >> "$GITHUB_OUTPUT" + + - name: Set container metadata + uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5 + id: docker-metadata + with: + images: ${{ env.GH_RUNNER_IMAGE }} + labels: | + org.opencontainers.image.source=${{ github.repositoryUrl }} + org.opencontainers.image.description="This is a container for the Stacklok GitHub Runner image" + org.opencontainers.image.title="Stacklok GitHub Runner Image" + org.opencontainers.image.vendor="Stacklok Inc." + org.opencontainers.image.version=${{ github.sha }} + flavor: | + latest=true + # Even if tags are floating, it's handy and user-friendly to have a + # matching tag for each build. This way, we can search for the digest + # and verify that it's the same as the digest in the Helm chart. + tags: | + type=raw,value=${{ steps.version-string.outputs.tag }} + + - name: Build image + id: image-build + uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5 + with: + context: . + platforms: linux/arm64 + push: true + file: ./images/gh-runner/Dockerfile + tags: ${{ steps.docker-metadata.outputs.tags }} + labels: ${{ steps.docker-metadata.outputs.labels }} \ No newline at end of file diff --git a/.tests/actions/.github/workflows/gh-runner-image.yaml b/.tests/actions/.github/workflows/gh-runner-image.yaml new file mode 100644 index 0000000..0b6ccea --- /dev/null +++ b/.tests/actions/.github/workflows/gh-runner-image.yaml @@ -0,0 +1,24 @@ +--- +name: GH Runner image + +on: + pull_request: + +jobs: + build: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4.1.4 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3.3.0 + + - name: Test build on arm64 + id: docker_build + uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5 + with: + context: . + file: ./images/gh-runner/Dockerfile + platforms: linux/arm64 + push: false # Only attempt to build, to verify the Dockerfile is working diff --git a/.tests/actions/.github/workflows/krakend-check.yaml b/.tests/actions/.github/workflows/krakend-check.yaml new file mode 100644 index 0000000..be0a31d --- /dev/null +++ b/.tests/actions/.github/workflows/krakend-check.yaml @@ -0,0 +1,44 @@ +--- +name: Krakend lint + +on: + pull_request: + paths: + - 'argocd/production/cluster-config/krakend.yaml' + - 'argocd/staging2/cluster-config/krakend.yaml' + +jobs: + lint: + runs-on: ubuntu-latest + strategy: + fail-fast: false + matrix: + krakend-app: + - argocd/production/cluster-config/krakend.yaml + - argocd/staging2/cluster-config/krakend.yaml + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Go + uses: actions/setup-go@v5 + with: + check-latest: true + + - name: Get krakend config from Argo App + uses: mikefarah/yq@master + id: krakendcfg + with: + cmd: yq '.spec.source.helm.valuesObject.krakend.config' ${{ matrix.krakend-app }} + + - name: persist krakend config in temp + run: | + cat < /tmp/krakend.json + ${{ steps.krakendcfg.outputs.result }} + EOF + + - name: Run krakend check + run: | + # krakend check needs the --config flag + # docker run -it devopsfaith/krakend + docker run -i -v /tmp/krakend.json:/tmp/krakend.json devopsfaith/krakend check --config /tmp/krakend.json diff --git a/.tests/actions/.github/workflows/security.yaml b/.tests/actions/.github/workflows/security.yaml new file mode 100644 index 0000000..003bcda --- /dev/null +++ b/.tests/actions/.github/workflows/security.yaml @@ -0,0 +1,24 @@ +--- +name: "Security" + +on: + push: + branches: [main] + pull_request: + +jobs: + security-scan: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Security Scan + uses: aquasecurity/trivy-action@0.20.0 + with: + scan-type: "fs" + scanners: vuln,secret,config + severity: HIGH,CRITICAL + skip-dirs: "argocd/staging/database-provisioning,argocd/production/database-provisioning,argocd/staging2/database-provisioning,argocd/sandbox/database-provisioning" + exit-code: 1 + ignore-unfixed: true diff --git a/.tests/actions/.github/workflows/terraform-apply.yaml b/.tests/actions/.github/workflows/terraform-apply.yaml new file mode 100644 index 0000000..47fe5ab --- /dev/null +++ b/.tests/actions/.github/workflows/terraform-apply.yaml @@ -0,0 +1,87 @@ +name: "Terraform Apply" + +on: + pull_request: + types: + - closed + branches: + - main + + schedule: + # 15:00 UTC = 6PM/7PM Eastern Europe, 8AM/9AM Pacific + - cron: "0 15 * * *" + + workflow_dispatch: + +jobs: + apply_on_merge: + if: | + github.event_name == 'schedule' || + github.event_name == 'workflow_dispatch' || + (github.event_name == 'pull_request' && github.event.pull_request.merged == true) + # We tried the following addition to the if statement to only run on + # merged PRs, but not on closed (cancelled) PRs, but GitHub stopped + # running the workflow on merge: https://github.com/stacklok/infra/issues/77 + # + # github.ref == 'main' && github.event.pull_request.merged == true) + name: "Terraform Apply" + runs-on: ubuntu-latest + permissions: + contents: read + id-token: write + + strategy: + # Don't stop / cancel remaining applies if one apply fails + fail-fast: false + matrix: + # Note that these run in-order, so "iam" will start first. + # We could set `max-parallel` to 1 to get strict ordering + # TODO: can we get this list from a directory listing? + component: + [ + "iam", + "sandbox", + "dashboards", + "dns", + "telemetry", + "staging2", + "production", + ] + + env: + CONFIG_DIRECTORY: "./${{ matrix.component }}" + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + + - name: Set up AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + # Role includes the account ID as an ARN + role-to-assume: ${{ secrets.AWS_ROLE_TO_APPLY }} + aws-region: us-east-1 + + - name: Init + id: init + run: | + cd $CONFIG_DIRECTORY + terraform init + + - name: Validate + id: validate + run: | + set -e + + cd $CONFIG_DIRECTORY + terraform validate -no-color + + - name: Apply + id: apply + run: | + set -e + cd $CONFIG_DIRECTORY + terraform apply -no-color -input=false -auto-approve diff --git a/.tests/actions/.github/workflows/terraform-preview.yaml b/.tests/actions/.github/workflows/terraform-preview.yaml new file mode 100644 index 0000000..725b144 --- /dev/null +++ b/.tests/actions/.github/workflows/terraform-preview.yaml @@ -0,0 +1,141 @@ +name: "Terraform Plan" + +on: + pull_request: + branches: + - main + paths: + - "**/*.tf" + - "**/*.tfvars" + - "**/*.hcl" + - ".github/workflows/preview.yaml" + +jobs: + plan: + name: "Terraform Plan" + runs-on: ubuntu-latest + permissions: + contents: read + pull-requests: write + id-token: write + + strategy: + matrix: + # TODO: can we get this from a directory listing? + component: + [ + "dashboards", + "dns", + "iam", + "staging2", + "production", + "sandbox", + "telemetry", + ] + + defaults: + run: + working-directory: ${{ matrix.component }} + env: + PLAN_OUTPUT: "./tfplan.txt" + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Terraform + uses: hashicorp/setup-terraform@v3 + + - name: Set up AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + # Role includes the account ID as an ARN + role-to-assume: ${{ secrets.AWS_ROLE_TO_PLAN }} + aws-region: us-east-1 + + - name: Terraform fmt + id: fmt + run: terraform fmt -check + continue-on-error: true + + - name: Init + id: init + run: | + terraform init + + - name: Validate + id: validate + run: | + terraform validate -no-color + + - name: Plan + id: plan + run: | + terraform plan -no-color -lock=false -input=false -out=tfplan + continue-on-error: true + + - name: Show Plan + id: show_plan + run: | + terraform show -no-color tfplan >> "$PLAN_OUTPUT" + + - name: Report + uses: actions/github-script@v7 + id: plan-comment + if: github.event_name == 'pull_request' + with: + github-token: ${{ secrets.GITHUB_TOKEN }} + script: | + const { readFileSync } = require('node:fs'); + // 1. Determine output format; large diffs need additional summarization + const planFormat = readFileSync('${{ matrix.component }}/' + process.env.PLAN_OUTPUT, 'utf8'); + let planComment = planFormat; + if (planFormat.length > 65535) { + planComment = planFormat.split('\n').filter( + (line) => (line.startsWith(' # ') || line.startsWith('Plan:'))).join('\n'); + planComment = '# Plan output too large to display, summarized:\n\n' + planComment; + } + + // 1. Retrieve existing bot comments for the PR + + const output = ` + ## Terraform Plan \`${{ matrix.component }}\` + #### Terraform Fmt \`${{ steps.fmt.outcome }}\` + + #### Terraform Init \`${{ steps.init.outcome }}\` + + #### Terraform Validate \`${{ steps.validate.outcome }}\` + + #### Terraform Plan \`${{ steps.plan.outcome }}\` + #### Terraform Plan Output + \`\`\` + ` + planComment + ` + \`\`\` + `; + + const { data: comments } = await github.rest.issues.listComments({ + owner: context.repo.owner, + repo: context.repo.repo, + issue_number: context.issue.number, + }); + const botComment = comments.find(comment => { + return comment.user.type === 'Bot' && comment.body.includes('') + }); + if (botComment) { + github.rest.issues.deleteComment({ + owner: context.repo.owner, + repo: context.repo.repo, + comment_id: botComment.id, + }); + } + github.rest.issues.createComment({ + issue_number: context.issue.number, + owner: context.repo.owner, + repo: context.repo.repo, + body: output + }); + + - name: Fail if continue steps failed + id: check-failure + if: steps.plan.outcome != 'success' || steps.fmt.outcome != 'success' + run: exit 1 diff --git a/.tests/containers/Dockerfile b/.tests/containers/Dockerfile new file mode 100644 index 0000000..cd260e6 --- /dev/null +++ b/.tests/containers/Dockerfile @@ -0,0 +1,62 @@ +# +# Copyright 2023 Stacklok, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM golang:1.22.2@sha256:aca60c1f21de99aa3a34e653f0cdc8c8ea8fe6480359229809d5bcb974f599ec AS builder +ENV APP_ROOT=/opt/app-root +ENV GOPATH=$APP_ROOT +FROM golang:1.22.2 AS builder +FROM golang:latest AS builder +FROM golang AS builder +FROM ubuntu + +WORKDIR $APP_ROOT/src/ +ADD go.mod go.sum $APP_ROOT/src/ +RUN go mod download + +# Add source code +ADD ./ $APP_ROOT/src/ + +RUN CGO_ENABLED=0 go build -trimpath -o minder-server ./cmd/server + +# Create a "nobody" non-root user for the next image by crafting an /etc/passwd +# file that the next image can copy in. This is necessary since the next image +# is based on scratch, which doesn't have adduser, cat, echo, or even sh. +RUN echo "nobody:x:65534:65534:Nobody:/:" > /etc_passwd + +RUN mkdir -p /app + +FROM scratch + +COPY --chown=65534:65534 --from=builder /app /app + +WORKDIR /app + +# Copy database directory and config. This is needed for the migration sub-command to work. +ADD --chown=65534:65534 ./cmd/server/kodata/server-config.yaml /app + +COPY --from=builder /opt/app-root/src/minder-server /usr/bin/minder-server + +# Copy the certs from the builder stage +COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ + +# Copy the /etc_passwd file we created in the builder stage into /etc/passwd in +# the target stage. This creates a new non-root user as a security best +# practice. +COPY --from=builder /etc_passwd /etc/passwd + +USER nobody + +# Set the binary as the entrypoint of the container +ENTRYPOINT ["/usr/bin/minder-server"] diff --git a/.tests/containers/docker-compose.yaml b/.tests/containers/docker-compose.yaml new file mode 100644 index 0000000..78faaec --- /dev/null +++ b/.tests/containers/docker-compose.yaml @@ -0,0 +1,192 @@ +# +# Copyright 2023 Stacklok, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +version: '3.2' +services: + minder: + container_name: minder_server + build: + context: . + dockerfile: ./docker/minder/Dockerfile + image: minder:latest + + command: [ + "serve", + "--grpc-host=0.0.0.0", + "--http-host=0.0.0.0", + "--metric-host=0.0.0.0", + "--db-host=postgres", + "--issuer-url=http://keycloak:8080", + "--config=/app/server-config.yaml", + # If you don't want to store your GitHub client ID and secret in the main + # config file, point to them here: + # "--github-client-id-file=/secrets/github_client_id", + # "--github-client-secret-file=/secrets/github_client_secret", + ] + restart: always # keep the server running + read_only: true + ports: + - "8080:8080" + - "8090:8090" + - "9090:9090" + volumes: + - ./server-config.yaml:/app/server-config.yaml:z + - ./flags-config.yaml:/app/flags-config.yaml:z + # If you don't want to store your GitHub client ID and secret in the main + # config file, point to them here: + # - ./.github_client_id:/secrets/github_client_id:z + # - ./.github_client_secret:/secrets/github_client_secret:z + # If you're using a GitHub App, you'll need to provide the private key: + - ./.secrets/:/app/.secrets/:z + - ./.ssh:/app/.ssh:z + environment: + - KO_DATA_PATH=/app/ + # Use viper environment variables to set specific paths to keys; + # these values are relative paths in server-config.yaml, but it's not clear + # what they are relative _to_... + - MINDER_AUTH_ACCESS_TOKEN_PRIVATE_KEY=/app/.ssh/access_token_rsa + - MINDER_AUTH_ACCESS_TOKEN_PUBLIC_KEY=/app/.ssh/access_token_rsa.pub + - MINDER_AUTH_REFRESH_TOKEN_PRIVATE_KEY=/app/.ssh/refresh_token_rsa + - MINDER_AUTH_REFRESH_TOKEN_PUBLIC_KEY=/app/.ssh/refresh_token_rsa.pub + - MINDER_AUTH_TOKEN_KEY=/app/.ssh/token_key_passphrase + - MINDER_UNSTABLE_TRUSTY_ENDPOINT=https://api.trustypkg.dev + - MINDER_PROVIDER_GITHUB_APP_PRIVATE_KEY=/app/.secrets/github-app.pem + - MINDER_FLAGS_GO_FEATURE_FILE_PATH=/app/flags-config.yaml + - MINDER_LOG_GITHUB_REQUESTS=1 + networks: + - app_net + depends_on: + postgres: + condition: service_healthy + keycloak: + condition: service_healthy + openfga: + condition: service_healthy + migrate: + condition: service_completed_successfully + keycloak-config: + condition: service_completed_successfully + migrate: + container_name: minder_migrate_up + build: + context: . + dockerfile: ./docker/minder/Dockerfile + image: minder:latest + + command: [ + "migrate", + "up", + "--yes", + "--db-host=postgres", + "--config=/app/server-config.yaml", + ] + volumes: + - ./server-config.yaml:/app/server-config.yaml:z + - ./database/migrations:/app/database/migrations:z + environment: + - KO_DATA_PATH=/app/ + networks: + - app_net + deploy: + restart_policy: + condition: on-failure + delay: 5s + max_attempts: 3 + window: 120s + depends_on: + postgres: + condition: service_healthy + openfga: + condition: service_healthy + postgres: + container_name: postgres_container + image: postgres:16.2-alpine + restart: always + user: postgres + environment: + POSTGRES_USER: postgres + POSTGRES_PASSWORD: postgres + POSTGRES_DB: minder + ports: + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready"] + interval: 10s + timeout: 5s + retries: 5 + networks: + - app_net + + keycloak: + container_name: keycloak_container + image: quay.io/keycloak/keycloak:23.0 + command: ["start-dev"] + environment: + KEYCLOAK_ADMIN: admin + KEYCLOAK_ADMIN_PASSWORD: admin + KC_HEALTH_ENABLED: "true" + healthcheck: + test: ["CMD", "/opt/keycloak/bin/kcadm.sh", "config", "credentials", "--server", "http://localhost:8080", "--realm", "master", "--user", "admin", "--password", "admin"] + interval: 10s + timeout: 5s + retries: 10 + ports: + - "8081:8080" + volumes: + - ./identity/themes:/opt/keycloak/themes:z + networks: + - app_net + + keycloak-config: + container_name: keycloak_config + image: bitnami/keycloak-config-cli:5.10.0 + entrypoint: ["java", "-jar", "/opt/bitnami/keycloak-config-cli/keycloak-config-cli.jar"] + environment: + KEYCLOAK_URL: http://keycloak:8080 + KEYCLOAK_USER: admin + KEYCLOAK_PASSWORD: admin + KC_MINDER_SERVER_SECRET: secret + IMPORT_VARSUBSTITUTION_ENABLED: "true" + IMPORT_FILES_LOCATIONS: /config/*.yaml + volumes: + - ./identity/config:/config:z + networks: + - app_net + + depends_on: + keycloak: + condition: service_healthy + + openfga: + container_name: openfga + image: openfga/openfga:v1.5.0 + command: [ + "run", + "--playground-port=8085" + ] + healthcheck: + test: + - CMD + - grpc_health_probe + - "-addr=:8081" + ports: + - 8082:8080 + - 8083:8081 + - 8085:8085 + networks: + - app_net +networks: + app_net: + driver: bridge diff --git a/.tests/containers/k8s.yaml b/.tests/containers/k8s.yaml new file mode 100644 index 0000000..6e62fd7 --- /dev/null +++ b/.tests/containers/k8s.yaml @@ -0,0 +1,19 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: nginx-deployment +spec: + replicas: 3 + selector: + matchLabels: + app: nginx + template: + metadata: + labels: + app: nginx + spec: + containers: + - name: nginx + image: nginx:latest + ports: + - containerPort: 80 diff --git a/cmd/action/action.go b/cmd/action/action.go index 8a8300a..3f753fb 100644 --- a/cmd/action/action.go +++ b/cmd/action/action.go @@ -17,10 +17,12 @@ package action import ( + "fmt" "github.com/stacklok/frizbee/internal/cli" "github.com/stacklok/frizbee/pkg/config" "github.com/stacklok/frizbee/pkg/replacer" "os" + "path/filepath" "github.com/spf13/cobra" ) @@ -35,28 +37,28 @@ with the latest commit hash of the referenced tag or branch. Example: - $ frizbee action -d .github/workflows + $ frizbee action <.github/workflows> or This will replace all tag or branch references in all GitHub Actions workflows -for the given directory. +for the given directory. Supports both directories and single references. ` + cli.TokenHelpText + "\n", Aliases: []string{"ghactions"}, // backwards compatibility RunE: replaceCmd, SilenceUsage: true, + Args: cobra.ExactArgs(1), } // flags - cli.DeclareFrizbeeFlags(cmd, ".github/workflows") + cli.DeclareFrizbeeFlags(cmd, false) // sub-commands - cmd.AddCommand(CmdOne()) cmd.AddCommand(CmdList()) return cmd } -func replaceCmd(cmd *cobra.Command, _ []string) error { +func replaceCmd(cmd *cobra.Command, args []string) error { // Extract the CLI flags from the cobra command cliFlags, err := cli.NewHelper(cmd) if err != nil { @@ -74,12 +76,22 @@ func replaceCmd(cmd *cobra.Command, _ []string) error { WithUserRegex(cliFlags.Regex). WithGitHubClient(os.Getenv(cli.GitHubTokenEnvKey)) - // Replace the tags in the given directory - res, err := r.ParseGitHubActions(cmd.Context(), cliFlags.Dir) - if err != nil { - return err + if cli.IsPath(args[0]) { + dir := filepath.Clean(args[0]) + // Replace the tags in the given directory + res, err := r.ParseGitHubActions(cmd.Context(), dir) + if err != nil { + return err + } + // Process the output files + return cliFlags.ProcessOutput(dir, res.Processed, res.Modified) + } else { + // Replace the passed reference + res, err := r.ParseSingleGitHubAction(cmd.Context(), args[0]) + if err != nil { + return err + } + fmt.Fprintln(cmd.OutOrStdout(), res) + return nil } - - // Process the output files - return cliFlags.ProcessOutput(res.Processed, res.Modified) } diff --git a/cmd/action/list.go b/cmd/action/list.go index 8afee25..e22783c 100644 --- a/cmd/action/list.go +++ b/cmd/action/list.go @@ -24,6 +24,7 @@ import ( "github.com/stacklok/frizbee/pkg/config" "github.com/stacklok/frizbee/pkg/replacer" "os" + "path/filepath" "strconv" ) @@ -40,15 +41,19 @@ Example: Aliases: []string{"ls"}, RunE: list, SilenceUsage: true, + Args: cobra.ExactArgs(1), } - cli.DeclareFrizbeeFlags(cmd, ".github/workflows") - cmd.Flags().StringP("output", "o", "table", "output format. Can be 'json' or 'table'") + cli.DeclareFrizbeeFlags(cmd, true) return cmd } -func list(cmd *cobra.Command, _ []string) error { +func list(cmd *cobra.Command, args []string) error { + dir := filepath.Clean(args[0]) + if !cli.IsPath(dir) { + return fmt.Errorf("the provided argument is not a path") + } // Extract the CLI flags from the cobra command cliFlags, err := cli.NewHelper(cmd) if err != nil { @@ -67,7 +72,7 @@ func list(cmd *cobra.Command, _ []string) error { WithGitHubClient(os.Getenv(cli.GitHubTokenEnvKey)) // List the references in the directory - res, err := r.ListGitHibActions(cliFlags.Dir) + res, err := r.ListGitHibActions(dir) if err != nil { return err } diff --git a/cmd/action/one.go b/cmd/action/one.go deleted file mode 100644 index 62d30f4..0000000 --- a/cmd/action/one.go +++ /dev/null @@ -1,81 +0,0 @@ -// -// Copyright 2024 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package action - -import ( - "fmt" - "github.com/stacklok/frizbee/internal/cli" - "github.com/stacklok/frizbee/pkg/config" - "github.com/stacklok/frizbee/pkg/replacer" - "os" - - "github.com/spf13/cobra" -) - -// CmdOne represents the one sub-command -func CmdOne() *cobra.Command { - cmd := &cobra.Command{ - Use: "one", - Short: "Replace the tag in GitHub Action reference", - Long: `This utility replaces a tag or branch reference in a GitHub Action reference -with the latest commit hash of the referenced tag or branch. - -Example: - - $ frizbee action one actions/checkout@v4.1.1 - -This will replace the tag or branch reference for the commit hash of the -referenced tag or branch. - -` + cli.TokenHelpText + "\n", - Args: cobra.ExactArgs(1), - RunE: replaceOne, - SilenceUsage: true, - } - cli.DeclareFrizbeeFlags(cmd, "") - - return cmd -} - -func replaceOne(cmd *cobra.Command, args []string) error { - ref := args[0] - - // Extract the CLI flags from the cobra command - cliFlags, err := cli.NewHelper(cmd) - if err != nil { - return err - } - - // Set up the config - cfg, err := config.FromCommand(cmd) - if err != nil { - return err - } - - // Create a new replacer - r := replacer.New(cfg). - WithUserRegex(cliFlags.Regex). - WithGitHubClient(os.Getenv(cli.GitHubTokenEnvKey)) - - // Replace the passed reference - res, err := r.ParseSingleGitHubAction(cmd.Context(), ref) - if err != nil { - return err - } - - fmt.Fprintln(cmd.OutOrStdout(), res) - return nil -} diff --git a/cmd/image/image.go b/cmd/image/image.go index 292b233..6f0511d 100644 --- a/cmd/image/image.go +++ b/cmd/image/image.go @@ -17,10 +17,12 @@ package image import ( + "fmt" "github.com/spf13/cobra" "github.com/stacklok/frizbee/internal/cli" "github.com/stacklok/frizbee/pkg/config" "github.com/stacklok/frizbee/pkg/replacer" + "path/filepath" ) // CmdContainerImage represents the containers command @@ -33,26 +35,26 @@ with the latest commit hash of the referenced tag or branch. Example: - $ frizbee image -d + $ frizbee image or This will replace all tag or branch references in all yaml files for the given directory. `, RunE: replaceCmd, SilenceUsage: true, Aliases: []string{"containerimage", "dockercompose", "compose"}, // backwards compatibility + Args: cobra.ExactArgs(1), } // flags - cli.DeclareFrizbeeFlags(cmd, ".") + cli.DeclareFrizbeeFlags(cmd, false) // sub-commands - cmd.AddCommand(CmdOne()) cmd.AddCommand(CmdList()) return cmd } -func replaceCmd(cmd *cobra.Command, _ []string) error { +func replaceCmd(cmd *cobra.Command, args []string) error { // Extract the CLI flags from the cobra command cliFlags, err := cli.NewHelper(cmd) if err != nil { @@ -69,12 +71,22 @@ func replaceCmd(cmd *cobra.Command, _ []string) error { r := replacer.New(cfg). WithUserRegex(cliFlags.Regex) - // Replace the tags in the directory - res, err := r.ParseContainerImages(cmd.Context(), cliFlags.Dir) - if err != nil { - return err + if cli.IsPath(args[0]) { + dir := filepath.Clean(args[0]) + // Replace the tags in the directory + res, err := r.ParseContainerImages(cmd.Context(), dir) + if err != nil { + return err + } + // Process the output files + return cliFlags.ProcessOutput(dir, res.Processed, res.Modified) + } else { + // Replace the passed reference + res, err := r.ParseSingleContainerImage(cmd.Context(), args[0]) + if err != nil { + return err + } + fmt.Fprintln(cmd.OutOrStdout(), res) + return nil } - - // Process the output files - return cliFlags.ProcessOutput(res.Processed, res.Modified) } diff --git a/cmd/image/list.go b/cmd/image/list.go index dd1983f..584dc29 100644 --- a/cmd/image/list.go +++ b/cmd/image/list.go @@ -23,6 +23,7 @@ import ( "github.com/stacklok/frizbee/internal/cli" "github.com/stacklok/frizbee/pkg/config" "github.com/stacklok/frizbee/pkg/replacer" + "path/filepath" "strconv" ) @@ -41,13 +42,16 @@ Example: SilenceUsage: true, } - cli.DeclareFrizbeeFlags(cmd, ".") - cmd.Flags().StringP("output", "o", "table", "output format. Can be 'json' or 'table'") + cli.DeclareFrizbeeFlags(cmd, true) return cmd } -func list(cmd *cobra.Command, _ []string) error { +func list(cmd *cobra.Command, args []string) error { + dir := filepath.Clean(args[0]) + if !cli.IsPath(dir) { + return fmt.Errorf("the provided argument is not a path") + } // Extract the CLI flags from the cobra command cliFlags, err := cli.NewHelper(cmd) if err != nil { @@ -65,7 +69,7 @@ func list(cmd *cobra.Command, _ []string) error { WithUserRegex(cliFlags.Regex) // List the references in the directory - res, err := r.ListContainerImages(cliFlags.Dir) + res, err := r.ListContainerImages(dir) if err != nil { return err } diff --git a/cmd/image/one.go b/cmd/image/one.go deleted file mode 100644 index 2b6870f..0000000 --- a/cmd/image/one.go +++ /dev/null @@ -1,77 +0,0 @@ -// -// Copyright 2024 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package image - -import ( - "fmt" - "github.com/spf13/cobra" - "github.com/stacklok/frizbee/internal/cli" - "github.com/stacklok/frizbee/pkg/config" - "github.com/stacklok/frizbee/pkg/replacer" -) - -// CmdOne represents the one sub-command -func CmdOne() *cobra.Command { - cmd := &cobra.Command{ - Use: "one", - Short: "Replace the tag with a digest reference", - Long: `This utility replaces a tag of a container reference -with the corresponding digest. - -Example: - - $ frizbee image one ghcr.io/stacklok/minder/server:latest - -This will replace a tag of the container reference with the corresponding digest. - -`, - Args: cobra.ExactArgs(1), - RunE: replaceOne, - SilenceUsage: true, - } - cli.DeclareFrizbeeFlags(cmd, "") - - return cmd -} - -func replaceOne(cmd *cobra.Command, args []string) error { - ref := args[0] - - // Extract the CLI flags from the cobra command - cliFlags, err := cli.NewHelper(cmd) - if err != nil { - return err - } - - // Set up the config - cfg, err := config.FromCommand(cmd) - if err != nil { - return err - } - - // Create a new replacer - r := replacer.New(cfg). - WithUserRegex(cliFlags.Regex) - - // Replace the passed reference - res, err := r.ParseSingleContainerImage(cmd.Context(), ref) - if err != nil { - return err - } - - fmt.Fprintf(cmd.OutOrStdout(), res) - return nil -} diff --git a/internal/cli/cli.go b/internal/cli/cli.go index e1676a8..9f703e4 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -51,7 +51,6 @@ Dirty: {{.Modified}} // Helper is a common struct for implementing a CLI command that replaces // files. type Helper struct { - Dir string DryRun bool Quiet bool ErrOnModified bool @@ -117,10 +116,6 @@ func (vvs *versionInfo) String() string { } func NewHelper(cmd *cobra.Command) (*Helper, error) { - dir := "." - if cmd.Flags().Lookup("dir") != nil { - dir = removeTrailingSlash(cmd.Flag("dir").Value.String()) - } dryRun, err := cmd.Flags().GetBool("dry-run") if err != nil { return nil, fmt.Errorf("failed to get dry-run flag: %w", err) @@ -140,7 +135,6 @@ func NewHelper(cmd *cobra.Command) (*Helper, error) { return &Helper{ Cmd: cmd, - Dir: dir, DryRun: dryRun, ErrOnModified: errOnModified, Quiet: quiet, @@ -149,17 +143,15 @@ func NewHelper(cmd *cobra.Command) (*Helper, error) { } // DeclareFrizbeeFlags declares the flags common to all replacer commands. -func DeclareFrizbeeFlags(cmd *cobra.Command, defaultDir string) { +func DeclareFrizbeeFlags(cmd *cobra.Command, enableOutput bool) { cmd.Flags().BoolP("dry-run", "n", false, "don't modify files") cmd.Flags().BoolP("quiet", "q", false, "don't print anything") cmd.Flags().BoolP("error", "e", false, "exit with error code if any file is modified") cmd.Flags().StringP("regex", "r", "", "regex to match artifact references") cmd.Flags().StringP("platform", "p", "", "platform to match artifact references, e.g. linux/amd64") - - if defaultDir != "" { - cmd.Flags().StringP("dir", "d", defaultDir, "directory path to parse") + if enableOutput { + cmd.Flags().StringP("output", "o", "table", "output format. Can be 'json' or 'table'") } - } // Logf logs the given message to the given command's stderr if the command is @@ -174,8 +166,8 @@ func (r *Helper) Logf(format string, args ...interface{}) { // If the command is quiet, the output is discarded. // If the command is a dry run, the output is written to the command's stdout. // Otherwise, the output is written to the given filesystem. -func (r *Helper) ProcessOutput(processed []string, modified map[string]string) error { - basedir := filepath.Dir(r.Dir) +func (r *Helper) ProcessOutput(path string, processed []string, modified map[string]string) error { + basedir := filepath.Dir(path) bfs := osfs.New(basedir, osfs.WithBoundOS()) var out io.Writer for _, path := range processed { @@ -214,14 +206,10 @@ func (r *Helper) ProcessOutput(processed []string, modified map[string]string) e return nil } -// removeTrailingSlash processes the given directory name for use with -// go-billy filesystems. -func removeTrailingSlash(dir string) string { - // remove trailing / from dir. This doesn't play well with - // the go-billy filesystem and walker we use. - if dir[len(dir)-1] == '/' { - return dir[:len(dir)-1] +func IsPath(pathOrRef string) bool { + _, err := os.Stat(pathOrRef) + if err == nil { + return true } - - return dir + return false } diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go new file mode 100644 index 0000000..475bdf5 --- /dev/null +++ b/pkg/errors/errors.go @@ -0,0 +1,24 @@ +// +// Copyright 2024 Stacklok, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package errors provides error values, constants and functions. +package errors + +import "errors" + +var ( + // ErrReferenceSkipped is returned when the reference is skipped. + ErrReferenceSkipped = errors.New("reference skipped") +) diff --git a/pkg/interfaces/interfaces.go b/pkg/interfaces/interfaces.go index 8b17a2c..90fa340 100644 --- a/pkg/interfaces/interfaces.go +++ b/pkg/interfaces/interfaces.go @@ -24,14 +24,16 @@ import ( // EntityRef represents an action reference. type EntityRef struct { - Name string `json:"name"` - Ref string `json:"ref"` - Type string `json:"type"` + Name string `json:"name"` + Ref string `json:"ref"` + Type string `json:"type"` + Tag string `json:"tag"` + Prefix string `json:"prefix"` } type Parser interface { GetRegex() string - Replace(ctx context.Context, matchedLine string, restIf REST, cfg config.Config, cache store.RefCacher, keepPrefix bool) (string, error) + Replace(ctx context.Context, matchedLine string, restIf REST, cfg config.Config, cache store.RefCacher) (*EntityRef, error) ConvertToEntityRef(reference string) (*EntityRef, error) } diff --git a/pkg/replacer/action/action.go b/pkg/replacer/action/action.go index a8a3355..5b6c37a 100644 --- a/pkg/replacer/action/action.go +++ b/pkg/replacer/action/action.go @@ -20,6 +20,7 @@ import ( "fmt" "github.com/stacklok/frizbee/internal/store" "github.com/stacklok/frizbee/pkg/config" + ferrors "github.com/stacklok/frizbee/pkg/errors" "github.com/stacklok/frizbee/pkg/interfaces" "github.com/stacklok/frizbee/pkg/replacer/image" "strings" @@ -50,106 +51,102 @@ func (p *Parser) GetRegex() string { return p.regex } -func (p *Parser) Replace(ctx context.Context, matchedLine string, restIf interfaces.REST, cfg config.Config, cache store.RefCacher, keepPrefix bool) (string, error) { +func (p *Parser) Replace(ctx context.Context, matchedLine string, restIf interfaces.REST, cfg config.Config, cache store.RefCacher) (*interfaces.EntityRef, error) { var err error + var actionRef *interfaces.EntityRef // Trim the uses prefix - actionRef := strings.TrimPrefix(matchedLine, prefixUses) + trimmedRef := strings.TrimPrefix(matchedLine, prefixUses) // Determine if the action reference has a docker prefix - if strings.Contains(actionRef, prefixDocker) { - actionRef, err = p.replaceDocker(ctx, actionRef, restIf, cfg, cache, keepPrefix) + if strings.Contains(trimmedRef, prefixDocker) { + actionRef, err = p.replaceDocker(ctx, trimmedRef, restIf, cfg, cache) } else { - actionRef, err = p.replaceAction(ctx, actionRef, restIf, cfg, cache, keepPrefix) + actionRef, err = p.replaceAction(ctx, trimmedRef, restIf, cfg, cache) } if err != nil { - return "", err + return nil, err } - // Add back the uses prefix, if needed - if keepPrefix { - actionRef = fmt.Sprintf("%s%s", prefixUses, actionRef) - } + // Add back the uses prefix + actionRef.Prefix = fmt.Sprintf("%s%s", prefixUses, actionRef.Prefix) // Return the new action reference return actionRef, nil } -func (p *Parser) replaceAction(ctx context.Context, matchedLine string, restIf interfaces.REST, cfg config.Config, cache store.RefCacher, keepPrefix bool) (string, error) { - actionRef := matchedLine +func (p *Parser) replaceAction(ctx context.Context, matchedLine string, restIf interfaces.REST, cfg config.Config, cache store.RefCacher) (*interfaces.EntityRef, error) { // If the value is a local path or should be excluded, skip it - if isLocal(actionRef) || shouldExclude(&cfg.GHActions, actionRef) { - return matchedLine, nil + if isLocal(matchedLine) || shouldExclude(&cfg.GHActions, matchedLine) { + return nil, fmt.Errorf("%w: %s", ferrors.ErrReferenceSkipped, matchedLine) } // Parse the action reference - act, ref, err := ParseActionReference(actionRef) + act, ref, err := ParseActionReference(matchedLine) if err != nil { - return matchedLine, nil + return nil, fmt.Errorf("failed to parse action reference '%s': %w", matchedLine, err) } // Check if the parsed reference should be excluded if shouldExclude(&cfg.GHActions, act) { - return matchedLine, nil + return nil, fmt.Errorf("%w: %s", ferrors.ErrReferenceSkipped, matchedLine) } var sum string // Check if we have a cache if cache != nil { // Check if we have a cached value - if val, ok := cache.Load(actionRef); ok { + if val, ok := cache.Load(matchedLine); ok { sum = val } else { // Get the checksum for the action reference sum, err = GetChecksum(ctx, restIf, act, ref) if err != nil { - return matchedLine, nil + return nil, fmt.Errorf("failed to get checksum for action '%s': %w", matchedLine, err) } // Store the checksum in the cache - cache.Store(actionRef, sum) + cache.Store(matchedLine, sum) } } else { // Get the checksum for the action reference sum, err = GetChecksum(ctx, restIf, act, ref) if err != nil { - return matchedLine, nil + return nil, fmt.Errorf("failed to get checksum for action '%s': %w", matchedLine, err) } } - // If the checksum is different from the reference, update the reference - // Otherwise, return the original line - if ref == sum { - return matchedLine, nil - } - return fmt.Sprintf("%s@%s # %s", act, sum, ref), nil + return &interfaces.EntityRef{ + Name: act, + Ref: sum, + Type: ReferenceType, + Tag: ref, + }, nil } -func (p *Parser) replaceDocker(ctx context.Context, matchedLine string, _ interfaces.REST, cfg config.Config, cache store.RefCacher, keepPrefix bool) (string, error) { - var err error +func (p *Parser) replaceDocker(ctx context.Context, matchedLine string, _ interfaces.REST, cfg config.Config, cache store.RefCacher) (*interfaces.EntityRef, error) { // Trim the docker prefix - actionRef := strings.TrimPrefix(matchedLine, prefixDocker) + trimmedRef := strings.TrimPrefix(matchedLine, prefixDocker) // If the value is a local path or should be excluded, skip it - if isLocal(actionRef) || shouldExclude(&cfg.GHActions, actionRef) { - return matchedLine, nil + if isLocal(trimmedRef) || shouldExclude(&cfg.GHActions, trimmedRef) { + return nil, fmt.Errorf("%w: %s", ferrors.ErrReferenceSkipped, matchedLine) } // Get the digest of the docker:// image reference - actionRef, err = image.GetImageDigestFromRef(ctx, actionRef, cfg.Platform, cache, false) + actionRef, err := image.GetImageDigestFromRef(ctx, trimmedRef, cfg.Platform, cache) if err != nil { - return "", err + return nil, err } // Check if the parsed reference should be excluded - if shouldExclude(&cfg.GHActions, actionRef) { - return matchedLine, nil + if shouldExclude(&cfg.GHActions, actionRef.Name) { + return nil, fmt.Errorf("%w: %s", ferrors.ErrReferenceSkipped, matchedLine) } - // Add back the docker prefix, if needed - if keepPrefix { - actionRef = fmt.Sprintf("%s%s", prefixDocker, actionRef) - } + // Add back the docker prefix + actionRef.Prefix = fmt.Sprintf("%s%s", prefixDocker, actionRef.Prefix) + return actionRef, nil } diff --git a/pkg/replacer/image/image.go b/pkg/replacer/image/image.go index 24be3f5..39f4cad 100644 --- a/pkg/replacer/image/image.go +++ b/pkg/replacer/image/image.go @@ -20,13 +20,14 @@ import ( "fmt" "github.com/stacklok/frizbee/internal/store" "github.com/stacklok/frizbee/pkg/config" + ferrors "github.com/stacklok/frizbee/pkg/errors" "github.com/stacklok/frizbee/pkg/interfaces" "strings" ) const ( // ContainerImageRegex is regular expression pattern to match container image usage in YAML - ContainerImageRegex = `image\s*:\s*["']?([^\s"']+/[^\s"']+(:[^\s"']+)?(@[^\s"']+)?)["']?|FROM\s+([^\s]+(/[^\s]+)?(:[^\s]+)?(@[^\s]+)?)` // `\b(image|FROM)\s*:?(\s*([^\s]+))?` + ContainerImageRegex = `image\s*:\s*["']?([^\s"']+/[^\s"']+|[^\s"']+)(:[^\s"']+)?(@[^\s"']+)?["']?|FROM\s+([^\s]+(/[^\s]+)?(:[^\s]+)?(@[^\s]+)?)` // `image\s*:\s*["']?([^\s"']+/[^\s"']+(:[^\s"']+)?(@[^\s"']+)?)["']?|FROM\s+([^\s]+(/[^\s]+)?(:[^\s]+)?(@[^\s]+)?)` // `\b(image|FROM)\s*:?(\s*([^\s]+))?` prefixFROM = "FROM " prefixImage = "image: " ReferenceType = "container" @@ -49,42 +50,37 @@ func (p *Parser) GetRegex() string { return p.regex } -func (p *Parser) Replace(ctx context.Context, matchedLine string, _ interfaces.REST, cfg config.Config, cache store.RefCacher, keepPrefix bool) (string, error) { +func (p *Parser) Replace(ctx context.Context, matchedLine string, _ interfaces.REST, cfg config.Config, cache store.RefCacher) (*interfaces.EntityRef, error) { // Trim the prefix hasFROMPrefix := false - imageRef := matchedLine // Check if the image reference has the FROM prefix, i.e. Dockerfile - if strings.HasPrefix(imageRef, prefixFROM) { - imageRef = strings.TrimPrefix(imageRef, prefixFROM) + if strings.HasPrefix(matchedLine, prefixFROM) { + matchedLine = strings.TrimPrefix(matchedLine, prefixFROM) // Check if the image reference should be excluded, i.e. scratch - if shouldExclude(imageRef) { - return matchedLine, nil + if shouldExclude(matchedLine) { + return nil, fmt.Errorf("image reference %s should be excluded - %w", matchedLine, ferrors.ErrReferenceSkipped) } hasFROMPrefix = true - } else if strings.HasPrefix(imageRef, prefixImage) { + } else if strings.HasPrefix(matchedLine, prefixImage) { // Check if the image reference has the image prefix, i.e. Kubernetes or Docker Compose YAML - imageRef = strings.TrimPrefix(imageRef, prefixImage) + matchedLine = strings.TrimPrefix(matchedLine, prefixImage) } // Get the digest of the image reference - imageRefWithDigest, err := GetImageDigestFromRef(ctx, imageRef, cfg.Platform, cache, hasFROMPrefix) + imageRefWithDigest, err := GetImageDigestFromRef(ctx, matchedLine, cfg.Platform, cache) if err != nil { - return "", err + return nil, err } - // Add the prefix back, if needed - if keepPrefix { - if hasFROMPrefix { - imageRefWithDigest = prefixFROM + imageRefWithDigest - } else { - imageRefWithDigest = prefixImage + imageRefWithDigest - } - // Return the modified line with the prefix - return imageRefWithDigest, nil - + // Add the prefix back + if hasFROMPrefix { + imageRefWithDigest.Prefix = fmt.Sprintf("%s%s", prefixFROM, imageRefWithDigest.Prefix) + } else { + imageRefWithDigest.Prefix = fmt.Sprintf("%s%s", prefixImage, imageRefWithDigest.Prefix) } - // Return the modified line without the prefix + + // Return the reference return imageRefWithDigest, nil } diff --git a/pkg/replacer/image/utils.go b/pkg/replacer/image/utils.go index 154ef26..68b9857 100644 --- a/pkg/replacer/image/utils.go +++ b/pkg/replacer/image/utils.go @@ -24,16 +24,18 @@ import ( "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/stacklok/frizbee/internal/cli" "github.com/stacklok/frizbee/internal/store" + ferrors "github.com/stacklok/frizbee/pkg/errors" + "github.com/stacklok/frizbee/pkg/interfaces" "strings" ) // GetImageDigestFromRef returns the digest of a container image reference // from a name.Reference. -func GetImageDigestFromRef(ctx context.Context, imageRef, platform string, cache store.RefCacher, isDockerfileRef bool) (string, error) { +func GetImageDigestFromRef(ctx context.Context, imageRef, platform string, cache store.RefCacher) (*interfaces.EntityRef, error) { // Parse the image reference ref, err := name.ParseReference(imageRef) if err != nil { - return "", err + return nil, err } opts := []remote.Option{ remote.WithContext(ctx), @@ -45,7 +47,7 @@ func GetImageDigestFromRef(ctx context.Context, imageRef, platform string, cache if platform != "" { platformSplit := strings.Split(platform, "/") if len(platformSplit) != 2 { - return "", fmt.Errorf("platform must be in the format os/arch") + return nil, fmt.Errorf("platform must be in the format os/arch") } opts = append(opts, remote.WithPlatform(v1.Platform{ OS: platformSplit[0], @@ -62,28 +64,29 @@ func GetImageDigestFromRef(ctx context.Context, imageRef, platform string, cache } desc, err := remote.Get(ref, opts...) if err != nil { - return "", err + return nil, err } digest = desc.Digest.String() cache.Store(imageRef, digest) } else { desc, err := remote.Get(ref, opts...) if err != nil { - return "", err + return nil, err } digest = desc.Digest.String() } // Compare the digest with the reference and return the original reference if they already match if digest == ref.Identifier() { - return imageRef, nil + return nil, fmt.Errorf("image already referenced by digest: %s %w", imageRef, ferrors.ErrReferenceSkipped) } - // Return the image reference with the digest differently if it is a Dockerfile reference - if isDockerfileRef { - return fmt.Sprintf("%s:%s@%s", ref.Context().Name(), ref.Identifier(), digest), nil - } - return fmt.Sprintf("%s@%s # %s", ref.Context().Name(), digest, ref.Identifier()), nil + return &interfaces.EntityRef{ + Name: ref.Context().Name(), + Ref: digest, + Type: ReferenceType, + Tag: ref.Identifier(), + }, nil } func shouldExclude(ref string) bool { diff --git a/pkg/replacer/replacer.go b/pkg/replacer/replacer.go index 488fb30..1f5c7ff 100644 --- a/pkg/replacer/replacer.go +++ b/pkg/replacer/replacer.go @@ -42,10 +42,10 @@ type ParserType string // Replacer replaces container image references in YAML files type Replacer struct { + regex string parser interfaces.Parser interfaces.REST config.Config - regex string } type ReplaceResult struct { @@ -58,12 +58,14 @@ type ListResult struct { Entities []interfaces.EntityRef } +// WithGitHubClient creates an authatenticated GitHub client func (r *Replacer) WithGitHubClient(token string) *Replacer { client := ghrest.NewClient(token) r.REST = client return r } +// WithUserRegex sets a user-provided regex for the parser func (r *Replacer) WithUserRegex(regex string) *Replacer { r.regex = regex return r @@ -78,31 +80,45 @@ func New(cfg *config.Config) *Replacer { } } +// ParseSingleGitHubAction parses and returns the entity reference pinned by its digest func (r *Replacer) ParseSingleGitHubAction(ctx context.Context, entityRef string) (string, error) { r.parser = action.New(r.regex) - return r.parser.Replace(ctx, entityRef, r.REST, r.Config, nil, false) + ret, err := r.parser.Replace(ctx, entityRef, r.REST, r.Config, nil) + if err != nil { + return "", err + } + return ret.Ref, nil } +// ParseGitHubActions parses and replaces all GitHub actions references in yaml/yml files present the provided directory func (r *Replacer) ParseGitHubActions(ctx context.Context, dir string) (*ReplaceResult, error) { r.parser = action.New(r.regex) return r.parsePath(ctx, dir) } +// ListGitHibActions lists all GitHub actions references in yaml/yml files present the provided directory func (r *Replacer) ListGitHibActions(dir string) (*ListResult, error) { r.parser = action.New(r.regex) return r.listReferences(dir) } +// ParseSingleContainerImage parses and returns the entity reference pinned by its digest func (r *Replacer) ParseSingleContainerImage(ctx context.Context, entityRef string) (string, error) { r.parser = image.New(r.regex) - return r.parser.Replace(ctx, entityRef, r.REST, r.Config, nil, false) + ret, err := r.parser.Replace(ctx, entityRef, r.REST, r.Config, nil) + if err != nil { + return "", err + } + return ret.Ref, nil } +// ParseContainerImages parses and replaces all container image references in yaml, yml and dockerfiles present the provided directory func (r *Replacer) ParseContainerImages(ctx context.Context, dir string) (*ReplaceResult, error) { r.parser = image.New(r.regex) return r.parsePath(ctx, dir) } +// ListContainerImages lists all container image references in yaml, yml and dockerfiles present the provided directory func (r *Replacer) ListContainerImages(dir string) (*ListResult, error) { r.parser = image.New(r.regex) return r.listReferences(dir) @@ -133,21 +149,20 @@ func (r *Replacer) parsePath(ctx context.Context, dir string) (*ReplaceResult, e // nolint:errcheck // ignore error defer file.Close() - // Store the file name to the processed batch - res.Processed = append(res.Processed, path) - // Parse the content of the file and update the matching references modified, updatedFile, err := r.parseAndReplaceReferencesInFile(ctx, file, cache) if err != nil { return fmt.Errorf("failed to modify references in %s: %w", path, err) } + mu.Lock() + // Store the file name to the processed batch + res.Processed = append(res.Processed, path) // Store the updated file content if it was modified if modified { - mu.Lock() res.Modified[path] = updatedFile - mu.Unlock() } + mu.Unlock() // All good return nil @@ -226,11 +241,12 @@ func (r *Replacer) listReferences(dir string) (*ListResult, error) { } // parseAndReplaceReferencesInFile takes the given file reader and returns its content -// after replacing all references to tags with the checksum of the tag. -// The function returns an empty string and an error if it fails to process the file +// after replacing all references to tags with the respective digests. // It also uses the provided cache to store the checksums. func (r *Replacer) parseAndReplaceReferencesInFile(ctx context.Context, f io.Reader, cache store.RefCacher) (bool, string, error) { var contentBuilder strings.Builder + var ret *interfaces.EntityRef + modified := false // Compile the regular expression @@ -242,22 +258,22 @@ func (r *Replacer) parseAndReplaceReferencesInFile(ctx context.Context, f io.Rea // Read the file line by line scanner := bufio.NewScanner(f) for scanner.Scan() { - var err error - var resultingLine string line := scanner.Text() - // See if we can match an entity reference in the line - newLine := re.ReplaceAllStringFunc(line, func(entityRef string) string { + newLine := re.ReplaceAllStringFunc(line, func(matchedLine string) string { // Modify the reference in the line - resultingLine, err = r.parser.Replace(ctx, entityRef, r.REST, r.Config, cache, true) - return resultingLine + ret, err = r.parser.Replace(ctx, matchedLine, r.REST, r.Config, cache) + if err != nil { + // Return the original line as we don't want to update it in case something errored out + return matchedLine + } + // Construct the new line + if strings.Contains(matchedLine, "FROM") { + return fmt.Sprintf("%s%s:%s@%s", ret.Prefix, ret.Name, ret.Tag, ret.Ref) + } + return fmt.Sprintf("%s%s@%s # %s", ret.Prefix, ret.Name, ret.Ref, ret.Tag) }) - // Handle the case where the reference could not be modified, i.e. failed to parse it correctly - if err != nil { - return false, "", fmt.Errorf("failed to modify reference in line: %s: %w", line, err) - } - // Check if the line was modified and set the modified flag to true if it was if newLine != line { modified = true @@ -276,10 +292,8 @@ func (r *Replacer) parseAndReplaceReferencesInFile(ctx context.Context, f io.Rea return modified, contentBuilder.String(), nil } -// listReferencesInFile takes the given file reader and returns its content -// after replacing all references to tags with the checksum of the tag. -// The function returns an empty string and an error if it fails to process the file -// It also uses the provided cache to store the checksums. +// listReferencesInFile takes the given file reader and returns a map of all +// references, action or images, it found. func (r *Replacer) listReferencesInFile(f io.Reader) (mapset.Set[interfaces.EntityRef], error) { found := mapset.NewSet[interfaces.EntityRef]() diff --git a/pkg/replacer/replacer_test.go b/pkg/replacer/replacer_test.go index 42c1760..53ff0a5 100644 --- a/pkg/replacer/replacer_test.go +++ b/pkg/replacer/replacer_test.go @@ -14,3 +14,194 @@ // limitations under the License. package replacer + +import ( + "context" + "github.com/stacklok/frizbee/pkg/config" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "os" + "testing" + "time" +) + +func TestReplacer_ParseSingleContainerImage(t *testing.T) { + t.Parallel() + + type args struct { + refstr string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "valid 1", + args: args{ + refstr: "ghcr.io/stacklok/minder/helm/minder:0.20231123.829_ref.26ca90b", + }, + want: "ghcr.io/stacklok/minder/helm/minder@sha256:a29f8a8d28f0af7f70a4b3dd3e33c8c8cc5cf9e88e802e2700cf272a0b6140ec # 0.20231123.829_ref.26ca90b", + }, + { + name: "valid 2", + args: args{ + refstr: "devopsfaith/krakend:2.5.0", + }, + want: "index.docker.io/devopsfaith/krakend@sha256:6a3c8e5e1a4948042bfb364ed6471e16b4a26d0afb6c3c01ebcb88b3fa551036 # 2.5.0", + }, + { + name: "invalid ref string", + args: args{ + refstr: "ghcr.io/stacklok/minder/helm/minder!", + }, + wantErr: true, + }, + { + name: "nonexistent container in nonexistent registry", + args: args{ + refstr: "beeeeer.io/ipa/toppling-goliath/king-sue:1.0.0", + }, + wantErr: true, + }, + } + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + r := New(&config.Config{}) + got, err := r.ParseSingleContainerImage(ctx, tt.args.refstr) + if tt.wantErr { + assert.Error(t, err) + assert.Empty(t, got) + return + } + + assert.NoError(t, err) + assert.Equal(t, tt.want, got) + }) + } +} + +func TestReplacer_ParseSingleGitHubAction(t *testing.T) { + t.Parallel() + + type args struct { + action string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "actions/checkout with v4.1.1", + args: args{ + action: "actions/checkout@v4.1.1", + }, + want: "actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1", + wantErr: false, + }, + { + name: "actions/checkout with v3.6.0", + args: args{ + action: "actions/checkout@v3.6.0", + }, + want: "actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0", + wantErr: false, + }, + { + name: "actions/checkout with checksum returns checksum", + args: args{ + action: "actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f", + }, + want: "1d96c772d19495a3b5c517cd2bc0cb401ea0529f", + wantErr: false, + }, + { + name: "aquasecurity/trivy-action with 0.14.0", + args: args{ + action: "aquasecurity/trivy-action@0.14.0", + }, + want: "2b6a709cf9c4025c5438138008beaddbb02086f0", + wantErr: false, + }, + { + name: "aquasecurity/trivy-action with branch returns checksum", + args: args{ + action: "aquasecurity/trivy-action@bump-trivy", + }, + want: "fb5e1b36be448e92ca98648c661bd7e9da1f1317", + wantErr: false, + }, + { + name: "actions/checkout with invalid tag returns error", + args: args{ + action: "actions/checkout@v4.1.1.1", + }, + want: "", + wantErr: true, + }, + { + name: "actions/checkout with invalid action returns error", + args: args{ + action: "invalid-action@v4.1.1", + }, + want: "", + wantErr: true, + }, + { + name: "actions/checkout with empty action returns error", + args: args{ + action: "@v4.1.1", + }, + want: "", + wantErr: true, + }, + { + name: "actions/checkout with empty tag returns error", + args: args{ + action: "actions/checkout", + }, + want: "", + wantErr: true, + }, + { + name: "bufbuild/buf-setup-action with v1 is an array", + args: args{ + action: "bufbuild/buf-setup-action@v1", + }, + want: "480a0ee8a588045b52a847b48138c6f377a89519", + }, + { + name: "anchore/sbom-action/download-syft with a sub-action works", + args: args{ + action: "anchore/sbom-action/download-syft@v0.14.3", + }, + want: "78fc58e266e87a38d4194b2137a3d4e9bcaf7ca1", + }, + } + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + r := New(&config.Config{}).WithGitHubClient(os.Getenv("GITHUB_TOKEN")) + got, err := r.ParseSingleGitHubAction(context.Background(), tt.args.action) + if tt.wantErr { + require.Error(t, err, "Wanted error, got none") + require.Empty(t, got, "Wanted empty string, got %v", got) + return + } + require.NoError(t, err, "Wanted no error, got %v", err) + require.Equal(t, tt.want, got, "Wanted %v, got %v", tt.want, got) + }) + } +} From cbff99f86a918c3563273d48d0b56e63609365ff Mon Sep 17 00:00:00 2001 From: Radoslav Dimitrov Date: Tue, 28 May 2024 03:10:39 +0300 Subject: [PATCH 04/16] Update the unit tests Signed-off-by: Radoslav Dimitrov --- cmd/action/action.go | 8 +- cmd/action/list.go | 4 +- cmd/image/image.go | 6 +- cmd/image/list.go | 4 +- pkg/replacer/action/action.go | 21 ++- pkg/replacer/image/image.go | 5 +- pkg/replacer/replacer.go | 100 ++++++++---- pkg/replacer/replacer_test.go | 297 +++++++++++++++++++++++++++++++--- 8 files changed, 376 insertions(+), 69 deletions(-) diff --git a/cmd/action/action.go b/cmd/action/action.go index 3f753fb..cfda45c 100644 --- a/cmd/action/action.go +++ b/cmd/action/action.go @@ -43,7 +43,7 @@ This will replace all tag or branch references in all GitHub Actions workflows for the given directory. Supports both directories and single references. ` + cli.TokenHelpText + "\n", - Aliases: []string{"ghactions"}, // backwards compatibility + Aliases: []string{"ghactions", "actions"}, // backwards compatibility RunE: replaceCmd, SilenceUsage: true, Args: cobra.ExactArgs(1), @@ -79,7 +79,7 @@ func replaceCmd(cmd *cobra.Command, args []string) error { if cli.IsPath(args[0]) { dir := filepath.Clean(args[0]) // Replace the tags in the given directory - res, err := r.ParseGitHubActions(cmd.Context(), dir) + res, err := r.ParseGitHubActionsInPath(cmd.Context(), dir) if err != nil { return err } @@ -87,11 +87,11 @@ func replaceCmd(cmd *cobra.Command, args []string) error { return cliFlags.ProcessOutput(dir, res.Processed, res.Modified) } else { // Replace the passed reference - res, err := r.ParseSingleGitHubAction(cmd.Context(), args[0]) + res, err := r.ParseGitHubActionString(cmd.Context(), args[0]) if err != nil { return err } - fmt.Fprintln(cmd.OutOrStdout(), res) + fmt.Fprintln(cmd.OutOrStdout(), fmt.Sprintf("%s@%s", res.Name, res.Ref)) return nil } } diff --git a/cmd/action/list.go b/cmd/action/list.go index e22783c..011aa4f 100644 --- a/cmd/action/list.go +++ b/cmd/action/list.go @@ -72,7 +72,7 @@ func list(cmd *cobra.Command, args []string) error { WithGitHubClient(os.Getenv(cli.GitHubTokenEnvKey)) // List the references in the directory - res, err := r.ListGitHibActions(dir) + res, err := r.ListGitHibActionsInPath(dir) if err != nil { return err } @@ -86,7 +86,7 @@ func list(cmd *cobra.Command, args []string) error { } jsonString := string(jsonBytes) - fmt.Fprintf(cmd.OutOrStdout(), "%s\n", jsonString) + fmt.Fprintln(cmd.OutOrStdout(), jsonString) return nil case "table": table := tablewriter.NewWriter(cmd.OutOrStdout()) diff --git a/cmd/image/image.go b/cmd/image/image.go index 6f0511d..b477e0e 100644 --- a/cmd/image/image.go +++ b/cmd/image/image.go @@ -74,7 +74,7 @@ func replaceCmd(cmd *cobra.Command, args []string) error { if cli.IsPath(args[0]) { dir := filepath.Clean(args[0]) // Replace the tags in the directory - res, err := r.ParseContainerImages(cmd.Context(), dir) + res, err := r.ParseContainerImagesInPath(cmd.Context(), dir) if err != nil { return err } @@ -82,11 +82,11 @@ func replaceCmd(cmd *cobra.Command, args []string) error { return cliFlags.ProcessOutput(dir, res.Processed, res.Modified) } else { // Replace the passed reference - res, err := r.ParseSingleContainerImage(cmd.Context(), args[0]) + res, err := r.ParseContainerImageString(cmd.Context(), args[0]) if err != nil { return err } - fmt.Fprintln(cmd.OutOrStdout(), res) + fmt.Fprintln(cmd.OutOrStdout(), fmt.Sprintf("%s@%s", res.Name, res.Ref)) return nil } } diff --git a/cmd/image/list.go b/cmd/image/list.go index 584dc29..b426dc4 100644 --- a/cmd/image/list.go +++ b/cmd/image/list.go @@ -69,7 +69,7 @@ func list(cmd *cobra.Command, args []string) error { WithUserRegex(cliFlags.Regex) // List the references in the directory - res, err := r.ListContainerImages(dir) + res, err := r.ListContainerImagesInPath(dir) if err != nil { return err } @@ -82,7 +82,7 @@ func list(cmd *cobra.Command, args []string) error { return err } jsonString := string(jsonBytes) - fmt.Fprintf(cmd.OutOrStdout(), "%s\n", jsonString) + fmt.Fprintln(cmd.OutOrStdout(), jsonString) return nil case "table": table := tablewriter.NewWriter(cmd.OutOrStdout()) diff --git a/pkg/replacer/action/action.go b/pkg/replacer/action/action.go index 5b6c37a..bf2a301 100644 --- a/pkg/replacer/action/action.go +++ b/pkg/replacer/action/action.go @@ -54,22 +54,27 @@ func (p *Parser) GetRegex() string { func (p *Parser) Replace(ctx context.Context, matchedLine string, restIf interfaces.REST, cfg config.Config, cache store.RefCacher) (*interfaces.EntityRef, error) { var err error var actionRef *interfaces.EntityRef + hasUsesPrefix := false // Trim the uses prefix - trimmedRef := strings.TrimPrefix(matchedLine, prefixUses) - + if strings.HasPrefix(matchedLine, prefixUses) { + matchedLine = strings.TrimPrefix(matchedLine, prefixUses) + hasUsesPrefix = true + } // Determine if the action reference has a docker prefix - if strings.Contains(trimmedRef, prefixDocker) { - actionRef, err = p.replaceDocker(ctx, trimmedRef, restIf, cfg, cache) + if strings.HasPrefix(matchedLine, prefixDocker) { + actionRef, err = p.replaceDocker(ctx, matchedLine, restIf, cfg, cache) } else { - actionRef, err = p.replaceAction(ctx, trimmedRef, restIf, cfg, cache) + actionRef, err = p.replaceAction(ctx, matchedLine, restIf, cfg, cache) } if err != nil { return nil, err } // Add back the uses prefix - actionRef.Prefix = fmt.Sprintf("%s%s", prefixUses, actionRef.Prefix) + if hasUsesPrefix { + actionRef.Prefix = fmt.Sprintf("%s%s", prefixUses, actionRef.Prefix) + } // Return the new action reference return actionRef, nil @@ -145,7 +150,9 @@ func (p *Parser) replaceDocker(ctx context.Context, matchedLine string, _ interf } // Add back the docker prefix - actionRef.Prefix = fmt.Sprintf("%s%s", prefixDocker, actionRef.Prefix) + if strings.HasPrefix(matchedLine, prefixDocker) { + actionRef.Prefix = fmt.Sprintf("%s%s", prefixDocker, actionRef.Prefix) + } return actionRef, nil } diff --git a/pkg/replacer/image/image.go b/pkg/replacer/image/image.go index 39f4cad..989361f 100644 --- a/pkg/replacer/image/image.go +++ b/pkg/replacer/image/image.go @@ -53,7 +53,7 @@ func (p *Parser) GetRegex() string { func (p *Parser) Replace(ctx context.Context, matchedLine string, _ interfaces.REST, cfg config.Config, cache store.RefCacher) (*interfaces.EntityRef, error) { // Trim the prefix hasFROMPrefix := false - + hasImagePrefix := false // Check if the image reference has the FROM prefix, i.e. Dockerfile if strings.HasPrefix(matchedLine, prefixFROM) { matchedLine = strings.TrimPrefix(matchedLine, prefixFROM) @@ -65,6 +65,7 @@ func (p *Parser) Replace(ctx context.Context, matchedLine string, _ interfaces.R } else if strings.HasPrefix(matchedLine, prefixImage) { // Check if the image reference has the image prefix, i.e. Kubernetes or Docker Compose YAML matchedLine = strings.TrimPrefix(matchedLine, prefixImage) + hasImagePrefix = true } // Get the digest of the image reference @@ -76,7 +77,7 @@ func (p *Parser) Replace(ctx context.Context, matchedLine string, _ interfaces.R // Add the prefix back if hasFROMPrefix { imageRefWithDigest.Prefix = fmt.Sprintf("%s%s", prefixFROM, imageRefWithDigest.Prefix) - } else { + } else if hasImagePrefix { imageRefWithDigest.Prefix = fmt.Sprintf("%s%s", prefixImage, imageRefWithDigest.Prefix) } diff --git a/pkg/replacer/replacer.go b/pkg/replacer/replacer.go index 1f5c7ff..e53b12c 100644 --- a/pkg/replacer/replacer.go +++ b/pkg/replacer/replacer.go @@ -58,7 +58,7 @@ type ListResult struct { Entities []interfaces.EntityRef } -// WithGitHubClient creates an authatenticated GitHub client +// WithGitHubClient creates an authenticated GitHub client func (r *Replacer) WithGitHubClient(token string) *Replacer { client := ghrest.NewClient(token) r.REST = client @@ -71,8 +71,7 @@ func (r *Replacer) WithUserRegex(regex string) *Replacer { return r } -// New creates a new FileReplacer from the given -// command-line arguments and options +// New creates a new Replacer func New(cfg *config.Config) *Replacer { // Return the replacer return &Replacer{ @@ -80,50 +79,92 @@ func New(cfg *config.Config) *Replacer { } } -// ParseSingleGitHubAction parses and returns the entity reference pinned by its digest -func (r *Replacer) ParseSingleGitHubAction(ctx context.Context, entityRef string) (string, error) { +// ParseGitHubActionString parses and returns the referenced entity pinned by its digest +func (r *Replacer) ParseGitHubActionString(ctx context.Context, entityRef string) (*interfaces.EntityRef, error) { r.parser = action.New(r.regex) - ret, err := r.parser.Replace(ctx, entityRef, r.REST, r.Config, nil) - if err != nil { - return "", err - } - return ret.Ref, nil + return r.parser.Replace(ctx, entityRef, r.REST, r.Config, nil) } -// ParseGitHubActions parses and replaces all GitHub actions references in yaml/yml files present the provided directory -func (r *Replacer) ParseGitHubActions(ctx context.Context, dir string) (*ReplaceResult, error) { +// ParseGitHubActionsInPath parses and replaces all GitHub actions references in the provided directory +func (r *Replacer) ParseGitHubActionsInPath(ctx context.Context, dir string) (*ReplaceResult, error) { r.parser = action.New(r.regex) return r.parsePath(ctx, dir) } -// ListGitHibActions lists all GitHub actions references in yaml/yml files present the provided directory -func (r *Replacer) ListGitHibActions(dir string) (*ListResult, error) { +// ParseGitHubActionsInFile parses and replaces all GitHub actions references in the provided file +func (r *Replacer) ParseGitHubActionsInFile(ctx context.Context, f io.Reader, cache store.RefCacher) (bool, string, error) { + r.parser = action.New(r.regex) + return r.parseAndReplaceReferencesInFile(ctx, f, cache) +} + +// ListGitHibActionsInPath lists all GitHub actions references in the provided directory +func (r *Replacer) ListGitHibActionsInPath(dir string) (*ListResult, error) { r.parser = action.New(r.regex) return r.listReferences(dir) } -// ParseSingleContainerImage parses and returns the entity reference pinned by its digest -func (r *Replacer) ParseSingleContainerImage(ctx context.Context, entityRef string) (string, error) { - r.parser = image.New(r.regex) - ret, err := r.parser.Replace(ctx, entityRef, r.REST, r.Config, nil) +// ListGitHibActionsInFile lists all GitHub actions references in the provided file +func (r *Replacer) ListGitHibActionsInFile(f io.Reader) (*ListResult, error) { + r.parser = action.New(r.regex) + found, err := r.listReferencesInFile(f) if err != nil { - return "", err + return nil, err } - return ret.Ref, nil + res := &ListResult{} + res.Entities = found.ToSlice() + + // Sort the slice + sort.Slice(res.Entities, func(i, j int) bool { + return res.Entities[i].Name < res.Entities[j].Name + }) + + // All good + return res, nil +} + +// ParseContainerImageString parses and returns the referenced entity pinned by its digest +func (r *Replacer) ParseContainerImageString(ctx context.Context, entityRef string) (*interfaces.EntityRef, error) { + r.parser = image.New(r.regex) + return r.parser.Replace(ctx, entityRef, r.REST, r.Config, nil) } -// ParseContainerImages parses and replaces all container image references in yaml, yml and dockerfiles present the provided directory -func (r *Replacer) ParseContainerImages(ctx context.Context, dir string) (*ReplaceResult, error) { +// ParseContainerImagesInPath parses and replaces all container image references in the provided directory +func (r *Replacer) ParseContainerImagesInPath(ctx context.Context, dir string) (*ReplaceResult, error) { r.parser = image.New(r.regex) return r.parsePath(ctx, dir) } -// ListContainerImages lists all container image references in yaml, yml and dockerfiles present the provided directory -func (r *Replacer) ListContainerImages(dir string) (*ListResult, error) { +// ParseContainerImagesInFile parses and replaces all container image references in the provided file +func (r *Replacer) ParseContainerImagesInFile(ctx context.Context, f io.Reader, cache store.RefCacher) (bool, string, error) { + r.parser = image.New(r.regex) + return r.parseAndReplaceReferencesInFile(ctx, f, cache) +} + +// ListContainerImagesInPath lists all container image references in yaml, yml and dockerfiles present the provided directory +func (r *Replacer) ListContainerImagesInPath(dir string) (*ListResult, error) { r.parser = image.New(r.regex) return r.listReferences(dir) } +// ListContainerImagesInFile lists all container image references in yaml, yml or dockerfile +func (r *Replacer) ListContainerImagesInFile(f io.Reader) (*ListResult, error) { + r.parser = image.New(r.regex) + found, err := r.listReferencesInFile(f) + if err != nil { + return nil, err + } + res := &ListResult{} + res.Entities = found.ToSlice() + + // Sort the slice + sort.Slice(res.Entities, func(i, j int) bool { + return res.Entities[i].Name < res.Entities[j].Name + }) + + // All good + return res, nil +} + func (r *Replacer) parsePath(ctx context.Context, dir string) (*ReplaceResult, error) { var eg errgroup.Group var mu sync.Mutex @@ -196,7 +237,7 @@ func (r *Replacer) listReferences(dir string) (*ListResult, error) { found := mapset.NewSet[interfaces.EntityRef]() - // Traverse all YAML/YML files in dir + // Traverse all related files err := traverse.TraverseYAMLDockerfiles(bfs, base, func(path string) error { eg.Go(func() error { file, err := bfs.Open(path) @@ -206,17 +247,18 @@ func (r *Replacer) listReferences(dir string) (*ListResult, error) { // nolint:errcheck // ignore error defer file.Close() - // Store the file name to the processed batch - res.Processed = append(res.Processed, path) - // Parse the content of the file and listReferences the matching references foundRefs, err := r.listReferencesInFile(file) if err != nil { return fmt.Errorf("failed to listReferences references in %s: %w", path, err) } + + // Store the file name to the processed batch mu.Lock() + res.Processed = append(res.Processed, path) found = found.Union(foundRefs) mu.Unlock() + // All good return nil }) @@ -231,7 +273,7 @@ func (r *Replacer) listReferences(dir string) (*ListResult, error) { } res.Entities = found.ToSlice() - // Sort the slice by the Name field using sort.Slice + // Sort the slice sort.Slice(res.Entities, func(i, j int) bool { return res.Entities[i].Name < res.Entities[j].Name }) diff --git a/pkg/replacer/replacer_test.go b/pkg/replacer/replacer_test.go index 53ff0a5..2ee91d9 100644 --- a/pkg/replacer/replacer_test.go +++ b/pkg/replacer/replacer_test.go @@ -17,15 +17,20 @@ package replacer import ( "context" + "github.com/stacklok/frizbee/internal/cli" "github.com/stacklok/frizbee/pkg/config" + "github.com/stacklok/frizbee/pkg/interfaces" + "github.com/stacklok/frizbee/pkg/replacer/action" + "github.com/stacklok/frizbee/pkg/replacer/image" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "os" + "strings" "testing" "time" ) -func TestReplacer_ParseSingleContainerImage(t *testing.T) { +func TestReplacer_ParseContainerImageString(t *testing.T) { t.Parallel() type args struct { @@ -34,28 +39,73 @@ func TestReplacer_ParseSingleContainerImage(t *testing.T) { tests := []struct { name string args args - want string + want *interfaces.EntityRef wantErr bool }{ + { + name: "dockerfile - tag", + args: args{ + refstr: "FROM golang:1.22.2", + }, + want: &interfaces.EntityRef{ + Name: "index.docker.io/library/golang", + Ref: "sha256:d5302d40dc5fbbf38ec472d1848a9d2391a13f93293a6a5b0b87c99dc0eaa6ae", + Type: image.ReferenceType, + Tag: "1.22.2", + Prefix: "FROM ", + }, + wantErr: false, + }, + { + name: "dockerfile - already by digest", + args: args{ + refstr: "FROM golang:1.22.2@sha256:aca60c1f21de99aa3a34e653f0cdc8c8ea8fe6480359229809d5bcb974f599ec", + }, + want: nil, + wantErr: true, + }, + { + name: "dockerfile - scratch", + args: args{ + refstr: "FROM scratch", + }, + want: nil, + wantErr: true, + }, { name: "valid 1", args: args{ refstr: "ghcr.io/stacklok/minder/helm/minder:0.20231123.829_ref.26ca90b", }, - want: "ghcr.io/stacklok/minder/helm/minder@sha256:a29f8a8d28f0af7f70a4b3dd3e33c8c8cc5cf9e88e802e2700cf272a0b6140ec # 0.20231123.829_ref.26ca90b", + want: &interfaces.EntityRef{ + Name: "ghcr.io/stacklok/minder/helm/minder", + Ref: "sha256:a29f8a8d28f0af7f70a4b3dd3e33c8c8cc5cf9e88e802e2700cf272a0b6140ec", + Type: image.ReferenceType, + Tag: "0.20231123.829_ref.26ca90b", + Prefix: "", + }, + wantErr: false, }, { name: "valid 2", args: args{ refstr: "devopsfaith/krakend:2.5.0", }, - want: "index.docker.io/devopsfaith/krakend@sha256:6a3c8e5e1a4948042bfb364ed6471e16b4a26d0afb6c3c01ebcb88b3fa551036 # 2.5.0", + want: &interfaces.EntityRef{ + Name: "index.docker.io/devopsfaith/krakend", + Ref: "sha256:6a3c8e5e1a4948042bfb364ed6471e16b4a26d0afb6c3c01ebcb88b3fa551036", + Type: image.ReferenceType, + Tag: "2.5.0", + Prefix: "", + }, + wantErr: false, }, { name: "invalid ref string", args: args{ refstr: "ghcr.io/stacklok/minder/helm/minder!", }, + want: nil, wantErr: true, }, { @@ -63,6 +113,7 @@ func TestReplacer_ParseSingleContainerImage(t *testing.T) { args: args{ refstr: "beeeeer.io/ipa/toppling-goliath/king-sue:1.0.0", }, + want: nil, wantErr: true, }, } @@ -75,7 +126,7 @@ func TestReplacer_ParseSingleContainerImage(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() r := New(&config.Config{}) - got, err := r.ParseSingleContainerImage(ctx, tt.args.refstr) + got, err := r.ParseContainerImageString(ctx, tt.args.refstr) if tt.wantErr { assert.Error(t, err) assert.Empty(t, got) @@ -88,7 +139,7 @@ func TestReplacer_ParseSingleContainerImage(t *testing.T) { } } -func TestReplacer_ParseSingleGitHubAction(t *testing.T) { +func TestReplacer_ParseGitHubActionString(t *testing.T) { t.Parallel() type args struct { @@ -97,23 +148,49 @@ func TestReplacer_ParseSingleGitHubAction(t *testing.T) { tests := []struct { name string args args - want string + want *interfaces.EntityRef wantErr bool }{ + { + name: "action using a container via docker://avtodev/markdown-lint:v1", + args: args{ + action: "uses: docker://avtodev/markdown-lint:v1", + }, + want: &interfaces.EntityRef{ + Name: "index.docker.io/avtodev/markdown-lint", + Ref: "sha256:6aeedc2f49138ce7a1cd0adffc1b1c0321b841dc2102408967d9301c031949ee", + Type: image.ReferenceType, + Tag: "v1", + Prefix: "uses: docker://", + }, + wantErr: false, + }, { name: "actions/checkout with v4.1.1", args: args{ action: "actions/checkout@v4.1.1", }, - want: "actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1", + want: &interfaces.EntityRef{ + Name: "actions/checkout", + Ref: "b4ffde65f46336ab88eb53be808477a3936bae11", + Type: action.ReferenceType, + Tag: "v4.1.1", + Prefix: "", + }, wantErr: false, }, { name: "actions/checkout with v3.6.0", args: args{ - action: "actions/checkout@v3.6.0", + action: "uses: actions/checkout@v3.6.0", + }, + want: &interfaces.EntityRef{ + Name: "actions/checkout", + Ref: "f43a0e5ff2bd294095638e18286ca9a3d1956744", + Type: action.ReferenceType, + Tag: "v3.6.0", + Prefix: "uses: ", }, - want: "actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # v3.6.0", wantErr: false, }, { @@ -121,7 +198,13 @@ func TestReplacer_ParseSingleGitHubAction(t *testing.T) { args: args{ action: "actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f", }, - want: "1d96c772d19495a3b5c517cd2bc0cb401ea0529f", + want: &interfaces.EntityRef{ + Name: "actions/checkout", + Ref: "1d96c772d19495a3b5c517cd2bc0cb401ea0529f", + Type: action.ReferenceType, + Tag: "1d96c772d19495a3b5c517cd2bc0cb401ea0529f", + Prefix: "", + }, wantErr: false, }, { @@ -129,7 +212,13 @@ func TestReplacer_ParseSingleGitHubAction(t *testing.T) { args: args{ action: "aquasecurity/trivy-action@0.14.0", }, - want: "2b6a709cf9c4025c5438138008beaddbb02086f0", + want: &interfaces.EntityRef{ + Name: "aquasecurity/trivy-action", + Ref: "2b6a709cf9c4025c5438138008beaddbb02086f0", + Type: action.ReferenceType, + Tag: "0.14.0", + Prefix: "", + }, wantErr: false, }, { @@ -137,7 +226,13 @@ func TestReplacer_ParseSingleGitHubAction(t *testing.T) { args: args{ action: "aquasecurity/trivy-action@bump-trivy", }, - want: "fb5e1b36be448e92ca98648c661bd7e9da1f1317", + want: &interfaces.EntityRef{ + Name: "aquasecurity/trivy-action", + Ref: "fb5e1b36be448e92ca98648c661bd7e9da1f1317", + Type: action.ReferenceType, + Tag: "bump-trivy", + Prefix: "", + }, wantErr: false, }, { @@ -145,7 +240,7 @@ func TestReplacer_ParseSingleGitHubAction(t *testing.T) { args: args{ action: "actions/checkout@v4.1.1.1", }, - want: "", + want: nil, wantErr: true, }, { @@ -153,7 +248,7 @@ func TestReplacer_ParseSingleGitHubAction(t *testing.T) { args: args{ action: "invalid-action@v4.1.1", }, - want: "", + want: nil, wantErr: true, }, { @@ -161,7 +256,7 @@ func TestReplacer_ParseSingleGitHubAction(t *testing.T) { args: args{ action: "@v4.1.1", }, - want: "", + want: nil, wantErr: true, }, { @@ -169,7 +264,7 @@ func TestReplacer_ParseSingleGitHubAction(t *testing.T) { args: args{ action: "actions/checkout", }, - want: "", + want: nil, wantErr: true, }, { @@ -177,14 +272,26 @@ func TestReplacer_ParseSingleGitHubAction(t *testing.T) { args: args{ action: "bufbuild/buf-setup-action@v1", }, - want: "480a0ee8a588045b52a847b48138c6f377a89519", + want: &interfaces.EntityRef{ + Name: "bufbuild/buf-setup-action", + Ref: "f0475db2e1b1b2e8d121066b59dfb7f7bd6c4dc4", + Type: action.ReferenceType, + Tag: "v1", + Prefix: "", + }, }, { name: "anchore/sbom-action/download-syft with a sub-action works", args: args{ action: "anchore/sbom-action/download-syft@v0.14.3", }, - want: "78fc58e266e87a38d4194b2137a3d4e9bcaf7ca1", + want: &interfaces.EntityRef{ + Name: "anchore/sbom-action/download-syft", + Ref: "78fc58e266e87a38d4194b2137a3d4e9bcaf7ca1", + Type: action.ReferenceType, + Tag: "v0.14.3", + Prefix: "", + }, }, } for _, tt := range tests { @@ -194,7 +301,7 @@ func TestReplacer_ParseSingleGitHubAction(t *testing.T) { t.Parallel() r := New(&config.Config{}).WithGitHubClient(os.Getenv("GITHUB_TOKEN")) - got, err := r.ParseSingleGitHubAction(context.Background(), tt.args.action) + got, err := r.ParseGitHubActionString(context.Background(), tt.args.action) if tt.wantErr { require.Error(t, err, "Wanted error, got none") require.Empty(t, got, "Wanted empty string, got %v", got) @@ -205,3 +312,153 @@ func TestReplacer_ParseSingleGitHubAction(t *testing.T) { }) } } + +func TestReplacer_ParseContainerImagesInFile(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + before string + expected string + modified bool + wantErr bool + }{ + { + name: "Replace image reference", + before: ` +version: v1 +services: + - name: kube-apiserver + image: registry.k8s.io/kube-apiserver:v1.20.0 + - name: kube-controller-manager + image: registry.k8s.io/kube-controller-manager:v1.15.0 + - name: minder-app + image: minder:latest +`, + expected: ` +version: v1 +services: + - name: kube-apiserver + image: registry.k8s.io/kube-apiserver@sha256:8b8125d7a6e4225b08f04f65ca947b27d0cc86380bf09fab890cc80408230114 # v1.20.0 + - name: kube-controller-manager + image: registry.k8s.io/kube-controller-manager@sha256:835f32a5cdb30e86f35675dd91f9c7df01d48359ab8b51c1df866a2c7ea2e870 # v1.15.0 + - name: minder-app + image: minder:latest +`, + modified: true, + }, + // Add more test cases as needed + } + + // Define a regular expression to match YAML tags containing "image" + for _, tt := range testCases { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + r := New(&config.Config{}) + modified, newContent, err := r.ParseContainerImagesInFile(ctx, strings.NewReader(tt.before), nil) + + if tt.modified { + assert.True(t, modified) + assert.NotEmpty(t, newContent) + } else { + assert.False(t, modified) + assert.Empty(t, newContent) + } + + if tt.wantErr { + assert.False(t, modified) + assert.Empty(t, newContent) + assert.Error(t, err) + return + } + + assert.NoError(t, err) + assert.Equal(t, tt.expected, newContent) + }) + } +} + +func TestReplacer_ParseGitHubActionsInFile(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + before string + expected string + modified bool + wantErr bool + }{ + { + name: "Replace image reference", + before: ` +name: Linter +on: pull_request +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: ./minder/server.yml # this should not be replaced + - uses: actions/checkout@v2 + - uses: xt0rted/markdownlint-problem-matcher@v1 + - name: "Run Markdown linter" + uses: docker://avtodev/markdown-lint:v1 + with: + args: src/*.md +`, + expected: ` +name: Linter +on: pull_request +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: ./minder/server.yml # this should not be replaced + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2 + - uses: xt0rted/markdownlint-problem-matcher@c17ca40d1376f60aba7e7d38a8674a3f22f7f5b0 # v1 + - name: "Run Markdown linter" + uses: docker://index.docker.io/avtodev/markdown-lint@sha256:6aeedc2f49138ce7a1cd0adffc1b1c0321b841dc2102408967d9301c031949ee # v1 + with: + args: src/*.md +`, + modified: true, + }, + // Add more test cases as needed + } + + // Define a regular expression to match YAML tags containing "image" + for _, tt := range testCases { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + r := New(&config.Config{}).WithGitHubClient(os.Getenv(cli.GitHubTokenEnvKey)) + modified, newContent, err := r.ParseGitHubActionsInFile(ctx, strings.NewReader(tt.before), nil) + + if tt.modified { + assert.True(t, modified) + assert.NotEmpty(t, newContent) + } else { + assert.False(t, modified) + assert.Empty(t, newContent) + } + + if tt.wantErr { + assert.False(t, modified) + assert.Empty(t, newContent) + assert.Error(t, err) + return + } + + assert.NoError(t, err) + assert.Equal(t, tt.expected, newContent) + }) + } +} From 54d843a6c393abdd0e6f26e52d00f8e0a74b8c90 Mon Sep 17 00:00:00 2001 From: Radoslav Dimitrov Date: Tue, 28 May 2024 03:32:02 +0300 Subject: [PATCH 05/16] Fix linting errors Signed-off-by: Radoslav Dimitrov --- .github/workflows/test.yml | 2 +- .tests/actions/.github/dependabot.yml | 45 ---- .../.github/workflows/argo-helm-diff.yaml | 179 ---------------- .tests/actions/.github/workflows/docker.yaml | 12 -- .../workflows/gh-runner-image-publish.yaml | 67 ------ .../.github/workflows/gh-runner-image.yaml | 24 --- .../.github/workflows/krakend-check.yaml | 44 ---- .../actions/.github/workflows/security.yaml | 24 --- .../.github/workflows/terraform-apply.yaml | 87 -------- .../.github/workflows/terraform-preview.yaml | 141 ------------- .tests/containers/Dockerfile | 62 ------ .tests/containers/docker-compose.yaml | 192 ------------------ .tests/containers/k8s.yaml | 19 -- README.md | 53 ++++- cmd/{action/action.go => actions/actions.go} | 49 +++-- cmd/{action => actions}/list.go | 27 ++- cmd/image/image.go | 24 ++- cmd/image/list.go | 14 +- cmd/root.go | 6 +- cmd/version/version.go | 1 + internal/cli/cli.go | 19 +- internal/store/cache.go | 1 + internal/traverse/traverse.go | 36 ++-- pkg/config/config.go | 2 +- {internal => pkg}/ghrest/ghrest.go | 0 pkg/interfaces/interfaces.go | 5 +- .../{action/action.go => actions/actions.go} | 45 +++- pkg/replacer/{action => actions}/utils.go | 10 +- pkg/replacer/actions/utils_test.go | 143 +++++++++++++ pkg/replacer/image/image.go | 20 +- pkg/replacer/image/utils.go | 20 +- pkg/replacer/image/utils_test.go | 71 +++++++ pkg/replacer/replacer.go | 72 ++++--- pkg/replacer/replacer_test.go | 38 ++-- 34 files changed, 503 insertions(+), 1051 deletions(-) delete mode 100644 .tests/actions/.github/dependabot.yml delete mode 100644 .tests/actions/.github/workflows/argo-helm-diff.yaml delete mode 100644 .tests/actions/.github/workflows/docker.yaml delete mode 100644 .tests/actions/.github/workflows/gh-runner-image-publish.yaml delete mode 100644 .tests/actions/.github/workflows/gh-runner-image.yaml delete mode 100644 .tests/actions/.github/workflows/krakend-check.yaml delete mode 100644 .tests/actions/.github/workflows/security.yaml delete mode 100644 .tests/actions/.github/workflows/terraform-apply.yaml delete mode 100644 .tests/actions/.github/workflows/terraform-preview.yaml delete mode 100644 .tests/containers/Dockerfile delete mode 100644 .tests/containers/docker-compose.yaml delete mode 100644 .tests/containers/k8s.yaml rename cmd/{action/action.go => actions/actions.go} (69%) rename cmd/{action => actions}/list.go (86%) rename {internal => pkg}/ghrest/ghrest.go (100%) rename pkg/replacer/{action/action.go => actions/actions.go} (79%) rename pkg/replacer/{action => actions}/utils.go (99%) create mode 100644 pkg/replacer/actions/utils_test.go create mode 100644 pkg/replacer/image/utils_test.go diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index d5020c3..99d4530 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -121,6 +121,6 @@ jobs: - name: Frizbee run: |- - bin/frizbee ghactions --dry-run --error + bin/frizbee actions --dry-run --error env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.tests/actions/.github/dependabot.yml b/.tests/actions/.github/dependabot.yml deleted file mode 100644 index 22a54de..0000000 --- a/.tests/actions/.github/dependabot.yml +++ /dev/null @@ -1,45 +0,0 @@ -# -# Copyright 2023 Stacklok, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: 2 -updates: - - package-ecosystem: "github-actions" - directory: "/" - schedule: - interval: "daily" - - package-ecosystem: "terraform" - directory: "/dashboards" - schedule: - interval: "daily" - - package-ecosystem: "terraform" - directory: "/dns" - schedule: - interval: "daily" - - package-ecosystem: "terraform" - directory: "/iam" - schedule: - interval: "daily" - - package-ecosystem: "terraform" - directory: "/production" - schedule: - interval: "daily" - - package-ecosystem: "terraform" - directory: "/sandbox" - schedule: - interval: "daily" - - package-ecosystem: "terraform" - directory: "/staging" - schedule: - interval: "daily" diff --git a/.tests/actions/.github/workflows/argo-helm-diff.yaml b/.tests/actions/.github/workflows/argo-helm-diff.yaml deleted file mode 100644 index 574be8c..0000000 --- a/.tests/actions/.github/workflows/argo-helm-diff.yaml +++ /dev/null @@ -1,179 +0,0 @@ -name: Argo Helm Diff - -on: - pull_request: - branches: - - main - paths: - - "**/*.yaml" - types: - - synchronize - - opened - - reopened - -jobs: - diff-helm-template: - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: write - # charts published to private repos - # Note you also need to grant the infra repo access to those packages: - # https://docs.github.com/en/packages/learn-github-packages/configuring-a-packages-access-control-and-visibility#about-setting-visibility-and-access-permissions-for-packages - packages: read - - strategy: - matrix: - # TODO: can we get this from a directory listing? - environment: ["staging2", "production", "sandbox"] - - env: - # This finds "Application" kinds, then extracts the chart coordinates (repo, version, etc) - # and the valuesObject, and outputs them into a merged YAML document. This assumes no collisions - # on the value keys `installName`, `chartName`, `chartRepo`, and `chartVersion`. - YQ_QUERY: 'select(.kind == "Application" and .spec.source.chart) | - .spec.source.targetRevision as $version | .spec.source.chart as $chart | .spec.source.repoURL as $repo | - .metadata.name as $name | .spec.destination.namespace as $namespace | - .spec.source.helm.valuesObject | - .installName = $name | .installNamespace = $namespace | .chartName = $chart | .chartRepo = $repo | .chartVersion = $version' - - steps: - - name: Checkout base branch - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.base.sha }} - path: base - - - name: Checkout pull request branch - uses: actions/checkout@v4 - with: - ref: ${{ github.event.pull_request.head.sha }} - path: pr - - - name: Setup Helm - uses: azure/setup-helm@v4 - with: - version: "3.14.0" - - - name: Log in to read private GitHub packages - run: helm registry login $BASE_REPO --username ${{ github.repository_owner }} --password ${{ secrets.GITHUB_TOKEN }} - env: - BASE_REPO: ghcr.io/stacklok - - - name: create directories - run: | - mkdir -p base-tmp - mkdir -p pr-tmp - - - name: Extract base helm values with yq - # TODO: would be nice to be able to install instead of using a container - uses: mikefarah/yq@master - with: - cmd: | - if [[ -e "base/argocd/${{ matrix.environment }}/cluster-config/" ]]; then - yq -s '"base-tmp/" + (.chartName | sub("/", "_")) + "." + (.chartRepo | sub("/", "_")) + ".yaml"' "$YQ_QUERY" base/argocd/${{ matrix.environment }}/cluster-config/*.yaml - fi - - - name: Compute helm templates for base - run: | - for f in base-tmp/*.yaml; do - INSTALL_NAME=$(grep installName: $f | cut -d: -f2 | tr -d ' ' | sed 's/#.*//') - INSTALL_NS=$(grep installNamespace: $f | cut -d: -f2 | tr -d ' ' | sed 's/#.*//') - CHART_NAME=$(grep chartName: $f | cut -d: -f2 | tr -d ' ' | sed 's/#.*//') - CHART_REPO=$(grep chartRepo: $f | cut -d: -f2- | tr -d " '\"" | sed 's/#.*//') - CHART_VERSION=$(grep chartVersion: $f | cut -d: -f2- | tr -d " '\"" | sed 's/#.*//') - # Bare (OCI format) repos have a different format in helm template - if [[ "$CHART_REPO" != *":"* ]]; then - CHART_NAME="oci://$CHART_REPO/$CHART_NAME" - CHART_REPO="" - else - CHART_REPO="--repo $CHART_REPO" - fi - helm template --version "$CHART_VERSION" --namespace "$INSTALL_NS" $CHART_REPO $INSTALL_NAME $CHART_NAME -f $f > ${f%.yaml}.out || true # OCI fetch not yet working - echo "Expanding $f: $?" - done - - - name: Extract PR helm values with yq - # TODO: would be nice to be able to install instead of using a container - uses: mikefarah/yq@master - with: - cmd: yq -s '"pr-tmp/" + (.chartName | sub("/", "_")) + "." + (.chartRepo | sub("/", "_")) + ".yaml"' "$YQ_QUERY" pr/argocd/${{ matrix.environment }}/cluster-config/*.yaml - - - name: Compute helm templates for PR - run: | - for f in pr-tmp/*.yaml; do - INSTALL_NAME=$(grep installName: $f | cut -d: -f2 | tr -d ' ' | sed 's/#.*//') - INSTALL_NS=$(grep installNamespace: $f | cut -d: -f2 | tr -d ' ' | sed 's/#.*//') - CHART_NAME=$(grep chartName: $f | cut -d: -f2 | tr -d ' ' | sed 's/#.*//') - CHART_REPO=$(grep chartRepo: $f | cut -d: -f2- | tr -d " '\"" | sed 's/#.*//') - CHART_VERSION=$(grep chartVersion: $f | cut -d: -f2- | tr -d " '\"" | sed 's/#.*//') - # Bare (OCI format) repos have a different format in helm template - if [[ "$CHART_REPO" != *":"* ]]; then - CHART_NAME="oci://$CHART_REPO/$CHART_NAME" - CHART_REPO="" - else - CHART_REPO="--repo $CHART_REPO" - fi - helm template --version "$CHART_VERSION" --namespace "$INSTALL_NS" $CHART_REPO $INSTALL_NAME $CHART_NAME -f $f > ${f%.yaml}.out || true # OCI fetch not yet working - echo "Expanding $f: $?" - done - - - name: Diff template output - id: diff - run: | - # Remove *.yaml to only leave *.out - rm -f base-tmp/*.yaml pr-tmp/*.yaml - - # We use a file rather than a GitHub output, because sometimes the diff is too big. - # See https://github.com/actions/github-script/issues/266#issuecomment-1158990264 for - # an example from a different project. - (diff -Nu base-tmp pr-tmp || echo "no diff" ) > yaml.diff - - echo "Diff:" - cat yaml.diff - - - name: Present diff - uses: actions/github-script@v7 - id: diff-comment - if: github.event_name == 'pull_request' - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const { readFile } = require("fs/promises"); - let planComment = await readFile("yaml.diff", "utf8"); - - if (planComment.length > 65535) { - planComment = "TRUNCATED DIFF, see 'Diff template output' for full output:\n\n" + planComment.substring(0, 60000); - } - - // 1. Retrieve existing bot comments for the PR - - const output = ` - ## Helm diffs \`${{ matrix.environment }}\` - - \`\`\` - ` + planComment + ` - \`\`\` - `; - - const { data: comments } = await github.rest.issues.listComments({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - }); - const botComment = comments.find(comment => { - return comment.user.type === 'Bot' && comment.body.includes('') - }); - if (botComment) { - github.rest.issues.deleteComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: botComment.id, - }); - } - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: output - }); diff --git a/.tests/actions/.github/workflows/docker.yaml b/.tests/actions/.github/workflows/docker.yaml deleted file mode 100644 index d995899..0000000 --- a/.tests/actions/.github/workflows/docker.yaml +++ /dev/null @@ -1,12 +0,0 @@ -name: Linter -on: pull_request -jobs: - build: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - uses: xt0rted/markdownlint-problem-matcher@v1 - - name: "Run Markdown linter" - uses: docker://avtodev/markdown-lint:v1 - with: - args: src/*.md \ No newline at end of file diff --git a/.tests/actions/.github/workflows/gh-runner-image-publish.yaml b/.tests/actions/.github/workflows/gh-runner-image-publish.yaml deleted file mode 100644 index bf9400b..0000000 --- a/.tests/actions/.github/workflows/gh-runner-image-publish.yaml +++ /dev/null @@ -1,67 +0,0 @@ ---- -name: GH Runner image publish - -on: - push: - branches: - - main - workflow_dispatch: - -jobs: - publish: - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - env: - GH_RUNNER_IMAGE: ghcr.io/stacklok/gh-runner - steps: - - name: Checkout - uses: actions/checkout@v4.1.4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.3.0 - - - name: Login to ghcr.io - uses: docker/login-action@v3.1.0 - with: - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} - - - name: Compute version number - id: version-string - run: | - DATE="$(date +%Y%m%d)" - COMMIT="$(git rev-parse --short HEAD)" - echo "tag=0.$DATE.$GITHUB_RUN_NUMBER+ref.$COMMIT" >> "$GITHUB_OUTPUT" - - - name: Set container metadata - uses: docker/metadata-action@8e5442c4ef9f78752691e2d8f8d19755c6f78e81 # v5 - id: docker-metadata - with: - images: ${{ env.GH_RUNNER_IMAGE }} - labels: | - org.opencontainers.image.source=${{ github.repositoryUrl }} - org.opencontainers.image.description="This is a container for the Stacklok GitHub Runner image" - org.opencontainers.image.title="Stacklok GitHub Runner Image" - org.opencontainers.image.vendor="Stacklok Inc." - org.opencontainers.image.version=${{ github.sha }} - flavor: | - latest=true - # Even if tags are floating, it's handy and user-friendly to have a - # matching tag for each build. This way, we can search for the digest - # and verify that it's the same as the digest in the Helm chart. - tags: | - type=raw,value=${{ steps.version-string.outputs.tag }} - - - name: Build image - id: image-build - uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5 - with: - context: . - platforms: linux/arm64 - push: true - file: ./images/gh-runner/Dockerfile - tags: ${{ steps.docker-metadata.outputs.tags }} - labels: ${{ steps.docker-metadata.outputs.labels }} \ No newline at end of file diff --git a/.tests/actions/.github/workflows/gh-runner-image.yaml b/.tests/actions/.github/workflows/gh-runner-image.yaml deleted file mode 100644 index 0b6ccea..0000000 --- a/.tests/actions/.github/workflows/gh-runner-image.yaml +++ /dev/null @@ -1,24 +0,0 @@ ---- -name: GH Runner image - -on: - pull_request: - -jobs: - build: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4.1.4 - - - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v3.3.0 - - - name: Test build on arm64 - id: docker_build - uses: docker/build-push-action@2cdde995de11925a030ce8070c3d77a52ffcf1c0 # v5 - with: - context: . - file: ./images/gh-runner/Dockerfile - platforms: linux/arm64 - push: false # Only attempt to build, to verify the Dockerfile is working diff --git a/.tests/actions/.github/workflows/krakend-check.yaml b/.tests/actions/.github/workflows/krakend-check.yaml deleted file mode 100644 index be0a31d..0000000 --- a/.tests/actions/.github/workflows/krakend-check.yaml +++ /dev/null @@ -1,44 +0,0 @@ ---- -name: Krakend lint - -on: - pull_request: - paths: - - 'argocd/production/cluster-config/krakend.yaml' - - 'argocd/staging2/cluster-config/krakend.yaml' - -jobs: - lint: - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - krakend-app: - - argocd/production/cluster-config/krakend.yaml - - argocd/staging2/cluster-config/krakend.yaml - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Go - uses: actions/setup-go@v5 - with: - check-latest: true - - - name: Get krakend config from Argo App - uses: mikefarah/yq@master - id: krakendcfg - with: - cmd: yq '.spec.source.helm.valuesObject.krakend.config' ${{ matrix.krakend-app }} - - - name: persist krakend config in temp - run: | - cat < /tmp/krakend.json - ${{ steps.krakendcfg.outputs.result }} - EOF - - - name: Run krakend check - run: | - # krakend check needs the --config flag - # docker run -it devopsfaith/krakend - docker run -i -v /tmp/krakend.json:/tmp/krakend.json devopsfaith/krakend check --config /tmp/krakend.json diff --git a/.tests/actions/.github/workflows/security.yaml b/.tests/actions/.github/workflows/security.yaml deleted file mode 100644 index 003bcda..0000000 --- a/.tests/actions/.github/workflows/security.yaml +++ /dev/null @@ -1,24 +0,0 @@ ---- -name: "Security" - -on: - push: - branches: [main] - pull_request: - -jobs: - security-scan: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Security Scan - uses: aquasecurity/trivy-action@0.20.0 - with: - scan-type: "fs" - scanners: vuln,secret,config - severity: HIGH,CRITICAL - skip-dirs: "argocd/staging/database-provisioning,argocd/production/database-provisioning,argocd/staging2/database-provisioning,argocd/sandbox/database-provisioning" - exit-code: 1 - ignore-unfixed: true diff --git a/.tests/actions/.github/workflows/terraform-apply.yaml b/.tests/actions/.github/workflows/terraform-apply.yaml deleted file mode 100644 index 47fe5ab..0000000 --- a/.tests/actions/.github/workflows/terraform-apply.yaml +++ /dev/null @@ -1,87 +0,0 @@ -name: "Terraform Apply" - -on: - pull_request: - types: - - closed - branches: - - main - - schedule: - # 15:00 UTC = 6PM/7PM Eastern Europe, 8AM/9AM Pacific - - cron: "0 15 * * *" - - workflow_dispatch: - -jobs: - apply_on_merge: - if: | - github.event_name == 'schedule' || - github.event_name == 'workflow_dispatch' || - (github.event_name == 'pull_request' && github.event.pull_request.merged == true) - # We tried the following addition to the if statement to only run on - # merged PRs, but not on closed (cancelled) PRs, but GitHub stopped - # running the workflow on merge: https://github.com/stacklok/infra/issues/77 - # - # github.ref == 'main' && github.event.pull_request.merged == true) - name: "Terraform Apply" - runs-on: ubuntu-latest - permissions: - contents: read - id-token: write - - strategy: - # Don't stop / cancel remaining applies if one apply fails - fail-fast: false - matrix: - # Note that these run in-order, so "iam" will start first. - # We could set `max-parallel` to 1 to get strict ordering - # TODO: can we get this list from a directory listing? - component: - [ - "iam", - "sandbox", - "dashboards", - "dns", - "telemetry", - "staging2", - "production", - ] - - env: - CONFIG_DIRECTORY: "./${{ matrix.component }}" - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Terraform - uses: hashicorp/setup-terraform@v3 - - - name: Set up AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - # Role includes the account ID as an ARN - role-to-assume: ${{ secrets.AWS_ROLE_TO_APPLY }} - aws-region: us-east-1 - - - name: Init - id: init - run: | - cd $CONFIG_DIRECTORY - terraform init - - - name: Validate - id: validate - run: | - set -e - - cd $CONFIG_DIRECTORY - terraform validate -no-color - - - name: Apply - id: apply - run: | - set -e - cd $CONFIG_DIRECTORY - terraform apply -no-color -input=false -auto-approve diff --git a/.tests/actions/.github/workflows/terraform-preview.yaml b/.tests/actions/.github/workflows/terraform-preview.yaml deleted file mode 100644 index 725b144..0000000 --- a/.tests/actions/.github/workflows/terraform-preview.yaml +++ /dev/null @@ -1,141 +0,0 @@ -name: "Terraform Plan" - -on: - pull_request: - branches: - - main - paths: - - "**/*.tf" - - "**/*.tfvars" - - "**/*.hcl" - - ".github/workflows/preview.yaml" - -jobs: - plan: - name: "Terraform Plan" - runs-on: ubuntu-latest - permissions: - contents: read - pull-requests: write - id-token: write - - strategy: - matrix: - # TODO: can we get this from a directory listing? - component: - [ - "dashboards", - "dns", - "iam", - "staging2", - "production", - "sandbox", - "telemetry", - ] - - defaults: - run: - working-directory: ${{ matrix.component }} - env: - PLAN_OUTPUT: "./tfplan.txt" - - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: Setup Terraform - uses: hashicorp/setup-terraform@v3 - - - name: Set up AWS credentials - uses: aws-actions/configure-aws-credentials@v4 - with: - # Role includes the account ID as an ARN - role-to-assume: ${{ secrets.AWS_ROLE_TO_PLAN }} - aws-region: us-east-1 - - - name: Terraform fmt - id: fmt - run: terraform fmt -check - continue-on-error: true - - - name: Init - id: init - run: | - terraform init - - - name: Validate - id: validate - run: | - terraform validate -no-color - - - name: Plan - id: plan - run: | - terraform plan -no-color -lock=false -input=false -out=tfplan - continue-on-error: true - - - name: Show Plan - id: show_plan - run: | - terraform show -no-color tfplan >> "$PLAN_OUTPUT" - - - name: Report - uses: actions/github-script@v7 - id: plan-comment - if: github.event_name == 'pull_request' - with: - github-token: ${{ secrets.GITHUB_TOKEN }} - script: | - const { readFileSync } = require('node:fs'); - // 1. Determine output format; large diffs need additional summarization - const planFormat = readFileSync('${{ matrix.component }}/' + process.env.PLAN_OUTPUT, 'utf8'); - let planComment = planFormat; - if (planFormat.length > 65535) { - planComment = planFormat.split('\n').filter( - (line) => (line.startsWith(' # ') || line.startsWith('Plan:'))).join('\n'); - planComment = '# Plan output too large to display, summarized:\n\n' + planComment; - } - - // 1. Retrieve existing bot comments for the PR - - const output = ` - ## Terraform Plan \`${{ matrix.component }}\` - #### Terraform Fmt \`${{ steps.fmt.outcome }}\` - - #### Terraform Init \`${{ steps.init.outcome }}\` - - #### Terraform Validate \`${{ steps.validate.outcome }}\` - - #### Terraform Plan \`${{ steps.plan.outcome }}\` - #### Terraform Plan Output - \`\`\` - ` + planComment + ` - \`\`\` - `; - - const { data: comments } = await github.rest.issues.listComments({ - owner: context.repo.owner, - repo: context.repo.repo, - issue_number: context.issue.number, - }); - const botComment = comments.find(comment => { - return comment.user.type === 'Bot' && comment.body.includes('') - }); - if (botComment) { - github.rest.issues.deleteComment({ - owner: context.repo.owner, - repo: context.repo.repo, - comment_id: botComment.id, - }); - } - github.rest.issues.createComment({ - issue_number: context.issue.number, - owner: context.repo.owner, - repo: context.repo.repo, - body: output - }); - - - name: Fail if continue steps failed - id: check-failure - if: steps.plan.outcome != 'success' || steps.fmt.outcome != 'success' - run: exit 1 diff --git a/.tests/containers/Dockerfile b/.tests/containers/Dockerfile deleted file mode 100644 index cd260e6..0000000 --- a/.tests/containers/Dockerfile +++ /dev/null @@ -1,62 +0,0 @@ -# -# Copyright 2023 Stacklok, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -FROM golang:1.22.2@sha256:aca60c1f21de99aa3a34e653f0cdc8c8ea8fe6480359229809d5bcb974f599ec AS builder -ENV APP_ROOT=/opt/app-root -ENV GOPATH=$APP_ROOT -FROM golang:1.22.2 AS builder -FROM golang:latest AS builder -FROM golang AS builder -FROM ubuntu - -WORKDIR $APP_ROOT/src/ -ADD go.mod go.sum $APP_ROOT/src/ -RUN go mod download - -# Add source code -ADD ./ $APP_ROOT/src/ - -RUN CGO_ENABLED=0 go build -trimpath -o minder-server ./cmd/server - -# Create a "nobody" non-root user for the next image by crafting an /etc/passwd -# file that the next image can copy in. This is necessary since the next image -# is based on scratch, which doesn't have adduser, cat, echo, or even sh. -RUN echo "nobody:x:65534:65534:Nobody:/:" > /etc_passwd - -RUN mkdir -p /app - -FROM scratch - -COPY --chown=65534:65534 --from=builder /app /app - -WORKDIR /app - -# Copy database directory and config. This is needed for the migration sub-command to work. -ADD --chown=65534:65534 ./cmd/server/kodata/server-config.yaml /app - -COPY --from=builder /opt/app-root/src/minder-server /usr/bin/minder-server - -# Copy the certs from the builder stage -COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ - -# Copy the /etc_passwd file we created in the builder stage into /etc/passwd in -# the target stage. This creates a new non-root user as a security best -# practice. -COPY --from=builder /etc_passwd /etc/passwd - -USER nobody - -# Set the binary as the entrypoint of the container -ENTRYPOINT ["/usr/bin/minder-server"] diff --git a/.tests/containers/docker-compose.yaml b/.tests/containers/docker-compose.yaml deleted file mode 100644 index 78faaec..0000000 --- a/.tests/containers/docker-compose.yaml +++ /dev/null @@ -1,192 +0,0 @@ -# -# Copyright 2023 Stacklok, Inc. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -version: '3.2' -services: - minder: - container_name: minder_server - build: - context: . - dockerfile: ./docker/minder/Dockerfile - image: minder:latest - - command: [ - "serve", - "--grpc-host=0.0.0.0", - "--http-host=0.0.0.0", - "--metric-host=0.0.0.0", - "--db-host=postgres", - "--issuer-url=http://keycloak:8080", - "--config=/app/server-config.yaml", - # If you don't want to store your GitHub client ID and secret in the main - # config file, point to them here: - # "--github-client-id-file=/secrets/github_client_id", - # "--github-client-secret-file=/secrets/github_client_secret", - ] - restart: always # keep the server running - read_only: true - ports: - - "8080:8080" - - "8090:8090" - - "9090:9090" - volumes: - - ./server-config.yaml:/app/server-config.yaml:z - - ./flags-config.yaml:/app/flags-config.yaml:z - # If you don't want to store your GitHub client ID and secret in the main - # config file, point to them here: - # - ./.github_client_id:/secrets/github_client_id:z - # - ./.github_client_secret:/secrets/github_client_secret:z - # If you're using a GitHub App, you'll need to provide the private key: - - ./.secrets/:/app/.secrets/:z - - ./.ssh:/app/.ssh:z - environment: - - KO_DATA_PATH=/app/ - # Use viper environment variables to set specific paths to keys; - # these values are relative paths in server-config.yaml, but it's not clear - # what they are relative _to_... - - MINDER_AUTH_ACCESS_TOKEN_PRIVATE_KEY=/app/.ssh/access_token_rsa - - MINDER_AUTH_ACCESS_TOKEN_PUBLIC_KEY=/app/.ssh/access_token_rsa.pub - - MINDER_AUTH_REFRESH_TOKEN_PRIVATE_KEY=/app/.ssh/refresh_token_rsa - - MINDER_AUTH_REFRESH_TOKEN_PUBLIC_KEY=/app/.ssh/refresh_token_rsa.pub - - MINDER_AUTH_TOKEN_KEY=/app/.ssh/token_key_passphrase - - MINDER_UNSTABLE_TRUSTY_ENDPOINT=https://api.trustypkg.dev - - MINDER_PROVIDER_GITHUB_APP_PRIVATE_KEY=/app/.secrets/github-app.pem - - MINDER_FLAGS_GO_FEATURE_FILE_PATH=/app/flags-config.yaml - - MINDER_LOG_GITHUB_REQUESTS=1 - networks: - - app_net - depends_on: - postgres: - condition: service_healthy - keycloak: - condition: service_healthy - openfga: - condition: service_healthy - migrate: - condition: service_completed_successfully - keycloak-config: - condition: service_completed_successfully - migrate: - container_name: minder_migrate_up - build: - context: . - dockerfile: ./docker/minder/Dockerfile - image: minder:latest - - command: [ - "migrate", - "up", - "--yes", - "--db-host=postgres", - "--config=/app/server-config.yaml", - ] - volumes: - - ./server-config.yaml:/app/server-config.yaml:z - - ./database/migrations:/app/database/migrations:z - environment: - - KO_DATA_PATH=/app/ - networks: - - app_net - deploy: - restart_policy: - condition: on-failure - delay: 5s - max_attempts: 3 - window: 120s - depends_on: - postgres: - condition: service_healthy - openfga: - condition: service_healthy - postgres: - container_name: postgres_container - image: postgres:16.2-alpine - restart: always - user: postgres - environment: - POSTGRES_USER: postgres - POSTGRES_PASSWORD: postgres - POSTGRES_DB: minder - ports: - - "5432:5432" - healthcheck: - test: ["CMD-SHELL", "pg_isready"] - interval: 10s - timeout: 5s - retries: 5 - networks: - - app_net - - keycloak: - container_name: keycloak_container - image: quay.io/keycloak/keycloak:23.0 - command: ["start-dev"] - environment: - KEYCLOAK_ADMIN: admin - KEYCLOAK_ADMIN_PASSWORD: admin - KC_HEALTH_ENABLED: "true" - healthcheck: - test: ["CMD", "/opt/keycloak/bin/kcadm.sh", "config", "credentials", "--server", "http://localhost:8080", "--realm", "master", "--user", "admin", "--password", "admin"] - interval: 10s - timeout: 5s - retries: 10 - ports: - - "8081:8080" - volumes: - - ./identity/themes:/opt/keycloak/themes:z - networks: - - app_net - - keycloak-config: - container_name: keycloak_config - image: bitnami/keycloak-config-cli:5.10.0 - entrypoint: ["java", "-jar", "/opt/bitnami/keycloak-config-cli/keycloak-config-cli.jar"] - environment: - KEYCLOAK_URL: http://keycloak:8080 - KEYCLOAK_USER: admin - KEYCLOAK_PASSWORD: admin - KC_MINDER_SERVER_SECRET: secret - IMPORT_VARSUBSTITUTION_ENABLED: "true" - IMPORT_FILES_LOCATIONS: /config/*.yaml - volumes: - - ./identity/config:/config:z - networks: - - app_net - - depends_on: - keycloak: - condition: service_healthy - - openfga: - container_name: openfga - image: openfga/openfga:v1.5.0 - command: [ - "run", - "--playground-port=8085" - ] - healthcheck: - test: - - CMD - - grpc_health_probe - - "-addr=:8081" - ports: - - 8082:8080 - - 8083:8081 - - 8085:8085 - networks: - - app_net -networks: - app_net: - driver: bridge diff --git a/.tests/containers/k8s.yaml b/.tests/containers/k8s.yaml deleted file mode 100644 index 6e62fd7..0000000 --- a/.tests/containers/k8s.yaml +++ /dev/null @@ -1,19 +0,0 @@ -apiVersion: apps/v1 -kind: Deployment -metadata: - name: nginx-deployment -spec: - replicas: 3 - selector: - matchLabels: - app: nginx - template: - metadata: - labels: - app: nginx - spec: - containers: - - name: nginx - image: nginx:latest - ports: - - containerPort: 80 diff --git a/README.md b/README.md index 3fc82e5..f8575bb 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ brew install stacklok/tap/frizbee winget install stacklok.frizbee ``` -## Usage +## Usage - CLI ### GitHub Actions @@ -50,7 +50,7 @@ To quickly replace the GitHub Action references for your project, you can use the `action` command: ```bash -frizbee action -d path/to/your/repo/.github/workflows/ +frizbee action path/to/your/repo/.github/workflows/ ``` This will write all the replacements to the files in the directory provided. @@ -64,11 +64,11 @@ to stdout instead of writing them to the files. It also supports exiting with a non-zero exit code if any replacements are found. This is handy for CI/CD pipelines. -If you want to generate the replacement for a single GitHub Action, you can use -the `action one` command: +If you want to generate the replacement for a single GitHub Action, you can use the +same command: ```bash -frizbee action one metal-toolbox/container-push/.github/workflows/container-push.yml@main +frizbee action metal-toolbox/container-push/.github/workflows/container-push.yml@main ``` This is useful if you're developing and want to quickly test the replacement. @@ -83,17 +83,54 @@ To quickly replace the container image references for your project, you can use the `image` command: ```bash -frizbee image -d path/to/your/yaml/files/ +frizbee image path/to/your/yaml/files/ ``` -To get the digest for a single image tag, you can use the `image one` command: +To get the digest for a single image tag, you can use the same command: ```bash -frizbee image one quay.io/stacklok/frizbee:latest +frizbee image ghcr.io/stacklok/minder/server:latest ``` This will print the image reference with the digest for the image tag provided. +## Usage - Library + +Frizbee can also be used as a library. The library provides a set of functions +for working with tags and checksums. Here are a few examples of how you can use +the library: + +### GitHub Actions + +```go +// Create a new replacer +r := replacer.New(cfg) +... +// Parse a single GitHub Action reference +ret, err := r.ParseGitHubActionString(ctx, ghActionRef) +... +// Parse all GitHub Actions workflow yaml files in a given directory +res, err := r.ParseGitHubActionsInPath(ctx, dir) +... +// Parse a single yaml file referencing GitHub Actions +res, err := r.ParseGitHubActionsInFile(ctx, fileHandler) +``` + +### Container images + +```go +// Create a new replacer +r := replacer.New(cfg) +... +// Parse a single container image reference +ret, err := r.ParseContainerImageString(ctx, imageRef) +... +// Parse all yaml files referencing container images in a given directory (k8s, docker-compose, Dockerfile, etc) +res, err := r.ParseContainerImagesInPath(ctx, dir) +... +// Parse a single yaml file referencing container images +res, err := r.ParseContainerImagesInFile(ctx, fileHandler) +``` ## Contributing diff --git a/cmd/action/action.go b/cmd/actions/actions.go similarity index 69% rename from cmd/action/action.go rename to cmd/actions/actions.go index cfda45c..6a461ef 100644 --- a/cmd/action/action.go +++ b/cmd/actions/actions.go @@ -13,40 +13,43 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Package action provides command-line utilities to work with GitHub Actions. -package action +// Package actions provides command-line utilities to work with GitHub Actions. +package actions import ( + "errors" "fmt" - "github.com/stacklok/frizbee/internal/cli" - "github.com/stacklok/frizbee/pkg/config" - "github.com/stacklok/frizbee/pkg/replacer" "os" "path/filepath" "github.com/spf13/cobra" + + "github.com/stacklok/frizbee/internal/cli" + "github.com/stacklok/frizbee/pkg/config" + ferrors "github.com/stacklok/frizbee/pkg/errors" + "github.com/stacklok/frizbee/pkg/replacer" ) // CmdGHActions represents the actions command func CmdGHActions() *cobra.Command { cmd := &cobra.Command{ - Use: "action", + Use: "actions", Short: "Replace tags in GitHub Actions workflows", Long: `This utility replaces tag or branch references in GitHub Actions workflows with the latest commit hash of the referenced tag or branch. Example: - $ frizbee action <.github/workflows> or + $ frizbee actions <.github/workflows> or This will replace all tag or branch references in all GitHub Actions workflows for the given directory. Supports both directories and single references. ` + cli.TokenHelpText + "\n", - Aliases: []string{"ghactions", "actions"}, // backwards compatibility + Aliases: []string{"ghactions"}, // backwards compatibility RunE: replaceCmd, SilenceUsage: true, - Args: cobra.ExactArgs(1), + Args: cobra.MaximumNArgs(1), } // flags @@ -58,7 +61,14 @@ for the given directory. Supports both directories and single references. return cmd } +// nolint:errcheck func replaceCmd(cmd *cobra.Command, args []string) error { + // Set the default directory if not provided + pathOrRef := ".github/workflows" + if len(args) > 0 { + pathOrRef = args[0] + } + // Extract the CLI flags from the cobra command cliFlags, err := cli.NewHelper(cmd) if err != nil { @@ -76,8 +86,8 @@ func replaceCmd(cmd *cobra.Command, args []string) error { WithUserRegex(cliFlags.Regex). WithGitHubClient(os.Getenv(cli.GitHubTokenEnvKey)) - if cli.IsPath(args[0]) { - dir := filepath.Clean(args[0]) + if cli.IsPath(pathOrRef) { + dir := filepath.Clean(pathOrRef) // Replace the tags in the given directory res, err := r.ParseGitHubActionsInPath(cmd.Context(), dir) if err != nil { @@ -85,13 +95,16 @@ func replaceCmd(cmd *cobra.Command, args []string) error { } // Process the output files return cliFlags.ProcessOutput(dir, res.Processed, res.Modified) - } else { - // Replace the passed reference - res, err := r.ParseGitHubActionString(cmd.Context(), args[0]) - if err != nil { - return err + } + // Replace the passed reference + res, err := r.ParseGitHubActionString(cmd.Context(), pathOrRef) + if err != nil { + if errors.Is(err, ferrors.ErrReferenceSkipped) { + fmt.Fprintln(cmd.OutOrStdout(), pathOrRef) // nolint:errcheck + return nil } - fmt.Fprintln(cmd.OutOrStdout(), fmt.Sprintf("%s@%s", res.Name, res.Ref)) - return nil + return err } + fmt.Fprintf(cmd.OutOrStdout(), "%s@%s\n", res.Name, res.Ref) // nolint:errcheck + return nil } diff --git a/cmd/action/list.go b/cmd/actions/list.go similarity index 86% rename from cmd/action/list.go rename to cmd/actions/list.go index 011aa4f..93469af 100644 --- a/cmd/action/list.go +++ b/cmd/actions/list.go @@ -13,19 +13,22 @@ // See the License for the specific language governing permissions and // limitations under the License. -package action +package actions import ( "encoding/json" + "errors" "fmt" + "os" + "path/filepath" + "strconv" + "github.com/olekukonko/tablewriter" "github.com/spf13/cobra" + "github.com/stacklok/frizbee/internal/cli" "github.com/stacklok/frizbee/pkg/config" "github.com/stacklok/frizbee/pkg/replacer" - "os" - "path/filepath" - "strconv" ) // CmdList represents the one sub-command @@ -36,12 +39,12 @@ func CmdList() *cobra.Command { Long: `This utility lists all the github actions used in the workflows Example: - frizbee action list -d .github/workflows + frizbee action list .github/workflows `, Aliases: []string{"ls"}, RunE: list, SilenceUsage: true, - Args: cobra.ExactArgs(1), + Args: cobra.MaximumNArgs(1), } cli.DeclareFrizbeeFlags(cmd, true) @@ -50,9 +53,15 @@ Example: } func list(cmd *cobra.Command, args []string) error { - dir := filepath.Clean(args[0]) + // Set the default directory if not provided + dir := ".github/workflows" + if len(args) > 0 { + dir = args[0] + } + + dir = filepath.Clean(dir) if !cli.IsPath(dir) { - return fmt.Errorf("the provided argument is not a path") + return errors.New("the provided argument is not a path") } // Extract the CLI flags from the cobra command cliFlags, err := cli.NewHelper(cmd) @@ -86,7 +95,7 @@ func list(cmd *cobra.Command, args []string) error { } jsonString := string(jsonBytes) - fmt.Fprintln(cmd.OutOrStdout(), jsonString) + fmt.Fprintln(cmd.OutOrStdout(), jsonString) // nolint:errcheck return nil case "table": table := tablewriter.NewWriter(cmd.OutOrStdout()) diff --git a/cmd/image/image.go b/cmd/image/image.go index b477e0e..4aece3a 100644 --- a/cmd/image/image.go +++ b/cmd/image/image.go @@ -17,12 +17,16 @@ package image import ( + "errors" "fmt" + "path/filepath" + "github.com/spf13/cobra" + "github.com/stacklok/frizbee/internal/cli" "github.com/stacklok/frizbee/pkg/config" + ferrors "github.com/stacklok/frizbee/pkg/errors" "github.com/stacklok/frizbee/pkg/replacer" - "path/filepath" ) // CmdContainerImage represents the containers command @@ -80,13 +84,17 @@ func replaceCmd(cmd *cobra.Command, args []string) error { } // Process the output files return cliFlags.ProcessOutput(dir, res.Processed, res.Modified) - } else { - // Replace the passed reference - res, err := r.ParseContainerImageString(cmd.Context(), args[0]) - if err != nil { - return err + } + // Replace the passed reference + res, err := r.ParseContainerImageString(cmd.Context(), args[0]) + if err != nil { + if errors.Is(err, ferrors.ErrReferenceSkipped) { + fmt.Fprintln(cmd.OutOrStdout(), args[0]) // nolint:errcheck + return nil } - fmt.Fprintln(cmd.OutOrStdout(), fmt.Sprintf("%s@%s", res.Name, res.Ref)) - return nil + return err } + fmt.Fprintf(cmd.OutOrStdout(), "%s@%s\n", res.Name, res.Ref) // nolint:errcheck + return nil + } diff --git a/cmd/image/list.go b/cmd/image/list.go index b426dc4..538dc97 100644 --- a/cmd/image/list.go +++ b/cmd/image/list.go @@ -17,14 +17,17 @@ package image import ( "encoding/json" + "errors" "fmt" + "path/filepath" + "strconv" + "github.com/olekukonko/tablewriter" "github.com/spf13/cobra" + "github.com/stacklok/frizbee/internal/cli" "github.com/stacklok/frizbee/pkg/config" "github.com/stacklok/frizbee/pkg/replacer" - "path/filepath" - "strconv" ) // CmdList represents the one sub-command @@ -35,11 +38,12 @@ func CmdList() *cobra.Command { Long: `This utility lists all container images used in the files in the directory Example: - frizbee image list -d + frizbee image list `, Aliases: []string{"ls"}, RunE: list, SilenceUsage: true, + Args: cobra.ExactArgs(1), } cli.DeclareFrizbeeFlags(cmd, true) @@ -50,7 +54,7 @@ Example: func list(cmd *cobra.Command, args []string) error { dir := filepath.Clean(args[0]) if !cli.IsPath(dir) { - return fmt.Errorf("the provided argument is not a path") + return errors.New("the provided argument is not a path") } // Extract the CLI flags from the cobra command cliFlags, err := cli.NewHelper(cmd) @@ -82,7 +86,7 @@ func list(cmd *cobra.Command, args []string) error { return err } jsonString := string(jsonBytes) - fmt.Fprintln(cmd.OutOrStdout(), jsonString) + fmt.Fprintln(cmd.OutOrStdout(), jsonString) // nolint:errcheck return nil case "table": table := tablewriter.NewWriter(cmd.OutOrStdout()) diff --git a/cmd/root.go b/cmd/root.go index 4e3c3b1..78f790c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -19,12 +19,12 @@ package cmd import ( "context" "fmt" - "github.com/stacklok/frizbee/cmd/action" - "github.com/stacklok/frizbee/cmd/image" "os" "github.com/spf13/cobra" + "github.com/stacklok/frizbee/cmd/actions" + "github.com/stacklok/frizbee/cmd/image" "github.com/stacklok/frizbee/cmd/version" "github.com/stacklok/frizbee/pkg/config" ) @@ -39,7 +39,7 @@ func Execute() { rootCmd.PersistentFlags().StringP("config", "c", ".frizbee.yml", "config file (default is .frizbee.yml)") - rootCmd.AddCommand(action.CmdGHActions()) + rootCmd.AddCommand(actions.CmdGHActions()) rootCmd.AddCommand(image.CmdContainerImage()) rootCmd.AddCommand(version.CmdVersion()) diff --git a/cmd/version/version.go b/cmd/version/version.go index 836c5ed..7cc5348 100644 --- a/cmd/version/version.go +++ b/cmd/version/version.go @@ -18,6 +18,7 @@ package version import ( "github.com/spf13/cobra" + "github.com/stacklok/frizbee/internal/cli" ) diff --git a/internal/cli/cli.go b/internal/cli/cli.go index 9f703e4..e8c54fb 100644 --- a/internal/cli/cli.go +++ b/internal/cli/cli.go @@ -13,18 +13,20 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package cli provides utilities to work with the command-line interface. package cli import ( "fmt" - "github.com/go-git/go-billy/v5/osfs" - "github.com/spf13/cobra" "io" "os" "path/filepath" "runtime/debug" "strings" "text/template" + + "github.com/go-git/go-billy/v5/osfs" + "github.com/spf13/cobra" ) const ( @@ -77,7 +79,7 @@ var ( VerboseCLIVersion = "" ) -// nolint:init +// nolint:gochecknoinits func init() { buildinfo, ok := debug.ReadBuildInfo() if !ok { @@ -115,6 +117,7 @@ func (vvs *versionInfo) String() string { return stringBuilder.String() } +// NewHelper creates a new CLI Helper struct. func NewHelper(cmd *cobra.Command) (*Helper, error) { dryRun, err := cmd.Flags().GetBool("dry-run") if err != nil { @@ -158,7 +161,7 @@ func DeclareFrizbeeFlags(cmd *cobra.Command, enableOutput bool) { // not quiet. func (r *Helper) Logf(format string, args ...interface{}) { if !r.Quiet { - fmt.Fprintf(r.Cmd.ErrOrStderr(), format, args...) + fmt.Fprintf(r.Cmd.ErrOrStderr(), format, args...) // nolint:errcheck } } @@ -188,7 +191,7 @@ func (r *Helper) ProcessOutput(path string, processed []string, modified map[str defer func() { if err := f.Close(); err != nil { - fmt.Fprintf(r.Cmd.ErrOrStderr(), "failed to close file %s: %v", path, err) + fmt.Fprintf(r.Cmd.ErrOrStderr(), "failed to close file %s: %v", path, err) // nolint:errcheck } }() @@ -206,10 +209,8 @@ func (r *Helper) ProcessOutput(path string, processed []string, modified map[str return nil } +// IsPath returns true if the given path is a file or directory. func IsPath(pathOrRef string) bool { _, err := os.Stat(pathOrRef) - if err == nil { - return true - } - return false + return err == nil } diff --git a/internal/store/cache.go b/internal/store/cache.go index b6993f3..f0b90f8 100644 --- a/internal/store/cache.go +++ b/internal/store/cache.go @@ -13,6 +13,7 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package store provides utilities to work with a cache store. package store import ( diff --git a/internal/traverse/traverse.go b/internal/traverse/traverse.go index 7ab8c78..6bb092c 100644 --- a/internal/traverse/traverse.go +++ b/internal/traverse/traverse.go @@ -13,27 +13,29 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package traverse provides utilities to traverse directories. package traverse import ( "fmt" - "github.com/go-git/go-billy/v5" "io/fs" "os" "path/filepath" "strings" + + "github.com/go-git/go-billy/v5" ) -// TraverseGHWFunc is a function that gets called with each file in a GitHub Actions workflow +// GhwFunc is a function that gets called with each file in a GitHub Actions workflow // directory. It receives the path to the file. -type TraverseGHWFunc func(path string) error +type GhwFunc func(path string) error -// TraverseFunc is a function that gets called with each file in a directory. -type TraverseFunc func(path string, info fs.FileInfo) error +// FuncTraverse is a function that gets called with each file in a directory. +type FuncTraverse func(path string, info fs.FileInfo) error -// TraverseYAMLDockerfiles traverses all yaml/yml in the given directory +// YamlDockerfiles traverses all yaml/yml in the given directory // and calls the given function with each workflow. -func TraverseYAMLDockerfiles(bfs billy.Filesystem, base string, fun TraverseGHWFunc) error { +func YamlDockerfiles(bfs billy.Filesystem, base string, fun GhwFunc) error { return Traverse(bfs, base, func(path string, info fs.FileInfo) error { if !isYAMLOrDockerfile(info) { return nil @@ -48,7 +50,7 @@ func TraverseYAMLDockerfiles(bfs billy.Filesystem, base string, fun TraverseGHWF } // Traverse traverses the given directory and calls the given function with each file. -func Traverse(bfs billy.Filesystem, base string, fun TraverseFunc) error { +func Traverse(bfs billy.Filesystem, base string, fun FuncTraverse) error { return Walk(bfs, base, func(path string, info fs.FileInfo, err error) error { if err != nil { return nil @@ -76,12 +78,12 @@ func isYAMLOrDockerfile(info fs.FileInfo) bool { // walk recursively descends path, calling walkFn // adapted from https://golang.org/src/path/filepath/path.go -func walk(fs billy.Filesystem, path string, info os.FileInfo, walkFn filepath.WalkFunc) error { +func walk(bfs billy.Filesystem, path string, info os.FileInfo, walkFn filepath.WalkFunc) error { if !info.IsDir() { return walkFn(path, info, nil) } - names, err := readDirNames(fs, path) + names, err := readDirNames(bfs, path) err1 := walkFn(path, info, err) // If err != nil, walk can't walk into this directory. // err1 != nil means walkFn want walk to skip this directory or stop walking. @@ -96,13 +98,13 @@ func walk(fs billy.Filesystem, path string, info os.FileInfo, walkFn filepath.Wa for _, name := range names { filename := filepath.Join(path, name) - fileInfo, err := fs.Lstat(filename) + fileInfo, err := bfs.Lstat(filename) if err != nil { if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir { return err } } else { - err = walk(fs, filename, fileInfo, walkFn) + err = walk(bfs, filename, fileInfo, walkFn) if err != nil { if !fileInfo.IsDir() || err != filepath.SkipDir { return err @@ -123,12 +125,12 @@ func walk(fs billy.Filesystem, path string, info os.FileInfo, walkFn filepath.Wa // to walk that directory. Walk does not follow symbolic links. // // Function adapted from https://github.com/golang/go/blob/3b770f2ccb1fa6fecc22ea822a19447b10b70c5c/src/path/filepath/path.go#L500 -func Walk(fs billy.Filesystem, root string, walkFn filepath.WalkFunc) error { - info, err := fs.Lstat(root) +func Walk(bfs billy.Filesystem, root string, walkFn filepath.WalkFunc) error { + info, err := bfs.Lstat(root) if err != nil { err = walkFn(root, nil, err) } else { - err = walk(fs, root, info, walkFn) + err = walk(bfs, root, info, walkFn) } if err == filepath.SkipDir { @@ -138,8 +140,8 @@ func Walk(fs billy.Filesystem, root string, walkFn filepath.WalkFunc) error { return err } -func readDirNames(fs billy.Filesystem, dir string) ([]string, error) { - files, err := fs.ReadDir(dir) +func readDirNames(bfs billy.Filesystem, dir string) ([]string, error) { + files, err := bfs.ReadDir(dir) if err != nil { return nil, err } diff --git a/pkg/config/config.go b/pkg/config/config.go index f34dc45..7fd8335 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -19,12 +19,12 @@ package config import ( "errors" "fmt" - "github.com/spf13/cobra" "os" "path/filepath" "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/osfs" + "github.com/spf13/cobra" "gopkg.in/yaml.v3" ) diff --git a/internal/ghrest/ghrest.go b/pkg/ghrest/ghrest.go similarity index 100% rename from internal/ghrest/ghrest.go rename to pkg/ghrest/ghrest.go diff --git a/pkg/interfaces/interfaces.go b/pkg/interfaces/interfaces.go index 90fa340..5d00faf 100644 --- a/pkg/interfaces/interfaces.go +++ b/pkg/interfaces/interfaces.go @@ -13,13 +13,15 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package interfaces provides interfaces for the frizbee package. package interfaces import ( "context" + "net/http" + "github.com/stacklok/frizbee/internal/store" "github.com/stacklok/frizbee/pkg/config" - "net/http" ) // EntityRef represents an action reference. @@ -31,6 +33,7 @@ type EntityRef struct { Prefix string `json:"prefix"` } +// Parser is an interface to replace references with digests type Parser interface { GetRegex() string Replace(ctx context.Context, matchedLine string, restIf REST, cfg config.Config, cache store.RefCacher) (*EntityRef, error) diff --git a/pkg/replacer/action/action.go b/pkg/replacer/actions/actions.go similarity index 79% rename from pkg/replacer/action/action.go rename to pkg/replacer/actions/actions.go index bf2a301..51180c9 100644 --- a/pkg/replacer/action/action.go +++ b/pkg/replacer/actions/actions.go @@ -13,17 +13,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -package action +// Package actions provides utilities to work with GitHub Actions. +package actions import ( "context" "fmt" + "strings" + "github.com/stacklok/frizbee/internal/store" "github.com/stacklok/frizbee/pkg/config" ferrors "github.com/stacklok/frizbee/pkg/errors" "github.com/stacklok/frizbee/pkg/interfaces" "github.com/stacklok/frizbee/pkg/replacer/image" - "strings" ) const ( @@ -31,13 +33,16 @@ const ( GitHubActionsRegex = `uses:\s*[^\s]+/[^\s]+@[^\s]+|uses:\s*docker://[^\s]+:[^\s]+` prefixUses = "uses: " prefixDocker = "docker://" - ReferenceType = "action" + // ReferenceType is the type of the reference + ReferenceType = "action" ) +// Parser is a struct to replace action references with digests type Parser struct { regex string } +// New creates a new Parser func New(regex string) *Parser { if regex == "" { regex = GitHubActionsRegex @@ -47,11 +52,19 @@ func New(regex string) *Parser { } } +// GetRegex returns the regular expression pattern to match GitHub Actions usage func (p *Parser) GetRegex() string { return p.regex } -func (p *Parser) Replace(ctx context.Context, matchedLine string, restIf interfaces.REST, cfg config.Config, cache store.RefCacher) (*interfaces.EntityRef, error) { +// Replace replaces the action reference with the digest +func (p *Parser) Replace( + ctx context.Context, + matchedLine string, + restIf interfaces.REST, + cfg config.Config, + cache store.RefCacher, +) (*interfaces.EntityRef, error) { var err error var actionRef *interfaces.EntityRef hasUsesPrefix := false @@ -80,7 +93,13 @@ func (p *Parser) Replace(ctx context.Context, matchedLine string, restIf interfa return actionRef, nil } -func (p *Parser) replaceAction(ctx context.Context, matchedLine string, restIf interfaces.REST, cfg config.Config, cache store.RefCacher) (*interfaces.EntityRef, error) { +func (_ *Parser) replaceAction( + ctx context.Context, + matchedLine string, + restIf interfaces.REST, + cfg config.Config, + cache store.RefCacher, +) (*interfaces.EntityRef, error) { // If the value is a local path or should be excluded, skip it if isLocal(matchedLine) || shouldExclude(&cfg.GHActions, matchedLine) { @@ -121,6 +140,11 @@ func (p *Parser) replaceAction(ctx context.Context, matchedLine string, restIf i } } + // Compare the digest with the reference and return the original reference if they already match + if ref == sum { + return nil, fmt.Errorf("image already referenced by digest: %s %w", matchedLine, ferrors.ErrReferenceSkipped) + } + return &interfaces.EntityRef{ Name: act, Ref: sum, @@ -129,7 +153,13 @@ func (p *Parser) replaceAction(ctx context.Context, matchedLine string, restIf i }, nil } -func (p *Parser) replaceDocker(ctx context.Context, matchedLine string, _ interfaces.REST, cfg config.Config, cache store.RefCacher) (*interfaces.EntityRef, error) { +func (_ *Parser) replaceDocker( + ctx context.Context, + matchedLine string, + _ interfaces.REST, + cfg config.Config, + cache store.RefCacher, +) (*interfaces.EntityRef, error) { // Trim the docker prefix trimmedRef := strings.TrimPrefix(matchedLine, prefixDocker) @@ -157,7 +187,8 @@ func (p *Parser) replaceDocker(ctx context.Context, matchedLine string, _ interf return actionRef, nil } -func (p *Parser) ConvertToEntityRef(reference string) (*interfaces.EntityRef, error) { +// ConvertToEntityRef converts an action reference to an EntityRef +func (_ *Parser) ConvertToEntityRef(reference string) (*interfaces.EntityRef, error) { reference = strings.TrimPrefix(reference, prefixUses) refType := ReferenceType separator := "@" diff --git a/pkg/replacer/action/utils.go b/pkg/replacer/actions/utils.go similarity index 99% rename from pkg/replacer/action/utils.go rename to pkg/replacer/actions/utils.go index 02e9826..d299fbf 100644 --- a/pkg/replacer/action/utils.go +++ b/pkg/replacer/actions/utils.go @@ -13,19 +13,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -package action +package actions import ( "context" "encoding/json" "errors" "fmt" - "github.com/google/go-github/v61/github" - "github.com/stacklok/frizbee/pkg/config" - "github.com/stacklok/frizbee/pkg/interfaces" "net/http" "net/url" "strings" + + "github.com/google/go-github/v61/github" + + "github.com/stacklok/frizbee/pkg/config" + "github.com/stacklok/frizbee/pkg/interfaces" ) var ( diff --git a/pkg/replacer/actions/utils_test.go b/pkg/replacer/actions/utils_test.go new file mode 100644 index 0000000..fa0065a --- /dev/null +++ b/pkg/replacer/actions/utils_test.go @@ -0,0 +1,143 @@ +package actions + +import ( + "context" + "os" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/stacklok/frizbee/pkg/ghrest" +) + +func TestGetChecksum(t *testing.T) { + t.Parallel() + + tok := os.Getenv("GITHUB_TOKEN") + + type args struct { + action string + ref string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "actions/checkout with v4.1.1", + args: args{ + action: "actions/checkout", + ref: "v4.1.1", + }, + want: "b4ffde65f46336ab88eb53be808477a3936bae11", + wantErr: false, + }, + { + name: "actions/checkout with v3.6.0", + args: args{ + action: "actions/checkout", + ref: "v3.6.0", + }, + want: "f43a0e5ff2bd294095638e18286ca9a3d1956744", + wantErr: false, + }, + { + name: "actions/checkout with checksum returns checksum", + args: args{ + action: "actions/checkout", + ref: "1d96c772d19495a3b5c517cd2bc0cb401ea0529f", + }, + want: "1d96c772d19495a3b5c517cd2bc0cb401ea0529f", + wantErr: false, + }, + { + name: "aquasecurity/trivy-action with 0.14.0", + args: args{ + action: "aquasecurity/trivy-action", + ref: "0.14.0", + }, + want: "2b6a709cf9c4025c5438138008beaddbb02086f0", + wantErr: false, + }, + { + name: "aquasecurity/trivy-action with branch returns checksum", + args: args{ + action: "aquasecurity/trivy-action", + ref: "bump-trivy", + }, + want: "fb5e1b36be448e92ca98648c661bd7e9da1f1317", + wantErr: false, + }, + { + name: "actions/checkout with invalid tag returns error", + args: args{ + action: "actions/checkout", + ref: "v4.1.1.1", + }, + want: "", + wantErr: true, + }, + { + name: "actions/checkout with invalid action returns error", + args: args{ + action: "invalid-action", + ref: "v4.1.1", + }, + want: "", + wantErr: true, + }, + { + name: "actions/checkout with empty action returns error", + args: args{ + action: "", + ref: "v4.1.1", + }, + want: "", + wantErr: true, + }, + { + name: "actions/checkout with empty tag returns error", + args: args{ + action: "actions/checkout", + ref: "", + }, + want: "", + wantErr: true, + }, + { + name: "bufbuild/buf-setup-action with v1 is an array", + args: args{ + action: "bufbuild/buf-setup-action", + ref: "v1", + }, + want: "f0475db2e1b1b2e8d121066b59dfb7f7bd6c4dc4", + }, + { + name: "anchore/sbom-action/download-syft with a sub-action works", + args: args{ + action: "anchore/sbom-action/download-syft", + ref: "v0.14.3", + }, + want: "78fc58e266e87a38d4194b2137a3d4e9bcaf7ca1", + }, + } + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ghcli := ghrest.NewClient(tok) + got, err := GetChecksum(context.Background(), ghcli, tt.args.action, tt.args.ref) + if tt.wantErr { + require.Error(t, err, "Wanted error, got none") + require.Empty(t, got, "Wanted empty string, got %v", got) + return + } + require.NoError(t, err, "Wanted no error, got %v", err) + require.Equal(t, tt.want, got, "Wanted %v, got %v", tt.want, got) + }) + } +} diff --git a/pkg/replacer/image/image.go b/pkg/replacer/image/image.go index 989361f..564a0ed 100644 --- a/pkg/replacer/image/image.go +++ b/pkg/replacer/image/image.go @@ -13,30 +13,36 @@ // See the License for the specific language governing permissions and // limitations under the License. +// Package image provides utilities to work with container images. package image import ( "context" "fmt" + "strings" + "github.com/stacklok/frizbee/internal/store" "github.com/stacklok/frizbee/pkg/config" ferrors "github.com/stacklok/frizbee/pkg/errors" "github.com/stacklok/frizbee/pkg/interfaces" - "strings" ) const ( // ContainerImageRegex is regular expression pattern to match container image usage in YAML - ContainerImageRegex = `image\s*:\s*["']?([^\s"']+/[^\s"']+|[^\s"']+)(:[^\s"']+)?(@[^\s"']+)?["']?|FROM\s+([^\s]+(/[^\s]+)?(:[^\s]+)?(@[^\s]+)?)` // `image\s*:\s*["']?([^\s"']+/[^\s"']+(:[^\s"']+)?(@[^\s"']+)?)["']?|FROM\s+([^\s]+(/[^\s]+)?(:[^\s]+)?(@[^\s]+)?)` // `\b(image|FROM)\s*:?(\s*([^\s]+))?` + // nolint:lll + ContainerImageRegex = `image\s*:\s*["']?([^\s"']+/[^\s"']+|[^\s"']+)(:[^\s"']+)?(@[^\s"']+)?["']?|FROM\s+([^\s]+(/[^\s]+)?(:[^\s]+)?(@[^\s]+)?)` prefixFROM = "FROM " prefixImage = "image: " - ReferenceType = "container" + // ReferenceType is the type of the reference + ReferenceType = "container" ) +// Parser is a struct to replace container image references with digests type Parser struct { regex string } +// New creates a new Parser func New(regex string) *Parser { if regex == "" { regex = ContainerImageRegex @@ -46,11 +52,14 @@ func New(regex string) *Parser { } } +// GetRegex returns the regular expression pattern to match container image usage func (p *Parser) GetRegex() string { return p.regex } -func (p *Parser) Replace(ctx context.Context, matchedLine string, _ interfaces.REST, cfg config.Config, cache store.RefCacher) (*interfaces.EntityRef, error) { +// Replace replaces the container image reference with the digest +func (_ *Parser) Replace(ctx context.Context, + matchedLine string, _ interfaces.REST, cfg config.Config, cache store.RefCacher) (*interfaces.EntityRef, error) { // Trim the prefix hasFROMPrefix := false hasImagePrefix := false @@ -85,7 +94,8 @@ func (p *Parser) Replace(ctx context.Context, matchedLine string, _ interfaces.R return imageRefWithDigest, nil } -func (p *Parser) ConvertToEntityRef(reference string) (*interfaces.EntityRef, error) { +// ConvertToEntityRef converts a container image reference to an EntityRef +func (_ *Parser) ConvertToEntityRef(reference string) (*interfaces.EntityRef, error) { reference = strings.TrimPrefix(reference, prefixImage) reference = strings.TrimPrefix(reference, prefixFROM) var sep string diff --git a/pkg/replacer/image/utils.go b/pkg/replacer/image/utils.go index 68b9857..68ee149 100644 --- a/pkg/replacer/image/utils.go +++ b/pkg/replacer/image/utils.go @@ -17,16 +17,19 @@ package image import ( "context" + "errors" "fmt" + "strings" + "github.com/google/go-containerregistry/pkg/authn" "github.com/google/go-containerregistry/pkg/name" v1 "github.com/google/go-containerregistry/pkg/v1" "github.com/google/go-containerregistry/pkg/v1/remote" + "github.com/stacklok/frizbee/internal/cli" "github.com/stacklok/frizbee/internal/store" ferrors "github.com/stacklok/frizbee/pkg/errors" "github.com/stacklok/frizbee/pkg/interfaces" - "strings" ) // GetImageDigestFromRef returns the digest of a container image reference @@ -47,7 +50,7 @@ func GetImageDigestFromRef(ctx context.Context, imageRef, platform string, cache if platform != "" { platformSplit := strings.Split(platform, "/") if len(platformSplit) != 2 { - return nil, fmt.Errorf("platform must be in the format os/arch") + return nil, errors.New("platform must be in the format os/arch") } opts = append(opts, remote.WithPlatform(v1.Platform{ OS: platformSplit[0], @@ -61,13 +64,14 @@ func GetImageDigestFromRef(ctx context.Context, imageRef, platform string, cache if cache != nil { if d, ok := cache.Load(imageRef); ok { digest = d + } else { + desc, err := remote.Get(ref, opts...) + if err != nil { + return nil, err + } + digest = desc.Digest.String() + cache.Store(imageRef, digest) } - desc, err := remote.Get(ref, opts...) - if err != nil { - return nil, err - } - digest = desc.Digest.String() - cache.Store(imageRef, digest) } else { desc, err := remote.Get(ref, opts...) if err != nil { diff --git a/pkg/replacer/image/utils_test.go b/pkg/replacer/image/utils_test.go new file mode 100644 index 0000000..8943fd2 --- /dev/null +++ b/pkg/replacer/image/utils_test.go @@ -0,0 +1,71 @@ +package image + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" +) + +func TestGetImageDigestFromRef(t *testing.T) { + t.Parallel() + + type args struct { + refstr string + } + tests := []struct { + name string + args args + want string + wantErr bool + }{ + { + name: "valid 1", + args: args{ + refstr: "ghcr.io/stacklok/minder/helm/minder:0.20231123.829_ref.26ca90b", + }, + want: "sha256:a29f8a8d28f0af7f70a4b3dd3e33c8c8cc5cf9e88e802e2700cf272a0b6140ec", + }, + { + name: "valid 2", + args: args{ + refstr: "devopsfaith/krakend:2.5.0", + }, + want: "sha256:6a3c8e5e1a4948042bfb364ed6471e16b4a26d0afb6c3c01ebcb88b3fa551036", + }, + { + name: "invalid ref string", + args: args{ + refstr: "ghcr.io/stacklok/minder/helm/minder!", + }, + wantErr: true, + }, + { + name: "unexistent container in unexistent registry", + args: args{ + refstr: "beeeeer.io/ipa/toppling-goliath/king-sue:1.0.0", + }, + wantErr: true, + }, + } + for _, tt := range tests { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) + defer cancel() + got, err := GetImageDigestFromRef(ctx, tt.args.refstr, "", nil) + if tt.wantErr { + assert.Error(t, err) + assert.Nil(t, got) + return + } + + assert.NoError(t, err) + assert.Equal(t, tt.want, got.Ref) + }) + } +} diff --git a/pkg/replacer/replacer.go b/pkg/replacer/replacer.go index e53b12c..17887ca 100644 --- a/pkg/replacer/replacer.go +++ b/pkg/replacer/replacer.go @@ -20,26 +20,27 @@ import ( "bufio" "context" "fmt" + "io" + "path/filepath" + "regexp" + "sort" + "strings" + "sync" + mapset "github.com/deckarep/golang-set/v2" + "github.com/go-git/go-billy/v5" "github.com/go-git/go-billy/v5/osfs" - "github.com/stacklok/frizbee/internal/ghrest" + "golang.org/x/sync/errgroup" + "github.com/stacklok/frizbee/internal/store" "github.com/stacklok/frizbee/internal/traverse" "github.com/stacklok/frizbee/pkg/config" + "github.com/stacklok/frizbee/pkg/ghrest" "github.com/stacklok/frizbee/pkg/interfaces" - "github.com/stacklok/frizbee/pkg/replacer/action" + "github.com/stacklok/frizbee/pkg/replacer/actions" "github.com/stacklok/frizbee/pkg/replacer/image" - "golang.org/x/sync/errgroup" - "io" - "path/filepath" - "regexp" - "sort" - "strings" - "sync" ) -type ParserType string - // Replacer replaces container image references in YAML files type Replacer struct { regex string @@ -48,11 +49,13 @@ type Replacer struct { config.Config } +// ReplaceResult holds a slice of all processed files along with a map of their modified content type ReplaceResult struct { Processed []string Modified map[string]string } +// ListResult holds the result of the list methods type ListResult struct { Processed []string Entities []interfaces.EntityRef @@ -81,31 +84,37 @@ func New(cfg *config.Config) *Replacer { // ParseGitHubActionString parses and returns the referenced entity pinned by its digest func (r *Replacer) ParseGitHubActionString(ctx context.Context, entityRef string) (*interfaces.EntityRef, error) { - r.parser = action.New(r.regex) + r.parser = actions.New(r.regex) return r.parser.Replace(ctx, entityRef, r.REST, r.Config, nil) } // ParseGitHubActionsInPath parses and replaces all GitHub actions references in the provided directory func (r *Replacer) ParseGitHubActionsInPath(ctx context.Context, dir string) (*ReplaceResult, error) { - r.parser = action.New(r.regex) - return r.parsePath(ctx, dir) + r.parser = actions.New(r.regex) + return r.parsePathInFS(ctx, osfs.New(filepath.Dir(dir), osfs.WithBoundOS()), filepath.Base(dir)) +} + +// ParseGitHubActionsInFS parses and replaces all container image references in the provided file system +func (r *Replacer) ParseGitHubActionsInFS(ctx context.Context, bfs billy.Filesystem, base string) (*ReplaceResult, error) { + r.parser = actions.New(r.regex) + return r.parsePathInFS(ctx, bfs, base) } // ParseGitHubActionsInFile parses and replaces all GitHub actions references in the provided file func (r *Replacer) ParseGitHubActionsInFile(ctx context.Context, f io.Reader, cache store.RefCacher) (bool, string, error) { - r.parser = action.New(r.regex) + r.parser = actions.New(r.regex) return r.parseAndReplaceReferencesInFile(ctx, f, cache) } // ListGitHibActionsInPath lists all GitHub actions references in the provided directory func (r *Replacer) ListGitHibActionsInPath(dir string) (*ListResult, error) { - r.parser = action.New(r.regex) + r.parser = actions.New(r.regex) return r.listReferences(dir) } // ListGitHibActionsInFile lists all GitHub actions references in the provided file func (r *Replacer) ListGitHibActionsInFile(f io.Reader) (*ListResult, error) { - r.parser = action.New(r.regex) + r.parser = actions.New(r.regex) found, err := r.listReferencesInFile(f) if err != nil { return nil, err @@ -131,7 +140,13 @@ func (r *Replacer) ParseContainerImageString(ctx context.Context, entityRef stri // ParseContainerImagesInPath parses and replaces all container image references in the provided directory func (r *Replacer) ParseContainerImagesInPath(ctx context.Context, dir string) (*ReplaceResult, error) { r.parser = image.New(r.regex) - return r.parsePath(ctx, dir) + return r.parsePathInFS(ctx, osfs.New(filepath.Dir(dir), osfs.WithBoundOS()), filepath.Base(dir)) +} + +// ParseContainerImagesInFS parses and replaces all container image references in the provided file system +func (r *Replacer) ParseContainerImagesInFS(ctx context.Context, bfs billy.Filesystem, base string) (*ReplaceResult, error) { + r.parser = image.New(r.regex) + return r.parsePathInFS(ctx, bfs, base) } // ParseContainerImagesInFile parses and replaces all container image references in the provided file @@ -165,14 +180,10 @@ func (r *Replacer) ListContainerImagesInFile(f io.Reader) (*ListResult, error) { return res, nil } -func (r *Replacer) parsePath(ctx context.Context, dir string) (*ReplaceResult, error) { +func (r *Replacer) parsePathInFS(ctx context.Context, bfs billy.Filesystem, base string) (*ReplaceResult, error) { var eg errgroup.Group var mu sync.Mutex - basedir := filepath.Dir(dir) - base := filepath.Base(dir) - bfs := osfs.New(basedir, osfs.WithBoundOS()) - cache := store.NewRefCacher() res := ReplaceResult{ @@ -181,7 +192,7 @@ func (r *Replacer) parsePath(ctx context.Context, dir string) (*ReplaceResult, e } // Traverse all YAML/YML files in dir - err := traverse.TraverseYAMLDockerfiles(bfs, base, func(path string) error { + err := traverse.YamlDockerfiles(bfs, base, func(path string) error { eg.Go(func() error { file, err := bfs.Open(path) if err != nil { @@ -238,7 +249,7 @@ func (r *Replacer) listReferences(dir string) (*ListResult, error) { found := mapset.NewSet[interfaces.EntityRef]() // Traverse all related files - err := traverse.TraverseYAMLDockerfiles(bfs, base, func(path string) error { + err := traverse.YamlDockerfiles(bfs, base, func(path string) error { eg.Go(func() error { file, err := bfs.Open(path) if err != nil { @@ -282,10 +293,10 @@ func (r *Replacer) listReferences(dir string) (*ListResult, error) { return &res, nil } -// parseAndReplaceReferencesInFile takes the given file reader and returns its content -// after replacing all references to tags with the respective digests. -// It also uses the provided cache to store the checksums. -func (r *Replacer) parseAndReplaceReferencesInFile(ctx context.Context, f io.Reader, cache store.RefCacher) (bool, string, error) { +func (r *Replacer) parseAndReplaceReferencesInFile( + ctx context.Context, + f io.Reader, cache store.RefCacher, +) (bool, string, error) { var contentBuilder strings.Builder var ret *interfaces.EntityRef @@ -309,7 +320,7 @@ func (r *Replacer) parseAndReplaceReferencesInFile(ctx context.Context, f io.Rea // Return the original line as we don't want to update it in case something errored out return matchedLine } - // Construct the new line + // Construct the new line, comments in dockerfiles are handled differently than yml files if strings.Contains(matchedLine, "FROM") { return fmt.Sprintf("%s%s:%s@%s", ret.Prefix, ret.Name, ret.Tag, ret.Ref) } @@ -352,6 +363,7 @@ func (r *Replacer) listReferencesInFile(f io.Reader) (mapset.Set[interfaces.Enti // See if we can match an entity reference in the line foundEntries := re.FindAllString(line, -1) + // nolint:gosimple if foundEntries != nil { for _, entry := range foundEntries { e, err := r.parser.ConvertToEntityRef(entry) diff --git a/pkg/replacer/replacer_test.go b/pkg/replacer/replacer_test.go index 2ee91d9..5656ee0 100644 --- a/pkg/replacer/replacer_test.go +++ b/pkg/replacer/replacer_test.go @@ -17,17 +17,19 @@ package replacer import ( "context" - "github.com/stacklok/frizbee/internal/cli" - "github.com/stacklok/frizbee/pkg/config" - "github.com/stacklok/frizbee/pkg/interfaces" - "github.com/stacklok/frizbee/pkg/replacer/action" - "github.com/stacklok/frizbee/pkg/replacer/image" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" "os" "strings" "testing" "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stacklok/frizbee/internal/cli" + "github.com/stacklok/frizbee/pkg/config" + "github.com/stacklok/frizbee/pkg/interfaces" + "github.com/stacklok/frizbee/pkg/replacer/actions" + "github.com/stacklok/frizbee/pkg/replacer/image" ) func TestReplacer_ParseContainerImageString(t *testing.T) { @@ -173,7 +175,7 @@ func TestReplacer_ParseGitHubActionString(t *testing.T) { want: &interfaces.EntityRef{ Name: "actions/checkout", Ref: "b4ffde65f46336ab88eb53be808477a3936bae11", - Type: action.ReferenceType, + Type: actions.ReferenceType, Tag: "v4.1.1", Prefix: "", }, @@ -187,7 +189,7 @@ func TestReplacer_ParseGitHubActionString(t *testing.T) { want: &interfaces.EntityRef{ Name: "actions/checkout", Ref: "f43a0e5ff2bd294095638e18286ca9a3d1956744", - Type: action.ReferenceType, + Type: actions.ReferenceType, Tag: "v3.6.0", Prefix: "uses: ", }, @@ -198,14 +200,8 @@ func TestReplacer_ParseGitHubActionString(t *testing.T) { args: args{ action: "actions/checkout@1d96c772d19495a3b5c517cd2bc0cb401ea0529f", }, - want: &interfaces.EntityRef{ - Name: "actions/checkout", - Ref: "1d96c772d19495a3b5c517cd2bc0cb401ea0529f", - Type: action.ReferenceType, - Tag: "1d96c772d19495a3b5c517cd2bc0cb401ea0529f", - Prefix: "", - }, - wantErr: false, + want: nil, + wantErr: true, }, { name: "aquasecurity/trivy-action with 0.14.0", @@ -215,7 +211,7 @@ func TestReplacer_ParseGitHubActionString(t *testing.T) { want: &interfaces.EntityRef{ Name: "aquasecurity/trivy-action", Ref: "2b6a709cf9c4025c5438138008beaddbb02086f0", - Type: action.ReferenceType, + Type: actions.ReferenceType, Tag: "0.14.0", Prefix: "", }, @@ -229,7 +225,7 @@ func TestReplacer_ParseGitHubActionString(t *testing.T) { want: &interfaces.EntityRef{ Name: "aquasecurity/trivy-action", Ref: "fb5e1b36be448e92ca98648c661bd7e9da1f1317", - Type: action.ReferenceType, + Type: actions.ReferenceType, Tag: "bump-trivy", Prefix: "", }, @@ -275,7 +271,7 @@ func TestReplacer_ParseGitHubActionString(t *testing.T) { want: &interfaces.EntityRef{ Name: "bufbuild/buf-setup-action", Ref: "f0475db2e1b1b2e8d121066b59dfb7f7bd6c4dc4", - Type: action.ReferenceType, + Type: actions.ReferenceType, Tag: "v1", Prefix: "", }, @@ -288,7 +284,7 @@ func TestReplacer_ParseGitHubActionString(t *testing.T) { want: &interfaces.EntityRef{ Name: "anchore/sbom-action/download-syft", Ref: "78fc58e266e87a38d4194b2137a3d4e9bcaf7ca1", - Type: action.ReferenceType, + Type: actions.ReferenceType, Tag: "v0.14.3", Prefix: "", }, From c2aa963eb53b0d3e3cfa5b33e166cdfcf5bcafe1 Mon Sep 17 00:00:00 2001 From: Radoslav Dimitrov Date: Tue, 28 May 2024 18:18:41 +0300 Subject: [PATCH 06/16] Group up the methods for parsing actions and images Signed-off-by: Radoslav Dimitrov --- cmd/actions/actions.go | 6 +- cmd/actions/list.go | 4 +- cmd/image/image.go | 6 +- cmd/image/list.go | 4 +- pkg/interfaces/interfaces.go | 4 +- pkg/replacer/actions/actions.go | 38 ++++--- pkg/replacer/image/image.go | 29 ++++-- pkg/replacer/replacer.go | 176 +++++++++++++------------------- pkg/replacer/replacer_test.go | 16 +-- 9 files changed, 137 insertions(+), 146 deletions(-) diff --git a/cmd/actions/actions.go b/cmd/actions/actions.go index 6a461ef..5a3b8e5 100644 --- a/cmd/actions/actions.go +++ b/cmd/actions/actions.go @@ -82,14 +82,14 @@ func replaceCmd(cmd *cobra.Command, args []string) error { } // Create a new replacer - r := replacer.New(cfg). + r := replacer.NewActionsReplacer(cfg). WithUserRegex(cliFlags.Regex). WithGitHubClient(os.Getenv(cli.GitHubTokenEnvKey)) if cli.IsPath(pathOrRef) { dir := filepath.Clean(pathOrRef) // Replace the tags in the given directory - res, err := r.ParseGitHubActionsInPath(cmd.Context(), dir) + res, err := r.ParsePath(cmd.Context(), dir) if err != nil { return err } @@ -97,7 +97,7 @@ func replaceCmd(cmd *cobra.Command, args []string) error { return cliFlags.ProcessOutput(dir, res.Processed, res.Modified) } // Replace the passed reference - res, err := r.ParseGitHubActionString(cmd.Context(), pathOrRef) + res, err := r.ParseString(cmd.Context(), pathOrRef) if err != nil { if errors.Is(err, ferrors.ErrReferenceSkipped) { fmt.Fprintln(cmd.OutOrStdout(), pathOrRef) // nolint:errcheck diff --git a/cmd/actions/list.go b/cmd/actions/list.go index 93469af..2098158 100644 --- a/cmd/actions/list.go +++ b/cmd/actions/list.go @@ -76,12 +76,12 @@ func list(cmd *cobra.Command, args []string) error { } // Create a new replacer - r := replacer.New(cfg). + r := replacer.NewActionsReplacer(cfg). WithUserRegex(cliFlags.Regex). WithGitHubClient(os.Getenv(cli.GitHubTokenEnvKey)) // List the references in the directory - res, err := r.ListGitHibActionsInPath(dir) + res, err := r.ListPath(dir) if err != nil { return err } diff --git a/cmd/image/image.go b/cmd/image/image.go index 4aece3a..519963a 100644 --- a/cmd/image/image.go +++ b/cmd/image/image.go @@ -72,13 +72,13 @@ func replaceCmd(cmd *cobra.Command, args []string) error { } // Create a new replacer - r := replacer.New(cfg). + r := replacer.NewImageReplacer(cfg). WithUserRegex(cliFlags.Regex) if cli.IsPath(args[0]) { dir := filepath.Clean(args[0]) // Replace the tags in the directory - res, err := r.ParseContainerImagesInPath(cmd.Context(), dir) + res, err := r.ParsePath(cmd.Context(), dir) if err != nil { return err } @@ -86,7 +86,7 @@ func replaceCmd(cmd *cobra.Command, args []string) error { return cliFlags.ProcessOutput(dir, res.Processed, res.Modified) } // Replace the passed reference - res, err := r.ParseContainerImageString(cmd.Context(), args[0]) + res, err := r.ParseString(cmd.Context(), args[0]) if err != nil { if errors.Is(err, ferrors.ErrReferenceSkipped) { fmt.Fprintln(cmd.OutOrStdout(), args[0]) // nolint:errcheck diff --git a/cmd/image/list.go b/cmd/image/list.go index 538dc97..c04bccd 100644 --- a/cmd/image/list.go +++ b/cmd/image/list.go @@ -69,11 +69,11 @@ func list(cmd *cobra.Command, args []string) error { } // Create a new replacer - r := replacer.New(cfg). + r := replacer.NewImageReplacer(cfg). WithUserRegex(cliFlags.Regex) // List the references in the directory - res, err := r.ListContainerImagesInPath(dir) + res, err := r.ListPath(dir) if err != nil { return err } diff --git a/pkg/interfaces/interfaces.go b/pkg/interfaces/interfaces.go index 5d00faf..03a59e3 100644 --- a/pkg/interfaces/interfaces.go +++ b/pkg/interfaces/interfaces.go @@ -35,8 +35,10 @@ type EntityRef struct { // Parser is an interface to replace references with digests type Parser interface { + SetCache(cache store.RefCacher) + SetRegex(regex string) GetRegex() string - Replace(ctx context.Context, matchedLine string, restIf REST, cfg config.Config, cache store.RefCacher) (*EntityRef, error) + Replace(ctx context.Context, matchedLine string, restIf REST, cfg config.Config) (*EntityRef, error) ConvertToEntityRef(reference string) (*EntityRef, error) } diff --git a/pkg/replacer/actions/actions.go b/pkg/replacer/actions/actions.go index 51180c9..bf69428 100644 --- a/pkg/replacer/actions/actions.go +++ b/pkg/replacer/actions/actions.go @@ -40,18 +40,27 @@ const ( // Parser is a struct to replace action references with digests type Parser struct { regex string + cache store.RefCacher } // New creates a new Parser -func New(regex string) *Parser { - if regex == "" { - regex = GitHubActionsRegex - } +func New() *Parser { return &Parser{ - regex: regex, + regex: GitHubActionsRegex, + cache: store.NewRefCacher(), } } +// SetCache returns the regular expression pattern to match GitHub Actions usage +func (p *Parser) SetCache(cache store.RefCacher) { + p.cache = cache +} + +// SetRegex returns the regular expression pattern to match GitHub Actions usage +func (p *Parser) SetRegex(regex string) { + p.regex = regex +} + // GetRegex returns the regular expression pattern to match GitHub Actions usage func (p *Parser) GetRegex() string { return p.regex @@ -63,7 +72,6 @@ func (p *Parser) Replace( matchedLine string, restIf interfaces.REST, cfg config.Config, - cache store.RefCacher, ) (*interfaces.EntityRef, error) { var err error var actionRef *interfaces.EntityRef @@ -76,9 +84,9 @@ func (p *Parser) Replace( } // Determine if the action reference has a docker prefix if strings.HasPrefix(matchedLine, prefixDocker) { - actionRef, err = p.replaceDocker(ctx, matchedLine, restIf, cfg, cache) + actionRef, err = p.replaceDocker(ctx, matchedLine, restIf, cfg) } else { - actionRef, err = p.replaceAction(ctx, matchedLine, restIf, cfg, cache) + actionRef, err = p.replaceAction(ctx, matchedLine, restIf, cfg) } if err != nil { return nil, err @@ -93,12 +101,11 @@ func (p *Parser) Replace( return actionRef, nil } -func (_ *Parser) replaceAction( +func (p *Parser) replaceAction( ctx context.Context, matchedLine string, restIf interfaces.REST, cfg config.Config, - cache store.RefCacher, ) (*interfaces.EntityRef, error) { // If the value is a local path or should be excluded, skip it @@ -119,9 +126,9 @@ func (_ *Parser) replaceAction( var sum string // Check if we have a cache - if cache != nil { + if p.cache != nil { // Check if we have a cached value - if val, ok := cache.Load(matchedLine); ok { + if val, ok := p.cache.Load(matchedLine); ok { sum = val } else { // Get the checksum for the action reference @@ -130,7 +137,7 @@ func (_ *Parser) replaceAction( return nil, fmt.Errorf("failed to get checksum for action '%s': %w", matchedLine, err) } // Store the checksum in the cache - cache.Store(matchedLine, sum) + p.cache.Store(matchedLine, sum) } } else { // Get the checksum for the action reference @@ -153,12 +160,11 @@ func (_ *Parser) replaceAction( }, nil } -func (_ *Parser) replaceDocker( +func (p *Parser) replaceDocker( ctx context.Context, matchedLine string, _ interfaces.REST, cfg config.Config, - cache store.RefCacher, ) (*interfaces.EntityRef, error) { // Trim the docker prefix trimmedRef := strings.TrimPrefix(matchedLine, prefixDocker) @@ -169,7 +175,7 @@ func (_ *Parser) replaceDocker( } // Get the digest of the docker:// image reference - actionRef, err := image.GetImageDigestFromRef(ctx, trimmedRef, cfg.Platform, cache) + actionRef, err := image.GetImageDigestFromRef(ctx, trimmedRef, cfg.Platform, p.cache) if err != nil { return nil, err } diff --git a/pkg/replacer/image/image.go b/pkg/replacer/image/image.go index 564a0ed..b049666 100644 --- a/pkg/replacer/image/image.go +++ b/pkg/replacer/image/image.go @@ -40,26 +40,39 @@ const ( // Parser is a struct to replace container image references with digests type Parser struct { regex string + cache store.RefCacher } // New creates a new Parser -func New(regex string) *Parser { - if regex == "" { - regex = ContainerImageRegex - } +func New() *Parser { return &Parser{ - regex: regex, + regex: ContainerImageRegex, + cache: store.NewRefCacher(), } } +// SetCache sets the cache to store the image references +func (p *Parser) SetCache(cache store.RefCacher) { + p.cache = cache +} + +// SetRegex sets the regular expression pattern to match container image usage +func (p *Parser) SetRegex(regex string) { + p.regex = regex +} + // GetRegex returns the regular expression pattern to match container image usage func (p *Parser) GetRegex() string { return p.regex } // Replace replaces the container image reference with the digest -func (_ *Parser) Replace(ctx context.Context, - matchedLine string, _ interfaces.REST, cfg config.Config, cache store.RefCacher) (*interfaces.EntityRef, error) { +func (p *Parser) Replace( + ctx context.Context, + matchedLine string, + _ interfaces.REST, + cfg config.Config, +) (*interfaces.EntityRef, error) { // Trim the prefix hasFROMPrefix := false hasImagePrefix := false @@ -78,7 +91,7 @@ func (_ *Parser) Replace(ctx context.Context, } // Get the digest of the image reference - imageRefWithDigest, err := GetImageDigestFromRef(ctx, matchedLine, cfg.Platform, cache) + imageRefWithDigest, err := GetImageDigestFromRef(ctx, matchedLine, cfg.Platform, p.cache) if err != nil { return nil, err } diff --git a/pkg/replacer/replacer.go b/pkg/replacer/replacer.go index 17887ca..67b9cc2 100644 --- a/pkg/replacer/replacer.go +++ b/pkg/replacer/replacer.go @@ -32,7 +32,6 @@ import ( "github.com/go-git/go-billy/v5/osfs" "golang.org/x/sync/errgroup" - "github.com/stacklok/frizbee/internal/store" "github.com/stacklok/frizbee/internal/traverse" "github.com/stacklok/frizbee/pkg/config" "github.com/stacklok/frizbee/pkg/ghrest" @@ -41,14 +40,6 @@ import ( "github.com/stacklok/frizbee/pkg/replacer/image" ) -// Replacer replaces container image references in YAML files -type Replacer struct { - regex string - parser interfaces.Parser - interfaces.REST - config.Config -} - // ReplaceResult holds a slice of all processed files along with a map of their modified content type ReplaceResult struct { Processed []string @@ -61,110 +52,78 @@ type ListResult struct { Entities []interfaces.EntityRef } -// WithGitHubClient creates an authenticated GitHub client -func (r *Replacer) WithGitHubClient(token string) *Replacer { - client := ghrest.NewClient(token) - r.REST = client - return r -} - -// WithUserRegex sets a user-provided regex for the parser -func (r *Replacer) WithUserRegex(regex string) *Replacer { - r.regex = regex - return r +// Replacer replaces container image references in YAML files +type Replacer struct { + parser interfaces.Parser + rest interfaces.REST + cfg config.Config } -// New creates a new Replacer -func New(cfg *config.Config) *Replacer { - // Return the replacer +// NewActionsReplacer creates a new replacer for GitHub actions +func NewActionsReplacer(cfg *config.Config) *Replacer { return &Replacer{ - Config: *cfg, + cfg: *cfg, + parser: actions.New(), } } -// ParseGitHubActionString parses and returns the referenced entity pinned by its digest -func (r *Replacer) ParseGitHubActionString(ctx context.Context, entityRef string) (*interfaces.EntityRef, error) { - r.parser = actions.New(r.regex) - return r.parser.Replace(ctx, entityRef, r.REST, r.Config, nil) -} - -// ParseGitHubActionsInPath parses and replaces all GitHub actions references in the provided directory -func (r *Replacer) ParseGitHubActionsInPath(ctx context.Context, dir string) (*ReplaceResult, error) { - r.parser = actions.New(r.regex) - return r.parsePathInFS(ctx, osfs.New(filepath.Dir(dir), osfs.WithBoundOS()), filepath.Base(dir)) -} - -// ParseGitHubActionsInFS parses and replaces all container image references in the provided file system -func (r *Replacer) ParseGitHubActionsInFS(ctx context.Context, bfs billy.Filesystem, base string) (*ReplaceResult, error) { - r.parser = actions.New(r.regex) - return r.parsePathInFS(ctx, bfs, base) -} - -// ParseGitHubActionsInFile parses and replaces all GitHub actions references in the provided file -func (r *Replacer) ParseGitHubActionsInFile(ctx context.Context, f io.Reader, cache store.RefCacher) (bool, string, error) { - r.parser = actions.New(r.regex) - return r.parseAndReplaceReferencesInFile(ctx, f, cache) +// NewImageReplacer creates a new replacer for container images +func NewImageReplacer(cfg *config.Config) *Replacer { + return &Replacer{ + cfg: *cfg, + parser: image.New(), + } } -// ListGitHibActionsInPath lists all GitHub actions references in the provided directory -func (r *Replacer) ListGitHibActionsInPath(dir string) (*ListResult, error) { - r.parser = actions.New(r.regex) - return r.listReferences(dir) +// WithGitHubClient creates an authenticated GitHub client +func (r *Replacer) WithGitHubClient(token string) *Replacer { + client := ghrest.NewClient(token) + r.rest = client + return r } -// ListGitHibActionsInFile lists all GitHub actions references in the provided file -func (r *Replacer) ListGitHibActionsInFile(f io.Reader) (*ListResult, error) { - r.parser = actions.New(r.regex) - found, err := r.listReferencesInFile(f) - if err != nil { - return nil, err +// WithUserRegex sets a user-provided regex for the parser +func (r *Replacer) WithUserRegex(regex string) *Replacer { + if r.parser != nil { + r.parser.SetRegex(regex) } - res := &ListResult{} - res.Entities = found.ToSlice() - - // Sort the slice - sort.Slice(res.Entities, func(i, j int) bool { - return res.Entities[i].Name < res.Entities[j].Name - }) + return r +} - // All good - return res, nil +// WithCacheDisabled disables caching +func (r *Replacer) WithCacheDisabled() *Replacer { + r.parser.SetCache(nil) + return r } -// ParseContainerImageString parses and returns the referenced entity pinned by its digest -func (r *Replacer) ParseContainerImageString(ctx context.Context, entityRef string) (*interfaces.EntityRef, error) { - r.parser = image.New(r.regex) - return r.parser.Replace(ctx, entityRef, r.REST, r.Config, nil) +// ParseString parses and returns the referenced entity pinned by its digest +func (r *Replacer) ParseString(ctx context.Context, entityRef string) (*interfaces.EntityRef, error) { + return r.parser.Replace(ctx, entityRef, r.rest, r.cfg) } -// ParseContainerImagesInPath parses and replaces all container image references in the provided directory -func (r *Replacer) ParseContainerImagesInPath(ctx context.Context, dir string) (*ReplaceResult, error) { - r.parser = image.New(r.regex) - return r.parsePathInFS(ctx, osfs.New(filepath.Dir(dir), osfs.WithBoundOS()), filepath.Base(dir)) +// ParsePath parses and replaces all entity references in the provided directory +func (r *Replacer) ParsePath(ctx context.Context, dir string) (*ReplaceResult, error) { + return parsePathInFS(ctx, r.parser, r.rest, r.cfg, osfs.New(filepath.Dir(dir), osfs.WithBoundOS()), filepath.Base(dir)) } -// ParseContainerImagesInFS parses and replaces all container image references in the provided file system -func (r *Replacer) ParseContainerImagesInFS(ctx context.Context, bfs billy.Filesystem, base string) (*ReplaceResult, error) { - r.parser = image.New(r.regex) - return r.parsePathInFS(ctx, bfs, base) +// ParsePathInFS parses and replaces all entity references in the provided file system +func (r *Replacer) ParsePathInFS(ctx context.Context, bfs billy.Filesystem, base string) (*ReplaceResult, error) { + return parsePathInFS(ctx, r.parser, r.rest, r.cfg, bfs, base) } -// ParseContainerImagesInFile parses and replaces all container image references in the provided file -func (r *Replacer) ParseContainerImagesInFile(ctx context.Context, f io.Reader, cache store.RefCacher) (bool, string, error) { - r.parser = image.New(r.regex) - return r.parseAndReplaceReferencesInFile(ctx, f, cache) +// ParseFile parses and replaces all entity references in the provided file +func (r *Replacer) ParseFile(ctx context.Context, f io.Reader) (bool, string, error) { + return parseAndReplaceReferencesInFile(ctx, f, r.parser, r.rest, r.cfg) } -// ListContainerImagesInPath lists all container image references in yaml, yml and dockerfiles present the provided directory -func (r *Replacer) ListContainerImagesInPath(dir string) (*ListResult, error) { - r.parser = image.New(r.regex) - return r.listReferences(dir) +// ListPath lists all entity references in the provided directory +func (r *Replacer) ListPath(dir string) (*ListResult, error) { + return listReferences(dir, r.parser) } -// ListContainerImagesInFile lists all container image references in yaml, yml or dockerfile -func (r *Replacer) ListContainerImagesInFile(f io.Reader) (*ListResult, error) { - r.parser = image.New(r.regex) - found, err := r.listReferencesInFile(f) +// ListInFile lists all entity in the provided file +func (r *Replacer) ListInFile(f io.Reader) (*ListResult, error) { + found, err := listReferencesInFile(f, r.parser) if err != nil { return nil, err } @@ -180,12 +139,17 @@ func (r *Replacer) ListContainerImagesInFile(f io.Reader) (*ListResult, error) { return res, nil } -func (r *Replacer) parsePathInFS(ctx context.Context, bfs billy.Filesystem, base string) (*ReplaceResult, error) { +func parsePathInFS( + ctx context.Context, + parser interfaces.Parser, + rest interfaces.REST, + cfg config.Config, + bfs billy.Filesystem, + base string, +) (*ReplaceResult, error) { var eg errgroup.Group var mu sync.Mutex - cache := store.NewRefCacher() - res := ReplaceResult{ Processed: make([]string, 0), Modified: make(map[string]string), @@ -202,7 +166,7 @@ func (r *Replacer) parsePathInFS(ctx context.Context, bfs billy.Filesystem, base defer file.Close() // Parse the content of the file and update the matching references - modified, updatedFile, err := r.parseAndReplaceReferencesInFile(ctx, file, cache) + modified, updatedFile, err := parseAndReplaceReferencesInFile(ctx, file, parser, rest, cfg) if err != nil { return fmt.Errorf("failed to modify references in %s: %w", path, err) } @@ -233,7 +197,7 @@ func (r *Replacer) parsePathInFS(ctx context.Context, bfs billy.Filesystem, base return &res, nil } -func (r *Replacer) listReferences(dir string) (*ListResult, error) { +func listReferences(dir string, parser interfaces.Parser) (*ListResult, error) { var eg errgroup.Group var mu sync.Mutex @@ -258,8 +222,8 @@ func (r *Replacer) listReferences(dir string) (*ListResult, error) { // nolint:errcheck // ignore error defer file.Close() - // Parse the content of the file and listReferences the matching references - foundRefs, err := r.listReferencesInFile(file) + // Parse the content of the file and list the matching references + foundRefs, err := listReferencesInFile(file, parser) if err != nil { return fmt.Errorf("failed to listReferences references in %s: %w", path, err) } @@ -293,9 +257,12 @@ func (r *Replacer) listReferences(dir string) (*ListResult, error) { return &res, nil } -func (r *Replacer) parseAndReplaceReferencesInFile( +func parseAndReplaceReferencesInFile( ctx context.Context, - f io.Reader, cache store.RefCacher, + f io.Reader, + parser interfaces.Parser, + rest interfaces.REST, + cfg config.Config, ) (bool, string, error) { var contentBuilder strings.Builder var ret *interfaces.EntityRef @@ -303,7 +270,7 @@ func (r *Replacer) parseAndReplaceReferencesInFile( modified := false // Compile the regular expression - re, err := regexp.Compile(r.parser.GetRegex()) + re, err := regexp.Compile(parser.GetRegex()) if err != nil { return false, "", err } @@ -315,7 +282,7 @@ func (r *Replacer) parseAndReplaceReferencesInFile( // See if we can match an entity reference in the line newLine := re.ReplaceAllStringFunc(line, func(matchedLine string) string { // Modify the reference in the line - ret, err = r.parser.Replace(ctx, matchedLine, r.REST, r.Config, cache) + ret, err = parser.Replace(ctx, matchedLine, rest, cfg) if err != nil { // Return the original line as we don't want to update it in case something errored out return matchedLine @@ -347,11 +314,14 @@ func (r *Replacer) parseAndReplaceReferencesInFile( // listReferencesInFile takes the given file reader and returns a map of all // references, action or images, it found. -func (r *Replacer) listReferencesInFile(f io.Reader) (mapset.Set[interfaces.EntityRef], error) { +func listReferencesInFile( + f io.Reader, + parser interfaces.Parser, +) (mapset.Set[interfaces.EntityRef], error) { found := mapset.NewSet[interfaces.EntityRef]() // Compile the regular expression - re, err := regexp.Compile(r.parser.GetRegex()) + re, err := regexp.Compile(parser.GetRegex()) if err != nil { return nil, err } @@ -366,7 +336,7 @@ func (r *Replacer) listReferencesInFile(f io.Reader) (mapset.Set[interfaces.Enti // nolint:gosimple if foundEntries != nil { for _, entry := range foundEntries { - e, err := r.parser.ConvertToEntityRef(entry) + e, err := parser.ConvertToEntityRef(entry) if err != nil { continue } diff --git a/pkg/replacer/replacer_test.go b/pkg/replacer/replacer_test.go index 5656ee0..68d9519 100644 --- a/pkg/replacer/replacer_test.go +++ b/pkg/replacer/replacer_test.go @@ -127,8 +127,8 @@ func TestReplacer_ParseContainerImageString(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - r := New(&config.Config{}) - got, err := r.ParseContainerImageString(ctx, tt.args.refstr) + r := NewImageReplacer(&config.Config{}) + got, err := r.ParseString(ctx, tt.args.refstr) if tt.wantErr { assert.Error(t, err) assert.Empty(t, got) @@ -296,8 +296,8 @@ func TestReplacer_ParseGitHubActionString(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - r := New(&config.Config{}).WithGitHubClient(os.Getenv("GITHUB_TOKEN")) - got, err := r.ParseGitHubActionString(context.Background(), tt.args.action) + r := NewActionsReplacer(&config.Config{}).WithGitHubClient(os.Getenv("GITHUB_TOKEN")) + got, err := r.ParseString(context.Background(), tt.args.action) if tt.wantErr { require.Error(t, err, "Wanted error, got none") require.Empty(t, got, "Wanted empty string, got %v", got) @@ -355,8 +355,8 @@ services: ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - r := New(&config.Config{}) - modified, newContent, err := r.ParseContainerImagesInFile(ctx, strings.NewReader(tt.before), nil) + r := NewImageReplacer(&config.Config{}) + modified, newContent, err := r.ParseFile(ctx, strings.NewReader(tt.before)) if tt.modified { assert.True(t, modified) @@ -435,8 +435,8 @@ jobs: ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - r := New(&config.Config{}).WithGitHubClient(os.Getenv(cli.GitHubTokenEnvKey)) - modified, newContent, err := r.ParseGitHubActionsInFile(ctx, strings.NewReader(tt.before), nil) + r := NewActionsReplacer(&config.Config{}).WithGitHubClient(os.Getenv(cli.GitHubTokenEnvKey)) + modified, newContent, err := r.ParseFile(ctx, strings.NewReader(tt.before)) if tt.modified { assert.True(t, modified) From e5da0cf831676262889bc23519c4cd872611b4d4 Mon Sep 17 00:00:00 2001 From: Radoslav Dimitrov Date: Wed, 29 May 2024 01:05:17 +0300 Subject: [PATCH 07/16] Add test case for custom regex and tidy up the package layout Signed-off-by: Radoslav Dimitrov --- cmd/actions/actions.go | 8 +-- cmd/actions/list.go | 4 +- cmd/image/image.go | 8 +-- cmd/image/list.go | 4 +- cmd/root.go | 2 +- pkg/errors/errors.go | 24 --------- pkg/interfaces/interfaces.go | 10 +++- pkg/replacer/actions/actions.go | 15 +++--- pkg/replacer/actions/utils.go | 2 +- pkg/replacer/actions/utils_test.go | 4 +- pkg/replacer/image/image.go | 7 ++- pkg/replacer/image/utils.go | 5 +- pkg/replacer/replacer.go | 37 +++++++------- pkg/replacer/replacer_test.go | 71 +++++++++++++++++++++----- pkg/{ => utils}/config/config.go | 0 pkg/{ => utils}/ghrest/ghrest.go | 0 {internal => pkg/utils}/store/cache.go | 0 17 files changed, 111 insertions(+), 90 deletions(-) delete mode 100644 pkg/errors/errors.go rename pkg/{ => utils}/config/config.go (100%) rename pkg/{ => utils}/ghrest/ghrest.go (100%) rename {internal => pkg/utils}/store/cache.go (100%) diff --git a/cmd/actions/actions.go b/cmd/actions/actions.go index 5a3b8e5..2833f4d 100644 --- a/cmd/actions/actions.go +++ b/cmd/actions/actions.go @@ -25,9 +25,9 @@ import ( "github.com/spf13/cobra" "github.com/stacklok/frizbee/internal/cli" - "github.com/stacklok/frizbee/pkg/config" - ferrors "github.com/stacklok/frizbee/pkg/errors" + "github.com/stacklok/frizbee/pkg/interfaces" "github.com/stacklok/frizbee/pkg/replacer" + "github.com/stacklok/frizbee/pkg/utils/config" ) // CmdGHActions represents the actions command @@ -82,7 +82,7 @@ func replaceCmd(cmd *cobra.Command, args []string) error { } // Create a new replacer - r := replacer.NewActionsReplacer(cfg). + r := replacer.NewGitHubActionsReplacer(cfg). WithUserRegex(cliFlags.Regex). WithGitHubClient(os.Getenv(cli.GitHubTokenEnvKey)) @@ -99,7 +99,7 @@ func replaceCmd(cmd *cobra.Command, args []string) error { // Replace the passed reference res, err := r.ParseString(cmd.Context(), pathOrRef) if err != nil { - if errors.Is(err, ferrors.ErrReferenceSkipped) { + if errors.Is(err, interfaces.ErrReferenceSkipped) { fmt.Fprintln(cmd.OutOrStdout(), pathOrRef) // nolint:errcheck return nil } diff --git a/cmd/actions/list.go b/cmd/actions/list.go index 2098158..d384d39 100644 --- a/cmd/actions/list.go +++ b/cmd/actions/list.go @@ -27,8 +27,8 @@ import ( "github.com/spf13/cobra" "github.com/stacklok/frizbee/internal/cli" - "github.com/stacklok/frizbee/pkg/config" "github.com/stacklok/frizbee/pkg/replacer" + "github.com/stacklok/frizbee/pkg/utils/config" ) // CmdList represents the one sub-command @@ -76,7 +76,7 @@ func list(cmd *cobra.Command, args []string) error { } // Create a new replacer - r := replacer.NewActionsReplacer(cfg). + r := replacer.NewGitHubActionsReplacer(cfg). WithUserRegex(cliFlags.Regex). WithGitHubClient(os.Getenv(cli.GitHubTokenEnvKey)) diff --git a/cmd/image/image.go b/cmd/image/image.go index 519963a..7e4bd9e 100644 --- a/cmd/image/image.go +++ b/cmd/image/image.go @@ -24,9 +24,9 @@ import ( "github.com/spf13/cobra" "github.com/stacklok/frizbee/internal/cli" - "github.com/stacklok/frizbee/pkg/config" - ferrors "github.com/stacklok/frizbee/pkg/errors" + "github.com/stacklok/frizbee/pkg/interfaces" "github.com/stacklok/frizbee/pkg/replacer" + "github.com/stacklok/frizbee/pkg/utils/config" ) // CmdContainerImage represents the containers command @@ -72,7 +72,7 @@ func replaceCmd(cmd *cobra.Command, args []string) error { } // Create a new replacer - r := replacer.NewImageReplacer(cfg). + r := replacer.NewContainerImagesReplacer(cfg). WithUserRegex(cliFlags.Regex) if cli.IsPath(args[0]) { @@ -88,7 +88,7 @@ func replaceCmd(cmd *cobra.Command, args []string) error { // Replace the passed reference res, err := r.ParseString(cmd.Context(), args[0]) if err != nil { - if errors.Is(err, ferrors.ErrReferenceSkipped) { + if errors.Is(err, interfaces.ErrReferenceSkipped) { fmt.Fprintln(cmd.OutOrStdout(), args[0]) // nolint:errcheck return nil } diff --git a/cmd/image/list.go b/cmd/image/list.go index c04bccd..474dfb8 100644 --- a/cmd/image/list.go +++ b/cmd/image/list.go @@ -26,8 +26,8 @@ import ( "github.com/spf13/cobra" "github.com/stacklok/frizbee/internal/cli" - "github.com/stacklok/frizbee/pkg/config" "github.com/stacklok/frizbee/pkg/replacer" + "github.com/stacklok/frizbee/pkg/utils/config" ) // CmdList represents the one sub-command @@ -69,7 +69,7 @@ func list(cmd *cobra.Command, args []string) error { } // Create a new replacer - r := replacer.NewImageReplacer(cfg). + r := replacer.NewContainerImagesReplacer(cfg). WithUserRegex(cliFlags.Regex) // List the references in the directory diff --git a/cmd/root.go b/cmd/root.go index 78f790c..926475f 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -26,7 +26,7 @@ import ( "github.com/stacklok/frizbee/cmd/actions" "github.com/stacklok/frizbee/cmd/image" "github.com/stacklok/frizbee/cmd/version" - "github.com/stacklok/frizbee/pkg/config" + "github.com/stacklok/frizbee/pkg/utils/config" ) // Execute runs the root command. diff --git a/pkg/errors/errors.go b/pkg/errors/errors.go deleted file mode 100644 index 475bdf5..0000000 --- a/pkg/errors/errors.go +++ /dev/null @@ -1,24 +0,0 @@ -// -// Copyright 2024 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package errors provides error values, constants and functions. -package errors - -import "errors" - -var ( - // ErrReferenceSkipped is returned when the reference is skipped. - ErrReferenceSkipped = errors.New("reference skipped") -) diff --git a/pkg/interfaces/interfaces.go b/pkg/interfaces/interfaces.go index 03a59e3..4006451 100644 --- a/pkg/interfaces/interfaces.go +++ b/pkg/interfaces/interfaces.go @@ -18,10 +18,16 @@ package interfaces import ( "context" + "errors" "net/http" - "github.com/stacklok/frizbee/internal/store" - "github.com/stacklok/frizbee/pkg/config" + "github.com/stacklok/frizbee/pkg/utils/config" + "github.com/stacklok/frizbee/pkg/utils/store" +) + +var ( + // ErrReferenceSkipped is returned when the reference is skipped. + ErrReferenceSkipped = errors.New("reference skipped") ) // EntityRef represents an action reference. diff --git a/pkg/replacer/actions/actions.go b/pkg/replacer/actions/actions.go index bf69428..9ed8e7c 100644 --- a/pkg/replacer/actions/actions.go +++ b/pkg/replacer/actions/actions.go @@ -21,11 +21,10 @@ import ( "fmt" "strings" - "github.com/stacklok/frizbee/internal/store" - "github.com/stacklok/frizbee/pkg/config" - ferrors "github.com/stacklok/frizbee/pkg/errors" "github.com/stacklok/frizbee/pkg/interfaces" "github.com/stacklok/frizbee/pkg/replacer/image" + "github.com/stacklok/frizbee/pkg/utils/config" + "github.com/stacklok/frizbee/pkg/utils/store" ) const ( @@ -110,7 +109,7 @@ func (p *Parser) replaceAction( // If the value is a local path or should be excluded, skip it if isLocal(matchedLine) || shouldExclude(&cfg.GHActions, matchedLine) { - return nil, fmt.Errorf("%w: %s", ferrors.ErrReferenceSkipped, matchedLine) + return nil, fmt.Errorf("%w: %s", interfaces.ErrReferenceSkipped, matchedLine) } // Parse the action reference @@ -121,7 +120,7 @@ func (p *Parser) replaceAction( // Check if the parsed reference should be excluded if shouldExclude(&cfg.GHActions, act) { - return nil, fmt.Errorf("%w: %s", ferrors.ErrReferenceSkipped, matchedLine) + return nil, fmt.Errorf("%w: %s", interfaces.ErrReferenceSkipped, matchedLine) } var sum string @@ -149,7 +148,7 @@ func (p *Parser) replaceAction( // Compare the digest with the reference and return the original reference if they already match if ref == sum { - return nil, fmt.Errorf("image already referenced by digest: %s %w", matchedLine, ferrors.ErrReferenceSkipped) + return nil, fmt.Errorf("image already referenced by digest: %s %w", matchedLine, interfaces.ErrReferenceSkipped) } return &interfaces.EntityRef{ @@ -171,7 +170,7 @@ func (p *Parser) replaceDocker( // If the value is a local path or should be excluded, skip it if isLocal(trimmedRef) || shouldExclude(&cfg.GHActions, trimmedRef) { - return nil, fmt.Errorf("%w: %s", ferrors.ErrReferenceSkipped, matchedLine) + return nil, fmt.Errorf("%w: %s", interfaces.ErrReferenceSkipped, matchedLine) } // Get the digest of the docker:// image reference @@ -182,7 +181,7 @@ func (p *Parser) replaceDocker( // Check if the parsed reference should be excluded if shouldExclude(&cfg.GHActions, actionRef.Name) { - return nil, fmt.Errorf("%w: %s", ferrors.ErrReferenceSkipped, matchedLine) + return nil, fmt.Errorf("%w: %s", interfaces.ErrReferenceSkipped, matchedLine) } // Add back the docker prefix diff --git a/pkg/replacer/actions/utils.go b/pkg/replacer/actions/utils.go index d299fbf..69b7eec 100644 --- a/pkg/replacer/actions/utils.go +++ b/pkg/replacer/actions/utils.go @@ -26,8 +26,8 @@ import ( "github.com/google/go-github/v61/github" - "github.com/stacklok/frizbee/pkg/config" "github.com/stacklok/frizbee/pkg/interfaces" + "github.com/stacklok/frizbee/pkg/utils/config" ) var ( diff --git a/pkg/replacer/actions/utils_test.go b/pkg/replacer/actions/utils_test.go index fa0065a..06e2949 100644 --- a/pkg/replacer/actions/utils_test.go +++ b/pkg/replacer/actions/utils_test.go @@ -7,7 +7,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/stacklok/frizbee/pkg/ghrest" + "github.com/stacklok/frizbee/pkg/utils/ghrest" ) func TestGetChecksum(t *testing.T) { @@ -112,7 +112,7 @@ func TestGetChecksum(t *testing.T) { action: "bufbuild/buf-setup-action", ref: "v1", }, - want: "f0475db2e1b1b2e8d121066b59dfb7f7bd6c4dc4", + want: "dde0b9351db90fbf78e345f41a57de8514bf1091", }, { name: "anchore/sbom-action/download-syft with a sub-action works", diff --git a/pkg/replacer/image/image.go b/pkg/replacer/image/image.go index b049666..e4c42e3 100644 --- a/pkg/replacer/image/image.go +++ b/pkg/replacer/image/image.go @@ -21,10 +21,9 @@ import ( "fmt" "strings" - "github.com/stacklok/frizbee/internal/store" - "github.com/stacklok/frizbee/pkg/config" - ferrors "github.com/stacklok/frizbee/pkg/errors" "github.com/stacklok/frizbee/pkg/interfaces" + "github.com/stacklok/frizbee/pkg/utils/config" + "github.com/stacklok/frizbee/pkg/utils/store" ) const ( @@ -81,7 +80,7 @@ func (p *Parser) Replace( matchedLine = strings.TrimPrefix(matchedLine, prefixFROM) // Check if the image reference should be excluded, i.e. scratch if shouldExclude(matchedLine) { - return nil, fmt.Errorf("image reference %s should be excluded - %w", matchedLine, ferrors.ErrReferenceSkipped) + return nil, fmt.Errorf("image reference %s should be excluded - %w", matchedLine, interfaces.ErrReferenceSkipped) } hasFROMPrefix = true } else if strings.HasPrefix(matchedLine, prefixImage) { diff --git a/pkg/replacer/image/utils.go b/pkg/replacer/image/utils.go index 68ee149..b69dd80 100644 --- a/pkg/replacer/image/utils.go +++ b/pkg/replacer/image/utils.go @@ -27,9 +27,8 @@ import ( "github.com/google/go-containerregistry/pkg/v1/remote" "github.com/stacklok/frizbee/internal/cli" - "github.com/stacklok/frizbee/internal/store" - ferrors "github.com/stacklok/frizbee/pkg/errors" "github.com/stacklok/frizbee/pkg/interfaces" + "github.com/stacklok/frizbee/pkg/utils/store" ) // GetImageDigestFromRef returns the digest of a container image reference @@ -82,7 +81,7 @@ func GetImageDigestFromRef(ctx context.Context, imageRef, platform string, cache // Compare the digest with the reference and return the original reference if they already match if digest == ref.Identifier() { - return nil, fmt.Errorf("image already referenced by digest: %s %w", imageRef, ferrors.ErrReferenceSkipped) + return nil, fmt.Errorf("image already referenced by digest: %s %w", imageRef, interfaces.ErrReferenceSkipped) } return &interfaces.EntityRef{ diff --git a/pkg/replacer/replacer.go b/pkg/replacer/replacer.go index 67b9cc2..b450472 100644 --- a/pkg/replacer/replacer.go +++ b/pkg/replacer/replacer.go @@ -33,11 +33,11 @@ import ( "golang.org/x/sync/errgroup" "github.com/stacklok/frizbee/internal/traverse" - "github.com/stacklok/frizbee/pkg/config" - "github.com/stacklok/frizbee/pkg/ghrest" "github.com/stacklok/frizbee/pkg/interfaces" "github.com/stacklok/frizbee/pkg/replacer/actions" "github.com/stacklok/frizbee/pkg/replacer/image" + "github.com/stacklok/frizbee/pkg/utils/config" + "github.com/stacklok/frizbee/pkg/utils/ghrest" ) // ReplaceResult holds a slice of all processed files along with a map of their modified content @@ -52,23 +52,23 @@ type ListResult struct { Entities []interfaces.EntityRef } -// Replacer replaces container image references in YAML files +// Replacer is an object with methods to replace references with digests type Replacer struct { parser interfaces.Parser rest interfaces.REST cfg config.Config } -// NewActionsReplacer creates a new replacer for GitHub actions -func NewActionsReplacer(cfg *config.Config) *Replacer { +// NewGitHubActionsReplacer creates a new replacer for GitHub actions +func NewGitHubActionsReplacer(cfg *config.Config) *Replacer { return &Replacer{ cfg: *cfg, parser: actions.New(), } } -// NewImageReplacer creates a new replacer for container images -func NewImageReplacer(cfg *config.Config) *Replacer { +// NewContainerImagesReplacer creates a new replacer for container images +func NewContainerImagesReplacer(cfg *config.Config) *Replacer { return &Replacer{ cfg: *cfg, parser: image.New(), @@ -118,10 +118,15 @@ func (r *Replacer) ParseFile(ctx context.Context, f io.Reader) (bool, string, er // ListPath lists all entity references in the provided directory func (r *Replacer) ListPath(dir string) (*ListResult, error) { - return listReferences(dir, r.parser) + return listReferencesInFS(r.parser, osfs.New(filepath.Dir(dir), osfs.WithBoundOS()), filepath.Base(dir)) } -// ListInFile lists all entity in the provided file +// ListPathInFS lists all entity references in the provided file system +func (r *Replacer) ListPathInFS(bfs billy.Filesystem, base string) (*ListResult, error) { + return listReferencesInFS(r.parser, bfs, base) +} + +// ListInFile lists all entities in the provided file func (r *Replacer) ListInFile(f io.Reader) (*ListResult, error) { found, err := listReferencesInFile(f, r.parser) if err != nil { @@ -197,14 +202,10 @@ func parsePathInFS( return &res, nil } -func listReferences(dir string, parser interfaces.Parser) (*ListResult, error) { +func listReferencesInFS(parser interfaces.Parser, bfs billy.Filesystem, base string) (*ListResult, error) { var eg errgroup.Group var mu sync.Mutex - basedir := filepath.Dir(dir) - base := filepath.Base(dir) - bfs := osfs.New(basedir, osfs.WithBoundOS()) - res := ListResult{ Processed: make([]string, 0), Entities: make([]interfaces.EntityRef, 0), @@ -219,13 +220,12 @@ func listReferences(dir string, parser interfaces.Parser) (*ListResult, error) { if err != nil { return fmt.Errorf("failed to open file %s: %w", path, err) } - // nolint:errcheck // ignore error - defer file.Close() + defer file.Close() // nolint:errcheck // Parse the content of the file and list the matching references foundRefs, err := listReferencesInFile(file, parser) if err != nil { - return fmt.Errorf("failed to listReferences references in %s: %w", path, err) + return fmt.Errorf("failed to list references in %s: %w", path, err) } // Store the file name to the processed batch @@ -312,8 +312,7 @@ func parseAndReplaceReferencesInFile( return modified, contentBuilder.String(), nil } -// listReferencesInFile takes the given file reader and returns a map of all -// references, action or images, it found. +// listReferencesInFile takes the given file reader and returns a map of all references, action or images it finds func listReferencesInFile( f io.Reader, parser interfaces.Parser, diff --git a/pkg/replacer/replacer_test.go b/pkg/replacer/replacer_test.go index 68d9519..4f42277 100644 --- a/pkg/replacer/replacer_test.go +++ b/pkg/replacer/replacer_test.go @@ -26,10 +26,10 @@ import ( "github.com/stretchr/testify/require" "github.com/stacklok/frizbee/internal/cli" - "github.com/stacklok/frizbee/pkg/config" "github.com/stacklok/frizbee/pkg/interfaces" "github.com/stacklok/frizbee/pkg/replacer/actions" "github.com/stacklok/frizbee/pkg/replacer/image" + "github.com/stacklok/frizbee/pkg/utils/config" ) func TestReplacer_ParseContainerImageString(t *testing.T) { @@ -127,7 +127,7 @@ func TestReplacer_ParseContainerImageString(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - r := NewImageReplacer(&config.Config{}) + r := NewContainerImagesReplacer(&config.Config{}) got, err := r.ParseString(ctx, tt.args.refstr) if tt.wantErr { assert.Error(t, err) @@ -270,7 +270,7 @@ func TestReplacer_ParseGitHubActionString(t *testing.T) { }, want: &interfaces.EntityRef{ Name: "bufbuild/buf-setup-action", - Ref: "f0475db2e1b1b2e8d121066b59dfb7f7bd6c4dc4", + Ref: "dde0b9351db90fbf78e345f41a57de8514bf1091", Type: actions.ReferenceType, Tag: "v1", Prefix: "", @@ -296,7 +296,7 @@ func TestReplacer_ParseGitHubActionString(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - r := NewActionsReplacer(&config.Config{}).WithGitHubClient(os.Getenv("GITHUB_TOKEN")) + r := NewGitHubActionsReplacer(&config.Config{}).WithGitHubClient(os.Getenv("GITHUB_TOKEN")) got, err := r.ParseString(context.Background(), tt.args.action) if tt.wantErr { require.Error(t, err, "Wanted error, got none") @@ -355,7 +355,7 @@ services: ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - r := NewImageReplacer(&config.Config{}) + r := NewContainerImagesReplacer(&config.Config{}) modified, newContent, err := r.ParseFile(ctx, strings.NewReader(tt.before)) if tt.modified { @@ -383,11 +383,13 @@ func TestReplacer_ParseGitHubActionsInFile(t *testing.T) { t.Parallel() testCases := []struct { - name string - before string - expected string - modified bool - wantErr bool + name string + before string + expected string + regex string + modified bool + wantErr bool + useCustomRegex bool }{ { name: "Replace image reference", @@ -422,6 +424,44 @@ jobs: args: src/*.md `, modified: true, + wantErr: false, + }, + { + name: "Fail with custom regex", + before: ` +name: Linter +on: pull_request +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: ./minder/server.yml # this should not be replaced + - uses: actions/checkout@v2 + - uses: xt0rted/markdownlint-problem-matcher@v1 + - name: "Run Markdown linter" + uses: docker://avtodev/markdown-lint:v1 + with: + args: src/*.md +`, + expected: ` +name: Linter +on: pull_request +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: ./minder/server.yml # this should not be replaced + - uses: actions/checkout@v2 + - uses: xt0rted/markdownlint-problem-matcher@v1 + - name: "Run Markdown linter" + uses: docker://avtodev/markdown-lint:v1 + with: + args: src/*.md +`, + modified: false, + wantErr: false, + regex: "invalid-regexp", + useCustomRegex: true, }, // Add more test cases as needed } @@ -435,20 +475,23 @@ jobs: ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) defer cancel() - r := NewActionsReplacer(&config.Config{}).WithGitHubClient(os.Getenv(cli.GitHubTokenEnvKey)) + r := NewGitHubActionsReplacer(&config.Config{}).WithGitHubClient(os.Getenv(cli.GitHubTokenEnvKey)) + if tt.useCustomRegex { + r = r.WithUserRegex(tt.regex) + } modified, newContent, err := r.ParseFile(ctx, strings.NewReader(tt.before)) if tt.modified { assert.True(t, modified) - assert.NotEmpty(t, newContent) + assert.Equal(t, tt.expected, newContent) } else { assert.False(t, modified) - assert.Empty(t, newContent) + assert.Equal(t, tt.before, newContent) } if tt.wantErr { assert.False(t, modified) - assert.Empty(t, newContent) + assert.Equal(t, tt.before, newContent) assert.Error(t, err) return } diff --git a/pkg/config/config.go b/pkg/utils/config/config.go similarity index 100% rename from pkg/config/config.go rename to pkg/utils/config/config.go diff --git a/pkg/ghrest/ghrest.go b/pkg/utils/ghrest/ghrest.go similarity index 100% rename from pkg/ghrest/ghrest.go rename to pkg/utils/ghrest/ghrest.go diff --git a/internal/store/cache.go b/pkg/utils/store/cache.go similarity index 100% rename from internal/store/cache.go rename to pkg/utils/store/cache.go From 5f07ebe5a4aa917c7adedf4e98ddb9e4a5cb16b6 Mon Sep 17 00:00:00 2001 From: Radoslav Dimitrov Date: Wed, 29 May 2024 01:52:06 +0300 Subject: [PATCH 08/16] Update README.md --- README.md | 73 +++++++++++++++++++++++++++++++++++++++++++------------ 1 file changed, 57 insertions(+), 16 deletions(-) diff --git a/README.md b/README.md index f8575bb..80107af 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,10 @@ -# Frizbee +![image](https://github.com/stacklok/frizbee/assets/16540482/35034046-d962-475d-b8e2-67b7625f2a60) +--- [![Coverage Status](https://coveralls.io/repos/github/stacklok/frizbee/badge.svg?branch=main)](https://coveralls.io/github/stacklok/frizbee?branch=main) | [![License: Apache 2.0](https://img.shields.io/badge/License-Apache2.0-brightgreen.svg)](https://opensource.org/licenses/Apache-2.0) | [![](https://dcbadge.vercel.app/api/server/RkzVuTp3WK?logo=discord&label=Discord&color=5865&style=flat)](https://discord.gg/RkzVuTp3WK) --- +# Frizbee Frizbee is a tool you may throw a tag at and it comes back with a checksum. @@ -14,12 +16,13 @@ It also includes a set of libraries for working with tags and checksums. ## Table of Contents - [Installation](#installation) -- [Usage](#usage) +- [Usage - CLI](#usage---cli) + - [GitHub Actions](#github-actions) + - [Container Images](#container-images) +- [Usage - Library](#usage---library) - [GitHub Actions](#github-actions) - [Container Images](#container-images) - [Configuration](#configuration) -- [Commands](#commands) -- [Autocompletion](#autocompletion) - [Contributing](#contributing) - [License](#license) @@ -47,10 +50,10 @@ Frizbee can be used to generate checksums for GitHub Actions. This is useful for verifying that the contents of a GitHub Action have not changed. To quickly replace the GitHub Action references for your project, you can use -the `action` command: +the `actions` command: ```bash -frizbee action path/to/your/repo/.github/workflows/ +frizbee actions path/to/your/repo/.github/workflows/ ``` This will write all the replacements to the files in the directory provided. @@ -68,7 +71,7 @@ If you want to generate the replacement for a single GitHub Action, you can use same command: ```bash -frizbee action metal-toolbox/container-push/.github/workflows/container-push.yml@main +frizbee actions metal-toolbox/container-push/.github/workflows/container-push.yml@main ``` This is useful if you're developing and want to quickly test the replacement. @@ -104,32 +107,70 @@ the library: ```go // Create a new replacer -r := replacer.New(cfg) +r := replacer.NewGitHubActionsReplacer(cfg) ... // Parse a single GitHub Action reference -ret, err := r.ParseGitHubActionString(ctx, ghActionRef) +ret, err := r.ParseString(ctx, ghActionRef) ... // Parse all GitHub Actions workflow yaml files in a given directory -res, err := r.ParseGitHubActionsInPath(ctx, dir) +res, err := r.ParsePath(ctx, dir) +... +// Parse and replace all GitHub Actions references in the provided file system +res, err := r.ParsePathInFS(ctx, bfs, base) ... // Parse a single yaml file referencing GitHub Actions -res, err := r.ParseGitHubActionsInFile(ctx, fileHandler) +res, err := r.ParseFile(ctx, fileHandler) +... +// List all GitHub Actions referenced in the given directory +res, err := r.ListPath(dir) +... +// List all GitHub Actions referenced in the provided file system +res, err := r.ListPathInFS(bfs, base) +... +// List all GitHub Actions referenced in the provided file +res, err := r.ListFile(fileHandler) ``` ### Container images ```go // Create a new replacer -r := replacer.New(cfg) +r := replacer.NewContainerImagesReplacer(cfg) ... // Parse a single container image reference -ret, err := r.ParseContainerImageString(ctx, imageRef) +ret, err := r.ParseString(ctx, ghActionRef) ... -// Parse all yaml files referencing container images in a given directory (k8s, docker-compose, Dockerfile, etc) -res, err := r.ParseContainerImagesInPath(ctx, dir) +// Parse all files containing container image references in a given directory +res, err := r.ParsePath(ctx, dir) +... +// Parse and replace all container image references in the provided file system +res, err := r.ParsePathInFS(ctx, bfs, base) ... // Parse a single yaml file referencing container images -res, err := r.ParseContainerImagesInFile(ctx, fileHandler) +res, err := r.ParseFile(ctx, fileHandler) +... +// List all container images referenced in the given directory +res, err := r.ListPath(dir) +... +// List all container images referenced in the provided file system +res, err := r.ListPathInFS(bfs, base) +... +// List all container images referenced in the provided file +res, err := r.ListFile(fileHandler) +``` + +## Configuration + +Frizbee can be configured by setting up a `.frizbee.yml` file. +You can configure Frizbee to skip processing certain actions, i.e. + +```yml +ghactions: + exclude: + # Exclude the SLSA GitHub Generator workflow. + # See https://github.com/slsa-framework/slsa-github-generator/issues/2993 + - slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml + ``` ## Contributing From b5f98fb57eb004e0bcec5057ab5723e481e93f37 Mon Sep 17 00:00:00 2001 From: Radoslav Dimitrov Date: Wed, 29 May 2024 01:59:39 +0300 Subject: [PATCH 09/16] Tidy up the image and actions packages Signed-off-by: Radoslav Dimitrov --- pkg/replacer/actions/actions.go | 139 ++++++++++++++- .../{utils_test.go => actions_test.go} | 0 pkg/replacer/actions/utils.go | 161 ------------------ pkg/replacer/image/image.go | 72 ++++++++ .../image/{utils_test.go => image_test.go} | 0 pkg/replacer/image/utils.go | 97 ----------- 6 files changed, 209 insertions(+), 260 deletions(-) rename pkg/replacer/actions/{utils_test.go => actions_test.go} (100%) delete mode 100644 pkg/replacer/actions/utils.go rename pkg/replacer/image/{utils_test.go => image_test.go} (100%) delete mode 100644 pkg/replacer/image/utils.go diff --git a/pkg/replacer/actions/actions.go b/pkg/replacer/actions/actions.go index 9ed8e7c..6f63a01 100644 --- a/pkg/replacer/actions/actions.go +++ b/pkg/replacer/actions/actions.go @@ -18,9 +18,15 @@ package actions import ( "context" + "encoding/json" + "errors" "fmt" + "net/http" + "net/url" "strings" + "github.com/google/go-github/v61/github" + "github.com/stacklok/frizbee/pkg/interfaces" "github.com/stacklok/frizbee/pkg/replacer/image" "github.com/stacklok/frizbee/pkg/utils/config" @@ -28,14 +34,21 @@ import ( ) const ( + prefixUses = "uses: " + prefixDocker = "docker://" // GitHubActionsRegex is regular expression pattern to match GitHub Actions usage GitHubActionsRegex = `uses:\s*[^\s]+/[^\s]+@[^\s]+|uses:\s*docker://[^\s]+:[^\s]+` - prefixUses = "uses: " - prefixDocker = "docker://" // ReferenceType is the type of the reference ReferenceType = "action" ) +var ( + // ErrInvalidAction is returned when parsing the action fails. + ErrInvalidAction = errors.New("invalid action") + // ErrInvalidActionReference is returned when parsing the action reference fails. + ErrInvalidActionReference = errors.New("action reference is not a tag nor branch") +) + // Parser is a struct to replace action references with digests type Parser struct { regex string @@ -216,3 +229,125 @@ func (_ *Parser) ConvertToEntityRef(reference string) (*interfaces.EntityRef, er Type: refType, }, nil } + +// isLocal returns true if the input is a local path. +func isLocal(input string) bool { + return strings.HasPrefix(input, "./") || strings.HasPrefix(input, "../") +} + +func shouldExclude(cfg *config.GHActions, input string) bool { + for _, e := range cfg.Exclude { + if e == input { + return true + } + } + return false +} + +// ParseActionReference parses an action reference into action and reference. +func ParseActionReference(input string) (action string, reference string, err error) { + frags := strings.Split(input, "@") + if len(frags) != 2 { + return "", "", fmt.Errorf("invalid action reference: %s", input) + } + + return frags[0], frags[1], nil +} + +// GetChecksum returns the checksum for a given action and tag. +func GetChecksum(ctx context.Context, restIf interfaces.REST, action, ref string) (string, error) { + owner, repo, err := parseActionFragments(action) + if err != nil { + return "", err + } + + // Check if we're using a checksum + if isChecksum(ref) { + return ref, nil + } + + res, err := getCheckSumForTag(ctx, restIf, owner, repo, ref) + if err != nil { + return "", fmt.Errorf("failed to get checksum for tag: %w", err) + } else if res != "" { + return res, nil + } + + // check branch + res, err = getCheckSumForBranch(ctx, restIf, owner, repo, ref) + if err != nil { + return "", fmt.Errorf("failed to get checksum for branch: %w", err) + } else if res != "" { + return res, nil + } + + return "", ErrInvalidActionReference +} + +func parseActionFragments(action string) (owner string, repo string, err error) { + frags := strings.Split(action, "/") + + // if we have more than 2 fragments, we're probably dealing with + // sub-actions, so we take the first two fragments as the owner and repo + if len(frags) < 2 { + return "", "", fmt.Errorf("%w: '%s' reference is incorrect", ErrInvalidAction, action) + } + + return frags[0], frags[1], nil +} + +// isChecksum returns true if the input is a checksum. +func isChecksum(ref string) bool { + return len(ref) == 40 +} + +func getCheckSumForTag(ctx context.Context, restIf interfaces.REST, owner, repo, tag string) (string, error) { + path, err := url.JoinPath("repos", owner, repo, "git", "refs", "tags", tag) + if err != nil { + return "", fmt.Errorf("failed to join path: %w", err) + } + + return doGetReference(ctx, restIf, path) +} + +func getCheckSumForBranch(ctx context.Context, restIf interfaces.REST, owner, repo, branch string) (string, error) { + path, err := url.JoinPath("repos", owner, repo, "git", "refs", "heads", branch) + if err != nil { + return "", fmt.Errorf("failed to join path: %w", err) + } + + return doGetReference(ctx, restIf, path) +} + +func doGetReference(ctx context.Context, restIf interfaces.REST, path string) (string, error) { + req, err := restIf.NewRequest(http.MethodGet, path, nil) + if err != nil { + return "", fmt.Errorf("cannot create REST request: %w", err) + } + + resp, err := restIf.Do(ctx, req) + + if resp != nil { + defer func() { + _ = resp.Body.Close() + }() + } + + if err != nil && resp.StatusCode != http.StatusNotFound { + return "", fmt.Errorf("failed to do API request: %w", err) + } else if resp.StatusCode == http.StatusNotFound { + // No error, but no tag found + return "", nil + } + + var t github.Reference + err = json.NewDecoder(resp.Body).Decode(&t) + if err != nil && strings.Contains(err.Error(), "cannot unmarshal array into Go value of type") { + // This is a branch, not a tag + return "", nil + } else if err != nil { + return "", fmt.Errorf("canont decode response: %w", err) + } + + return t.GetObject().GetSHA(), nil +} diff --git a/pkg/replacer/actions/utils_test.go b/pkg/replacer/actions/actions_test.go similarity index 100% rename from pkg/replacer/actions/utils_test.go rename to pkg/replacer/actions/actions_test.go diff --git a/pkg/replacer/actions/utils.go b/pkg/replacer/actions/utils.go deleted file mode 100644 index 69b7eec..0000000 --- a/pkg/replacer/actions/utils.go +++ /dev/null @@ -1,161 +0,0 @@ -// -// Copyright 2024 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package actions - -import ( - "context" - "encoding/json" - "errors" - "fmt" - "net/http" - "net/url" - "strings" - - "github.com/google/go-github/v61/github" - - "github.com/stacklok/frizbee/pkg/interfaces" - "github.com/stacklok/frizbee/pkg/utils/config" -) - -var ( - // ErrInvalidAction is returned when parsing the action fails. - ErrInvalidAction = errors.New("invalid action") - - // ErrInvalidActionReference is returned when parsing the action reference fails. - ErrInvalidActionReference = errors.New("action reference is not a tag nor branch") -) - -// isLocal returns true if the input is a local path. -func isLocal(input string) bool { - return strings.HasPrefix(input, "./") || strings.HasPrefix(input, "../") -} - -func shouldExclude(cfg *config.GHActions, input string) bool { - for _, e := range cfg.Exclude { - if e == input { - return true - } - } - return false -} - -// ParseActionReference parses an action reference into action and reference. -func ParseActionReference(input string) (action string, reference string, err error) { - frags := strings.Split(input, "@") - if len(frags) != 2 { - return "", "", fmt.Errorf("invalid action reference: %s", input) - } - - return frags[0], frags[1], nil -} - -// GetChecksum returns the checksum for a given action and tag. -func GetChecksum(ctx context.Context, restIf interfaces.REST, action, ref string) (string, error) { - owner, repo, err := parseActionFragments(action) - if err != nil { - return "", err - } - - // Check if we're using a checksum - if isChecksum(ref) { - return ref, nil - } - - res, err := getCheckSumForTag(ctx, restIf, owner, repo, ref) - if err != nil { - return "", fmt.Errorf("failed to get checksum for tag: %w", err) - } else if res != "" { - return res, nil - } - - // check branch - res, err = getCheckSumForBranch(ctx, restIf, owner, repo, ref) - if err != nil { - return "", fmt.Errorf("failed to get checksum for branch: %w", err) - } else if res != "" { - return res, nil - } - - return "", ErrInvalidActionReference -} - -func parseActionFragments(action string) (owner string, repo string, err error) { - frags := strings.Split(action, "/") - - // if we have more than 2 fragments, we're probably dealing with - // sub-actions, so we take the first two fragments as the owner and repo - if len(frags) < 2 { - return "", "", fmt.Errorf("%w: '%s' reference is incorrect", ErrInvalidAction, action) - } - - return frags[0], frags[1], nil -} - -// isChecksum returns true if the input is a checksum. -func isChecksum(ref string) bool { - return len(ref) == 40 -} - -func getCheckSumForTag(ctx context.Context, restIf interfaces.REST, owner, repo, tag string) (string, error) { - path, err := url.JoinPath("repos", owner, repo, "git", "refs", "tags", tag) - if err != nil { - return "", fmt.Errorf("failed to join path: %w", err) - } - - return doGetReference(ctx, restIf, path) -} - -func getCheckSumForBranch(ctx context.Context, restIf interfaces.REST, owner, repo, branch string) (string, error) { - path, err := url.JoinPath("repos", owner, repo, "git", "refs", "heads", branch) - if err != nil { - return "", fmt.Errorf("failed to join path: %w", err) - } - - return doGetReference(ctx, restIf, path) -} - -func doGetReference(ctx context.Context, restIf interfaces.REST, path string) (string, error) { - req, err := restIf.NewRequest(http.MethodGet, path, nil) - if err != nil { - return "", fmt.Errorf("cannot create REST request: %w", err) - } - - resp, err := restIf.Do(ctx, req) - - if resp != nil { - defer func() { - _ = resp.Body.Close() - }() - } - - if err != nil && resp.StatusCode != http.StatusNotFound { - return "", fmt.Errorf("failed to do API request: %w", err) - } else if resp.StatusCode == http.StatusNotFound { - // No error, but no tag found - return "", nil - } - - var t github.Reference - err = json.NewDecoder(resp.Body).Decode(&t) - if err != nil && strings.Contains(err.Error(), "cannot unmarshal array into Go value of type") { - // This is a branch, not a tag - return "", nil - } else if err != nil { - return "", fmt.Errorf("canont decode response: %w", err) - } - - return t.GetObject().GetSHA(), nil -} diff --git a/pkg/replacer/image/image.go b/pkg/replacer/image/image.go index e4c42e3..853f8fe 100644 --- a/pkg/replacer/image/image.go +++ b/pkg/replacer/image/image.go @@ -18,9 +18,16 @@ package image import ( "context" + "errors" "fmt" "strings" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/remote" + + "github.com/stacklok/frizbee/internal/cli" "github.com/stacklok/frizbee/pkg/interfaces" "github.com/stacklok/frizbee/pkg/utils/config" "github.com/stacklok/frizbee/pkg/utils/store" @@ -133,3 +140,68 @@ func (_ *Parser) ConvertToEntityRef(reference string) (*interfaces.EntityRef, er Type: ReferenceType, }, nil } + +// GetImageDigestFromRef returns the digest of a container image reference +// from a name.Reference. +func GetImageDigestFromRef(ctx context.Context, imageRef, platform string, cache store.RefCacher) (*interfaces.EntityRef, error) { + // Parse the image reference + ref, err := name.ParseReference(imageRef) + if err != nil { + return nil, err + } + opts := []remote.Option{ + remote.WithContext(ctx), + remote.WithUserAgent(cli.UserAgent), + remote.WithAuthFromKeychain(authn.DefaultKeychain), + } + + // Set the platform if provided + if platform != "" { + platformSplit := strings.Split(platform, "/") + if len(platformSplit) != 2 { + return nil, errors.New("platform must be in the format os/arch") + } + opts = append(opts, remote.WithPlatform(v1.Platform{ + OS: platformSplit[0], + Architecture: platformSplit[1], + })) + } + + // Get the digest of the image reference + var digest string + + if cache != nil { + if d, ok := cache.Load(imageRef); ok { + digest = d + } else { + desc, err := remote.Get(ref, opts...) + if err != nil { + return nil, err + } + digest = desc.Digest.String() + cache.Store(imageRef, digest) + } + } else { + desc, err := remote.Get(ref, opts...) + if err != nil { + return nil, err + } + digest = desc.Digest.String() + } + + // Compare the digest with the reference and return the original reference if they already match + if digest == ref.Identifier() { + return nil, fmt.Errorf("image already referenced by digest: %s %w", imageRef, interfaces.ErrReferenceSkipped) + } + + return &interfaces.EntityRef{ + Name: ref.Context().Name(), + Ref: digest, + Type: ReferenceType, + Tag: ref.Identifier(), + }, nil +} + +func shouldExclude(ref string) bool { + return ref == "scratch" +} diff --git a/pkg/replacer/image/utils_test.go b/pkg/replacer/image/image_test.go similarity index 100% rename from pkg/replacer/image/utils_test.go rename to pkg/replacer/image/image_test.go diff --git a/pkg/replacer/image/utils.go b/pkg/replacer/image/utils.go deleted file mode 100644 index b69dd80..0000000 --- a/pkg/replacer/image/utils.go +++ /dev/null @@ -1,97 +0,0 @@ -// -// Copyright 2024 Stacklok, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package image - -import ( - "context" - "errors" - "fmt" - "strings" - - "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/name" - v1 "github.com/google/go-containerregistry/pkg/v1" - "github.com/google/go-containerregistry/pkg/v1/remote" - - "github.com/stacklok/frizbee/internal/cli" - "github.com/stacklok/frizbee/pkg/interfaces" - "github.com/stacklok/frizbee/pkg/utils/store" -) - -// GetImageDigestFromRef returns the digest of a container image reference -// from a name.Reference. -func GetImageDigestFromRef(ctx context.Context, imageRef, platform string, cache store.RefCacher) (*interfaces.EntityRef, error) { - // Parse the image reference - ref, err := name.ParseReference(imageRef) - if err != nil { - return nil, err - } - opts := []remote.Option{ - remote.WithContext(ctx), - remote.WithUserAgent(cli.UserAgent), - remote.WithAuthFromKeychain(authn.DefaultKeychain), - } - - // Set the platform if provided - if platform != "" { - platformSplit := strings.Split(platform, "/") - if len(platformSplit) != 2 { - return nil, errors.New("platform must be in the format os/arch") - } - opts = append(opts, remote.WithPlatform(v1.Platform{ - OS: platformSplit[0], - Architecture: platformSplit[1], - })) - } - - // Get the digest of the image reference - var digest string - - if cache != nil { - if d, ok := cache.Load(imageRef); ok { - digest = d - } else { - desc, err := remote.Get(ref, opts...) - if err != nil { - return nil, err - } - digest = desc.Digest.String() - cache.Store(imageRef, digest) - } - } else { - desc, err := remote.Get(ref, opts...) - if err != nil { - return nil, err - } - digest = desc.Digest.String() - } - - // Compare the digest with the reference and return the original reference if they already match - if digest == ref.Identifier() { - return nil, fmt.Errorf("image already referenced by digest: %s %w", imageRef, interfaces.ErrReferenceSkipped) - } - - return &interfaces.EntityRef{ - Name: ref.Context().Name(), - Ref: digest, - Type: ReferenceType, - Tag: ref.Identifier(), - }, nil -} - -func shouldExclude(ref string) bool { - return ref == "scratch" -} From f5908a018f65d0dd85182cde6bcc0639a0f5d3d1 Mon Sep 17 00:00:00 2001 From: Radoslav Dimitrov Date: Wed, 29 May 2024 03:12:02 +0300 Subject: [PATCH 10/16] Add more unit tests - cache, config, ghrest, cli, action, image Signed-off-by: Radoslav Dimitrov --- go.mod | 2 + go.sum | 5 + internal/cli/cli_test.go | 223 +++++++++++++++++++ internal/traverse/traverse_test.go | 264 +++++++++++++++++++++++ pkg/replacer/actions/actions_test.go | 310 +++++++++++++++++++++------ pkg/replacer/image/image_test.go | 196 ++++++++++++++--- pkg/replacer/replacer_test.go | 60 +++--- pkg/utils/config/config.go | 8 +- pkg/utils/config/config_test.go | 147 +++++++++++++ pkg/utils/ghrest/ghrest_test.go | 114 ++++++++++ pkg/utils/store/cache_test.go | 119 ++++++++++ 11 files changed, 1316 insertions(+), 132 deletions(-) create mode 100644 internal/cli/cli_test.go create mode 100644 internal/traverse/traverse_test.go create mode 100644 pkg/utils/config/config_test.go create mode 100644 pkg/utils/ghrest/ghrest_test.go create mode 100644 pkg/utils/store/cache_test.go diff --git a/go.mod b/go.mod index a9e8b7b..43ab306 100644 --- a/go.mod +++ b/go.mod @@ -24,6 +24,7 @@ require ( github.com/docker/docker v24.0.9+incompatible // indirect github.com/docker/docker-credential-helpers v0.7.0 // indirect github.com/google/go-querystring v1.1.0 // indirect + github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/klauspost/compress v1.16.5 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect @@ -36,4 +37,5 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/vbatts/tar-split v0.11.3 // indirect golang.org/x/sys v0.15.0 // indirect + gopkg.in/h2non/gock.v1 v1.1.2 // indirect ) diff --git a/go.sum b/go.sum index 2853a50..12429b6 100644 --- a/go.sum +++ b/go.sum @@ -29,6 +29,8 @@ github.com/google/go-github/v61 v61.0.0 h1:VwQCBwhyE9JclCI+22/7mLB1PuU9eowCXKY5p github.com/google/go-github/v61 v61.0.0/go.mod h1:0WR+KmsWX75G2EbpyGsGmradjo3IiciuI4BmdVCobQY= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= +github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= @@ -41,6 +43,7 @@ github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/Qd github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= @@ -91,6 +94,8 @@ golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8T gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= +gopkg.in/h2non/gock.v1 v1.1.2 h1:jBbHXgGBK/AoPVfJh5x4r/WxIrElvbLel8TCZkkZJoY= +gopkg.in/h2non/gock.v1 v1.1.2/go.mod h1:n7UGz/ckNChHiK05rDoiC4MYSunEC/lyaUm2WWaDva0= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/internal/cli/cli_test.go b/internal/cli/cli_test.go new file mode 100644 index 0000000..f98217b --- /dev/null +++ b/internal/cli/cli_test.go @@ -0,0 +1,223 @@ +package cli + +import ( + "path/filepath" + "strings" + "testing" + + "github.com/go-git/go-billy/v5" + "github.com/go-git/go-billy/v5/memfs" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" +) + +func TestNewHelper(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + cmdArgs []string + expected *Helper + expectedError bool + }{ + { + name: "ValidFlags", + cmdArgs: []string{"--dry-run", "--quiet", "--error", "--regex", "test"}, + expected: &Helper{ + DryRun: true, + Quiet: true, + ErrOnModified: true, + Regex: "test", + }, + expectedError: false, + }, + { + name: "MissingFlags", + cmdArgs: []string{}, + expected: &Helper{}, + expectedError: false, + }, + { + name: "InvalidFlags", + cmdArgs: []string{"--nonexistent"}, + expected: nil, + expectedError: true, + }, + } + + for _, tt := range testCases { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + cmd := &cobra.Command{} + DeclareFrizbeeFlags(cmd, true) + cmd.SetArgs(tt.cmdArgs) + + if tt.expectedError { + assert.Error(t, cmd.Execute()) + return + } + + assert.NoError(t, cmd.Execute()) + + helper, err := NewHelper(cmd) + if tt.expectedError { + assert.Error(t, err) + assert.Nil(t, helper) + } else { + assert.NoError(t, err) + assert.NotNil(t, helper) + assert.Equal(t, tt.expected.DryRun, helper.DryRun) + assert.Equal(t, tt.expected.Quiet, helper.Quiet) + assert.Equal(t, tt.expected.ErrOnModified, helper.ErrOnModified) + assert.Equal(t, tt.expected.Regex, helper.Regex) + } + }) + } +} + +func TestProcessOutput(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + helper *Helper + path string + processed []string + modified map[string]string + expectedOutput string + expectError bool + }{ + { + name: "QuietMode", + helper: &Helper{ + Quiet: true, + Cmd: &cobra.Command{}, + }, + path: "test/path", + processed: []string{"file1.txt", "file2.txt"}, + modified: map[string]string{"file1.txt": "new content"}, + expectedOutput: "", + expectError: false, + }, + { + name: "DryRunMode", + helper: &Helper{ + Quiet: false, + DryRun: true, + Cmd: &cobra.Command{}, + }, + path: "test/path", + processed: []string{"file1.txt"}, + modified: map[string]string{"file1.txt": "new content"}, + expectedOutput: "Processed: file1.txt\nModified: file1.txt\nnew content", + expectError: false, + }, + { + name: "ErrorOpeningFile", + helper: &Helper{ + Quiet: false, + Cmd: &cobra.Command{}, + }, + path: "invalid/path", + modified: map[string]string{"invalid/path": "new content"}, + expectedOutput: "", + expectError: true, + }, + } + + for _, tt := range testCases { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + // Set up command output + var output strings.Builder + tt.helper.Cmd.SetOut(&output) + tt.helper.Cmd.SetErr(&output) + + // Create in-memory filesystem and add files + fs := memfs.New() + for path, content := range tt.modified { + dir := filepath.Join(tt.path, filepath.Dir(path)) + assert.NoError(t, fs.MkdirAll(dir, 0755)) + file, err := fs.Create(filepath.Join(tt.path, path)) + if err == nil { + _, _ = file.Write([]byte(content)) + assert.NoError(t, file.Close()) + } + } + + // Process the output using the in-memory filesystem + err := tt.helper.ProcessOutput(tt.path, tt.processed, tt.modified) + if tt.expectError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.Contains(t, output.String(), tt.expectedOutput) + } + }) + } +} + +func TestIsPath(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + setup func(fs billy.Filesystem) + path string + expected bool + }{ + { + name: "ExistingFile", + setup: func(fs billy.Filesystem) { + file, _ := fs.Create("testfile.txt") + assert.NoError(t, file.Close()) + }, + path: "testfile.txt", + expected: true, + }, + { + name: "NonExistentFile", + setup: func(_ billy.Filesystem) {}, + path: "nonexistent.txt", + expected: false, + }, + { + name: "ExistingDirectory", + setup: func(fs billy.Filesystem) { + assert.NoError(t, fs.MkdirAll("testdir", 0755)) + }, + path: "testdir", + expected: true, + }, + { + name: "NonExistentDirectory", + setup: func(_ billy.Filesystem) {}, + path: "nonexistentdir", + expected: false, + }, + } + + for _, tt := range testCases { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + // Use in-memory filesystem for testing + fs := memfs.New() + tt.setup(fs) + + // Check if the path exists in the in-memory filesystem + _, err := fs.Stat(tt.path) + result := err == nil + + assert.Equal(t, tt.expected, result) + }) + } +} diff --git a/internal/traverse/traverse_test.go b/internal/traverse/traverse_test.go new file mode 100644 index 0000000..701b838 --- /dev/null +++ b/internal/traverse/traverse_test.go @@ -0,0 +1,264 @@ +package traverse + +import ( + "errors" + "os" + "testing" + "time" + + "github.com/go-git/go-billy/v5/memfs" + "github.com/stretchr/testify/assert" +) + +func TestYamlDockerfiles(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + fsContent map[string]string + baseDir string + expected []string + expectError bool + }{ + { + name: "NoYAMLOrDockerfile", + fsContent: map[string]string{ + "base/file.txt": "content", + }, + baseDir: "base", + expected: []string{}, + expectError: false, + }, + { + name: "WithYAMLFiles", + fsContent: map[string]string{ + "base/file.yml": "content", + "base/file.yaml": "content", + "base/not_included.txt": "content", + }, + baseDir: "base", + expected: []string{ + "base/file.yml", + "base/file.yaml", + }, + expectError: false, + }, + { + name: "WithDockerfiles", + fsContent: map[string]string{ + "base/Dockerfile": "content", + "base/nested/dockerfile": "content", + "base/not_included.txt": "content", + }, + baseDir: "base", + expected: []string{ + "base/Dockerfile", + "base/nested/dockerfile", + }, + expectError: false, + }, + { + name: "MixedFiles", + fsContent: map[string]string{ + "base/file.yml": "content", + "base/Dockerfile": "content", + "base/nested/file.yaml": "content", + "base/nested/dockerfile": "content", + "base/not_included.txt": "content", + }, + baseDir: "base", + expected: []string{ + "base/file.yml", + "base/Dockerfile", + "base/nested/file.yaml", + "base/nested/dockerfile", + }, + expectError: false, + }, + { + name: "ErrorInProcessingFile", + fsContent: map[string]string{ + "base/file.yml": "content", + }, + baseDir: "base", + expectError: true, + }, + } + + for _, tt := range testCases { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + fs := memfs.New() + for name, content := range tt.fsContent { + f, _ := fs.Create(name) + _, _ = f.Write([]byte(content)) + assert.NoError(t, f.Close()) + } + + var processedFiles []string + err := YamlDockerfiles(fs, tt.baseDir, func(path string) error { + if tt.expectError { + return errors.New("error in processing file") + } + processedFiles = append(processedFiles, path) + return nil + }) + + if tt.expectError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.ElementsMatch(t, tt.expected, processedFiles) + } + }) + } +} + +func TestTraverse(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + fsContent map[string]string + baseDir string + expected []string + expectError bool + }{ + { + name: "TraverseFiles", + fsContent: map[string]string{ + "base/file1.txt": "content", + "base/file2.txt": "content", + "base/nested/file": "content", + }, + baseDir: "base", + expected: []string{ + "base", + "base/file1.txt", + "base/file2.txt", + "base/nested", + "base/nested/file", + }, + expectError: false, + }, + { + name: "TraverseWithError", + fsContent: map[string]string{ + "base/file.txt": "content", + }, + baseDir: "base", + expectError: true, + }, + } + + for _, tt := range testCases { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + fs := memfs.New() + for name, content := range tt.fsContent { + f, _ := fs.Create(name) + _, _ = f.Write([]byte(content)) + assert.NoError(t, f.Close()) + } + + var processedFiles []string + err := Traverse(fs, tt.baseDir, func(path string, _ os.FileInfo) error { + if tt.expectError { + return errors.New("error in traversing file") + } + processedFiles = append(processedFiles, path) + return nil + }) + + if tt.expectError { + assert.Error(t, err) + } else { + assert.NoError(t, err) + assert.ElementsMatch(t, tt.expected, processedFiles) + } + }) + } +} + +func TestIsYAMLOrDockerfile(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + fileName string + isDir bool + expected bool + }{ + { + name: "YAMLFile", + fileName: "config.yaml", + isDir: false, + expected: true, + }, + { + name: "YMLFile", + fileName: "config.yml", + isDir: false, + expected: true, + }, + { + name: "Dockerfile", + fileName: "Dockerfile", + isDir: false, + expected: true, + }, + { + name: "dockerfile", + fileName: "dockerfile", + isDir: false, + expected: true, + }, + { + name: "NonYAMLOrDockerfile", + fileName: "config.txt", + isDir: false, + expected: false, + }, + { + name: "Directory", + fileName: "config", + isDir: true, + expected: false, + }, + } + + for _, tt := range testCases { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + info := &fileInfoMock{ + name: tt.fileName, + dir: tt.isDir, + } + + result := isYAMLOrDockerfile(info) + assert.Equal(t, tt.expected, result) + }) + } +} + +// fileInfoMock is a mock implementation of os.FileInfo for testing. +type fileInfoMock struct { + name string + dir bool +} + +func (f *fileInfoMock) Name() string { return f.name } +func (_ *fileInfoMock) Size() int64 { return 0 } +func (_ *fileInfoMock) Mode() os.FileMode { return 0 } +func (_ *fileInfoMock) ModTime() time.Time { return time.Time{} } +func (f *fileInfoMock) IsDir() bool { return f.dir } +func (_ *fileInfoMock) Sys() interface{} { return nil } diff --git a/pkg/replacer/actions/actions_test.go b/pkg/replacer/actions/actions_test.go index 06e2949..318f7ad 100644 --- a/pkg/replacer/actions/actions_test.go +++ b/pkg/replacer/actions/actions_test.go @@ -7,130 +7,310 @@ import ( "github.com/stretchr/testify/require" + "github.com/stacklok/frizbee/pkg/utils/config" "github.com/stacklok/frizbee/pkg/utils/ghrest" + "github.com/stacklok/frizbee/pkg/utils/store" ) +func TestNewParser(t *testing.T) { + t.Parallel() + + parser := New() + require.NotNil(t, parser, "Parser should not be nil") + require.Equal(t, GitHubActionsRegex, parser.regex, "Default regex should be GitHubActionsRegex") + require.NotNil(t, parser.cache, "Cache should be initialized") +} + +func TestSetCache(t *testing.T) { + t.Parallel() + + parser := New() + cache := store.NewRefCacher() + parser.SetCache(cache) + require.Equal(t, cache, parser.cache, "Cache should be set correctly") +} + +func TestSetAndGetRegex(t *testing.T) { + t.Parallel() + + parser := New() + tests := []struct { + name string + newRegex string + }{ + { + name: "Set and get new regex", + newRegex: `new-regex`, + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + parser.SetRegex(tt.newRegex) + require.Equal(t, tt.newRegex, parser.GetRegex(), "Regex should be set and retrieved correctly") + }) + } +} + +func TestReplaceLocalPath(t *testing.T) { + t.Parallel() + + parser := New() + ctx := context.Background() + cfg := config.Config{} + restIf := &ghrest.Client{} + + tests := []struct { + name string + matchedLine string + }{ + { + name: "Replace local path", + matchedLine: "./local/path", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + _, err := parser.Replace(ctx, tt.matchedLine, restIf, cfg) + require.Error(t, err, "Should return error for local path") + require.Contains(t, err.Error(), "reference skipped", "Error should indicate reference skipped") + }) + } +} + +func TestReplaceExcludedPath(t *testing.T) { + t.Parallel() + + parser := New() + ctx := context.Background() + cfg := config.Config{GHActions: config.GHActions{Filter: config.Filter{Exclude: []string{"actions/checkout"}}}} + restIf := &ghrest.Client{} + + tests := []struct { + name string + matchedLine string + }{ + { + name: "Replace excluded path", + matchedLine: "uses: actions/checkout@v2", + }, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + _, err := parser.Replace(ctx, tt.matchedLine, restIf, cfg) + require.Error(t, err, "Should return error for excluded path") + require.Contains(t, err.Error(), "reference skipped", "Error should indicate reference skipped") + }) + } +} + +func TestConvertToEntityRef(t *testing.T) { + t.Parallel() + + parser := New() + + tests := []struct { + name string + reference string + wantErr bool + }{ + {"Valid action reference", "uses: actions/checkout@v2", false}, + {"Valid docker reference", "docker://mydocker/image:tag", false}, + {"Invalid reference format", "invalid-reference", true}, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ref, err := parser.ConvertToEntityRef(tt.reference) + if tt.wantErr { + require.Error(t, err, "Expected error but got none") + } else { + require.NoError(t, err, "Expected no error but got %v", err) + require.NotNil(t, ref, "EntityRef should not be nil") + } + }) + } +} + +func TestIsLocal(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + want bool + }{ + {"Local path with ./", "./local/path", true}, + {"Local path with ../", "../local/path", true}, + {"Non-local path", "non/local/path", false}, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + require.Equal(t, tt.want, isLocal(tt.input), "IsLocal should return correct value") + }) + } +} + +func TestShouldExclude(t *testing.T) { + t.Parallel() + + cfg := &config.GHActions{Filter: config.Filter{Exclude: []string{"actions/checkout", "actions/setup"}}} + + tests := []struct { + name string + input string + want bool + }{ + {"Excluded path", "actions/checkout", true}, + {"Non-excluded path", "actions/unknown", false}, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + require.Equal(t, tt.want, shouldExclude(cfg, tt.input), "ShouldExclude should return correct value") + }) + } +} + +func TestParseActionReference(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + input string + wantAction string + wantRef string + wantErr bool + }{ + {"Valid action reference", "actions/checkout@v2", "actions/checkout", "v2", false}, + {"Invalid reference format", "invalid-reference", "", "", true}, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + action, ref, err := ParseActionReference(tt.input) + if tt.wantErr { + require.Error(t, err, "Expected error but got none") + } else { + require.NoError(t, err, "Expected no error but got %v", err) + require.Equal(t, tt.wantAction, action, "Action should be parsed correctly") + require.Equal(t, tt.wantRef, ref, "Reference should be parsed correctly") + } + }) + } +} + func TestGetChecksum(t *testing.T) { t.Parallel() tok := os.Getenv("GITHUB_TOKEN") + ctx := context.Background() + ghcli := ghrest.NewClient(tok) - type args struct { - action string - ref string - } tests := []struct { name string - args args + args struct{ action, ref string } want string wantErr bool }{ { - name: "actions/checkout with v4.1.1", - args: args{ - action: "actions/checkout", - ref: "v4.1.1", - }, + name: "actions/checkout with v4.1.1", + args: struct{ action, ref string }{action: "actions/checkout", ref: "v4.1.1"}, want: "b4ffde65f46336ab88eb53be808477a3936bae11", wantErr: false, }, { - name: "actions/checkout with v3.6.0", - args: args{ - action: "actions/checkout", - ref: "v3.6.0", - }, + name: "actions/checkout with v3.6.0", + args: struct{ action, ref string }{action: "actions/checkout", ref: "v3.6.0"}, want: "f43a0e5ff2bd294095638e18286ca9a3d1956744", wantErr: false, }, { - name: "actions/checkout with checksum returns checksum", - args: args{ - action: "actions/checkout", - ref: "1d96c772d19495a3b5c517cd2bc0cb401ea0529f", - }, + name: "actions/checkout with checksum returns checksum", + args: struct{ action, ref string }{action: "actions/checkout", ref: "1d96c772d19495a3b5c517cd2bc0cb401ea0529f"}, want: "1d96c772d19495a3b5c517cd2bc0cb401ea0529f", wantErr: false, }, { - name: "aquasecurity/trivy-action with 0.14.0", - args: args{ - action: "aquasecurity/trivy-action", - ref: "0.14.0", - }, + name: "aquasecurity/trivy-action with 0.14.0", + args: struct{ action, ref string }{action: "aquasecurity/trivy-action", ref: "0.14.0"}, want: "2b6a709cf9c4025c5438138008beaddbb02086f0", wantErr: false, }, { - name: "aquasecurity/trivy-action with branch returns checksum", - args: args{ - action: "aquasecurity/trivy-action", - ref: "bump-trivy", - }, + name: "aquasecurity/trivy-action with branch returns checksum", + args: struct{ action, ref string }{action: "aquasecurity/trivy-action", ref: "bump-trivy"}, want: "fb5e1b36be448e92ca98648c661bd7e9da1f1317", wantErr: false, }, { - name: "actions/checkout with invalid tag returns error", - args: args{ - action: "actions/checkout", - ref: "v4.1.1.1", - }, + name: "actions/checkout with invalid tag returns error", + args: struct{ action, ref string }{action: "actions/checkout", ref: "v4.1.1.1"}, want: "", wantErr: true, }, { - name: "actions/checkout with invalid action returns error", - args: args{ - action: "invalid-action", - ref: "v4.1.1", - }, + name: "actions/checkout with invalid action returns error", + args: struct{ action, ref string }{action: "invalid-action", ref: "v4.1.1"}, want: "", wantErr: true, }, { - name: "actions/checkout with empty action returns error", - args: args{ - action: "", - ref: "v4.1.1", - }, + name: "actions/checkout with empty action returns error", + args: struct{ action, ref string }{action: "", ref: "v4.1.1"}, want: "", wantErr: true, }, { - name: "actions/checkout with empty tag returns error", - args: args{ - action: "actions/checkout", - ref: "", - }, + name: "actions/checkout with empty tag returns error", + args: struct{ action, ref string }{action: "actions/checkout", ref: ""}, want: "", wantErr: true, }, { - name: "bufbuild/buf-setup-action with v1 is an array", - args: args{ - action: "bufbuild/buf-setup-action", - ref: "v1", - }, - want: "dde0b9351db90fbf78e345f41a57de8514bf1091", + name: "bufbuild/buf-setup-action with v1 is an array", + args: struct{ action, ref string }{action: "bufbuild/buf-setup-action", ref: "v1"}, + want: "dde0b9351db90fbf78e345f41a57de8514bf1091", + wantErr: false, }, { - name: "anchore/sbom-action/download-syft with a sub-action works", - args: args{ - action: "anchore/sbom-action/download-syft", - ref: "v0.14.3", - }, - want: "78fc58e266e87a38d4194b2137a3d4e9bcaf7ca1", + name: "anchore/sbom-action/download-syft with a sub-action works", + args: struct{ action, ref string }{action: "anchore/sbom-action/download-syft", ref: "v0.14.3"}, + want: "78fc58e266e87a38d4194b2137a3d4e9bcaf7ca1", + wantErr: false, }, } + for _, tt := range tests { tt := tt - t.Run(tt.name, func(t *testing.T) { t.Parallel() - ghcli := ghrest.NewClient(tok) - got, err := GetChecksum(context.Background(), ghcli, tt.args.action, tt.args.ref) + got, err := GetChecksum(ctx, ghcli, tt.args.action, tt.args.ref) if tt.wantErr { require.Error(t, err, "Wanted error, got none") require.Empty(t, got, "Wanted empty string, got %v", got) diff --git a/pkg/replacer/image/image_test.go b/pkg/replacer/image/image_test.go index 8943fd2..5594798 100644 --- a/pkg/replacer/image/image_test.go +++ b/pkg/replacer/image/image_test.go @@ -3,69 +3,199 @@ package image import ( "context" "testing" - "time" - "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/stacklok/frizbee/pkg/utils/config" + "github.com/stacklok/frizbee/pkg/utils/store" ) -func TestGetImageDigestFromRef(t *testing.T) { +func TestNewParser(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + }{ + {"New parser initialization"}, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + parser := New() + require.NotNil(t, parser, "Parser should not be nil") + require.Equal(t, ContainerImageRegex, parser.regex, "Default regex should be ContainerImageRegex") + require.NotNil(t, parser.cache, "Cache should be initialized") + }) + } +} + +func TestSetCache(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + cache store.RefCacher + }{ + {"Set cache for parser", store.NewRefCacher()}, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + parser := New() + parser.SetCache(tt.cache) + require.Equal(t, tt.cache, parser.cache, "Cache should be set correctly") + }) + } +} + +func TestSetAndGetRegex(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + newRegex string + }{ + {"Set and get new regex", `new-regex`}, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + parser := New() + parser.SetRegex(tt.newRegex) + require.Equal(t, tt.newRegex, parser.GetRegex(), "Regex should be set and retrieved correctly") + }) + } +} + +func TestReplaceExcludedPath(t *testing.T) { + t.Parallel() + + parser := New() + ctx := context.Background() + cfg := config.Config{GHActions: config.GHActions{Filter: config.Filter{Exclude: []string{"scratch"}}}} + + tests := []struct { + name string + matchedLine string + }{ + {"Replace excluded path", "FROM scratch"}, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + _, err := parser.Replace(ctx, tt.matchedLine, nil, cfg) + require.Error(t, err, "Should return error for excluded path") + require.Contains(t, err.Error(), "reference skipped", "Error should indicate reference skipped") + }) + } +} + +func TestConvertToEntityRef(t *testing.T) { t.Parallel() - type args struct { - refstr string + parser := New() + + tests := []struct { + name string + reference string + wantErr bool + }{ + {"Valid container reference with tag", "ghcr.io/stacklok/minder/helm/minder:0.20231123.829_ref.26ca90b", false}, + {"Valid container reference with digest", "ghcr.io/stacklok/minder/helm/minder@sha256:a29f8a8d28f0af7f70a4b3dd3e33c8c8cc5cf9e88e802e2700cf272a0b6140ec", false}, + {"Invalid reference format", "invalid:reference:format", true}, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + ref, err := parser.ConvertToEntityRef(tt.reference) + if tt.wantErr { + require.Error(t, err, "Expected error but got none") + } else { + require.NoError(t, err, "Expected no error but got %v", err) + require.NotNil(t, ref, "EntityRef should not be nil") + } + }) } +} + +func TestGetImageDigestFromRef(t *testing.T) { + t.Parallel() + + ctx := context.Background() + tests := []struct { name string - args args + refstr string want string wantErr bool }{ { - name: "valid 1", - args: args{ - refstr: "ghcr.io/stacklok/minder/helm/minder:0.20231123.829_ref.26ca90b", - }, - want: "sha256:a29f8a8d28f0af7f70a4b3dd3e33c8c8cc5cf9e88e802e2700cf272a0b6140ec", + name: "Valid image reference 1", + refstr: "ghcr.io/stacklok/minder/helm/minder:0.20231123.829_ref.26ca90b", + want: "sha256:a29f8a8d28f0af7f70a4b3dd3e33c8c8cc5cf9e88e802e2700cf272a0b6140ec", }, { - name: "valid 2", - args: args{ - refstr: "devopsfaith/krakend:2.5.0", - }, - want: "sha256:6a3c8e5e1a4948042bfb364ed6471e16b4a26d0afb6c3c01ebcb88b3fa551036", + name: "Valid image reference 2", + refstr: "devopsfaith/krakend:2.5.0", + want: "sha256:6a3c8e5e1a4948042bfb364ed6471e16b4a26d0afb6c3c01ebcb88b3fa551036", }, { - name: "invalid ref string", - args: args{ - refstr: "ghcr.io/stacklok/minder/helm/minder!", - }, + name: "Invalid ref string", + refstr: "ghcr.io/stacklok/minder/helm/minder!", wantErr: true, }, { - name: "unexistent container in unexistent registry", - args: args{ - refstr: "beeeeer.io/ipa/toppling-goliath/king-sue:1.0.0", - }, + name: "Nonexistent container in nonexistent registry", + refstr: "beeeeer.io/ipa/toppling-goliath/king-sue:1.0.0", wantErr: true, }, } + for _, tt := range tests { tt := tt - t.Run(tt.name, func(t *testing.T) { t.Parallel() - - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() - got, err := GetImageDigestFromRef(ctx, tt.args.refstr, "", nil) + got, err := GetImageDigestFromRef(ctx, tt.refstr, "", nil) if tt.wantErr { - assert.Error(t, err) - assert.Nil(t, got) + require.Error(t, err) + require.Nil(t, got) return } - assert.NoError(t, err) - assert.Equal(t, tt.want, got.Ref) + require.NoError(t, err) + require.Equal(t, tt.want, got.Ref) + }) + } +} + +func TestShouldExclude(t *testing.T) { + t.Parallel() + + tests := []struct { + name string + ref string + want bool + }{ + {"Exclude scratch", "scratch", true}, + {"Do not exclude ubuntu", "ubuntu", false}, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + got := shouldExclude(tt.ref) + require.Equal(t, tt.want, got, "shouldExclude should return the correct exclusion status") }) } } diff --git a/pkg/replacer/replacer_test.go b/pkg/replacer/replacer_test.go index 4f42277..5194d41 100644 --- a/pkg/replacer/replacer_test.go +++ b/pkg/replacer/replacer_test.go @@ -20,9 +20,7 @@ import ( "os" "strings" "testing" - "time" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stacklok/frizbee/internal/cli" @@ -125,18 +123,17 @@ func TestReplacer_ParseContainerImageString(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() + ctx := context.Background() r := NewContainerImagesReplacer(&config.Config{}) got, err := r.ParseString(ctx, tt.args.refstr) if tt.wantErr { - assert.Error(t, err) - assert.Empty(t, got) + require.Error(t, err) + require.Empty(t, got) return } - assert.NoError(t, err) - assert.Equal(t, tt.want, got) + require.NoError(t, err) + require.Equal(t, tt.want, got) }) } } @@ -295,9 +292,10 @@ func TestReplacer_ParseGitHubActionString(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() + ctx := context.Background() r := NewGitHubActionsReplacer(&config.Config{}).WithGitHubClient(os.Getenv("GITHUB_TOKEN")) - got, err := r.ParseString(context.Background(), tt.args.action) + got, err := r.ParseString(ctx, tt.args.action) if tt.wantErr { require.Error(t, err, "Wanted error, got none") require.Empty(t, got, "Wanted empty string, got %v", got) @@ -353,28 +351,28 @@ services: t.Run(tt.name, func(t *testing.T) { t.Parallel() - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() + ctx := context.Background() + r := NewContainerImagesReplacer(&config.Config{}) modified, newContent, err := r.ParseFile(ctx, strings.NewReader(tt.before)) if tt.modified { - assert.True(t, modified) - assert.NotEmpty(t, newContent) + require.True(t, modified) + require.NotEmpty(t, newContent) } else { - assert.False(t, modified) - assert.Empty(t, newContent) + require.False(t, modified) + require.Empty(t, newContent) } if tt.wantErr { - assert.False(t, modified) - assert.Empty(t, newContent) - assert.Error(t, err) + require.False(t, modified) + require.Empty(t, newContent) + require.Error(t, err) return } - assert.NoError(t, err) - assert.Equal(t, tt.expected, newContent) + require.NoError(t, err) + require.Equal(t, tt.expected, newContent) }) } } @@ -473,8 +471,8 @@ jobs: t.Run(tt.name, func(t *testing.T) { t.Parallel() - ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second) - defer cancel() + ctx := context.Background() + r := NewGitHubActionsReplacer(&config.Config{}).WithGitHubClient(os.Getenv(cli.GitHubTokenEnvKey)) if tt.useCustomRegex { r = r.WithUserRegex(tt.regex) @@ -482,22 +480,22 @@ jobs: modified, newContent, err := r.ParseFile(ctx, strings.NewReader(tt.before)) if tt.modified { - assert.True(t, modified) - assert.Equal(t, tt.expected, newContent) + require.True(t, modified) + require.Equal(t, tt.expected, newContent) } else { - assert.False(t, modified) - assert.Equal(t, tt.before, newContent) + require.False(t, modified) + require.Equal(t, tt.before, newContent) } if tt.wantErr { - assert.False(t, modified) - assert.Equal(t, tt.before, newContent) - assert.Error(t, err) + require.False(t, modified) + require.Equal(t, tt.before, newContent) + require.Error(t, err) return } - assert.NoError(t, err) - assert.Equal(t, tt.expected, newContent) + require.NoError(t, err) + require.Equal(t, tt.expected, newContent) }) } } diff --git a/pkg/utils/config/config.go b/pkg/utils/config/config.go index 7fd8335..91977f1 100644 --- a/pkg/utils/config/config.go +++ b/pkg/utils/config/config.go @@ -19,6 +19,7 @@ package config import ( "errors" "fmt" + "io" "os" "path/filepath" @@ -89,12 +90,13 @@ func ParseConfigFileFromFS(fs billy.Filesystem, configfile string) (*Config, err return nil, fmt.Errorf("failed to open config file: %w", err) } - // nolint:errcheck // we don't care about the error here - defer cfgF.Close() + defer cfgF.Close() // nolint:errcheck dec := yaml.NewDecoder(cfgF) - if err := dec.Decode(cfg); err != nil { + if err == io.EOF { + return cfg, nil + } return nil, fmt.Errorf("failed to decode config file: %w", err) } diff --git a/pkg/utils/config/config_test.go b/pkg/utils/config/config_test.go new file mode 100644 index 0000000..c9a92f7 --- /dev/null +++ b/pkg/utils/config/config_test.go @@ -0,0 +1,147 @@ +package config + +import ( + "context" + "testing" + + "github.com/go-git/go-billy/v5/memfs" + "github.com/spf13/cobra" + "github.com/stretchr/testify/require" +) + +func TestFromCommand(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + contextCfg *Config + platformFlag string + expectedCfg *Config + expectError bool + }{ + { + name: "NoConfigInContext", + contextCfg: nil, + expectError: true, + }, + { + name: "WithConfigInContext", + contextCfg: &Config{Platform: "linux/arm64"}, + expectedCfg: &Config{Platform: "linux/arm64"}, + }, + { + name: "WithPlatformFlag", + contextCfg: &Config{Platform: "linux/amd64"}, + platformFlag: "windows/arm64", + expectedCfg: &Config{Platform: "windows/arm64"}, + }, + } + + for _, tt := range testCases { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + ctx := context.Background() + + cmd := &cobra.Command{} + if tt.contextCfg != nil { + ctx := context.WithValue(ctx, ContextConfigKey, tt.contextCfg) + cmd.SetContext(ctx) + } else { + cmd.SetContext(ctx) + } + if tt.platformFlag != "" { + cmd.Flags().String("platform", "", "platform") + require.NoError(t, cmd.Flags().Set("platform", tt.platformFlag)) + } + + cfg, err := FromCommand(cmd) + if tt.expectError { + require.Error(t, err) + require.Nil(t, cfg) + } else { + require.NoError(t, err) + require.Equal(t, tt.expectedCfg, cfg) + } + }) + } +} + +func TestParseConfigFile(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + fsContent map[string]string + fileName string + expectedResult *Config + expectError bool + }{ + { + name: "FileNotFound", + fileName: "nonexistent.yaml", + expectedResult: &Config{}, + }, + { + name: "InvalidYaml", + fileName: "invalid.yaml", + fsContent: map[string]string{"invalid.yaml": "invalid yaml content"}, + expectError: true, + }, + { + name: "ValidYaml", + fileName: "valid.yaml", + fsContent: map[string]string{ + "valid.yaml": ` +platform: linux/amd64 +ghactions: + exclude: + - pattern1 + - pattern2 +`, + }, + expectedResult: &Config{ + Platform: "linux/amd64", + GHActions: GHActions{ + Filter: Filter{ + Exclude: []string{"pattern1", "pattern2"}, + }, + }, + }, + }, + { + name: "EmptyFile", + fileName: "empty.yaml", + fsContent: map[string]string{"empty.yaml": ""}, + expectedResult: &Config{}, + }, + } + + for _, tt := range testCases { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + fs := memfs.New() + for name, content := range tt.fsContent { + f, _ := fs.Create(name) + _, _ = f.Write([]byte(content)) + require.NoError(t, f.Close()) + } + + cfg, err := ParseConfigFileFromFS(fs, tt.fileName) + if tt.expectError { + require.Error(t, err) + } else { + require.NoError(t, err) + require.Equal(t, tt.expectedResult.Platform, cfg.Platform) + if cfg.GHActions.Exclude != nil { + require.Equal(t, tt.expectedResult.GHActions.Exclude, cfg.GHActions.Exclude) + } + } + }) + } +} diff --git a/pkg/utils/ghrest/ghrest_test.go b/pkg/utils/ghrest/ghrest_test.go new file mode 100644 index 0000000..7150fbd --- /dev/null +++ b/pkg/utils/ghrest/ghrest_test.go @@ -0,0 +1,114 @@ +package ghrest + +import ( + "context" + "errors" + "io" + "net/http" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "gopkg.in/h2non/gock.v1" +) + +const ContextTimeout = 4 * time.Second + +// nolint:gocyclo +func TestClientFunctions(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + token string + method string + url string + mockResponse *gock.Response + expectedMethod string + expectedURL string + expectError bool + expectedStatus int + expectedBody string + }{ + { + name: "NewClient", + token: "test_token", + expectedMethod: "", + expectedURL: "", + }, + { + name: "NewRequest GET", + token: "", + method: "GET", + url: "test_url", + expectedMethod: http.MethodGet, + expectedURL: "https://api.github.com/test_url", + }, + { + name: "Do successful request", + token: "", + method: "GET", + url: "test", + mockResponse: gock.New("https://api.github.com").Get("/test").Reply(200).BodyString(`{"message": "hello world"}`), + expectedMethod: http.MethodGet, + expectedURL: "https://api.github.com/test", + expectedStatus: http.StatusOK, + expectedBody: `{"message": "hello world"}`, + }, + { + name: "Do failed request", + token: "", + method: "GET", + url: "test", + mockResponse: gock.New("https://api.github.com").Get("/test").ReplyError(errors.New("failed request")), + expectedMethod: http.MethodGet, + expectedURL: "https://api.github.com/test", + expectError: true, + }, + } + + for _, tt := range testCases { + tt := tt + + if tt.mockResponse != nil { + defer gock.Off() + //gock.DisableNetworking() + //t.Logf("Mock response configured for %s %s", tt.method, tt.url) + } + + client := NewClient(tt.token) + + if tt.name == "NewClient" { + assert.NotNil(t, client, "NewClient returned nil") + assert.NotNil(t, client.client, "NewClient returned client with nil GitHub client") + return + } + + req, err := client.NewRequest(tt.method, tt.url, nil) + require.NoError(t, err) + require.Equal(t, req.Method, tt.expectedMethod) + require.Equal(t, req.URL.String(), tt.expectedURL) + + if tt.name == "NewRequest GET" { + return + } + + ctx := context.Background() + + resp, err := client.Do(ctx, req) + if tt.expectError { + require.NotNil(t, err, "Expected error, got nil") + require.Nil(t, resp, "Expected nil response, got %v", resp) + return + } + require.Nil(t, err, "Expected no error, got %v", err) + + require.Equal(t, resp.StatusCode, tt.expectedStatus) + + body, err := io.ReadAll(resp.Body) + require.Nil(t, err) + require.Equal(t, string(body), tt.expectedBody) + defer resp.Body.Close() // nolint:errcheck + } +} diff --git a/pkg/utils/store/cache_test.go b/pkg/utils/store/cache_test.go new file mode 100644 index 0000000..4575c80 --- /dev/null +++ b/pkg/utils/store/cache_test.go @@ -0,0 +1,119 @@ +package store + +import ( + "fmt" + "testing" + + "github.com/stretchr/testify/require" +) + +type Cacher interface { + Store(key, value string) + Load(key string) (string, bool) +} + +// TestCacher tests the creation and basic functionality of both refCacher and unsafeCacher. +func TestCacher(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + cacher Cacher + key string + storeValue string + loadKey string + expectedVal string + expectFound bool + }{ + { + name: "RefCacher store and load existing key", + cacher: NewRefCacher(), + key: "key1", + storeValue: "value1", + loadKey: "key1", + expectedVal: "value1", + expectFound: true, + }, + { + name: "RefCacher load non-existing key", + cacher: NewRefCacher(), + key: "key1", + storeValue: "value1", + loadKey: "key2", + expectedVal: "", + expectFound: false, + }, + { + name: "UnsafeCacher store and load existing key", + cacher: NewUnsafeCacher(), + key: "key1", + storeValue: "value1", + loadKey: "key1", + expectedVal: "value1", + expectFound: true, + }, + { + name: "UnsafeCacher load non-existing key", + cacher: NewUnsafeCacher(), + key: "key1", + storeValue: "value1", + loadKey: "key2", + expectedVal: "", + expectFound: false, + }, + } + + for _, tt := range testCases { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + + tt.cacher.Store(tt.key, tt.storeValue) + val, ok := tt.cacher.Load(tt.loadKey) + require.Equal(t, tt.expectFound, ok) + require.Equal(t, tt.expectedVal, val) + }) + } +} + +// TestConcurrency tests the thread-safety of refCacher. +func TestConcurrency(t *testing.T) { + t.Parallel() + + cacher := NewRefCacher() + iterations := 1000 + done := make(chan bool) + + // Concurrently store values + for i := 0; i < iterations; i++ { + go func(i int) { + key := fmt.Sprintf("key%d", i) + value := fmt.Sprintf("value%d", i) + cacher.Store(key, value) + done <- true + }(i) + } + + // Wait for all goroutines to finish storing + for i := 0; i < iterations; i++ { + <-done + } + + // Concurrently load values + for i := 0; i < iterations; i++ { + go func(i int) { + key := fmt.Sprintf("key%d", i) + val, ok := cacher.Load(key) + expectedVal := fmt.Sprintf("value%d", i) + require.True(t, ok) + require.Equal(t, expectedVal, val) + done <- true + }(i) + } + + // Wait for all goroutines to finish loading + for i := 0; i < iterations; i++ { + <-done + } +} From 8a24357b89f484ef3027fa4ce1b351aa6d90ca05 Mon Sep 17 00:00:00 2001 From: Radoslav Dimitrov Date: Thu, 30 May 2024 11:07:11 +0300 Subject: [PATCH 11/16] Ignore processing commented lines in yaml Signed-off-by: Radoslav Dimitrov --- pkg/replacer/replacer.go | 13 ++ pkg/replacer/replacer_test.go | 218 +++++++++++++++++++++++++++++----- 2 files changed, 201 insertions(+), 30 deletions(-) diff --git a/pkg/replacer/replacer.go b/pkg/replacer/replacer.go index b450472..5f85304 100644 --- a/pkg/replacer/replacer.go +++ b/pkg/replacer/replacer.go @@ -279,6 +279,14 @@ func parseAndReplaceReferencesInFile( scanner := bufio.NewScanner(f) for scanner.Scan() { line := scanner.Text() + + // Skip commented lines + if strings.HasPrefix(strings.TrimLeft(line, " \t\n\r"), "#") { + // Write the line to the content builder buffer + contentBuilder.WriteString(line + "\n") + continue + } + // See if we can match an entity reference in the line newLine := re.ReplaceAllStringFunc(line, func(matchedLine string) string { // Modify the reference in the line @@ -330,6 +338,11 @@ func listReferencesInFile( for scanner.Scan() { line := scanner.Text() + // Skip commented lines + if strings.HasPrefix(strings.TrimLeft(line, " \t\n\r"), "#") { + continue + } + // See if we can match an entity reference in the line foundEntries := re.FindAllString(line, -1) // nolint:gosimple diff --git a/pkg/replacer/replacer_test.go b/pkg/replacer/replacer_test.go index 5194d41..b4b87da 100644 --- a/pkg/replacer/replacer_test.go +++ b/pkg/replacer/replacer_test.go @@ -116,13 +116,34 @@ func TestReplacer_ParseContainerImageString(t *testing.T) { want: nil, wantErr: true, }, + // TODO: Create a dedicated container image for this test and push it so that latest doesnt change + //{ + // name: "container reference with no tag or digest", + // args: args{ + // refstr: "nginx", + // }, + // want: &interfaces.EntityRef{ + // Name: "index.docker.io/library/nginx", + // Ref: "sha256:faef0b115e699b1e70b1f9a939ea2bc62c26485f6b72e91c8a7b236f1f8589c1", + // Type: image.ReferenceType, + // Tag: "latest", + // Prefix: "", + // }, + // wantErr: false, + //}, + { + name: "invalid reference with special characters", + args: args{ + refstr: "nginx@#$$%%^&*", + }, + want: nil, + wantErr: true, + }, } for _, tt := range tests { tt := tt - t.Run(tt.name, func(t *testing.T) { t.Parallel() - ctx := context.Background() r := NewContainerImagesReplacer(&config.Config{}) got, err := r.ParseString(ctx, tt.args.refstr) @@ -131,7 +152,6 @@ func TestReplacer_ParseContainerImageString(t *testing.T) { require.Empty(t, got) return } - require.NoError(t, err) require.Equal(t, tt.want, got) }) @@ -286,23 +306,45 @@ func TestReplacer_ParseGitHubActionString(t *testing.T) { Prefix: "", }, }, + { + name: "invalid action reference", + args: args{ + action: "invalid-reference", + }, + want: nil, + wantErr: true, + }, + { + name: "missing action tag", + args: args{ + action: "actions/checkout", + }, + want: nil, + wantErr: true, + }, + { + name: "action with special characters", + args: args{ + action: "actions/checkout@#$$%%^&*", + }, + want: nil, + wantErr: true, + }, } for _, tt := range tests { tt := tt - t.Run(tt.name, func(t *testing.T) { t.Parallel() ctx := context.Background() - r := NewGitHubActionsReplacer(&config.Config{}).WithGitHubClient(os.Getenv("GITHUB_TOKEN")) got, err := r.ParseString(ctx, tt.args.action) if tt.wantErr { - require.Error(t, err, "Wanted error, got none") - require.Empty(t, got, "Wanted empty string, got %v", got) + require.Error(t, err) + require.Empty(t, got) return } - require.NoError(t, err, "Wanted no error, got %v", err) - require.Equal(t, tt.want, got, "Wanted %v, got %v", tt.want, got) + require.NoError(t, err) + require.Equal(t, tt.want, got) }) } } @@ -341,36 +383,88 @@ services: `, modified: true, }, - // Add more test cases as needed + { + name: "No image reference modification", + before: ` +version: v1 +services: + - name: minder-app + image: minder:latest +`, + expected: ` +version: v1 +services: + - name: minder-app + image: minder:latest +`, + modified: false, + }, + { + name: "Invalid image reference format", + before: ` +version: v1 +services: + - name: invalid-service + image: invalid@@reference +`, + expected: ` +version: v1 +services: + - name: invalid-service + image: invalid@@reference +`, + modified: false, + wantErr: false, + }, + { + name: "Multiple valid image references", + before: ` +version: v1 +services: + - name: kube-apiserver + image: registry.k8s.io/kube-apiserver:v1.20.0 + - name: kube-controller-manager + image: registry.k8s.io/kube-controller-manager:v1.15.0 + - name: minder-app + image: minder:latest + # - name: nginx + # image: nginx:latest +`, + expected: ` +version: v1 +services: + - name: kube-apiserver + image: registry.k8s.io/kube-apiserver@sha256:8b8125d7a6e4225b08f04f65ca947b27d0cc86380bf09fab890cc80408230114 # v1.20.0 + - name: kube-controller-manager + image: registry.k8s.io/kube-controller-manager@sha256:835f32a5cdb30e86f35675dd91f9c7df01d48359ab8b51c1df866a2c7ea2e870 # v1.15.0 + - name: minder-app + image: minder:latest + # - name: nginx + # image: nginx:latest +`, + modified: true, + }, } - - // Define a regular expression to match YAML tags containing "image" for _, tt := range testCases { tt := tt - t.Run(tt.name, func(t *testing.T) { t.Parallel() - ctx := context.Background() - r := NewContainerImagesReplacer(&config.Config{}) modified, newContent, err := r.ParseFile(ctx, strings.NewReader(tt.before)) - if tt.modified { require.True(t, modified) - require.NotEmpty(t, newContent) + require.Equal(t, tt.expected, newContent) } else { require.False(t, modified) - require.Empty(t, newContent) + require.Equal(t, tt.before, newContent) } - if tt.wantErr { require.False(t, modified) - require.Empty(t, newContent) + require.Equal(t, tt.before, newContent) require.Error(t, err) return } - require.NoError(t, err) require.Equal(t, tt.expected, newContent) }) @@ -424,6 +518,79 @@ jobs: modified: true, wantErr: false, }, + { + name: "No action reference modification", + before: ` +name: Linter +on: pull_request +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: ./minder/server.yml # this should not be replaced + # - uses: actions/checkout@v2 +`, + expected: ` +name: Linter +on: pull_request +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: ./minder/server.yml # this should not be replaced + # - uses: actions/checkout@v2 +`, + modified: false, + }, + { + name: "Invalid action reference format", + before: ` +name: Linter +on: pull_request +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: invalid@@reference +`, + expected: ` +name: Linter +on: pull_request +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: invalid@@reference +`, + modified: false, + wantErr: false, + }, + { + name: "Multiple valid action references", + before: ` +name: Linter +on: pull_request +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: ./minder/server.yml # this should not be replaced + - uses: actions/checkout@v2 + - uses: xt0rted/markdownlint-problem-matcher@v1 +`, + expected: ` +name: Linter +on: pull_request +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: ./minder/server.yml # this should not be replaced + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2 + - uses: xt0rted/markdownlint-problem-matcher@c17ca40d1376f60aba7e7d38a8674a3f22f7f5b0 # v1 +`, + modified: true, + }, { name: "Fail with custom regex", before: ` @@ -461,24 +628,17 @@ jobs: regex: "invalid-regexp", useCustomRegex: true, }, - // Add more test cases as needed } - - // Define a regular expression to match YAML tags containing "image" for _, tt := range testCases { tt := tt - t.Run(tt.name, func(t *testing.T) { t.Parallel() - ctx := context.Background() - r := NewGitHubActionsReplacer(&config.Config{}).WithGitHubClient(os.Getenv(cli.GitHubTokenEnvKey)) if tt.useCustomRegex { r = r.WithUserRegex(tt.regex) } modified, newContent, err := r.ParseFile(ctx, strings.NewReader(tt.before)) - if tt.modified { require.True(t, modified) require.Equal(t, tt.expected, newContent) @@ -486,14 +646,12 @@ jobs: require.False(t, modified) require.Equal(t, tt.before, newContent) } - if tt.wantErr { require.False(t, modified) require.Equal(t, tt.before, newContent) require.Error(t, err) return } - require.NoError(t, err) require.Equal(t, tt.expected, newContent) }) From 2f04baff4fd434c4353e91d7d981db9aafcbe7b5 Mon Sep 17 00:00:00 2001 From: Radoslav Dimitrov Date: Thu, 30 May 2024 11:31:48 +0300 Subject: [PATCH 12/16] Add more replacer unit test coverage Signed-off-by: Radoslav Dimitrov --- pkg/replacer/replacer_test.go | 166 ++++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) diff --git a/pkg/replacer/replacer_test.go b/pkg/replacer/replacer_test.go index b4b87da..1cec784 100644 --- a/pkg/replacer/replacer_test.go +++ b/pkg/replacer/replacer_test.go @@ -21,6 +21,7 @@ import ( "strings" "testing" + "github.com/go-git/go-billy/v5/memfs" "github.com/stretchr/testify/require" "github.com/stacklok/frizbee/internal/cli" @@ -28,6 +29,7 @@ import ( "github.com/stacklok/frizbee/pkg/replacer/actions" "github.com/stacklok/frizbee/pkg/replacer/image" "github.com/stacklok/frizbee/pkg/utils/config" + "github.com/stacklok/frizbee/pkg/utils/ghrest" ) func TestReplacer_ParseContainerImageString(t *testing.T) { @@ -657,3 +659,167 @@ jobs: }) } } + +func TestReplacer_NewGitHubActionsReplacer(t *testing.T) { + t.Parallel() + + cfg := &config.Config{} + tests := []struct { + name string + cfg *config.Config + }{ + {name: "valid config", cfg: cfg}, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + r := NewGitHubActionsReplacer(tt.cfg) + require.NotNil(t, r) + require.IsType(t, &Replacer{}, r) + require.IsType(t, actions.New(), r.parser) + }) + } +} + +func TestReplacer_NewContainerImagesReplacer(t *testing.T) { + t.Parallel() + + cfg := &config.Config{} + tests := []struct { + name string + cfg *config.Config + }{ + {name: "valid config", cfg: cfg}, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + r := NewContainerImagesReplacer(tt.cfg) + require.NotNil(t, r) + require.IsType(t, &Replacer{}, r) + require.IsType(t, image.New(), r.parser) + }) + } +} + +func TestReplacer_WithGitHubClient(t *testing.T) { + t.Parallel() + + r := &Replacer{} + tests := []struct { + name string + token string + }{ + {name: "valid token", token: "valid_token"}, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + r = r.WithGitHubClient(tt.token) + require.NotNil(t, r) + require.IsType(t, ghrest.NewClient(tt.token), r.rest) + }) + } +} + +func TestReplacer_WithUserRegex(t *testing.T) { + t.Parallel() + + r := &Replacer{parser: actions.New()} + tests := []struct { + name string + regex string + }{ + {name: "valid regex", regex: `^test-regex$`}, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + r = r.WithUserRegex(tt.regex) + require.Equal(t, tt.regex, r.parser.GetRegex()) + }) + } +} + +func TestReplacer_WithCacheDisabled(t *testing.T) { + t.Parallel() + + r := &Replacer{parser: actions.New()} + tests := []struct { + name string + }{ + {name: "disable cache"}, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + r = r.WithCacheDisabled() + // we don't test if this passed here because it's an internal implementation detail + // but let's ensure we don't panic for some reason + }) + } +} + +func TestReplacer_ParsePathInFS(t *testing.T) { + t.Parallel() + + r := &Replacer{parser: actions.New(), cfg: config.Config{}} + fs := memfs.New() + tests := []struct { + name string + base string + wantErr bool + }{ + {name: "valid base", base: "some-base", wantErr: false}, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + _, err := r.ParsePathInFS(context.Background(), fs, tt.base) + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } +} + +func TestReplacer_ListPathInFS(t *testing.T) { + t.Parallel() + + r := &Replacer{parser: actions.New(), cfg: config.Config{}} + fs := memfs.New() + tests := []struct { + name string + base string + wantErr bool + }{ + {name: "valid base", base: "some-base", wantErr: false}, + } + + for _, tt := range tests { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + _, err := r.ListPathInFS(fs, tt.base) + if tt.wantErr { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } +} From 3a0d04b4a864dc9e5d5720a2c4ef129b8f8b2164 Mon Sep 17 00:00:00 2001 From: Radoslav Dimitrov Date: Fri, 31 May 2024 11:24:48 +0300 Subject: [PATCH 13/16] Add another option for passing an already created GitHub Client Signed-off-by: Radoslav Dimitrov --- cmd/actions/actions.go | 2 +- cmd/actions/list.go | 2 +- pkg/replacer/replacer.go | 10 ++++-- pkg/replacer/replacer_test.go | 55 ++++++++++++++++++++++++++------- pkg/utils/ghrest/ghrest_test.go | 3 -- 5 files changed, 53 insertions(+), 19 deletions(-) diff --git a/cmd/actions/actions.go b/cmd/actions/actions.go index 2833f4d..9673555 100644 --- a/cmd/actions/actions.go +++ b/cmd/actions/actions.go @@ -84,7 +84,7 @@ func replaceCmd(cmd *cobra.Command, args []string) error { // Create a new replacer r := replacer.NewGitHubActionsReplacer(cfg). WithUserRegex(cliFlags.Regex). - WithGitHubClient(os.Getenv(cli.GitHubTokenEnvKey)) + WithGitHubClientFromToken(os.Getenv(cli.GitHubTokenEnvKey)) if cli.IsPath(pathOrRef) { dir := filepath.Clean(pathOrRef) diff --git a/cmd/actions/list.go b/cmd/actions/list.go index d384d39..435c978 100644 --- a/cmd/actions/list.go +++ b/cmd/actions/list.go @@ -78,7 +78,7 @@ func list(cmd *cobra.Command, args []string) error { // Create a new replacer r := replacer.NewGitHubActionsReplacer(cfg). WithUserRegex(cliFlags.Regex). - WithGitHubClient(os.Getenv(cli.GitHubTokenEnvKey)) + WithGitHubClientFromToken(os.Getenv(cli.GitHubTokenEnvKey)) // List the references in the directory res, err := r.ListPath(dir) diff --git a/pkg/replacer/replacer.go b/pkg/replacer/replacer.go index 5f85304..2b4ac90 100644 --- a/pkg/replacer/replacer.go +++ b/pkg/replacer/replacer.go @@ -75,13 +75,19 @@ func NewContainerImagesReplacer(cfg *config.Config) *Replacer { } } -// WithGitHubClient creates an authenticated GitHub client -func (r *Replacer) WithGitHubClient(token string) *Replacer { +// WithGitHubClientFromToken creates an authenticated GitHub client from a token +func (r *Replacer) WithGitHubClientFromToken(token string) *Replacer { client := ghrest.NewClient(token) r.rest = client return r } +// WithGitHubClient sets the GitHub client to use +func (r *Replacer) WithGitHubClient(client interfaces.REST) *Replacer { + r.rest = client + return r +} + // WithUserRegex sets a user-provided regex for the parser func (r *Replacer) WithUserRegex(regex string) *Replacer { if r.parser != nil { diff --git a/pkg/replacer/replacer_test.go b/pkg/replacer/replacer_test.go index 1cec784..c69ffcf 100644 --- a/pkg/replacer/replacer_test.go +++ b/pkg/replacer/replacer_test.go @@ -338,7 +338,7 @@ func TestReplacer_ParseGitHubActionString(t *testing.T) { t.Run(tt.name, func(t *testing.T) { t.Parallel() ctx := context.Background() - r := NewGitHubActionsReplacer(&config.Config{}).WithGitHubClient(os.Getenv("GITHUB_TOKEN")) + r := NewGitHubActionsReplacer(&config.Config{}).WithGitHubClientFromToken(os.Getenv("GITHUB_TOKEN")) got, err := r.ParseString(ctx, tt.args.action) if tt.wantErr { require.Error(t, err) @@ -419,7 +419,7 @@ services: wantErr: false, }, { - name: "Multiple valid image references", + name: "Multiple valid image references with one commented", before: ` version: v1 services: @@ -446,6 +446,33 @@ services: `, modified: true, }, + { + name: "Valid image reference without specifying the tag", + before: ` +apiVersion: v1 +kind: Pod +metadata: + name: mount-host + namespace: playground +spec: + containers: + - name: mount-host + image: alpine + command: ["sleep"] + args: ["infinity"] + volumeMounts: + - name: host-root + mountPath: /host + readOnly: true + volumes: + - name: host-root + hostPath: + path: / + type: Directory +`, + expected: "", + modified: true, + }, } for _, tt := range testCases { tt := tt @@ -454,13 +481,6 @@ services: ctx := context.Background() r := NewContainerImagesReplacer(&config.Config{}) modified, newContent, err := r.ParseFile(ctx, strings.NewReader(tt.before)) - if tt.modified { - require.True(t, modified) - require.Equal(t, tt.expected, newContent) - } else { - require.False(t, modified) - require.Equal(t, tt.before, newContent) - } if tt.wantErr { require.False(t, modified) require.Equal(t, tt.before, newContent) @@ -468,7 +488,18 @@ services: return } require.NoError(t, err) - require.Equal(t, tt.expected, newContent) + if tt.modified { + require.True(t, modified) + if tt.expected != "" { + require.Equal(t, tt.expected, newContent) + } else { + require.NotEmpty(t, tt.before, newContent) + } + } else { + require.False(t, modified) + require.Equal(t, tt.before, newContent) + } + }) } } @@ -636,7 +667,7 @@ jobs: t.Run(tt.name, func(t *testing.T) { t.Parallel() ctx := context.Background() - r := NewGitHubActionsReplacer(&config.Config{}).WithGitHubClient(os.Getenv(cli.GitHubTokenEnvKey)) + r := NewGitHubActionsReplacer(&config.Config{}).WithGitHubClientFromToken(os.Getenv(cli.GitHubTokenEnvKey)) if tt.useCustomRegex { r = r.WithUserRegex(tt.regex) } @@ -721,7 +752,7 @@ func TestReplacer_WithGitHubClient(t *testing.T) { tt := tt t.Run(tt.name, func(t *testing.T) { t.Parallel() - r = r.WithGitHubClient(tt.token) + r = r.WithGitHubClientFromToken(tt.token) require.NotNil(t, r) require.IsType(t, ghrest.NewClient(tt.token), r.rest) }) diff --git a/pkg/utils/ghrest/ghrest_test.go b/pkg/utils/ghrest/ghrest_test.go index 7150fbd..e4962be 100644 --- a/pkg/utils/ghrest/ghrest_test.go +++ b/pkg/utils/ghrest/ghrest_test.go @@ -6,15 +6,12 @@ import ( "io" "net/http" "testing" - "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "gopkg.in/h2non/gock.v1" ) -const ContextTimeout = 4 * time.Second - // nolint:gocyclo func TestClientFunctions(t *testing.T) { t.Parallel() From 968d18d471fbf1f805953735277510fa443972ee Mon Sep 17 00:00:00 2001 From: Radoslav Dimitrov Date: Mon, 3 Jun 2024 16:54:24 +0300 Subject: [PATCH 14/16] Add tests for the list functionality Signed-off-by: Radoslav Dimitrov --- pkg/replacer/replacer_test.go | 317 ++++++++++++++++++++++++++++++++++ 1 file changed, 317 insertions(+) diff --git a/pkg/replacer/replacer_test.go b/pkg/replacer/replacer_test.go index c69ffcf..670eb9a 100644 --- a/pkg/replacer/replacer_test.go +++ b/pkg/replacer/replacer_test.go @@ -854,3 +854,320 @@ func TestReplacer_ListPathInFS(t *testing.T) { }) } } + +func TestReplacer_ListContainerImagesInFile(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + before string + expected *ListResult + regex string + wantErr bool + useCustomRegex bool + }{ + { + name: "Lust image reference", + before: ` +version: v1 +services: + - name: kube-apiserver + image: registry.k8s.io/kube-apiserver:v1.20.0 + - name: kube-controller-manager + image: registry.k8s.io/kube-controller-manager:v1.15.0 + - name: minder-app + image: minder:latest +`, + expected: &ListResult{ + Entities: []interfaces.EntityRef{ + { + Name: "registry.k8s.io/kube-apiserver", + Ref: "v1.20.0", + Type: image.ReferenceType, + }, + { + Name: "registry.k8s.io/kube-controller-manager", + Ref: "v1.15.0", + Type: image.ReferenceType, + }, + { + Name: "minder", + Ref: "latest", + Type: image.ReferenceType, + }, + }, + }, + wantErr: false, + }, + { + name: "No image reference modification", + before: ` + version: v1 + services: + - name: minder-app + # image: minder:latest + `, + expected: &ListResult{ + Entities: []interfaces.EntityRef{}, + }, + wantErr: false, + }, + { + name: "Invalid image reference format", + before: ` + version: v1 + services: + - name: invalid-service + image: invalid@@reference + `, + expected: &ListResult{ + Entities: []interfaces.EntityRef{}, + }, + wantErr: false, + }, + { + name: "Multiple valid image references with one commented", + before: ` + version: v1 + services: + - name: kube-apiserver + image: registry.k8s.io/kube-apiserver@sha256:8b8125d7a6e4225b08f04f65ca947b27d0cc86380bf09fab890cc80408230114 # v1.20.0 + - name: kube-controller-manager + image: registry.k8s.io/kube-controller-manager@sha256:835f32a5cdb30e86f35675dd91f9c7df01d48359ab8b51c1df866a2c7ea2e870 # v1.15.0 + - name: minder-app + image: minder:latest + # - name: nginx + # image: nginx:latest + `, + expected: &ListResult{ + Entities: []interfaces.EntityRef{ + { + Name: "registry.k8s.io/kube-apiserver", + Ref: "sha256:8b8125d7a6e4225b08f04f65ca947b27d0cc86380bf09fab890cc80408230114", + Type: image.ReferenceType, + }, + { + Name: "registry.k8s.io/kube-controller-manager", + Ref: "sha256:835f32a5cdb30e86f35675dd91f9c7df01d48359ab8b51c1df866a2c7ea2e870", + Type: image.ReferenceType, + }, + { + Name: "minder", + Ref: "latest", + Type: image.ReferenceType, + }, + }, + }, + }, + { + name: "Valid image reference without specifying the tag", + before: ` +apiVersion: v1 +kind: Pod +metadata: + name: mount-host + namespace: playground +spec: + containers: + - name: mount-host + image: alpine + command: ["sleep"] + args: ["infinity"] + volumeMounts: + - name: host-root + mountPath: /host + readOnly: true + volumes: + - name: host-root + hostPath: + path: / + type: Directory + `, + expected: &ListResult{ + Entities: []interfaces.EntityRef{ + { + Name: "alpine", + Ref: "latest", + Type: image.ReferenceType, + }, + }, + }, + }, + } + for _, tt := range testCases { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + r := NewContainerImagesReplacer(&config.Config{}) + listRes, err := r.ListInFile(strings.NewReader(tt.before)) + if tt.wantErr { + require.Nil(t, listRes) + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, len(tt.expected.Entities), len(listRes.Entities)) + for _, entity := range tt.expected.Entities { + require.Contains(t, listRes.Entities, entity) + } + }) + } +} + +func TestReplacer_ListGitHubActionsInFile(t *testing.T) { + t.Parallel() + + testCases := []struct { + name string + before string + expected *ListResult + regex string + wantErr bool + useCustomRegex bool + }{ + { + name: "List image reference", + before: ` +name: Linter +on: pull_request +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: ./minder/server.yml # this should not be listed + - uses: actions/checkout@v2 + - uses: xt0rted/markdownlint-problem-matcher@v1 + - name: "Run Markdown linter" + uses: docker://avtodev/markdown-lint:v1 + with: + args: src/*.md +`, + expected: &ListResult{ + Entities: []interfaces.EntityRef{ + { + Name: "actions/checkout", + Ref: "v2", + Type: actions.ReferenceType, + }, + { + Name: "xt0rted/markdownlint-problem-matcher", + Ref: "v1", + Type: actions.ReferenceType, + }, + { + Name: "avtodev/markdown-lint", + Ref: "v1", + Type: image.ReferenceType, + }, + }, + }, + wantErr: false, + }, + { + name: "No action references", + before: ` + name: Linter + on: pull_request + jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: ./minder/server.yml # this should not be replaced + # - uses: actions/checkout@v2 + `, + expected: &ListResult{ + Entities: []interfaces.EntityRef{}, + }, + wantErr: false, + }, + { + name: "Invalid action reference format", + before: ` + name: Linter + on: pull_request + jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: invalid@@reference + `, + expected: &ListResult{ + Entities: []interfaces.EntityRef{}, + }, + wantErr: false, + }, + { + name: "Multiple valid action references", + before: ` + name: Linter + on: pull_request + jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: ./minder/server.yml # this should not be replaced + - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # v2 + - uses: xt0rted/markdownlint-problem-matcher@c17ca40d1376f60aba7e7d38a8674a3f22f7f5b0 # v1 + `, + expected: &ListResult{ + Entities: []interfaces.EntityRef{ + { + Name: "actions/checkout", + Ref: "ee0669bd1cc54295c223e0bb666b733df41de1c5", + Type: actions.ReferenceType, + }, + { + Name: "xt0rted/markdownlint-problem-matcher", + Ref: "c17ca40d1376f60aba7e7d38a8674a3f22f7f5b0", + Type: actions.ReferenceType, + }, + }, + }, + }, + { + name: "Fail with custom regex", + before: ` + name: Linter + on: pull_request + jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: ./minder/server.yml # this should not be replaced + - uses: actions/checkout@v2 + - uses: xt0rted/markdownlint-problem-matcher@v1 + - name: "Run Markdown linter" + uses: docker://avtodev/markdown-lint:v1 + with: + args: src/*.md + `, + expected: &ListResult{ + Entities: []interfaces.EntityRef{}, + }, + wantErr: false, + regex: "invalid-regexp", + useCustomRegex: true, + }, + } + for _, tt := range testCases { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + r := NewGitHubActionsReplacer(&config.Config{}).WithGitHubClientFromToken(os.Getenv(cli.GitHubTokenEnvKey)) + if tt.useCustomRegex { + r = r.WithUserRegex(tt.regex) + } + listRes, err := r.ListInFile(strings.NewReader(tt.before)) + if tt.wantErr { + require.Nil(t, listRes) + require.Error(t, err) + return + } + require.NoError(t, err) + require.Equal(t, len(tt.expected.Entities), len(listRes.Entities)) + for _, entity := range tt.expected.Entities { + require.Contains(t, listRes.Entities, entity) + } + }) + } +} From 7522446012f771cef2d334b9372b4917bd07a370 Mon Sep 17 00:00:00 2001 From: Radoslav Dimitrov Date: Mon, 3 Jun 2024 17:05:56 +0300 Subject: [PATCH 15/16] Do not overwrite the parser's regex if the new value is an empty string Signed-off-by: Radoslav Dimitrov --- pkg/replacer/replacer.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/replacer/replacer.go b/pkg/replacer/replacer.go index 2b4ac90..24877fe 100644 --- a/pkg/replacer/replacer.go +++ b/pkg/replacer/replacer.go @@ -90,7 +90,7 @@ func (r *Replacer) WithGitHubClient(client interfaces.REST) *Replacer { // WithUserRegex sets a user-provided regex for the parser func (r *Replacer) WithUserRegex(regex string) *Replacer { - if r.parser != nil { + if r.parser != nil && regex != "" { r.parser.SetRegex(regex) } return r From 4ff1d926999c570575a3071741370d6cde0d56c4 Mon Sep 17 00:00:00 2001 From: Radoslav Dimitrov Date: Mon, 3 Jun 2024 19:28:14 +0300 Subject: [PATCH 16/16] Create a default GitHub REST client Signed-off-by: Radoslav Dimitrov --- pkg/replacer/replacer.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/replacer/replacer.go b/pkg/replacer/replacer.go index 24877fe..79dbf7e 100644 --- a/pkg/replacer/replacer.go +++ b/pkg/replacer/replacer.go @@ -64,6 +64,7 @@ func NewGitHubActionsReplacer(cfg *config.Config) *Replacer { return &Replacer{ cfg: *cfg, parser: actions.New(), + rest: ghrest.NewClient(""), } } @@ -72,6 +73,7 @@ func NewContainerImagesReplacer(cfg *config.Config) *Replacer { return &Replacer{ cfg: *cfg, parser: image.New(), + rest: ghrest.NewClient(""), } }