From 4c4ff86bd69f63f9f53d7bc77c99e3252de708d3 Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Tue, 28 Nov 2023 14:52:45 +0200 Subject: [PATCH 01/11] Add autogenerated 'test' docs. Signed-off-by: Ville Aikas --- docs/md/melange.md | 1 + docs/md/melange_test.md | 67 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 68 insertions(+) create mode 100644 docs/md/melange_test.md diff --git a/docs/md/melange.md b/docs/md/melange.md index 07d0dfad9..0b1f1e687 100644 --- a/docs/md/melange.md +++ b/docs/md/melange.md @@ -30,6 +30,7 @@ toc: true * [melange query](/docs/md/melange_query.md) - Query a Melange YAML file for information * [melange sign](/docs/md/melange_sign.md) - Sign an APK package * [melange sign-index](/docs/md/melange_sign-index.md) - Sign an APK index +* [melange test](/docs/md/melange_test.md) - Test a package with a YAML configuration file * [melange update-cache](/docs/md/melange_update-cache.md) - Update a source artifact cache * [melange version](/docs/md/melange_version.md) - Prints the version diff --git a/docs/md/melange_test.md b/docs/md/melange_test.md new file mode 100644 index 000000000..fd9eb37d1 --- /dev/null +++ b/docs/md/melange_test.md @@ -0,0 +1,67 @@ +--- +title: "melange test" +slug: melange_test +url: /docs/md/melange_test.md +draft: false +images: [] +type: "article" +toc: true +--- +## melange test + +Test a package with a YAML configuration file + +### Synopsis + +Test a package from a YAML configuration file containing a test pipeline. + +``` +melange test [flags] +``` + +### Examples + +``` + melange test +``` + +### Options + +``` + --apk-cache-dir string directory used for cached apk packages (default is system-defined cache directory) + --arch strings architectures to build for (e.g., x86_64,ppc64le,arm64) -- default is all, unless specified in config + --breakpoint-label string stop build execution at the specified label + --build-date string date used for the timestamps of the files inside the image + --build-option strings build options to enable + --cache-dir string directory used for cached inputs (default "./melange-cache/") + --cache-source string directory or bucket used for preloading the cache + --continue-label string continue build execution at the specified label + --create-build-log creates a package.log file containing a list of packages that were built by the command + --debug enables debug logging of build pipelines + --debug-runner when enabled, the builder pod will persist after the build succeeds or fails + --dependency-log string log dependencies to a specified file + --empty-workspace whether the build workspace should be empty + --env-file string file to use for preloaded environment variables + --fail-on-lint-warning turns linter warnings into failures + --generate-index whether to generate APKINDEX.tar.gz (default true) + --guest-dir string directory used for the build environment guest + -h, --help help for test + -k, --keyring-append strings path to extra keys to include in the build environment keyring + --log-policy strings logging policy to use (default [builtin:stderr]) + --namespace string namespace to use in package URLs in SBOM (eg wolfi, alpine) (default "unknown") + --out-dir string directory where packages will be output (default "./packages/") + --overlay-binsh string use specified file as /bin/sh overlay in build environment + --pipeline-dir string directory used to extend defined built-in pipelines + -r, --repository-append strings path to extra repositories to include in the build environment + --runner string which runner to use to enable running commands, default is based on your platform. Options are ["bubblewrap" "docker" "lima" "kubernetes"] (default "bubblewrap") + --signing-key string key to use for signing + --source-dir string directory used for included sources + --strip-origin-name whether origin names should be stripped (for bootstrap) + --vars-file string file to use for preloaded build configuration variables + --workspace-dir string directory used for the workspace at /home/build +``` + +### SEE ALSO + +* [melange](/docs/md/melange.md) - + From 044103b8d600643cb5f8b57fa8f52302ebb80db4 Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Tue, 28 Nov 2023 14:54:58 +0200 Subject: [PATCH 02/11] config struct changes for test. Signed-off-by: Ville Aikas --- pkg/config/config.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pkg/config/config.go b/pkg/config/config.go index 197b5571a..8c361558f 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -318,6 +318,8 @@ type Subpackage struct { Commit string `json:"commit,omitempty" yaml:"commit,omitempty"` // Optional: enabling, disabling, and configuration of build checks Checks Checks `json:"checks,omitempty" yaml:"checks,omitempty"` + // Test section for the subpackage. + Test Test `json:"test,omitempty" yaml:"test,omitempty"` } // PackageURL returns the package URL ("purl") for the subpackage. For more @@ -371,10 +373,24 @@ type Configuration struct { // Optional: Deviations to the build Options map[string]BuildOption `json:"options,omitempty" yaml:"options,omitempty"` + // Test section for the main package. + Test Test `json:"test,omitempty" yaml:"test,omitempty"` + // Parsed AST for this configuration root *yaml.Node } +type Test struct { + // Additional Environment necessary for test. + // Environment.Contents.Packages automatically get + // package.dependencies.runtime added to it. So, if your test needs + // no additional packages, you can leave it blank. + Environment apko_types.ImageConfiguration + + // Required: The list of pipelines that test the produced package. + Pipeline []Pipeline `json:"pipeline" yaml:"pipeline"` +} + // Name returns a name for the configuration, using the package name. func (cfg Configuration) Name() string { return cfg.Package.Name From 73d19438d5864b0f4fc525b063470794d6d72a2d Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Tue, 28 Nov 2023 14:59:11 +0200 Subject: [PATCH 03/11] Refactor so can be used with test and build. Signed-off-by: Ville Aikas --- pkg/build/build.go | 110 +++++++++++++++---------------- pkg/build/pipeline.go | 129 +++++++++++++++++++++++++------------ pkg/build/pipeline_test.go | 6 +- pkg/cli/build.go | 5 +- 4 files changed, 146 insertions(+), 104 deletions(-) diff --git a/pkg/build/build.go b/pkg/build/build.go index 1f2924192..ba1d9d5ca 100644 --- a/pkg/build/build.go +++ b/pkg/build/build.go @@ -52,49 +52,46 @@ import ( ) type Build struct { - Configuration config.Configuration - ConfigFile string - SourceDateEpoch time.Time - WorkspaceDir string - WorkspaceIgnore string - PipelineDir string - BuiltinPipelineDir string - SourceDir string - GuestDir string - SigningKey string - SigningPassphrase string - Namespace string - GenerateIndex bool - EmptyWorkspace bool - OutDir string - Logger apko_log.Logger - Arch apko_types.Architecture - ExtraKeys []string - ExtraRepos []string - DependencyLog string - BinShOverlay string - CreateBuildLog bool - ignorePatterns []*xignore.Pattern - CacheDir string - ApkCacheDir string - CacheSource string - BreakpointLabel string - ContinueLabel string - foundContinuation bool - StripOriginName bool - EnvFile string - VarsFile string - Runner container.Runner - RunnerName string - imgRef string - containerConfig *container.Config - Debug bool - DebugRunner bool - LogPolicy []string - FailOnLintWarning bool - DefaultCPU string - DefaultMemory string - DefaultTimeout time.Duration + Configuration config.Configuration + ConfigFile string + SourceDateEpoch time.Time + WorkspaceDir string + WorkspaceIgnore string + // Ordered directories where to find 'uses' pipelines. + PipelineDirs []string + SourceDir string + GuestDir string + SigningKey string + SigningPassphrase string + Namespace string + GenerateIndex bool + EmptyWorkspace bool + OutDir string + Logger apko_log.Logger + Arch apko_types.Architecture + ExtraKeys []string + ExtraRepos []string + DependencyLog string + BinShOverlay string + CreateBuildLog bool + ignorePatterns []*xignore.Pattern + CacheDir string + ApkCacheDir string + CacheSource string + BreakpointLabel string + ContinueLabel string + foundContinuation bool + StripOriginName bool + EnvFile string + VarsFile string + Runner container.Runner + RunnerName string + imgRef string + containerConfig *container.Config + Debug bool + DebugRunner bool + LogPolicy []string + FailOnLintWarning bool EnabledBuildOptions []string } @@ -312,18 +309,13 @@ func WithEmptyWorkspace(emptyWorkspace bool) Option { } } -// WithPipelineDir sets the pipeline directory to extend the built-in pipeline directory. +// WithPipelineDir sets the pipeline directory to extend the built-in pipeline +// directory. These are searched in order, so the first one found is used. func WithPipelineDir(pipelineDir string) Option { return func(b *Build) error { - b.PipelineDir = pipelineDir - return nil - } -} - -// WithBuiltinPipelineDirectory sets the pipeline directory to use. -func WithBuiltinPipelineDirectory(builtinPipelineDir string) Option { - return func(b *Build) error { - b.BuiltinPipelineDir = builtinPipelineDir + if pipelineDir != "" { + b.PipelineDirs = append(b.PipelineDirs, pipelineDir) + } return nil } } @@ -1007,7 +999,8 @@ func (b *Build) BuildPackage(ctx context.Context) error { b.Logger.Printf("evaluating pipelines for package requirements") for _, p := range b.Configuration.Pipeline { - pctx := NewPipelineContext(&p, b.Logger) + // fine to pass nil for config, since not running in container. + pctx := NewPipelineContext(&p, &b.Configuration.Environment, nil, b.PipelineDirs, b.Logger) if err := pctx.ApplyNeeds(&pb); err != nil { return fmt.Errorf("unable to apply pipeline requirements: %w", err) @@ -1018,7 +1011,8 @@ func (b *Build) BuildPackage(ctx context.Context) error { spkg := spkg pb.Subpackage = &spkg for _, p := range spkg.Pipeline { - pctx := NewPipelineContext(&p, b.Logger) + // fine to pass nil for config, since not running in container. + pctx := NewPipelineContext(&p, &b.Configuration.Environment, nil, b.PipelineDirs, b.Logger) if err := pctx.ApplyNeeds(&pb); err != nil { return fmt.Errorf("unable to apply pipeline requirements: %w", err) } @@ -1069,7 +1063,7 @@ func (b *Build) BuildPackage(ctx context.Context) error { // run the main pipeline b.Logger.Printf("running the main pipeline") for _, p := range b.Configuration.Pipeline { - pctx := NewPipelineContext(&p, b.Logger) + pctx := NewPipelineContext(&p, &b.Configuration.Environment, cfg, b.PipelineDirs, b.Logger) if _, err := pctx.Run(ctx, &pb); err != nil { return fmt.Errorf("unable to run pipeline: %w", err) } @@ -1104,7 +1098,7 @@ func (b *Build) BuildPackage(ctx context.Context) error { } for _, p := range sp.Pipeline { - pctx := NewPipelineContext(&p, b.Logger) + pctx := NewPipelineContext(&p, &b.Configuration.Environment, cfg, b.PipelineDirs, b.Logger) if _, err := pctx.Run(ctx, &pb); err != nil { return fmt.Errorf("unable to run pipeline: %w", err) } @@ -1126,7 +1120,7 @@ func (b *Build) BuildPackage(ctx context.Context) error { // Retrieve the post build workspace from the runner b.Logger.Infof("retrieving workspace from builder: %s", cfg.PodID) if err := b.RetrieveWorkspace(ctx); err != nil { - return fmt.Errorf("retrieving workspace: %v", err) + return fmt.Errorf("retrieving workspace: %w", err) } b.Logger.Printf("retrieved and wrote post-build workspace to: %s", b.WorkspaceDir) diff --git a/pkg/build/pipeline.go b/pkg/build/pipeline.go index 934e1f820..3279acbda 100644 --- a/pkg/build/pipeline.go +++ b/pkg/build/pipeline.go @@ -27,37 +27,65 @@ import ( "gopkg.in/yaml.v3" + apko_types "chainguard.dev/apko/pkg/build/types" apko_log "chainguard.dev/apko/pkg/log" "chainguard.dev/melange/pkg/cond" "chainguard.dev/melange/pkg/config" + "chainguard.dev/melange/pkg/container" "chainguard.dev/melange/pkg/logger" "chainguard.dev/melange/pkg/util" ) type PipelineContext struct { - Pipeline *config.Pipeline - logger apko_log.Logger - steps int + Pipeline *config.Pipeline + Environment *apko_types.ImageConfiguration + WorkspaceConfig *container.Config + // Ordered list of pipeline directories to search for pipelines + PipelineDirs []string + logger apko_log.Logger + steps int } -func NewPipelineContext(p *config.Pipeline, log apko_log.Logger) *PipelineContext { +func NewPipelineContext(p *config.Pipeline, environment *apko_types.ImageConfiguration, config *container.Config, pipelineDirs []string, log apko_log.Logger) *PipelineContext { if log == nil { log = logger.NopLogger{} } return &PipelineContext{ - Pipeline: p, - logger: log, - steps: 0, + Pipeline: p, + PipelineDirs: pipelineDirs, + Environment: environment, + WorkspaceConfig: config, + logger: log, + steps: 0, } } type PipelineBuild struct { Build *Build + Test *Test Package *config.Package Subpackage *config.Subpackage } +// GetConfiguration returns the configuration for the current pipeline. +// This is either for the Test or the Build +func (pb *PipelineBuild) GetConfiguration() *config.Configuration { + if pb.Test != nil { + return &pb.Test.Configuration + } + return &pb.Build.Configuration +} + +// GetRunner returns the runner for the current pipeline. +// This is either for the Test or the Build +func (pb *PipelineBuild) GetRunner() container.Runner { + if pb.Test != nil { + return pb.Test.Runner + } + return pb.Build.Runner +} + func (pctx *PipelineContext) Identity() string { if pctx.Pipeline.Name != "" { return pctx.Pipeline.Name @@ -98,21 +126,25 @@ func MutateWith(pb *PipelineBuild, with map[string]string) (map[string]string, e func substitutionMap(pb *PipelineBuild) (map[string]string, error) { nw := map[string]string{ - config.SubstitutionPackageName: pb.Package.Name, - config.SubstitutionPackageVersion: pb.Package.Version, - config.SubstitutionPackageEpoch: strconv.FormatUint(pb.Package.Epoch, 10), - config.SubstitutionPackageFullVersion: fmt.Sprintf("%s-r%s", config.SubstitutionPackageVersion, config.SubstitutionPackageEpoch), - config.SubstitutionTargetsDestdir: fmt.Sprintf("/home/build/melange-out/%s", pb.Package.Name), - config.SubstitutionTargetsContextdir: fmt.Sprintf("/home/build/melange-out/%s", pb.Package.Name), - config.SubstitutionHostTripletGnu: pb.Build.BuildTripletGnu(), - config.SubstitutionHostTripletRust: pb.Build.BuildTripletRust(), - config.SubstitutionCrossTripletGnuGlibc: pb.Build.Arch.ToTriplet("gnu"), - config.SubstitutionCrossTripletGnuMusl: pb.Build.Arch.ToTriplet("musl"), - config.SubstitutionBuildArch: pb.Build.Arch.ToAPK(), + config.SubstitutionPackageName: pb.Package.Name, + config.SubstitutionPackageVersion: pb.Package.Version, + config.SubstitutionPackageEpoch: strconv.FormatUint(pb.Package.Epoch, 10), + config.SubstitutionPackageFullVersion: fmt.Sprintf("%s-r%s", config.SubstitutionPackageVersion, config.SubstitutionPackageEpoch), + config.SubstitutionTargetsDestdir: fmt.Sprintf("/home/build/melange-out/%s", pb.Package.Name), + config.SubstitutionTargetsContextdir: fmt.Sprintf("/home/build/melange-out/%s", pb.Package.Name), + } + + // These are not really meaningful for Test, so only use them for build. + if pb.Build != nil { + nw[config.SubstitutionHostTripletGnu] = pb.Build.BuildTripletGnu() + nw[config.SubstitutionHostTripletRust] = pb.Build.BuildTripletRust() + nw[config.SubstitutionCrossTripletGnuGlibc] = pb.Build.Arch.ToTriplet("gnu") + nw[config.SubstitutionCrossTripletGnuMusl] = pb.Build.Arch.ToTriplet("musl") + nw[config.SubstitutionBuildArch] = pb.Build.Arch.ToAPK() } // Retrieve vars from config - subst_nw, err := pb.Build.Configuration.GetVarsFromConfig() + subst_nw, err := pb.GetConfiguration().GetVarsFromConfig() if err != nil { return nil, err } @@ -122,7 +154,7 @@ func substitutionMap(pb *PipelineBuild) (map[string]string, error) { } // Perform substitutions on current map - err = pb.Build.Configuration.PerformVarSubstitutions(nw) + err = pb.GetConfiguration().PerformVarSubstitutions(nw) if err != nil { return nil, err } @@ -133,7 +165,7 @@ func substitutionMap(pb *PipelineBuild) (map[string]string, error) { } packageNames := []string{pb.Package.Name} - for _, sp := range pb.Build.Configuration.Subpackages { + for _, sp := range pb.GetConfiguration().Subpackages { packageNames = append(packageNames, sp.Name) } @@ -142,14 +174,16 @@ func substitutionMap(pb *PipelineBuild) (map[string]string, error) { nw[k] = fmt.Sprintf("/home/build/melange-out/%s", pn) } - for k := range pb.Build.Configuration.Options { + for k := range pb.GetConfiguration().Options { nk := fmt.Sprintf("${{options.%s.enabled}}", k) nw[nk] = "false" } - for _, opt := range pb.Build.EnabledBuildOptions { - nk := fmt.Sprintf("${{options.%s.enabled}}", opt) - nw[nk] = "true" + if pb.Build != nil { + for _, opt := range pb.Build.EnabledBuildOptions { + nk := fmt.Sprintf("${{options.%s.enabled}}", opt) + nw[nk] = "true" + } } return nw, nil @@ -187,14 +221,25 @@ func loadPipelineData(dir string, uses string) ([]byte, error) { } func (pctx *PipelineContext) loadUse(pb *PipelineBuild, uses string, with map[string]string) error { - data, err := loadPipelineData(pb.Build.PipelineDir, uses) + var data []byte + // Set this to fail up front in case there are no pipeline dirs specified + // and we can't find them. + err := fmt.Errorf("could not find 'uses' pipeline %q", uses) + // See first if we can read from the specified pipeline dirs + // and if we can't, below we'll try from the embedded pipelines. + for _, pd := range pctx.PipelineDirs { + pctx.logger.Debugf("trying to load pipeline %q from %q", uses, pd) + data, err = loadPipelineData(pd, uses) + if err == nil { + pctx.logger.Printf("Found pipeline %s", string(data)) + break + } + } if err != nil { - data, err = loadPipelineData(pb.Build.BuiltinPipelineDir, uses) + pctx.logger.Debugf("trying to load pipeline %q from embedded fs pipelines/%q.yaml", uses, uses) + data, err = f.ReadFile("pipelines/" + uses + ".yaml") if err != nil { - data, err = f.ReadFile("pipelines/" + uses + ".yaml") - if err != nil { - return fmt.Errorf("unable to load pipeline: %w", err) - } + return fmt.Errorf("unable to load pipeline: %w", err) } } @@ -233,7 +278,7 @@ func (pctx *PipelineContext) dumpWith() { } func (pctx *PipelineContext) evalUse(ctx context.Context, pb *PipelineBuild) error { - spctx := NewPipelineContext(&config.Pipeline{}, pb.Build.Logger) + spctx := NewPipelineContext(&config.Pipeline{}, pctx.Environment, pctx.WorkspaceConfig, pctx.PipelineDirs, pctx.logger) if err := spctx.loadUse(pb, pctx.Pipeline.Uses, pctx.Pipeline.With); err != nil { return err @@ -282,7 +327,7 @@ func (pctx *PipelineContext) evalRun(ctx context.Context, pb *PipelineBuild) err pctx.dumpWith() debugOption := ' ' - if pb.Build.Debug { + if (pb.Build != nil && pb.Build.Debug) || (pb.Test != nil && pb.Test.Debug) { debugOption = 'x' } @@ -302,8 +347,7 @@ func (pctx *PipelineContext) evalRun(ctx context.Context, pb *PipelineBuild) err } command := pctx.buildEvalRunCommand(debugOption, sysPath, workdir, fragment) - config := pb.Build.WorkspaceConfig() - if err := pb.Build.Runner.Run(ctx, config, command...); err != nil { + if err := pb.GetRunner().Run(ctx, pctx.WorkspaceConfig, command...); err != nil { return err } @@ -335,6 +379,9 @@ func (pctx *PipelineContext) evaluateBranchConditional(pb *PipelineBuild) bool { } func (pctx *PipelineContext) isContinuationPoint(pb *PipelineBuild) bool { + if pb.Build == nil { + return true + } b := pb.Build if b.ContinueLabel == "" { @@ -397,7 +444,7 @@ func (pctx *PipelineContext) Run(ctx context.Context, pb *PipelineBuild) (bool, } for _, sp := range pctx.Pipeline.Pipeline { - spctx := NewPipelineContext(&sp, pb.Build.Logger) + spctx := NewPipelineContext(&sp, pctx.Environment, pctx.WorkspaceConfig, pctx.PipelineDirs, pctx.logger) if spctx.Pipeline.WorkDir == "" { spctx.Pipeline.WorkDir = pctx.Pipeline.WorkDir } @@ -423,15 +470,13 @@ func (pctx *PipelineContext) Run(ctx context.Context, pb *PipelineBuild) (bool, // TODO(kaniini): Precompile pipeline before running / evaluating its // needs. func (pctx *PipelineContext) ApplyNeeds(pb *PipelineBuild) error { - ic := &pb.Build.Configuration.Environment - for _, pkg := range pctx.Pipeline.Needs.Packages { pctx.logger.Printf(" adding package %q for pipeline %q", pkg, pctx.Identity()) - ic.Contents.Packages = append(ic.Contents.Packages, pkg) + pctx.Environment.Contents.Packages = append(pctx.Environment.Contents.Packages, pkg) } if pctx.Pipeline.Uses != "" { - spctx := NewPipelineContext(nil, pb.Build.Logger) + spctx := NewPipelineContext(nil, pctx.Environment, pctx.WorkspaceConfig, pctx.PipelineDirs, pctx.logger) if err := spctx.loadUse(pb, pctx.Pipeline.Uses, pctx.Pipeline.With); err != nil { return err @@ -442,10 +487,10 @@ func (pctx *PipelineContext) ApplyNeeds(pb *PipelineBuild) error { } } - ic.Contents.Packages = util.Dedup(ic.Contents.Packages) + pctx.Environment.Contents.Packages = util.Dedup(pctx.Environment.Contents.Packages) for _, sp := range pctx.Pipeline.Pipeline { - spctx := NewPipelineContext(&sp, pb.Build.Logger) + spctx := NewPipelineContext(&sp, pctx.Environment, pctx.WorkspaceConfig, pctx.PipelineDirs, pctx.logger) if err := spctx.ApplyNeeds(pb); err != nil { return err diff --git a/pkg/build/pipeline_test.go b/pkg/build/pipeline_test.go index c218e8a7f..9b88fc229 100644 --- a/pkg/build/pipeline_test.go +++ b/pkg/build/pipeline_test.go @@ -125,12 +125,12 @@ func Test_substitutionNeedPackages(t *testing.T) { }, } - pctx := NewPipelineContext(p, logger.NopLogger{}) + pctx := NewPipelineContext(p, nil, nil, []string{"pipelines"}, logger.NopLogger{}) pb := &PipelineBuild{ Package: pkg, Build: &Build{ - PipelineDir: "pipelines", + PipelineDirs: []string{"pipelines"}, Configuration: config.Configuration{ Pipeline: []config.Pipeline{ { @@ -156,7 +156,7 @@ func Test_buildEvalRunCommand(t *testing.T) { Environment: map[string]string{"FOO": "bar"}, } - pctx := NewPipelineContext(p, logger.NopLogger{}) + pctx := NewPipelineContext(p, nil, nil, []string{}, logger.NopLogger{}) debugOption := ' ' sysPath := "/foo" diff --git a/pkg/cli/build.go b/pkg/cli/build.go index e96073e23..68c325653 100644 --- a/pkg/cli/build.go +++ b/pkg/cli/build.go @@ -80,7 +80,10 @@ func Build() *cobra.Command { options := []build.Option{ build.WithBuildDate(buildDate), build.WithWorkspaceDir(workspaceDir), + // Order matters, so add any specified pipelineDir before + // builtin pipelines. build.WithPipelineDir(pipelineDir), + build.WithPipelineDir(BuiltinPipelineDir), build.WithCacheDir(cacheDir), build.WithCacheSource(cacheSource), build.WithPackageCacheDir(apkCacheDir), @@ -180,7 +183,7 @@ func BuildCmd(ctx context.Context, archs []apko_types.Architecture, baseOpts ... // https://github.com/distroless/nginx/runs/7219233843?check_suite_focus=true bcs := []*build.Build{} for _, arch := range archs { - opts := append(baseOpts, build.WithArch(arch), build.WithBuiltinPipelineDirectory(BuiltinPipelineDir)) + opts := append(baseOpts, build.WithArch(arch)) bc, err := build.New(ctx, opts...) if errors.Is(err, build.ErrSkipThisArch) { From c987b0f23397df4d2d1c360ff43ef640d09ae752 Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Tue, 28 Nov 2023 15:00:32 +0200 Subject: [PATCH 04/11] alphabetize commands, add test. Signed-off-by: Ville Aikas --- pkg/cli/commands.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pkg/cli/commands.go b/pkg/cli/commands.go index c6abfc9b3..1dd645938 100644 --- a/pkg/cli/commands.go +++ b/pkg/cli/commands.go @@ -35,18 +35,19 @@ func New() *cobra.Command { }, } - cmd.AddCommand(Completion()) cmd.AddCommand(Build()) cmd.AddCommand(Bump()) - cmd.AddCommand(Keygen()) + cmd.AddCommand(Completion()) + cmd.AddCommand(Convert()) cmd.AddCommand(Index()) + cmd.AddCommand(Keygen()) cmd.AddCommand(Lint()) + cmd.AddCommand(PackageVersion()) + cmd.AddCommand(Query()) cmd.AddCommand(Sign()) cmd.AddCommand(SignIndex()) + cmd.AddCommand(Test()) cmd.AddCommand(UpdateCache()) - cmd.AddCommand(Convert()) - cmd.AddCommand(PackageVersion()) - cmd.AddCommand(Query()) cmd.AddCommand(version.Version()) return cmd } From 01997dab6795254efdc0d8b82f63140b861d79e0 Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Tue, 28 Nov 2023 15:01:06 +0200 Subject: [PATCH 05/11] Add test command / implementation. Signed-off-by: Ville Aikas --- pkg/build/test.go | 854 ++++++++++++++++++++++++++++++++++++++++++++++ pkg/cli/test.go | 180 ++++++++++ 2 files changed, 1034 insertions(+) create mode 100644 pkg/build/test.go create mode 100644 pkg/cli/test.go diff --git a/pkg/build/test.go b/pkg/build/test.go new file mode 100644 index 000000000..ec510fac3 --- /dev/null +++ b/pkg/build/test.go @@ -0,0 +1,854 @@ +// Copyright 2023 Chainguard, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import ( + "context" + "errors" + "fmt" + "io" + "io/fs" + "os" + "path/filepath" + "runtime" + "strings" + + apko_build "chainguard.dev/apko/pkg/build" + apko_types "chainguard.dev/apko/pkg/build/types" + apko_iocomb "chainguard.dev/apko/pkg/iocomb" + apko_log "chainguard.dev/apko/pkg/log" + "cloud.google.com/go/storage" + apkofs "github.com/chainguard-dev/go-apk/pkg/fs" + "github.com/google/go-containerregistry/pkg/logs" + "github.com/yookoala/realpath" + "go.opentelemetry.io/otel" + "google.golang.org/api/iterator" + "google.golang.org/api/option" + + "chainguard.dev/melange/pkg/config" + "chainguard.dev/melange/pkg/container" +) + +type Test struct { + // Package to test. + Package string + Configuration config.Configuration + ConfigFile string + WorkspaceDir string + WorkspaceIgnore string + PipelineDir string + BuiltinPipelineDir string + // Ordered directories where to find 'uses' pipelines. + PipelineDirs []string + SourceDir string + GuestDir string + Namespace string + EmptyWorkspace bool + Logger apko_log.Logger + Arch apko_types.Architecture + ExtraKeys []string + ExtraRepos []string + DependencyLog string + BinShOverlay string + CacheDir string + ApkCacheDir string + CacheSource string + BreakpointLabel string + ContinueLabel string + EnvFile string + VarsFile string + Runner container.Runner + RunnerName string + Debug bool + DebugRunner bool + LogPolicy []string +} + +func NewTest(ctx context.Context, opts ...TestOption) (*Test, error) { + t := Test{ + WorkspaceIgnore: ".melangeignore", + //SourceDir: ".", + CacheDir: "./melange-cache/", + Arch: apko_types.ParseArchitecture(runtime.GOARCH), + LogPolicy: []string{"builtin:stderr"}, + } + + for _, opt := range opts { + if err := opt(&t); err != nil { + return nil, err + } + } + + writer, err := apko_iocomb.Combine(t.LogPolicy) + if err != nil { + return nil, err + } + + // Enable printing warnings and progress from GGCR. + logs.Warn.SetOutput(writer) + logs.Progress.SetOutput(writer) + + logger := &apko_log.Adapter{ + Out: writer, + Level: apko_log.InfoLevel, + } + + fields := apko_log.Fields{ + "arch": t.Arch.ToAPK(), + } + t.Logger = logger.WithFields(fields) + + // try to get the runner + runner, err := container.GetRunner(ctx, t.RunnerName, t.Logger) + if err != nil { + return nil, fmt.Errorf("unable to get runner %s: %w", t.RunnerName, err) + } + t.Runner = runner + + // If no workspace directory is explicitly requested, create a + // temporary directory for it. Otherwise, ensure we are in a + // subdir for this specific build context. + if t.WorkspaceDir != "" { + // If we are continuing the build, do not modify the workspace + // directory path. + // TODO(kaniini): Clean up the logic for this, perhaps by signalling + // multi-arch builds to the build context. + if t.ContinueLabel == "" { + t.WorkspaceDir = filepath.Join(t.WorkspaceDir, t.Arch.ToAPK()) + } + + // Get the absolute path to the workspace dir, which is needed for bind + // mounts. + absdir, err := filepath.Abs(t.WorkspaceDir) + if err != nil { + return nil, fmt.Errorf("unable to resolve path %s: %w", t.WorkspaceDir, err) + } + + t.WorkspaceDir = absdir + } else { + tmpdir, err := os.MkdirTemp(t.Runner.TempDir(), "melange-workspace-*") + if err != nil { + return nil, fmt.Errorf("unable to create workspace dir: %w", err) + } + t.WorkspaceDir = tmpdir + } + + // If no config file is explicitly requested for the test context + // we check if .melange.yaml or melange.yaml exist. + checks := []string{".melange.yaml", ".melange.yml", "melange.yaml", "melange.yml"} + if t.ConfigFile == "" { + for _, chk := range checks { + if _, err := os.Stat(chk); err == nil { + t.Logger.Printf("no configuration file provided -- using %s", chk) + t.ConfigFile = chk + break + } + } + } + + // If no config file could be automatically detected, error. + if t.ConfigFile == "" { + return nil, fmt.Errorf("melange.yaml is missing") + } + + parsedCfg, err := config.ParseConfiguration( + t.ConfigFile, + config.WithEnvFileForParsing(t.EnvFile), + config.WithLogger(t.Logger), + config.WithVarsFileForParsing(t.VarsFile)) + if err != nil { + return nil, fmt.Errorf("failed to load configuration: %w", err) + } + + t.Configuration = *parsedCfg + + // Check that we actually can run things in containers. + if !runner.TestUsability(ctx) { + return nil, fmt.Errorf("unable to run containers using %s, specify --runner and one of %s", runner.Name(), GetAllRunners()) + } + + return &t, nil +} + +type TestOption func(*Test) error + +// WithTestConfig sets the configuration file used for the package test context. +func WithTestConfig(configFile string) TestOption { + return func(t *Test) error { + t.ConfigFile = configFile + return nil + } +} + +// WithWorkspaceDir sets the workspace directory to use. +func WithTestWorkspaceDir(workspaceDir string) TestOption { + return func(t *Test) error { + t.WorkspaceDir = workspaceDir + return nil + } +} + +// WithGuestDir sets the guest directory to use. +func WithTestGuestDir(guestDir string) TestOption { + return func(t *Test) error { + t.GuestDir = guestDir + return nil + } +} + +// WithWorkspaceIgnore sets the workspace ignore rules file to use. +func WithTestWorkspaceIgnore(workspaceIgnore string) TestOption { + return func(t *Test) error { + t.WorkspaceIgnore = workspaceIgnore + return nil + } +} + +// WithEmptyWorkspace sets whether the workspace should be empty. +func WithTestEmptyWorkspace(emptyWorkspace bool) TestOption { + return func(t *Test) error { + t.EmptyWorkspace = emptyWorkspace + return nil + } +} + +// WithPipelineDir sets the pipeline directory to extend the built-in pipeline directory. +func WithTestPipelineDir(pipelineDir string) TestOption { + return func(t *Test) error { + t.PipelineDir = pipelineDir + return nil + } +} + +// WithBuiltinPipelineDirectory sets the pipeline directory to use. +func WithTestBuiltinPipelineDirectory(builtinPipelineDir string) TestOption { + return func(t *Test) error { + t.BuiltinPipelineDir = builtinPipelineDir + return nil + } +} + +// WithSourceDir sets the source directory to use. +func WithTestSourceDir(sourceDir string) TestOption { + return func(t *Test) error { + t.SourceDir = sourceDir + return nil + } +} + +// WithCacheDir sets the cache directory to use. +func WithTestCacheDir(cacheDir string) TestOption { + return func(t *Test) error { + t.CacheDir = cacheDir + return nil + } +} + +// WithCacheSource sets the cache source directory to use. The cache will be +// pre-populated from this source directory. +func WithTestCacheSource(sourceDir string) TestOption { + return func(t *Test) error { + t.CacheSource = sourceDir + return nil + } +} + +// WithTestArch sets the build architecture to use for this test context. +func WithTestArch(arch apko_types.Architecture) TestOption { + return func(t *Test) error { + t.Arch = arch + return nil + } +} + +// WithTestExtraKeys adds a set of extra keys to the test context. +func WithTestExtraKeys(extraKeys []string) TestOption { + return func(t *Test) error { + t.ExtraKeys = extraKeys + return nil + } +} + +// WithTestExtraRepos adds a set of extra repos to the build context. +func WithTestExtraRepos(extraRepos []string) TestOption { + return func(t *Test) error { + t.ExtraRepos = extraRepos + return nil + } +} + +// WithTestBinShOverlay sets a filename to copy from when installing /bin/sh +// into a build environment. +func WithTestBinShOverlay(binShOverlay string) TestOption { + return func(t *Test) error { + t.BinShOverlay = binShOverlay + return nil + } +} + +// WithTestDebugRunner indicates whether the runner should leave the build environment up on failures +func WithTestDebugRunner(debug bool) TestOption { + return func(t *Test) error { + t.DebugRunner = debug + return nil + } +} + +// WithTestLogPolicy sets the logging policy to use during tests. +func WithTestLogPolicy(policy []string) TestOption { + return func(t *Test) error { + t.LogPolicy = policy + return nil + } +} + +// WithTestRunner specifies what runner to use to wrap +// the test environment. +func WithTestRunner(runner string) TestOption { + return func(t *Test) error { + t.RunnerName = runner + return nil + } +} + +// WithTestPackage specifies the package to test. +func WithTestPackage(pkg string) TestOption { + return func(t *Test) error { + t.Package = pkg + return nil + } +} + +func WithTestPackageCacheDir(apkCacheDir string) TestOption { + return func(t *Test) error { + t.ApkCacheDir = apkCacheDir + return nil + } +} + +// BuildGuest invokes apko to create the test image for the guest environment. +// imgConfig specifies the environment for the test to run (e.g. packages to +// install). +// Returns the imgRef for the created image, or error. +func (t *Test) BuildGuest(ctx context.Context, imgConfig *apko_types.ImageConfiguration, suffix string) (string, error) { + ctx, span := otel.Tracer("melange").Start(ctx, "BuildGuest") + defer span.End() + + // Prepare workspace directory + if err := os.MkdirAll(t.WorkspaceDir, 0755); err != nil { + return "", fmt.Errorf("mkdir -p %s: %w", t.WorkspaceDir, err) + } + + // Prepare guest directory. Note that we customize this for each unique + // Test by having a suffix, so we get a clean guest directory for each of + // them. + guestDir := fmt.Sprintf("%s-%s", t.GuestDir, suffix) + if err := os.MkdirAll(guestDir, 0755); err != nil { + return "", fmt.Errorf("mkdir -p %s: %w", guestDir, err) + } + + t.Logger.Printf("building test workspace in: '%s' with apko", guestDir) + + guestFS := apkofs.DirFS(guestDir, apkofs.WithCreateDir()) + + bc, err := apko_build.New(ctx, guestFS, + apko_build.WithImageConfiguration(*imgConfig), + apko_build.WithArch(t.Arch), + apko_build.WithExtraKeys(t.ExtraKeys), + apko_build.WithExtraRepos(t.ExtraRepos), + apko_build.WithLogger(t.Logger), + apko_build.WithDebugLogging(true), + apko_build.WithCacheDir(t.ApkCacheDir, false), // TODO: Replace with real offline plumbing + ) + if err != nil { + return "", fmt.Errorf("unable to create build context: %w", err) + } + + bc.Summarize() + + // lay out the contents for the image in a directory. + if err := bc.BuildImage(ctx); err != nil { + return "", fmt.Errorf("unable to generate image: %w", err) + } + // if the runner needs an image, create an OCI image from the directory and load it. + loader := t.Runner.OCIImageLoader() + if loader == nil { + return "", fmt.Errorf("runner %s does not support OCI image loading", t.Runner.Name()) + } + layerTarGZ, layer, err := bc.ImageLayoutToLayer(ctx) + if err != nil { + return "", err + } + defer os.Remove(layerTarGZ) + + t.Logger.Printf("using %s for image layer", layerTarGZ) + + ref, err := loader.LoadImage(ctx, layer, t.Arch, bc) + if err != nil { + return "", err + } + + t.Logger.Printf("pushed %s as %v", layerTarGZ, ref) + t.Logger.Printf("successfully built workspace with apko") + + return ref, nil +} + +// ApplyBuildOption applies a patch described by a BuildOption to a package build. +func (t *Test) ApplyTestOption(to config.BuildOption) error { + // Patch the variables block. + if t.Configuration.Vars == nil { + t.Configuration.Vars = make(map[string]string) + } + + for k, v := range to.Vars { + t.Configuration.Vars[k] = v + } + + // Patch the test environment configuration. + lo := to.Environment.Contents.Packages + t.Configuration.Test.Environment.Contents.Packages = append(t.Configuration.Test.Environment.Contents.Packages, lo.Add...) + + for _, pkg := range lo.Remove { + pkgList := t.Configuration.Test.Environment.Contents.Packages + + for pos, ppkg := range pkgList { + if pkg == ppkg { + pkgList[pos] = pkgList[len(pkgList)-1] + pkgList = pkgList[:len(pkgList)-1] + } + } + + t.Configuration.Test.Environment.Contents.Packages = pkgList + } + + return nil +} + +func (t *Test) OverlayBinSh(suffix string) error { + if t.BinShOverlay == "" { + return nil + } + + guestDir := fmt.Sprintf("%s-%s", t.GuestDir, suffix) + + targetPath := filepath.Join(guestDir, "bin", "sh") + + inF, err := os.Open(t.BinShOverlay) + if err != nil { + return fmt.Errorf("copying overlay /bin/sh: %w", err) + } + defer inF.Close() + + // We unlink the target first because it might be a symlink. + if err := os.Remove(targetPath); err != nil { + return fmt.Errorf("copying overlay /bin/sh: %w", err) + } + + outF, err := os.Create(targetPath) + if err != nil { + return fmt.Errorf("copying overlay /bin/sh: %w", err) + } + defer outF.Close() + + if _, err := io.Copy(outF, inF); err != nil { + return fmt.Errorf("copying overlay /bin/sh: %w", err) + } + + if err := os.Chmod(targetPath, 0o755); err != nil { + return fmt.Errorf("setting overlay /bin/sh executable: %w", err) + } + + return nil +} + +func (t *Test) fetchBucket(ctx context.Context, cmm CacheMembershipMap) (string, error) { + tmp, err := os.MkdirTemp("", "melange-cache") + if err != nil { + return "", err + } + bucket, prefix, _ := strings.Cut(strings.TrimPrefix(t.CacheSource, "gs://"), "/") + + client, err := storage.NewClient(ctx) + if err != nil { + t.Logger.Printf("downgrading to anonymous mode: %s", err) + + client, err = storage.NewClient(ctx, option.WithoutAuthentication()) + if err != nil { + return "", fmt.Errorf("failed to get storage client: %w", err) + } + } + + bh := client.Bucket(bucket) + it := bh.Objects(ctx, &storage.Query{Prefix: prefix}) + for { + attrs, err := it.Next() + if err == iterator.Done { + break + } else if err != nil { + return tmp, fmt.Errorf("failed to get next remote cache object: %w", err) + } + on := attrs.Name + if !cmm[on] { + continue + } + rc, err := bh.Object(on).NewReader(ctx) + if err != nil { + return tmp, fmt.Errorf("failed to get reader for next remote cache object %s: %w", on, err) + } + w, err := os.Create(filepath.Join(tmp, on)) + if err != nil { + return tmp, err + } + if _, err := io.Copy(w, rc); err != nil { + return tmp, fmt.Errorf("failed to copy remote cache object %s: %w", on, err) + } + if err := rc.Close(); err != nil { + return tmp, fmt.Errorf("failed to close remote cache object %s: %w", on, err) + } + t.Logger.Printf("cached gs://%s/%s -> %s", bucket, on, w.Name()) + } + + return tmp, nil +} + +// IsTestless returns true if the test context does not actually do any +// testing. +func (t *Test) IsTestless() bool { + return len(t.Configuration.Test.Pipeline) == 0 +} + +func (t *Test) PopulateCache(ctx context.Context) error { + ctx, span := otel.Tracer("melange").Start(ctx, "PopulateCache") + defer span.End() + + if t.CacheDir == "" { + return nil + } + + cmm, err := cacheItemsForBuild(t.ConfigFile) + if err != nil { + return fmt.Errorf("while determining which objects to fetch: %w", err) + } + + t.Logger.Printf("populating cache from %s", t.CacheSource) + + // --cache-dir=gs://bucket/path/to/cache first pulls all found objects to a + // tmp dir which is subsequently used as the cache. + if strings.HasPrefix(t.CacheSource, "gs://") { + tmp, err := t.fetchBucket(ctx, cmm) + if err != nil { + return err + } + defer os.RemoveAll(tmp) + t.Logger.Printf("cache bucket copied to %s", tmp) + + fsys := os.DirFS(tmp) + + // mkdir /var/cache/melange + if err := os.MkdirAll(t.CacheDir, 0o755); err != nil { + return err + } + + // --cache-dir doesn't exist, nothing to do. + if _, err := fs.Stat(fsys, "."); errors.Is(err, fs.ErrNotExist) { + return nil + } + + return fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + fi, err := d.Info() + if err != nil { + return err + } + + mode := fi.Mode() + if !mode.IsRegular() { + return nil + } + + // Skip files in the cache that aren't named like sha256:... or sha512:... + // This is likely a bug, and won't be matched by any fetch. + base := filepath.Base(fi.Name()) + if !strings.HasPrefix(base, "sha256:") && + !strings.HasPrefix(base, "sha512:") { + return nil + } + + t.Logger.Debugf(" -> %s", path) + + return copyFile(tmp, path, t.CacheDir, mode.Perm()) + }) + } + + return nil +} + +func (t *Test) PopulateWorkspace(ctx context.Context) error { + _, span := otel.Tracer("melange").Start(ctx, "PopulateWorkspace") + defer span.End() + + if t.EmptyWorkspace { + t.Logger.Printf("empty workspace requested") + return nil + } + if t.SourceDir == "" { + t.Logger.Printf("No source directory specified, skipping workspace population") + return nil + } + + t.Logger.Printf("populating workspace %s from %s", t.WorkspaceDir, t.SourceDir) + + fsys := os.DirFS(t.SourceDir) + + return fs.WalkDir(fsys, ".", func(path string, d fs.DirEntry, err error) error { + if err != nil { + return err + } + + fi, err := d.Info() + if err != nil { + return err + } + + mode := fi.Mode() + if !mode.IsRegular() { + return nil + } + + t.Logger.Debugf(" -> %s", path) + + if err := copyFile(t.SourceDir, path, t.WorkspaceDir, mode.Perm()); err != nil { + return err + } + + return nil + }) +} + +func (t *Test) TestPackage(ctx context.Context) error { + ctx, span := otel.Tracer("melange").Start(ctx, "testPackage") + defer span.End() + + pkg := &t.Configuration.Package + + pb := PipelineBuild{ + Test: t, + Package: pkg, + } + + if t.GuestDir == "" { + guestDir, err := os.MkdirTemp(t.Runner.TempDir(), "melange-guest-*") + if err != nil { + return fmt.Errorf("unable to make guest directory: %w", err) + } + t.GuestDir = guestDir + } + + t.Logger.Printf("evaluating main pipeline for package requirements") + // Append the main test package to be installed. + t.Configuration.Test.Environment.Contents.Packages = append(t.Configuration.Test.Environment.Contents.Packages, pkg.Name) + for i := range t.Configuration.Test.Pipeline { + p := &t.Configuration.Test.Pipeline[i] + // fine to pass nil for config, since not running in container. + pctx := NewPipelineContext(p, &t.Configuration.Test.Environment, nil, t.PipelineDirs, t.Logger) + + if err := pctx.ApplyNeeds(&pb); err != nil { + return fmt.Errorf("unable to apply pipeline requirements: %w", err) + } + } + + imgRef := "" + var err error + + // If there are no 'main' test pipelines, we can skip building the guest. + if !t.IsTestless() { + imgRef, err = t.BuildGuest(ctx, &t.Configuration.Test.Environment, "main") + if err != nil { + return fmt.Errorf("unable to build guest: %w", err) + } + + // TODO(kaniini): Make overlay-binsh work with Docker and Kubernetes. + // Probably needs help from apko. + if err := t.OverlayBinSh(""); err != nil { + return fmt.Errorf("unable to install overlay /bin/sh: %w", err) + } + + if err := t.PopulateCache(ctx); err != nil { + return fmt.Errorf("unable to populate cache: %w", err) + } + } + + if err := t.PopulateWorkspace(ctx); err != nil { + return fmt.Errorf("unable to populate workspace: %w", err) + } + + cfg := t.buildWorkspaceConfig(imgRef, pkg.Name, t.Configuration.Environment.Environment) + if !t.IsTestless() { + cfg.Arch = t.Arch + if err := t.Runner.StartPod(ctx, cfg); err != nil { + return fmt.Errorf("unable to start pod: %w", err) + } + if !t.DebugRunner { + defer func() { + if err := t.Runner.TerminatePod(ctx, cfg); err != nil { + t.Logger.Warnf("unable to terminate pod: %s", err) + } + }() + } + + // run the main test pipeline + t.Logger.Printf("running the main test pipeline") + for i := range t.Configuration.Test.Pipeline { + p := &t.Configuration.Test.Pipeline[i] + pctx := NewPipelineContext(p, &t.Configuration.Test.Environment, cfg, t.PipelineDirs, t.Logger) + if _, err := pctx.Run(ctx, &pb); err != nil { + return fmt.Errorf("unable to run pipeline: %w", err) + } + } + } + + // Run any test pipelines for subpackages. + // Note that we create a fresh container for each subpackage to ensure + // that we don't keep adding packages to tests and hence mask any missing + // dependencies. + for i := range t.Configuration.Subpackages { + sp := &t.Configuration.Subpackages[i] + if len(sp.Test.Pipeline) > 0 { + // Append the subpackage that we're testing to be installed. + sp.Test.Environment.Contents.Packages = append(sp.Test.Environment.Contents.Packages, sp.Name) + + // See if there are any packages needed by the 'uses' pipelines, so + // they get built into the container. + for i := range sp.Test.Pipeline { + p := &sp.Test.Pipeline[i] + // fine to pass nil for config, since not running in container. + pctx := NewPipelineContext(p, &sp.Test.Environment, nil, t.PipelineDirs, t.Logger) + if err := pctx.ApplyNeeds(&pb); err != nil { + return fmt.Errorf("unable to apply pipeline requirements: %w", err) + } + } + + t.Logger.Printf("running test pipeline for subpackage %s", sp.Name) + pb.Subpackage = sp + + spImgRef, err := t.BuildGuest(ctx, &sp.Test.Environment, sp.Name) + if err != nil { + return fmt.Errorf("unable to build guest: %w", err) + } + if err := t.OverlayBinSh(sp.Name); err != nil { + return fmt.Errorf("unable to install overlay /bin/sh: %w", err) + } + subCfg := t.buildWorkspaceConfig(spImgRef, sp.Name, sp.Test.Environment.Environment) + subCfg.Arch = t.Arch + if err := t.Runner.StartPod(ctx, subCfg); err != nil { + return fmt.Errorf("unable to start subpackage test pod: %w", err) + } + if !t.DebugRunner { + defer func() { + if err := t.Runner.TerminatePod(ctx, subCfg); err != nil { + t.Logger.Warnf("unable to terminate subpackage test pod: %s", err) + } + }() + } + + result, err := pb.ShouldRun(*sp) + if err != nil { + return err + } + if !result { + continue + } + + for i := range sp.Test.Pipeline { + p := &sp.Test.Pipeline[i] + pctx := NewPipelineContext(p, &sp.Test.Environment, subCfg, t.PipelineDirs, t.Logger) + if _, err := pctx.Run(ctx, &pb); err != nil { + return fmt.Errorf("unable to run pipeline: %w", err) + } + } + } + pb.Subpackage = nil + + if err := os.MkdirAll(filepath.Join(t.WorkspaceDir, "melange-out", sp.Name), 0o755); err != nil { + return err + } + } + + // clean workspace dir + if err := os.RemoveAll(t.WorkspaceDir); err != nil { + t.Logger.Printf("WARNING: unable to clean workspace: %s", err) + } + return nil +} + +func (t *Test) SummarizePaths() { + t.Logger.Printf(" workspace dir: %s", t.WorkspaceDir) + + if t.GuestDir != "" { + t.Logger.Printf(" guest dir: %s", t.GuestDir) + } +} + +func (t *Test) Summarize() { + t.Logger.Printf("melange is testing:") + t.Logger.Printf(" configuration file: %s", t.ConfigFile) + t.SummarizePaths() +} + +func (t *Test) buildWorkspaceConfig(imgRef, pkgName string, env map[string]string) *container.Config { + mounts := []container.BindMount{ + {Source: t.WorkspaceDir, Destination: container.DefaultWorkspaceDir}, + {Source: "/etc/resolv.conf", Destination: container.DefaultResolvConfPath}, + } + + if t.CacheDir != "" { + if fi, err := os.Stat(t.CacheDir); err == nil && fi.IsDir() { + mountSource, err := realpath.Realpath(t.CacheDir) + if err != nil { + t.Logger.Printf("could not resolve path for --cache-dir: %s", err) + } + + mounts = append(mounts, container.BindMount{Source: mountSource, Destination: container.DefaultCacheDir}) + } else { + t.Logger.Printf("--cache-dir %s not a dir; skipping", t.CacheDir) + } + } + + // TODO(kaniini): Disable networking capability according to the pipeline requirements. + caps := container.Capabilities{ + Networking: true, + } + + cfg := container.Config{ + PackageName: pkgName, + Mounts: mounts, + Capabilities: caps, + Logger: t.Logger, + Environment: map[string]string{}, + } + + for k, v := range env { + cfg.Environment[k] = v + } + + cfg.ImgRef = imgRef + t.Logger.Printf("ImgRef = %s", cfg.ImgRef) + + return &cfg +} diff --git a/pkg/cli/test.go b/pkg/cli/test.go new file mode 100644 index 000000000..6af9d4672 --- /dev/null +++ b/pkg/cli/test.go @@ -0,0 +1,180 @@ +// Copyright 2023 Chainguard, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cli + +import ( + "context" + "errors" + "fmt" + "log" + + apko_types "chainguard.dev/apko/pkg/build/types" + "chainguard.dev/melange/pkg/build" + "github.com/spf13/cobra" + "go.opentelemetry.io/otel" + "golang.org/x/sync/errgroup" +) + +func Test() *cobra.Command { + var buildDate string + var workspaceDir string + var pipelineDir string + var sourceDir string + var cacheDir string + var cacheSource string + var apkCacheDir string + var guestDir string + var signingKey string + var generateIndex bool + var emptyWorkspace bool + var stripOriginName bool + var outDir string + var archstrs []string + var extraKeys []string + var extraRepos []string + var dependencyLog string + var overlayBinSh string + var breakpointLabel string + var continueLabel string + var envFile string + var varsFile string + var purlNamespace string + var buildOption []string + var logPolicy []string + var createBuildLog bool + var debug bool + var debugRunner bool + var runner string + var failOnLintWarning bool + + cmd := &cobra.Command{ + Use: "test", + Short: "Test a package with a YAML configuration file", + Long: `Test a package from a YAML configuration file containing a test pipeline.`, + Example: ` melange test `, + Args: cobra.MinimumNArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + archs := apko_types.ParseArchitectures(archstrs) + options := []build.TestOption{ + build.WithTestWorkspaceDir(workspaceDir), + build.WithTestPipelineDir(pipelineDir), + build.WithTestCacheDir(cacheDir), + build.WithTestCacheSource(cacheSource), + build.WithTestPackageCacheDir(apkCacheDir), + build.WithTestGuestDir(guestDir), + build.WithTestEmptyWorkspace(emptyWorkspace), + build.WithTestExtraKeys(extraKeys), + build.WithTestExtraRepos(extraRepos), + build.WithTestBinShOverlay(overlayBinSh), + build.WithTestRunner(runner), + } + + if len(args) > 0 { + options = append(options, build.WithTestConfig(args[0])) + options = append(options, build.WithTestPackage(args[1])) + } + + if sourceDir != "" { + options = append(options, build.WithTestSourceDir(sourceDir)) + } + + return TestCmd(cmd.Context(), archs, options...) + }, + } + + cmd.Flags().StringVar(&buildDate, "build-date", "", "date used for the timestamps of the files inside the image") + cmd.Flags().StringVar(&workspaceDir, "workspace-dir", "", "directory used for the workspace at /home/build") + cmd.Flags().StringVar(&pipelineDir, "pipeline-dir", "", "directory used to extend defined built-in pipelines") + cmd.Flags().StringVar(&sourceDir, "source-dir", "", "directory used for included sources") + cmd.Flags().StringVar(&cacheDir, "cache-dir", "./melange-cache/", "directory used for cached inputs") + cmd.Flags().StringVar(&cacheSource, "cache-source", "", "directory or bucket used for preloading the cache") + cmd.Flags().StringVar(&apkCacheDir, "apk-cache-dir", "", "directory used for cached apk packages (default is system-defined cache directory)") + cmd.Flags().StringVar(&guestDir, "guest-dir", "", "directory used for the build environment guest") + cmd.Flags().StringVar(&signingKey, "signing-key", "", "key to use for signing") + cmd.Flags().StringVar(&envFile, "env-file", "", "file to use for preloaded environment variables") + cmd.Flags().StringVar(&varsFile, "vars-file", "", "file to use for preloaded build configuration variables") + cmd.Flags().BoolVar(&generateIndex, "generate-index", true, "whether to generate APKINDEX.tar.gz") + cmd.Flags().BoolVar(&emptyWorkspace, "empty-workspace", false, "whether the build workspace should be empty") + cmd.Flags().BoolVar(&stripOriginName, "strip-origin-name", false, "whether origin names should be stripped (for bootstrap)") + cmd.Flags().StringVar(&outDir, "out-dir", "./packages/", "directory where packages will be output") + cmd.Flags().StringVar(&dependencyLog, "dependency-log", "", "log dependencies to a specified file") + cmd.Flags().StringVar(&overlayBinSh, "overlay-binsh", "", "use specified file as /bin/sh overlay in build environment") + cmd.Flags().StringVar(&breakpointLabel, "breakpoint-label", "", "stop build execution at the specified label") + cmd.Flags().StringVar(&continueLabel, "continue-label", "", "continue build execution at the specified label") + cmd.Flags().StringVar(&purlNamespace, "namespace", "unknown", "namespace to use in package URLs in SBOM (eg wolfi, alpine)") + cmd.Flags().StringSliceVar(&archstrs, "arch", nil, "architectures to build for (e.g., x86_64,ppc64le,arm64) -- default is all, unless specified in config") + cmd.Flags().StringSliceVar(&buildOption, "build-option", []string{}, "build options to enable") + cmd.Flags().StringSliceVar(&logPolicy, "log-policy", []string{"builtin:stderr"}, "logging policy to use") + cmd.Flags().StringVar(&runner, "runner", string(build.GetDefaultRunner()), fmt.Sprintf("which runner to use to enable running commands, default is based on your platform. Options are %q", build.GetAllRunners())) + cmd.Flags().StringSliceVarP(&extraKeys, "keyring-append", "k", []string{}, "path to extra keys to include in the build environment keyring") + cmd.Flags().StringSliceVarP(&extraRepos, "repository-append", "r", []string{}, "path to extra repositories to include in the build environment") + cmd.Flags().BoolVar(&createBuildLog, "create-build-log", false, "creates a package.log file containing a list of packages that were built by the command") + cmd.Flags().BoolVar(&debug, "debug", false, "enables debug logging of build pipelines") + cmd.Flags().BoolVar(&debugRunner, "debug-runner", false, "when enabled, the builder pod will persist after the build succeeds or fails") + cmd.Flags().BoolVar(&failOnLintWarning, "fail-on-lint-warning", false, "turns linter warnings into failures") + + return cmd +} + +func TestCmd(ctx context.Context, archs []apko_types.Architecture, baseOpts ...build.TestOption) error { + ctx, span := otel.Tracer("melange").Start(ctx, "TestCmd") + defer span.End() + + if len(archs) == 0 { + archs = apko_types.AllArchs + } + + // Set up the test contexts before running them. This avoids various + // race conditions and the possibility that a context may be garbage + // collected before it is actually run. + // + // Yes, this happens. Really. + // https://github.com/distroless/nginx/runs/7219233843?check_suite_focus=true + bcs := []*build.Test{} + for _, arch := range archs { + opts := append(baseOpts, build.WithTestArch(arch), build.WithTestBuiltinPipelineDirectory(BuiltinPipelineDir)) + + bc, err := build.NewTest(ctx, opts...) + if errors.Is(err, build.ErrSkipThisArch) { + log.Printf("skipping arch %s", arch) + continue + } else if err != nil { + return err + } + + bcs = append(bcs, bc) + } + + if len(bcs) == 0 { + log.Printf("WARNING: target-architecture and --arch do not overlap, nothing to build") + return nil + } + + var errg errgroup.Group + for _, bc := range bcs { + bc := bc + + errg.Go(func() error { + if err := bc.TestPackage(ctx); err != nil { + log.Printf("ERROR: failed to test package. the test environment has been preserved:") + bc.SummarizePaths() + + return fmt.Errorf("failed to build package: %w", err) + } + return nil + }) + } + return errg.Wait() +} From 4fbc1d47f8c5316019a030724c6d60d21914c483 Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Wed, 29 Nov 2023 10:59:20 +0200 Subject: [PATCH 06/11] checkpoint. Signed-off-by: Ville Aikas --- docs/TESTING.md | 172 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 172 insertions(+) create mode 100644 docs/TESTING.md diff --git a/docs/TESTING.md b/docs/TESTING.md new file mode 100644 index 000000000..23b3caa44 --- /dev/null +++ b/docs/TESTING.md @@ -0,0 +1,172 @@ +# Testing + +Melange provides an ability to test packages with a `test` command. Tests are +implemented using the same `pipeline`, and `subpackages` centric ways to define +tests, so it should be very familiar continuation from `build` to `test`. + +## Overview + +The keyword `test:` starts a new test block, and can be embedded either at the +top level, or inside a subpackage. You can test the 'main' package, and +subpackages, or any combination of them. Each `test` block has a section for +specifying the test configuration including the necessary packages, (partly to +ensure the minimal set of packages, and therefore testing runtime dependencies +definitions), as well as any environmental variables. This section again, looks +exactly like a build pipelines, and therefore should feel very familiar. + +### Test environment (workspace) + +Just like with the build, there is a single shared `workspace` that gets mounted +as the `CWD` for each of the `test` runs. You can add any test fixtures, for +example, if you are testing some python packages, you could create `foo-test.py` +file, and by using a `--source-dir` pointing to the directory, the files in that +directory will then be available for your tests in the current directory. For +example, say you are testing `py3-pandas` package, and would like to exercise +some data transformations, you could create a file +`/tmp/testfiles/pandas-test.py`: + +```python +import numpy as np +import pandas as pd +s = pd.Series([1,3,5,np.nan, 6, 8]) +dates = pd.date_range("20130101", periods=6) +df = pd.DataFrame(np.random.randn(6, 4), index=dates, columns=list("ABCD")) +``` + +Then you could make sure this file ends up in your workspace as +`./pandas-test.py` by specifying `--source-dir /tmp/testfiles` + +### Execution environment (guest) + +Unlike a `build` guest, each `test` will get their own "fresh" container built +using apko that only contains the Package Under Test (PUT), that is defaulted to +each container depending on the context, as well as any additional packages +specified in the `test.environment.contents.packages`. For example, the "main" +`test` pipeline will get the main package added by default. For subpackages, the +subpackage that has the `test` block gets added by default. + +For example, with a test environment like this for the main package +(for example, `py3-pandas`): +```yaml +package: + name: py3-pandas +# Stuff omitted for readability +test: + environment: + contents: + packages: + - busybox + - python-3 +``` + +Will get a test execution containing the following packages (and their +transitive dependencies as per apk solver): + * py3-pandas + * busybox + * python-3 + +And a subpackage test like this: +```yaml +package: + name: php-8.2-msgpack +# Again stuff omitted for readability +subpackages: + - name: ${{package.name}}-dev + description: PHP 8.2 msgpack development headers + test: + environment: + contents: + packages: + - busybox + - wolfi-base + pipeline: + - runs: | + # Just make sure the expected define is in the expected file + # location. + grep PHP_MSGPACK_VERSION /usr/include/php/ext/msgpack/php_msgpack.h +``` + +This will get a test execution containing the following packages (and their +transitive dependencies as per apk solver): + * php-8.2-msgpack + * busybox + * wolfi-base + +### Execution environment, repo configuration + +Because we use the apk solver, and apko to build the guest containers, it's easy +to configure if you are testing either local, or remote, or combination of the +packages by simply configuring the appropriate repos. By using these, you can +easily add tests to existing packages without having to rebuild them, or fetch +them directly and juggle them. You can also iterate on the package and tests at +the same time, by rebuilding your local package (and ofc adding the repo +configuration for it). Because we rely on the normal apk solver rules for +figuring out which package to install into the guest context, it is flexible +enough to test whatever combinations of packages that you want to test with. + +As discussed above, you can specify which packages are tested, as well as which +packages a particular test needs to perform the tests (for example, you could +try to `curl` a URL to test a package, so that would require curl). You can +configure these with `--keyring-append` as well as `--repository-append` +variables. As usual by default the "highest" one wins, so if you are testing +local changes to a package, you can build a local version, and by bumping the +epoch it will become the PUT, or you can configure/test PUT dependencies this +way (build a local copy, and it will be picked up by APK resolver). This is very +similar to how we build/test images with local versions of packages, so again, +this should feel very natural. + +### Where to define the tests? + +So, this is one open question, but the short answer is that you can add these +tests inline with the existing yaml files that specify the build, OR you can +define them in alternate location. Because of the way the melange configuration +parsing currently works, you may need to add some "placeholders" to satisfy +the configuration parser. For example, here's a simple test file that I've been +using to test things that has some "placeholder" fields that are not really +actually used, but will allow one to decouple the test/build file if that's the +direction we want to go: + +```yaml +package: + name: php-8.2-msgpack + version: 2.2.0 + epoch: 0 + description: "Tests for PHP extension msgpack" + copyright: + - license: BSD-3-Clause + +# This is mandatory, so just put an empty one there. Otherwise, config parsing +# will fail. +pipeline: + +test: + environment: + contents: + packages: + - wolfi-base + - apk-tools + pipeline: + - runs: | + # Stuff goes here. +``` + +## Full example + +Here's a full example invocation, where I'm testing with my local mac, so just +testing the aarch64 (hence `--arch aarch64`` flag), and I'm pulling in the +abovementioned `py3-pandas-test.yaml` file as specified from the current +directory, and I do want to test the py3-pandas package (second argument), and +the keyring/repository append flags pull in my local changes, so that I can +iterate on package building, as well as testing at the same time. If you are +only writing tests for existing packages, you could drop the "local" +keyring/repository, and then only released packages would be pulled in for +testing. + +```shell +melange test ./py3-pandas-test.yaml py3-pandas \ +--source-dir /tmp/testfiles --arch aarch64 \ +--keyring-append /Users/vaikas/projects/go/src/github.com/wolfi-dev/os/local-melange.rsa.pub \ +--repository-append /Users/vaikas/projects/go/src/github.com/wolfi-dev/os/packages \ +--repository-append https://packages.wolfi.dev/os \ +--keyring-append https://packages.wolfi.dev/os/wolfi-signing.rsa.pub +``` From c219511b07ae18ae7ea48f89ed23f478dff12e29 Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Thu, 30 Nov 2023 12:50:40 +0200 Subject: [PATCH 07/11] e2e tests for `test` command. Signed-off-by: Ville Aikas --- .github/workflows/melange-test-pipelines.yaml | 81 +++++++++++++++++++ e2e-tests/php-8.2-msgpack-test.yaml | 52 ++++++++++++ e2e-tests/py3-pandas-test.yaml | 19 +++++ e2e-tests/test-fixtures/py3-pandas-test.py | 6 ++ 4 files changed, 158 insertions(+) create mode 100644 .github/workflows/melange-test-pipelines.yaml create mode 100644 e2e-tests/php-8.2-msgpack-test.yaml create mode 100644 e2e-tests/py3-pandas-test.yaml create mode 100644 e2e-tests/test-fixtures/py3-pandas-test.py diff --git a/.github/workflows/melange-test-pipelines.yaml b/.github/workflows/melange-test-pipelines.yaml new file mode 100644 index 000000000..1f63130e0 --- /dev/null +++ b/.github/workflows/melange-test-pipelines.yaml @@ -0,0 +1,81 @@ +name: ci + +on: + pull_request: + push: + branches: + - 'main' + +jobs: + build-melange: + name: Build melange and add to artifact cache + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + + - uses: actions/setup-go@93397bea11091df50f3d7e59dc26a7711a8bcfbe # v4.1.0 + with: + go-version: '1.21' + check-latest: true + + - name: build + run: | + make melange + + - uses: actions/upload-artifact@a8a3f3ad30e3422c9c7b888a15615d19a852ae32 # v3.1.3 + with: + name: melange-${{github.run_id}} + path: ${{github.workspace}}/melange + retention-days: 1 + + build-packages: + name: Build packages + needs: + - build-melange + # TODO: Set up a larger runner for this. + runs-on: ubuntu-latest + + # This is a list of packages which covers basic and exotic uses of + # the built-in pipelines. Goal is to balance efficiency while also + # exercising Melange with real-world package builds. + # Feel free to add additional packages to this matrix which exercise + # Melange in new ways (e.g. new pipelines, etc.) + strategy: + fail-fast: false + matrix: + package: + - hello-wolfi + - glibc + - tini + - lzo + - bubblewrap + - gdk-pixbuf + - gitsign + - guac + - mdbook + - s3cmd + - perl-yaml-syck + - xmlto + - ncurses + + steps: + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + with: + repository: wolfi-dev/os + + - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 + with: + name: melange-${{github.run_id}} + path: ${{github.workspace}}/.melange-dir + + - run: | + sudo mv ${{github.workspace}}/.melange-dir/melange /usr/bin/melange + sudo chmod a+x /usr/bin/melange + melange version + + - run: | + sudo apt-get -y install bubblewrap + + - run: | + make MELANGE="sudo melange" BUILDWORLD="no" package/${{matrix.package}} diff --git a/e2e-tests/php-8.2-msgpack-test.yaml b/e2e-tests/php-8.2-msgpack-test.yaml new file mode 100644 index 000000000..775675d03 --- /dev/null +++ b/e2e-tests/php-8.2-msgpack-test.yaml @@ -0,0 +1,52 @@ +# This is an example test file that shows how 'melange test' works. +# It has been pulled into its own file, to try to clearly show what the +# test file looks like. +# Note, that these tests can also be baked into the package file itself. +# +package: + name: php-8.2-msgpack + version: 2.2.0 + epoch: 0 + description: "Tests for PHP extension msgpack" + copyright: + - license: BSD-3-Clause + +# This is mandatory, so just put an empty one there. Otherwise, config parsing +# will fail. +pipeline: + +test: + environment: + contents: + packages: + - wolfi-base + pipeline: + - runs: | + # Make sure msgpack is correctly loaded and listed by modules + php -m | grep msgpack + +subpackages: + - name: ${{package.name}}-config + description: PHP 8.2 msgpack tests + test: + environment: + contents: + packages: + - wolfi-base + - busybox + pipeline: + - runs: | + grep msgpack.so /etc/php/conf.d/msgpack.ini + + - name: ${{package.name}}-dev + description: PHP 8.2 msgpack development headers tests + test: + environment: + contents: + packages: + - wolfi-base + - busybox + pipeline: + - runs: | + # Just make sure this define is there. + grep PHP_MSGPACK_VERSION /usr/include/php/ext/msgpack/php_msgpack.h diff --git a/e2e-tests/py3-pandas-test.yaml b/e2e-tests/py3-pandas-test.yaml new file mode 100644 index 000000000..9cb94c683 --- /dev/null +++ b/e2e-tests/py3-pandas-test.yaml @@ -0,0 +1,19 @@ +package: + name: py3-pandas + version: 2.1.3 + epoch: 1 + description: Tests for py3-pandas + copyright: + - license: 'BSD-3-Clause' + +pipeline: + +test: + environment: + contents: + packages: + - busybox + - python-3 + pipeline: + - runs: | + python3 ./py3-pandas-test.py diff --git a/e2e-tests/test-fixtures/py3-pandas-test.py b/e2e-tests/test-fixtures/py3-pandas-test.py new file mode 100644 index 000000000..f30d8396b --- /dev/null +++ b/e2e-tests/test-fixtures/py3-pandas-test.py @@ -0,0 +1,6 @@ +import numpy as np +import pandas as pd +s = pd.Series([1,3,5,np.nan, 6, 8]) +dates = pd.date_range("20130101", periods=6) +df = pd.DataFrame(np.random.randn(6, 4), index=dates, columns=list("ABCD")) + From 9a017ccafe8386245ff56d476ea23e2f3932b266 Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Fri, 1 Dec 2023 12:14:59 +0200 Subject: [PATCH 08/11] Add tests, simplify code. Signed-off-by: Ville Aikas --- .github/workflows/melange-test-pipelines.yaml | 37 +-- docs/TESTING.md | 7 + pkg/build/build.go | 10 +- pkg/build/test.go | 246 ++++---------- pkg/build/test_test.go | 299 ++++++++++++++++++ .../py3-pandas.melange.yaml | 18 ++ .../range-subpackages.melange.yaml | 41 +++ pkg/cli/test.go | 58 ++-- pkg/config/config.go | 23 ++ 9 files changed, 480 insertions(+), 259 deletions(-) create mode 100644 pkg/build/test_test.go create mode 100644 pkg/build/testdata/test_configuration_load/py3-pandas.melange.yaml create mode 100644 pkg/build/testdata/test_configuration_load/range-subpackages.melange.yaml diff --git a/.github/workflows/melange-test-pipelines.yaml b/.github/workflows/melange-test-pipelines.yaml index 1f63130e0..a15bbb52b 100644 --- a/.github/workflows/melange-test-pipelines.yaml +++ b/.github/workflows/melange-test-pipelines.yaml @@ -1,4 +1,4 @@ -name: ci +name: Test melange test command on: pull_request: @@ -29,41 +29,27 @@ jobs: path: ${{github.workspace}}/melange retention-days: 1 - build-packages: - name: Build packages + test-packages: + name: Test packages needs: - build-melange # TODO: Set up a larger runner for this. runs-on: ubuntu-latest - # This is a list of packages which covers basic and exotic uses of - # the built-in pipelines. Goal is to balance efficiency while also - # exercising Melange with real-world package builds. + # This is a list of packages which we want to test against. # Feel free to add additional packages to this matrix which exercise - # Melange in new ways (e.g. new pipelines, etc.) + # Melange `test` in new ways (e.g. new pipelines, etc.) + # Each test file is of the form -test.yaml and gets + # constructed from the package name. strategy: fail-fast: false matrix: package: - - hello-wolfi - - glibc - - tini - - lzo - - bubblewrap - - gdk-pixbuf - - gitsign - - guac - - mdbook - - s3cmd - - perl-yaml-syck - - xmlto - - ncurses + - php-8.2-msgpack + - py3-pandas steps: - - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 - with: - repository: wolfi-dev/os - + # Grab the melange we uploaded above, and install it. - uses: actions/download-artifact@9bc31d5ccc31df68ecc42ccf4149144866c47d8a # v3.0.2 with: name: melange-${{github.run_id}} @@ -78,4 +64,5 @@ jobs: sudo apt-get -y install bubblewrap - run: | - make MELANGE="sudo melange" BUILDWORLD="no" package/${{matrix.package}} + testfile="${{matrix.package}}-test.yaml" + melange test --arch x86_64,arm64 --source-dir ./e2e-test/test-fixtures ./e2e-test/$testfile ${{matrix.package}} --repository-append https://packages.wolfi.dev/os --keyring-append https://packages.wolfi.dev/os/wolfi-signing.rsa.pub diff --git a/docs/TESTING.md b/docs/TESTING.md index 23b3caa44..a713341d0 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -150,6 +150,13 @@ test: # Stuff goes here. ``` +## Using pipelines + +Not surprisingly you can also use predefined pipelines, just like in the build +step by using `uses:` instead of `runs:`. You can specify the location of the +predefined pipelines using the `--pipeline-dir` to point to the directory where +the custom pipelines are located. + ## Full example Here's a full example invocation, where I'm testing with my local mac, so just diff --git a/pkg/build/build.go b/pkg/build/build.go index ba1d9d5ca..c33c0bba2 100644 --- a/pkg/build/build.go +++ b/pkg/build/build.go @@ -766,16 +766,16 @@ func (b *Build) OverlayBinSh() error { return nil } -func (b *Build) fetchBucket(ctx context.Context, cmm CacheMembershipMap) (string, error) { +func fetchBucket(ctx context.Context, cacheSource string, logger apko_log.Logger, cmm CacheMembershipMap) (string, error) { tmp, err := os.MkdirTemp("", "melange-cache") if err != nil { return "", err } - bucket, prefix, _ := strings.Cut(strings.TrimPrefix(b.CacheSource, "gs://"), "/") + bucket, prefix, _ := strings.Cut(strings.TrimPrefix(cacheSource, "gs://"), "/") client, err := storage.NewClient(ctx) if err != nil { - b.Logger.Printf("downgrading to anonymous mode: %s", err) + logger.Printf("downgrading to anonymous mode: %s", err) client, err = storage.NewClient(ctx, option.WithoutAuthentication()) if err != nil { @@ -810,7 +810,7 @@ func (b *Build) fetchBucket(ctx context.Context, cmm CacheMembershipMap) (string if err := rc.Close(); err != nil { return tmp, fmt.Errorf("failed to close remote cache object %s: %w", on, err) } - b.Logger.Printf("cached gs://%s/%s -> %s", bucket, on, w.Name()) + logger.Printf("cached gs://%s/%s -> %s", bucket, on, w.Name()) } return tmp, nil @@ -841,7 +841,7 @@ func (b *Build) PopulateCache(ctx context.Context) error { // --cache-dir=gs://bucket/path/to/cache first pulls all found objects to a // tmp dir which is subsequently used as the cache. if strings.HasPrefix(b.CacheSource, "gs://") { - tmp, err := b.fetchBucket(ctx, cmm) + tmp, err := fetchBucket(ctx, b.CacheSource, b.Logger, cmm) if err != nil { return err } diff --git a/pkg/build/test.go b/pkg/build/test.go index ec510fac3..34ba5a865 100644 --- a/pkg/build/test.go +++ b/pkg/build/test.go @@ -29,13 +29,10 @@ import ( apko_types "chainguard.dev/apko/pkg/build/types" apko_iocomb "chainguard.dev/apko/pkg/iocomb" apko_log "chainguard.dev/apko/pkg/log" - "cloud.google.com/go/storage" apkofs "github.com/chainguard-dev/go-apk/pkg/fs" "github.com/google/go-containerregistry/pkg/logs" "github.com/yookoala/realpath" "go.opentelemetry.io/otel" - "google.golang.org/api/iterator" - "google.golang.org/api/option" "chainguard.dev/melange/pkg/config" "chainguard.dev/melange/pkg/container" @@ -43,46 +40,35 @@ import ( type Test struct { // Package to test. - Package string - Configuration config.Configuration - ConfigFile string - WorkspaceDir string - WorkspaceIgnore string - PipelineDir string - BuiltinPipelineDir string + Package string + Configuration config.Configuration + ConfigFile string + WorkspaceDir string + WorkspaceIgnore string // Ordered directories where to find 'uses' pipelines. - PipelineDirs []string - SourceDir string - GuestDir string - Namespace string - EmptyWorkspace bool - Logger apko_log.Logger - Arch apko_types.Architecture - ExtraKeys []string - ExtraRepos []string - DependencyLog string - BinShOverlay string - CacheDir string - ApkCacheDir string - CacheSource string - BreakpointLabel string - ContinueLabel string - EnvFile string - VarsFile string - Runner container.Runner - RunnerName string - Debug bool - DebugRunner bool - LogPolicy []string + PipelineDirs []string + SourceDir string + GuestDir string + Logger apko_log.Logger + Arch apko_types.Architecture + ExtraKeys []string + ExtraRepos []string + BinShOverlay string + CacheDir string + ApkCacheDir string + CacheSource string + Runner container.Runner + RunnerName string + Debug bool + DebugRunner bool + LogPolicy []string } func NewTest(ctx context.Context, opts ...TestOption) (*Test, error) { t := Test{ WorkspaceIgnore: ".melangeignore", - //SourceDir: ".", - CacheDir: "./melange-cache/", - Arch: apko_types.ParseArchitecture(runtime.GOARCH), - LogPolicy: []string{"builtin:stderr"}, + Arch: apko_types.ParseArchitecture(runtime.GOARCH), + LogPolicy: []string{"builtin:stderr"}, } for _, opt := range opts { @@ -121,13 +107,7 @@ func NewTest(ctx context.Context, opts ...TestOption) (*Test, error) { // temporary directory for it. Otherwise, ensure we are in a // subdir for this specific build context. if t.WorkspaceDir != "" { - // If we are continuing the build, do not modify the workspace - // directory path. - // TODO(kaniini): Clean up the logic for this, perhaps by signalling - // multi-arch builds to the build context. - if t.ContinueLabel == "" { - t.WorkspaceDir = filepath.Join(t.WorkspaceDir, t.Arch.ToAPK()) - } + t.WorkspaceDir = filepath.Join(t.WorkspaceDir, t.Arch.ToAPK()) // Get the absolute path to the workspace dir, which is needed for bind // mounts. @@ -145,29 +125,9 @@ func NewTest(ctx context.Context, opts ...TestOption) (*Test, error) { t.WorkspaceDir = tmpdir } - // If no config file is explicitly requested for the test context - // we check if .melange.yaml or melange.yaml exist. - checks := []string{".melange.yaml", ".melange.yml", "melange.yaml", "melange.yml"} - if t.ConfigFile == "" { - for _, chk := range checks { - if _, err := os.Stat(chk); err == nil { - t.Logger.Printf("no configuration file provided -- using %s", chk) - t.ConfigFile = chk - break - } - } - } - - // If no config file could be automatically detected, error. - if t.ConfigFile == "" { - return nil, fmt.Errorf("melange.yaml is missing") - } - parsedCfg, err := config.ParseConfiguration( t.ConfigFile, - config.WithEnvFileForParsing(t.EnvFile), - config.WithLogger(t.Logger), - config.WithVarsFileForParsing(t.VarsFile)) + config.WithLogger(t.Logger)) if err != nil { return nil, fmt.Errorf("failed to load configuration: %w", err) } @@ -216,26 +176,10 @@ func WithTestWorkspaceIgnore(workspaceIgnore string) TestOption { } } -// WithEmptyWorkspace sets whether the workspace should be empty. -func WithTestEmptyWorkspace(emptyWorkspace bool) TestOption { - return func(t *Test) error { - t.EmptyWorkspace = emptyWorkspace - return nil - } -} - // WithPipelineDir sets the pipeline directory to extend the built-in pipeline directory. func WithTestPipelineDir(pipelineDir string) TestOption { return func(t *Test) error { - t.PipelineDir = pipelineDir - return nil - } -} - -// WithBuiltinPipelineDirectory sets the pipeline directory to use. -func WithTestBuiltinPipelineDirectory(builtinPipelineDir string) TestOption { - return func(t *Test) error { - t.BuiltinPipelineDir = builtinPipelineDir + t.PipelineDirs = append(t.PipelineDirs, pipelineDir) return nil } } @@ -281,27 +225,34 @@ func WithTestExtraKeys(extraKeys []string) TestOption { } } -// WithTestExtraRepos adds a set of extra repos to the build context. -func WithTestExtraRepos(extraRepos []string) TestOption { +// WithTestDebug indicates whether debug logging of pipelines should be enabled. +func WithTestDebug(debug bool) TestOption { return func(t *Test) error { - t.ExtraRepos = extraRepos + t.Debug = debug return nil } } -// WithTestBinShOverlay sets a filename to copy from when installing /bin/sh -// into a build environment. -func WithTestBinShOverlay(binShOverlay string) TestOption { +func WithTestDebugRunner(debugRunner bool) TestOption { return func(t *Test) error { - t.BinShOverlay = binShOverlay + t.DebugRunner = debugRunner return nil } } -// WithTestDebugRunner indicates whether the runner should leave the build environment up on failures -func WithTestDebugRunner(debug bool) TestOption { +// WithTestExtraRepos adds a set of extra repos to the test context. +func WithTestExtraRepos(extraRepos []string) TestOption { return func(t *Test) error { - t.DebugRunner = debug + t.ExtraRepos = extraRepos + return nil + } +} + +// WithTestBinShOverlay sets a filename to copy from when installing /bin/sh +// into a test environment. +func WithTestBinShOverlay(binShOverlay string) TestOption { + return func(t *Test) error { + t.BinShOverlay = binShOverlay return nil } } @@ -406,37 +357,6 @@ func (t *Test) BuildGuest(ctx context.Context, imgConfig *apko_types.ImageConfig return ref, nil } -// ApplyBuildOption applies a patch described by a BuildOption to a package build. -func (t *Test) ApplyTestOption(to config.BuildOption) error { - // Patch the variables block. - if t.Configuration.Vars == nil { - t.Configuration.Vars = make(map[string]string) - } - - for k, v := range to.Vars { - t.Configuration.Vars[k] = v - } - - // Patch the test environment configuration. - lo := to.Environment.Contents.Packages - t.Configuration.Test.Environment.Contents.Packages = append(t.Configuration.Test.Environment.Contents.Packages, lo.Add...) - - for _, pkg := range lo.Remove { - pkgList := t.Configuration.Test.Environment.Contents.Packages - - for pos, ppkg := range pkgList { - if pkg == ppkg { - pkgList[pos] = pkgList[len(pkgList)-1] - pkgList = pkgList[:len(pkgList)-1] - } - } - - t.Configuration.Test.Environment.Contents.Packages = pkgList - } - - return nil -} - func (t *Test) OverlayBinSh(suffix string) error { if t.BinShOverlay == "" { return nil @@ -474,56 +394,6 @@ func (t *Test) OverlayBinSh(suffix string) error { return nil } -func (t *Test) fetchBucket(ctx context.Context, cmm CacheMembershipMap) (string, error) { - tmp, err := os.MkdirTemp("", "melange-cache") - if err != nil { - return "", err - } - bucket, prefix, _ := strings.Cut(strings.TrimPrefix(t.CacheSource, "gs://"), "/") - - client, err := storage.NewClient(ctx) - if err != nil { - t.Logger.Printf("downgrading to anonymous mode: %s", err) - - client, err = storage.NewClient(ctx, option.WithoutAuthentication()) - if err != nil { - return "", fmt.Errorf("failed to get storage client: %w", err) - } - } - - bh := client.Bucket(bucket) - it := bh.Objects(ctx, &storage.Query{Prefix: prefix}) - for { - attrs, err := it.Next() - if err == iterator.Done { - break - } else if err != nil { - return tmp, fmt.Errorf("failed to get next remote cache object: %w", err) - } - on := attrs.Name - if !cmm[on] { - continue - } - rc, err := bh.Object(on).NewReader(ctx) - if err != nil { - return tmp, fmt.Errorf("failed to get reader for next remote cache object %s: %w", on, err) - } - w, err := os.Create(filepath.Join(tmp, on)) - if err != nil { - return tmp, err - } - if _, err := io.Copy(w, rc); err != nil { - return tmp, fmt.Errorf("failed to copy remote cache object %s: %w", on, err) - } - if err := rc.Close(); err != nil { - return tmp, fmt.Errorf("failed to close remote cache object %s: %w", on, err) - } - t.Logger.Printf("cached gs://%s/%s -> %s", bucket, on, w.Name()) - } - - return tmp, nil -} - // IsTestless returns true if the test context does not actually do any // testing. func (t *Test) IsTestless() bool { @@ -548,7 +418,7 @@ func (t *Test) PopulateCache(ctx context.Context) error { // --cache-dir=gs://bucket/path/to/cache first pulls all found objects to a // tmp dir which is subsequently used as the cache. if strings.HasPrefix(t.CacheSource, "gs://") { - tmp, err := t.fetchBucket(ctx, cmm) + tmp, err := fetchBucket(ctx, t.CacheSource, t.Logger, cmm) if err != nil { return err } @@ -603,10 +473,6 @@ func (t *Test) PopulateWorkspace(ctx context.Context) error { _, span := otel.Tracer("melange").Start(ctx, "PopulateWorkspace") defer span.End() - if t.EmptyWorkspace { - t.Logger.Printf("empty workspace requested") - return nil - } if t.SourceDir == "" { t.Logger.Printf("No source directory specified, skipping workspace population") return nil @@ -633,11 +499,7 @@ func (t *Test) PopulateWorkspace(ctx context.Context) error { t.Logger.Debugf(" -> %s", path) - if err := copyFile(t.SourceDir, path, t.WorkspaceDir, mode.Perm()); err != nil { - return err - } - - return nil + return copyFile(t.SourceDir, path, t.WorkspaceDir, mode.Perm()) }) } @@ -698,7 +560,10 @@ func (t *Test) TestPackage(ctx context.Context) error { return fmt.Errorf("unable to populate workspace: %w", err) } - cfg := t.buildWorkspaceConfig(imgRef, pkg.Name, t.Configuration.Environment.Environment) + cfg, err := t.buildWorkspaceConfig(imgRef, pkg.Name, t.Configuration.Environment.Environment) + if err != nil { + return fmt.Errorf("unable to build workspace config: %w", err) + } if !t.IsTestless() { cfg.Arch = t.Arch if err := t.Runner.StartPod(ctx, cfg); err != nil { @@ -754,7 +619,10 @@ func (t *Test) TestPackage(ctx context.Context) error { if err := t.OverlayBinSh(sp.Name); err != nil { return fmt.Errorf("unable to install overlay /bin/sh: %w", err) } - subCfg := t.buildWorkspaceConfig(spImgRef, sp.Name, sp.Test.Environment.Environment) + subCfg, err := t.buildWorkspaceConfig(spImgRef, sp.Name, sp.Test.Environment.Environment) + if err != nil { + return fmt.Errorf("unable to build workspace config: %w", err) + } subCfg.Arch = t.Arch if err := t.Runner.StartPod(ctx, subCfg); err != nil { return fmt.Errorf("unable to start subpackage test pod: %w", err) @@ -811,7 +679,7 @@ func (t *Test) Summarize() { t.SummarizePaths() } -func (t *Test) buildWorkspaceConfig(imgRef, pkgName string, env map[string]string) *container.Config { +func (t *Test) buildWorkspaceConfig(imgRef, pkgName string, env map[string]string) (*container.Config, error) { mounts := []container.BindMount{ {Source: t.WorkspaceDir, Destination: container.DefaultWorkspaceDir}, {Source: "/etc/resolv.conf", Destination: container.DefaultResolvConfPath}, @@ -821,12 +689,12 @@ func (t *Test) buildWorkspaceConfig(imgRef, pkgName string, env map[string]strin if fi, err := os.Stat(t.CacheDir); err == nil && fi.IsDir() { mountSource, err := realpath.Realpath(t.CacheDir) if err != nil { - t.Logger.Printf("could not resolve path for --cache-dir: %s", err) + return nil, fmt.Errorf("could not resolve path for --cache-dir: %s : %w", t.CacheDir, err) } mounts = append(mounts, container.BindMount{Source: mountSource, Destination: container.DefaultCacheDir}) } else { - t.Logger.Printf("--cache-dir %s not a dir; skipping", t.CacheDir) + return nil, fmt.Errorf("--cache-dir %s not a dir", t.CacheDir) } } @@ -850,5 +718,5 @@ func (t *Test) buildWorkspaceConfig(imgRef, pkgName string, env map[string]strin cfg.ImgRef = imgRef t.Logger.Printf("ImgRef = %s", cfg.ImgRef) - return &cfg + return &cfg, nil } diff --git a/pkg/build/test_test.go b/pkg/build/test_test.go new file mode 100644 index 000000000..92afa4423 --- /dev/null +++ b/pkg/build/test_test.go @@ -0,0 +1,299 @@ +// Copyright 2023 Chainguard, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package build + +import ( + "fmt" + "path/filepath" + "strings" + "testing" + + "chainguard.dev/apko/pkg/build/types" + "chainguard.dev/melange/pkg/config" + "chainguard.dev/melange/pkg/container" + "chainguard.dev/melange/pkg/logger" + "github.com/google/go-cmp/cmp" + "github.com/google/go-cmp/cmp/cmpopts" + "github.com/stretchr/testify/require" + "github.com/yookoala/realpath" +) + +const ( + testImgRef = "testImageRef" + testPkgName = "testPkgName" + testWorkspaceDir = "/workspace" + etcResolveConf = "/etc/resolv.conf" + homeBuild = "/home/build" +) + +func TestBuildWorkspaceConfig(t *testing.T) { + tmpDir := t.TempDir() + // realpath is used to get the real path of the temp dir + tmpDirReal, err := realpath.Realpath(tmpDir) + if err != nil { + t.Fatalf("unexpected error: %v", err) + } + + logger := logger.NopLogger{} + // Just define the base stuff here that we can then + // modify in the tests. + baseTest := Test{ + WorkspaceDir: testWorkspaceDir, + Logger: logger, + } + + // Just define the base stuff here that we can then + // modify in the tests. + wantBase := container.Config{ + Logger: logger, + Environment: map[string]string{}, + PackageName: testPkgName, + ImgRef: testImgRef, + Capabilities: container.Capabilities{Networking: true}, + Mounts: []container.BindMount{ + {Source: testWorkspaceDir, Destination: homeBuild}, + {Source: etcResolveConf, Destination: etcResolveConf}, + }, + } + + tests := []struct { + name string + env map[string]string + t *Test + wantErr string + want *container.Config + }{ + { + name: "test - no cache dir", + t: &baseTest, + want: func() *container.Config { + want := wantBase + return &want + }(), + }, { + name: "test - with cache dir, does not exist", + t: func() *Test { + cacheT := baseTest + cacheT.CacheDir = "/cache" + return &cacheT + }(), + wantErr: "--cache-dir /cache not a dir", + }, { + name: "test - with cache dir, exists", + t: func() *Test { + cacheT := baseTest + cacheT.CacheDir = tmpDir + return &cacheT + }(), + want: func() *container.Config { + want := wantBase + want.Mounts = append(want.Mounts, container.BindMount{Source: tmpDirReal, Destination: "/var/cache/melange"}) + return &want + }(), + }, { + name: "test - with cache dir, exists, environment", + t: func() *Test { + cacheT := baseTest + cacheT.CacheDir = tmpDir + return &cacheT + }(), + env: map[string]string{"FOO": "bar", "BAZ": "zzz"}, + want: func() *container.Config { + want := wantBase + want.Mounts = append(want.Mounts, container.BindMount{Source: tmpDirReal, Destination: "/var/cache/melange"}) + want.Environment = map[string]string{"FOO": "bar", "BAZ": "zzz"} + return &want + }(), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, gotErr := tt.t.buildWorkspaceConfig(testImgRef, testPkgName, tt.env) + if gotErr != nil { + if tt.wantErr == "" { + t.Fatalf("unexpected error: %v", gotErr) + } + if !strings.Contains(gotErr.Error(), tt.wantErr) { + t.Fatalf("expected error to contain %q, got %q", tt.wantErr, gotErr.Error()) + } + } else { + if tt.wantErr != "" { + t.Fatalf("expected error %q, got nil", tt.wantErr) + } + if !cmp.Equal(tt.want, got) { + t.Errorf("%s", cmp.Diff(tt.want, got)) + } + } + }) + } +} + +// TestConfigurationLoad is the main set of tests for loading a configuration +// file for tests. When in doubt, add your test here. +func TestConfigurationLoad(t *testing.T) { + tests := []struct { + name string + skipConfigCleanStep bool + requireErr require.ErrorAssertionFunc + expected *config.Configuration + }{ + { + name: "range-subpackages", + requireErr: require.NoError, + expected: &config.Configuration{ + Package: config.Package{ + Name: "hello", + Version: "world", + }, + Test: config.Test{ + Pipeline: []config.Pipeline{ + { + Name: "hello", + Runs: "world", + }, + }}, + Subpackages: []config.Subpackage{{ + Name: "cats", + Test: config.Test{ + Pipeline: []config.Pipeline{{ + Runs: "cats are angry", + }}}, + }, { + Name: "dogs", + Test: config.Test{ + Pipeline: []config.Pipeline{{ + Runs: "dogs are loyal", + }}}, + }, { + Name: "turtles", + Test: config.Test{ + Pipeline: []config.Pipeline{{ + Runs: "turtles are slow", + }}}, + }, { + Name: "donatello", + Test: config.Test{ + Pipeline: []config.Pipeline{ + { + Runs: "donatello's color is purple", + }, + { + Uses: "go/build", + With: map[string]string{"packages": "purple"}, + }, + }}, + }, { + Name: "leonardo", + Test: config.Test{Pipeline: []config.Pipeline{ + { + Runs: "leonardo's color is blue", + }, + { + Uses: "go/build", + With: map[string]string{"packages": "blue"}, + }, + }}, + }, { + Name: "michelangelo", + Test: config.Test{Pipeline: []config.Pipeline{ + { + Runs: "michelangelo's color is orange", + }, + { + Uses: "go/build", + With: map[string]string{"packages": "orange"}, + }, + }}, + }, { + Name: "raphael", + Test: config.Test{Pipeline: []config.Pipeline{ + { + Runs: "raphael's color is red", + }, + { + Uses: "go/build", + With: map[string]string{"packages": "red"}, + }, + }}, + }, { + Name: "simple", + Test: config.Test{Pipeline: []config.Pipeline{ + { + Runs: "simple-runs", + }, { + Uses: "simple-uses", + }, + }}, + }}, + }, + }, { + name: "py3-pandas", + requireErr: require.NoError, + expected: &config.Configuration{ + Package: config.Package{ + Name: "py3-pandas", + Version: "2.1.3", + }, + Test: config.Test{ + Environment: types.ImageConfiguration{ + Contents: types.ImageContents{ + Packages: []string{"busybox", "python-3"}, + }, + }, + Pipeline: []config.Pipeline{ + { + Runs: "python3 ./py3-pandas-test.py\n", + }, { + Uses: "test-uses", + With: map[string]string{"test-with": "test-with-value"}, + }, + }}, + }, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + log := logger.NopLogger{} + ctx := Test{ + ConfigFile: filepath.Join("testdata", "test_configuration_load", fmt.Sprintf("%s.melange.yaml", tt.name)), + Logger: log, + } + + cfg, err := config.ParseConfiguration( + ctx.ConfigFile, + config.WithLogger(ctx.Logger)) + tt.requireErr(t, err) + + if !tt.skipConfigCleanStep { + cleanTestConfig(cfg) + } + + if tt.expected == nil { + if cfg != nil { + t.Fatalf("actual didn't match expected (want nil, got config)") + } + } else { + if d := cmp.Diff( + *tt.expected, + *cfg, + cmpopts.IgnoreUnexported(config.Pipeline{}, config.Configuration{}), + ); d != "" { + t.Fatalf("actual didn't match expected (-want, +got): %s", d) + } + } + }) + } +} diff --git a/pkg/build/testdata/test_configuration_load/py3-pandas.melange.yaml b/pkg/build/testdata/test_configuration_load/py3-pandas.melange.yaml new file mode 100644 index 000000000..45deac07d --- /dev/null +++ b/pkg/build/testdata/test_configuration_load/py3-pandas.melange.yaml @@ -0,0 +1,18 @@ +package: + name: py3-pandas + version: 2.1.3 + +pipeline: + +test: + environment: + contents: + packages: + - busybox + - python-3 + pipeline: + - runs: | + python3 ./py3-pandas-test.py + - uses: test-uses + with: + test-with: test-with-value diff --git a/pkg/build/testdata/test_configuration_load/range-subpackages.melange.yaml b/pkg/build/testdata/test_configuration_load/range-subpackages.melange.yaml new file mode 100644 index 000000000..85fe4c82b --- /dev/null +++ b/pkg/build/testdata/test_configuration_load/range-subpackages.melange.yaml @@ -0,0 +1,41 @@ +package: + name: hello + version: world + +test: + pipeline: + - name: hello + runs: world + +data: + - name: ninja-turtles + items: + michelangelo: orange + raphael: red + leonardo: blue + donatello: purple + - name: animals + items: + dogs: loyal + cats: angry + turtles: slow + +subpackages: + - range: animals + name: ${{range.key}} + test: + pipeline: + - runs: ${{range.key}} are ${{range.value}} + - range: ninja-turtles + name: ${{range.key}} + test: + pipeline: + - runs: ${{range.key}}'s color is ${{range.value}} + - uses: go/build + with: + packages: ${{range.value}} + - name: simple + test: + pipeline: + - runs: simple-runs + - uses: simple-uses diff --git a/pkg/cli/test.go b/pkg/cli/test.go index 6af9d4672..0a30e6b24 100644 --- a/pkg/cli/test.go +++ b/pkg/cli/test.go @@ -28,57 +28,43 @@ import ( ) func Test() *cobra.Command { - var buildDate string var workspaceDir string - var pipelineDir string var sourceDir string var cacheDir string var cacheSource string var apkCacheDir string var guestDir string - var signingKey string - var generateIndex bool - var emptyWorkspace bool - var stripOriginName bool - var outDir string var archstrs []string + var pipelineDirs []string var extraKeys []string var extraRepos []string - var dependencyLog string var overlayBinSh string - var breakpointLabel string - var continueLabel string - var envFile string - var varsFile string - var purlNamespace string - var buildOption []string + var testOption []string var logPolicy []string - var createBuildLog bool var debug bool var debugRunner bool var runner string - var failOnLintWarning bool cmd := &cobra.Command{ Use: "test", Short: "Test a package with a YAML configuration file", Long: `Test a package from a YAML configuration file containing a test pipeline.`, - Example: ` melange test `, + Example: ` melange test `, Args: cobra.MinimumNArgs(2), RunE: func(cmd *cobra.Command, args []string) error { archs := apko_types.ParseArchitectures(archstrs) options := []build.TestOption{ build.WithTestWorkspaceDir(workspaceDir), - build.WithTestPipelineDir(pipelineDir), build.WithTestCacheDir(cacheDir), build.WithTestCacheSource(cacheSource), build.WithTestPackageCacheDir(apkCacheDir), build.WithTestGuestDir(guestDir), - build.WithTestEmptyWorkspace(emptyWorkspace), build.WithTestExtraKeys(extraKeys), build.WithTestExtraRepos(extraRepos), build.WithTestBinShOverlay(overlayBinSh), build.WithTestRunner(runner), + build.WithTestDebug(debug), + build.WithTestDebugRunner(debugRunner), } if len(args) > 0 { @@ -90,40 +76,31 @@ func Test() *cobra.Command { options = append(options, build.WithTestSourceDir(sourceDir)) } + for i := range pipelineDirs { + options = append(options, build.WithTestPipelineDir(pipelineDirs[i])) + } + options = append(options, build.WithTestPipelineDir(BuiltinPipelineDir)) + return TestCmd(cmd.Context(), archs, options...) }, } - cmd.Flags().StringVar(&buildDate, "build-date", "", "date used for the timestamps of the files inside the image") cmd.Flags().StringVar(&workspaceDir, "workspace-dir", "", "directory used for the workspace at /home/build") - cmd.Flags().StringVar(&pipelineDir, "pipeline-dir", "", "directory used to extend defined built-in pipelines") + cmd.Flags().StringSliceVar(&pipelineDirs, "pipeline-dirs", []string{}, "directories used to extend defined built-in pipelines") cmd.Flags().StringVar(&sourceDir, "source-dir", "", "directory used for included sources") - cmd.Flags().StringVar(&cacheDir, "cache-dir", "./melange-cache/", "directory used for cached inputs") + cmd.Flags().StringVar(&cacheDir, "cache-dir", "", "directory used for cached inputs") cmd.Flags().StringVar(&cacheSource, "cache-source", "", "directory or bucket used for preloading the cache") cmd.Flags().StringVar(&apkCacheDir, "apk-cache-dir", "", "directory used for cached apk packages (default is system-defined cache directory)") cmd.Flags().StringVar(&guestDir, "guest-dir", "", "directory used for the build environment guest") - cmd.Flags().StringVar(&signingKey, "signing-key", "", "key to use for signing") - cmd.Flags().StringVar(&envFile, "env-file", "", "file to use for preloaded environment variables") - cmd.Flags().StringVar(&varsFile, "vars-file", "", "file to use for preloaded build configuration variables") - cmd.Flags().BoolVar(&generateIndex, "generate-index", true, "whether to generate APKINDEX.tar.gz") - cmd.Flags().BoolVar(&emptyWorkspace, "empty-workspace", false, "whether the build workspace should be empty") - cmd.Flags().BoolVar(&stripOriginName, "strip-origin-name", false, "whether origin names should be stripped (for bootstrap)") - cmd.Flags().StringVar(&outDir, "out-dir", "./packages/", "directory where packages will be output") - cmd.Flags().StringVar(&dependencyLog, "dependency-log", "", "log dependencies to a specified file") cmd.Flags().StringVar(&overlayBinSh, "overlay-binsh", "", "use specified file as /bin/sh overlay in build environment") - cmd.Flags().StringVar(&breakpointLabel, "breakpoint-label", "", "stop build execution at the specified label") - cmd.Flags().StringVar(&continueLabel, "continue-label", "", "continue build execution at the specified label") - cmd.Flags().StringVar(&purlNamespace, "namespace", "unknown", "namespace to use in package URLs in SBOM (eg wolfi, alpine)") cmd.Flags().StringSliceVar(&archstrs, "arch", nil, "architectures to build for (e.g., x86_64,ppc64le,arm64) -- default is all, unless specified in config") - cmd.Flags().StringSliceVar(&buildOption, "build-option", []string{}, "build options to enable") + cmd.Flags().StringSliceVar(&testOption, "test-option", []string{}, "build options to enable") cmd.Flags().StringSliceVar(&logPolicy, "log-policy", []string{"builtin:stderr"}, "logging policy to use") cmd.Flags().StringVar(&runner, "runner", string(build.GetDefaultRunner()), fmt.Sprintf("which runner to use to enable running commands, default is based on your platform. Options are %q", build.GetAllRunners())) cmd.Flags().StringSliceVarP(&extraKeys, "keyring-append", "k", []string{}, "path to extra keys to include in the build environment keyring") - cmd.Flags().StringSliceVarP(&extraRepos, "repository-append", "r", []string{}, "path to extra repositories to include in the build environment") - cmd.Flags().BoolVar(&createBuildLog, "create-build-log", false, "creates a package.log file containing a list of packages that were built by the command") - cmd.Flags().BoolVar(&debug, "debug", false, "enables debug logging of build pipelines") + cmd.Flags().BoolVar(&debug, "debug", false, "enables debug logging of test pipelines (sets -x for steps)") cmd.Flags().BoolVar(&debugRunner, "debug-runner", false, "when enabled, the builder pod will persist after the build succeeds or fails") - cmd.Flags().BoolVar(&failOnLintWarning, "fail-on-lint-warning", false, "turns linter warnings into failures") + cmd.Flags().StringSliceVarP(&extraRepos, "repository-append", "r", []string{}, "path to extra repositories to include in the build environment") return cmd } @@ -144,7 +121,8 @@ func TestCmd(ctx context.Context, archs []apko_types.Architecture, baseOpts ...b // https://github.com/distroless/nginx/runs/7219233843?check_suite_focus=true bcs := []*build.Test{} for _, arch := range archs { - opts := append(baseOpts, build.WithTestArch(arch), build.WithTestBuiltinPipelineDirectory(BuiltinPipelineDir)) + opts := []build.TestOption{build.WithTestArch(arch)} + opts = append(opts, baseOpts...) bc, err := build.NewTest(ctx, opts...) if errors.Is(err, build.ErrSkipThisArch) { @@ -158,7 +136,7 @@ func TestCmd(ctx context.Context, archs []apko_types.Architecture, baseOpts ...b } if len(bcs) == 0 { - log.Printf("WARNING: target-architecture and --arch do not overlap, nothing to build") + log.Printf("WARNING: target-architecture and --arch do not overlap, nothing to test") return nil } diff --git a/pkg/config/config.go b/pkg/config/config.go index 8c361558f..90fe52cd5 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -774,6 +774,29 @@ func ParseConfiguration(configurationFilePath string, opts ...ConfigurationParsi // TODO: p.Pipeline? }) } + for _, p := range sp.Test.Pipeline { + // take a copy of the with map, so we can replace the values + replacedWith := make(map[string]string) + for key, value := range p.With { + replacedWith[key] = replacer.Replace(value) + } + + // if the map is empty, set it to nil to avoid serializing an empty map + if len(replacedWith) == 0 { + replacedWith = nil + } + + thingToAdd.Test.Pipeline = append(thingToAdd.Test.Pipeline, Pipeline{ + Name: p.Name, + Uses: p.Uses, + With: replacedWith, + Inputs: p.Inputs, + Needs: p.Needs, + Label: p.Label, + Runs: replacer.Replace(p.Runs), + // TODO: p.Pipeline? + }) + } subpackages = append(subpackages, thingToAdd) } } From 6f39b98dc61a321776369dab43e1d3d360ec0a4a Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Fri, 1 Dec 2023 12:22:37 +0200 Subject: [PATCH 09/11] argh, fix typo. Signed-off-by: Ville Aikas --- .github/workflows/melange-test-pipelines.yaml | 7 +++++- docs/md/melange_test.md | 24 ++++--------------- 2 files changed, 11 insertions(+), 20 deletions(-) diff --git a/.github/workflows/melange-test-pipelines.yaml b/.github/workflows/melange-test-pipelines.yaml index a15bbb52b..a36421d76 100644 --- a/.github/workflows/melange-test-pipelines.yaml +++ b/.github/workflows/melange-test-pipelines.yaml @@ -63,6 +63,11 @@ jobs: - run: | sudo apt-get -y install bubblewrap + # Make sure we have our tests files here. + - uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1 + - run: | testfile="${{matrix.package}}-test.yaml" - melange test --arch x86_64,arm64 --source-dir ./e2e-test/test-fixtures ./e2e-test/$testfile ${{matrix.package}} --repository-append https://packages.wolfi.dev/os --keyring-append https://packages.wolfi.dev/os/wolfi-signing.rsa.pub + echo "Testing $testfile" + + melange test --arch x86_64 --source-dir ./e2e-tests/test-fixtures ./e2e-tests/$testfile ${{matrix.package}} --repository-append https://packages.wolfi.dev/os --keyring-append https://packages.wolfi.dev/os/wolfi-signing.rsa.pub diff --git a/docs/md/melange_test.md b/docs/md/melange_test.md index fd9eb37d1..952c38d7a 100644 --- a/docs/md/melange_test.md +++ b/docs/md/melange_test.md @@ -22,7 +22,7 @@ melange test [flags] ### Examples ``` - melange test + melange test ``` ### Options @@ -30,34 +30,20 @@ melange test [flags] ``` --apk-cache-dir string directory used for cached apk packages (default is system-defined cache directory) --arch strings architectures to build for (e.g., x86_64,ppc64le,arm64) -- default is all, unless specified in config - --breakpoint-label string stop build execution at the specified label - --build-date string date used for the timestamps of the files inside the image - --build-option strings build options to enable - --cache-dir string directory used for cached inputs (default "./melange-cache/") + --cache-dir string directory used for cached inputs --cache-source string directory or bucket used for preloading the cache - --continue-label string continue build execution at the specified label - --create-build-log creates a package.log file containing a list of packages that were built by the command - --debug enables debug logging of build pipelines + --debug enables debug logging of test pipelines (sets -x for steps) --debug-runner when enabled, the builder pod will persist after the build succeeds or fails - --dependency-log string log dependencies to a specified file - --empty-workspace whether the build workspace should be empty - --env-file string file to use for preloaded environment variables - --fail-on-lint-warning turns linter warnings into failures - --generate-index whether to generate APKINDEX.tar.gz (default true) --guest-dir string directory used for the build environment guest -h, --help help for test -k, --keyring-append strings path to extra keys to include in the build environment keyring --log-policy strings logging policy to use (default [builtin:stderr]) - --namespace string namespace to use in package URLs in SBOM (eg wolfi, alpine) (default "unknown") - --out-dir string directory where packages will be output (default "./packages/") --overlay-binsh string use specified file as /bin/sh overlay in build environment - --pipeline-dir string directory used to extend defined built-in pipelines + --pipeline-dirs strings directories used to extend defined built-in pipelines -r, --repository-append strings path to extra repositories to include in the build environment --runner string which runner to use to enable running commands, default is based on your platform. Options are ["bubblewrap" "docker" "lima" "kubernetes"] (default "bubblewrap") - --signing-key string key to use for signing --source-dir string directory used for included sources - --strip-origin-name whether origin names should be stripped (for bootstrap) - --vars-file string file to use for preloaded build configuration variables + --test-option strings build options to enable --workspace-dir string directory used for the workspace at /home/build ``` From 84d4b667250211f5cd5a4935555939ba3f1fbedc Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Fri, 1 Dec 2023 14:11:48 +0200 Subject: [PATCH 10/11] Default to package.name, but allow overrides, add example docs for specifying which package, and version to test. Signed-off-by: Ville Aikas --- docs/TESTING.md | 30 ++++++++++++++++++++++++++++++ docs/md/melange_test.md | 2 +- pkg/build/test.go | 9 +++++++-- pkg/cli/test.go | 6 ++++-- 4 files changed, 42 insertions(+), 5 deletions(-) diff --git a/docs/TESTING.md b/docs/TESTING.md index a713341d0..9fdaba607 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -157,6 +157,36 @@ step by using `uses:` instead of `runs:`. You can specify the location of the predefined pipelines using the `--pipeline-dir` to point to the directory where the custom pipelines are located. +## Specifying package to test / reusing tests + +You can leave out the package name from the command line if you want, in which +case the PUT is pulled from the package.Name. However, for versioned packages, +say for example, php-8.1, php-8.2, php-8.3, it is beneficial to reuse some of +the tests. In those cases, you can specify the testfile and also which package +to use for testing by providing a second argument to the `test` command that is +the name of the package used for the tests. + +Lastly, if you want to test a specific version of the package, you can specify +the constraint in the argument. For example: + + * Use package.Name + + ```shell + melange test ./testfile.yaml + ``` + + * Use above testfile, but a different package to run tests against + + ```shell + melange test ./testfile.yaml mypackage + ``` + + * Use above testfile, but specify a particular version of the package + + ```shell + melange test ./testfile.yaml mypackage=2.2.0-r2 + ``` + ## Full example Here's a full example invocation, where I'm testing with my local mac, so just diff --git a/docs/md/melange_test.md b/docs/md/melange_test.md index 952c38d7a..3eec37a50 100644 --- a/docs/md/melange_test.md +++ b/docs/md/melange_test.md @@ -22,7 +22,7 @@ melange test [flags] ### Examples ``` - melange test + melange test [package-name] ``` ### Options diff --git a/pkg/build/test.go b/pkg/build/test.go index 34ba5a865..7d8b9b31c 100644 --- a/pkg/build/test.go +++ b/pkg/build/test.go @@ -523,8 +523,13 @@ func (t *Test) TestPackage(ctx context.Context) error { } t.Logger.Printf("evaluating main pipeline for package requirements") - // Append the main test package to be installed. - t.Configuration.Test.Environment.Contents.Packages = append(t.Configuration.Test.Environment.Contents.Packages, pkg.Name) + // Append the main test package to be installed unless explicitly specified + // by the command line. + if t.Package != "" { + t.Configuration.Test.Environment.Contents.Packages = append(t.Configuration.Test.Environment.Contents.Packages, t.Package) + } else { + t.Configuration.Test.Environment.Contents.Packages = append(t.Configuration.Test.Environment.Contents.Packages, pkg.Name) + } for i := range t.Configuration.Test.Pipeline { p := &t.Configuration.Test.Pipeline[i] // fine to pass nil for config, since not running in container. diff --git a/pkg/cli/test.go b/pkg/cli/test.go index 0a30e6b24..0da5309dc 100644 --- a/pkg/cli/test.go +++ b/pkg/cli/test.go @@ -49,8 +49,8 @@ func Test() *cobra.Command { Use: "test", Short: "Test a package with a YAML configuration file", Long: `Test a package from a YAML configuration file containing a test pipeline.`, - Example: ` melange test `, - Args: cobra.MinimumNArgs(2), + Example: ` melange test [package-name]`, + Args: cobra.MinimumNArgs(1), RunE: func(cmd *cobra.Command, args []string) error { archs := apko_types.ParseArchitectures(archstrs) options := []build.TestOption{ @@ -69,6 +69,8 @@ func Test() *cobra.Command { if len(args) > 0 { options = append(options, build.WithTestConfig(args[0])) + } + if len(args) > 1 { options = append(options, build.WithTestPackage(args[1])) } From cd69237a0dfe2e7b65e3795ab593390925058e72 Mon Sep 17 00:00:00 2001 From: Ville Aikas Date: Sat, 2 Dec 2023 14:18:02 +0200 Subject: [PATCH 11/11] fix bad merge. Signed-off-by: Ville Aikas --- pkg/build/build.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkg/build/build.go b/pkg/build/build.go index c33c0bba2..44ba95c0f 100644 --- a/pkg/build/build.go +++ b/pkg/build/build.go @@ -92,6 +92,9 @@ type Build struct { DebugRunner bool LogPolicy []string FailOnLintWarning bool + DefaultCPU string + DefaultMemory string + DefaultTimeout time.Duration EnabledBuildOptions []string }