Skip to content

Commit

Permalink
integrate buildah
Browse files Browse the repository at this point in the history
  • Loading branch information
justadogistaken committed Jul 28, 2022
1 parent 25b39ce commit b89013c
Show file tree
Hide file tree
Showing 2,571 changed files with 339,840 additions and 18,365 deletions.
89 changes: 42 additions & 47 deletions build/buildinstruction/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,23 +15,17 @@
package buildinstruction

import (
"context"
"fmt"
"os"
"path/filepath"
"strings"

"github.com/opencontainers/go-digest"
"github.com/sirupsen/logrus"
fsutil "github.com/tonistiigi/fsutil/copy"

"github.com/sealerio/sealer/common"
"github.com/sealerio/sealer/pkg/image"
"github.com/sealerio/sealer/pkg/image/cache"
v1 "github.com/sealerio/sealer/types/api/v1"
"github.com/sealerio/sealer/utils/archive"
"github.com/sealerio/sealer/utils/collector"
"github.com/sealerio/sealer/utils/os/fs"
"github.com/sirupsen/logrus"
)

func tryCache(parentID cache.ChainID,
Expand All @@ -57,46 +51,47 @@ func tryCache(parentID cache.ChainID,
}

func GenerateSourceFilesDigest(root, src string) (digest.Digest, error) {
m, err := fsutil.ResolveWildcards(root, src, true)
if err != nil {
return "", err
}

// wrong wildcards: no such file or directory
if len(m) == 0 {
return "", fmt.Errorf("%s not found", src)
}

if len(m) == 1 {
return generateDigest(filepath.Join(root, src))
}

tmp, err := fs.NewFilesystem().MkTmpdir()
if err != nil {
return "", fmt.Errorf("failed to create tmp dir %s:%v", tmp, err)
}

defer func() {
if err = os.RemoveAll(tmp); err != nil {
logrus.Warn(err)
}
}()

xattrErrorHandler := func(dst, src, key string, err error) error {
logrus.Warn(err)
return nil
}
opt := []fsutil.Opt{
fsutil.WithXAttrErrorHandler(xattrErrorHandler),
}

for _, s := range m {
if err := fsutil.Copy(context.TODO(), root, s, tmp, filepath.Base(s), opt...); err != nil {
return "", err
}
}

return generateDigest(tmp)
return "", nil
//m, err := fsutil.ResolveWildcards(root, src, true)
//if err != nil {
// return "", err
//}
//
//// wrong wildcards: no such file or directory
//if len(m) == 0 {
// return "", fmt.Errorf("%s not found", src)
//}
//
//if len(m) == 1 {
// return generateDigest(filepath.Join(root, src))
//}
//
//tmp, err := fs.NewFilesystem().MkTmpdir()
//if err != nil {
// return "", fmt.Errorf("failed to create tmp dir %s:%v", tmp, err)
//}
//
//defer func() {
// if err = os.RemoveAll(tmp); err != nil {
// logrus.Warn(err)
// }
//}()
//
//xattrErrorHandler := func(dst, src, key string, err error) error {
// logrus.Warn(err)
// return nil
//}
//opt := []fsutil.Opt{
// fsutil.WithXAttrErrorHandler(xattrErrorHandler),
//}
//
//for _, s := range m {
// if err := fsutil.Copy(context.TODO(), root, s, tmp, filepath.Base(s), opt...); err != nil {
// return "", err
// }
//}
//
//return generateDigest(tmp)
}

func generateDigest(path string) (digest.Digest, error) {
Expand Down
41 changes: 41 additions & 0 deletions build/util/util.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
package util

import (
"github.com/pkg/errors"
"os"
"path/filepath"
)

// DiscoverKubefile tries to find a Kubefile within the provided `path`.
func DiscoverKubefile(path string) (foundFile string, err error) {
// Test for existence of the file
target, err := os.Stat(path)
if err != nil {
return "", errors.Wrap(err, "discovering Kubefile")
}

switch mode := target.Mode(); {
case mode.IsDir():
// If the path is a real directory, we assume a Kubefile within it
kubefile := filepath.Join(path, "Kubefile")

// Test for existence of the Kubefile file
file, err := os.Stat(kubefile)
if err != nil {
return "", errors.Wrap(err, "cannot find Kubefile in context directory")
}

// The file exists, now verify the correct mode
if mode := file.Mode(); mode.IsRegular() {
foundFile = kubefile
} else {
return "", errors.Errorf("assumed Kubefile %q is not a file", kubefile)
}

case mode.IsRegular():
// If the context dir is a file, we assume this as Kubefile
foundFile = path
}

return foundFile, nil
}
68 changes: 26 additions & 42 deletions cmd/sealer/cmd/build.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,14 +15,11 @@
package cmd

import (
"os"

"github.com/containers/buildah/pkg/parse"
"github.com/sealerio/sealer/pkg/image_adaptor"
bc "github.com/sealerio/sealer/pkg/image_adaptor/common"
"github.com/sirupsen/logrus"
"github.com/spf13/cobra"

"github.com/sealerio/sealer/build"
"github.com/sealerio/sealer/utils/platform"
"github.com/sealerio/sealer/utils/strings"
)

type BuildFlag struct {
Expand All @@ -35,17 +32,17 @@ type BuildFlag struct {
Base bool
}

var buildConfig *BuildFlag
var buildFlags bc.BuildFlags = bc.BuildFlags{}

// buildCmd represents the build command
var buildCmd = &cobra.Command{
Use: "build [flags] PATH",
Short: "build a ClusterImage from a Kubefile",
Long: `build command is used to build a ClusterImage from specified Kubefile.
Long: `build command is used to generate a ClusterImage from specified Kubefile.
It organizes the specified Kubefile and input building context, and builds
a brand new ClusterImage.`,
Args: cobra.ExactArgs(1),
Example: `the current path is the context path, default build type is lite and use build cache
Args: cobra.MaximumNArgs(1),
Example: `the current path is the context path, default build type is lite and use image_adaptor cache
build:
sealer build -f Kubefile -t my-kubernetes:1.19.8 .
Expand All @@ -60,48 +57,35 @@ build with args:
sealer build -f Kubefile -t my-kubernetes:1.19.8 --build-arg MY_ARG=abc,PASSWORD=Sealer123 .
`,
RunE: func(cmd *cobra.Command, args []string) error {
buildContext := args[0]
builder, err := image_adaptor.NewAdaptor()
if err != nil {
logrus.Fatalf("failed to initiate a builder, %v", err)
}

targetPlatforms, err := platform.GetPlatform(buildConfig.Platform)
err = builder.Build(&buildFlags, args)
if err != nil {
logrus.Error(err)
return err
}
for _, tp := range targetPlatforms {
p := tp
conf := &build.Config{
BuildType: buildConfig.BuildType,
NoCache: buildConfig.NoCache,
ImageName: buildConfig.ImageName,
NoBase: !buildConfig.Base,
BuildArgs: strings.ConvertToMap(buildConfig.BuildArgs),
Platform: *p,
}
builder, err := build.NewBuilder(conf)
if err != nil {
return err
}
err = builder.Build(buildConfig.ImageName, buildContext, buildConfig.KubefileName)
if err != nil {
return err
}
}
return nil
},
}

func init() {
buildConfig = &BuildFlag{}
rootCmd.AddCommand(buildCmd)
buildCmd.Flags().StringVarP(&buildConfig.BuildType, "mode", "m", "lite", "ClusterImage build type, default is lite")
buildCmd.Flags().StringVarP(&buildConfig.KubefileName, "kubefile", "f", "Kubefile", "Kubefile filepath")
buildCmd.Flags().StringVarP(&buildConfig.ImageName, "imageName", "t", "", "the name of ClusterImage")
buildCmd.Flags().BoolVar(&buildConfig.NoCache, "no-cache", false, "build without cache")
buildCmd.Flags().BoolVar(&buildConfig.Base, "base", true, "build with base image, default value is true.")
buildCmd.Flags().StringSliceVar(&buildConfig.BuildArgs, "build-arg", []string{}, "set custom build args")
buildCmd.Flags().StringVar(&buildConfig.Platform, "platform", "", "set ClusterImage platform. If not set, keep same platform with runtime")

if err := buildCmd.MarkFlagRequired("imageName"); err != nil {
logrus.Errorf("failed to init flag: %v", err)
os.Exit(1)
buildCmd.Flags().StringVarP(&buildFlags.BuildType, "mode", "m", "lite", "ClusterImage build type, default is lite")
buildCmd.Flags().StringVarP(&buildFlags.Kubefile, "file", "f", "Kubefile", "Kubefile filepath")
buildCmd.Flags().BoolVar(&buildFlags.NoCache, "no-cache", false, "do not use existing cached images for building. Build from the start with a new set of cached layers.")
buildCmd.Flags().BoolVar(&buildFlags.Base, "base", true, "build with base image, default value is true.")
buildCmd.Flags().StringSliceVarP(&buildFlags.Tags, "tag", "t", []string{}, "specify a name for ClusterImage")
buildCmd.Flags().StringSliceVar(&buildFlags.BuildArgs, "build-arg", []string{}, "set custom image_adaptor args")
buildCmd.Flags().StringVar(&buildFlags.Platform, "platform", parse.DefaultPlatform(), "set the target platform, like linux/amd64 or linux/amd64/v7")

requiredFlags := []string{"tag"}
for _, flag := range requiredFlags {
if err := buildCmd.MarkFlagRequired(flag); err != nil {
logrus.Fatal(err)
}
}
}
5 changes: 5 additions & 0 deletions cmd/sealer/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,11 +15,16 @@
package main

import (
"github.com/containers/buildah"
"github.com/sealerio/sealer/cmd/sealer/boot"
"github.com/sealerio/sealer/cmd/sealer/cmd"
)

func main() {
if buildah.InitReexec() {
return
}

if err := boot.OnBoot(); err != nil {
panic(err)
}
Expand Down
15 changes: 15 additions & 0 deletions docs/design/build_implementation.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
# implementation of sealer build

## Abstract

Generally, the image generated from sealer build has no differences with container images. The image is compatible with OCI. Let's call it cluster image.
Sealer has some special operations based on the usual build of container images. Like (1) Adding a layer for storing containers
images automatically; (2) Saving cluster image information to the annotations from manifest of OCI v1 images. Sealer doesn't implement
the concrete building procedure. Sealer implements build over mature tools (we choose `buildah` currently). We will have an introduction for how the
sealer build implements next.

## Implement

### Adaptor

### Store Container Images
43 changes: 23 additions & 20 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -6,44 +6,49 @@ require (
github.com/Masterminds/semver/v3 v3.1.1
github.com/aliyun/alibaba-cloud-sdk-go v1.61.985
github.com/cavaliergopher/grab/v3 v3.0.1
github.com/containers/buildah v1.24.5
github.com/containers/common v0.47.5
github.com/containers/image/v5 v5.19.3
github.com/containers/ocicrypt v1.1.4
github.com/containers/storage v1.38.5
github.com/distribution/distribution/v3 v3.0.0-20211125133600-cc4627fc6e5f
github.com/docker/cli v20.10.7+incompatible
github.com/docker/distribution v2.7.1+incompatible
github.com/docker/docker v20.10.7+incompatible
github.com/docker/go-connections v0.4.0
github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 // indirect
github.com/docker/distribution v2.8.1+incompatible
github.com/docker/docker v20.10.17+incompatible
github.com/docker/go-connections v0.4.1-0.20210727194412-58542c764a11
github.com/go-git/go-git/v5 v5.4.2
github.com/google/uuid v1.2.0
github.com/google/uuid v1.3.0
github.com/imdario/mergo v0.3.12
github.com/lestrrat-go/file-rotatelogs v2.4.0+incompatible
github.com/lestrrat-go/strftime v1.0.6 // indirect
github.com/mitchellh/go-homedir v1.1.0
github.com/moby/buildkit v0.9.3
github.com/moby/term v0.0.0-20201216013528-df9cb8a40635
github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6
github.com/olekukonko/tablewriter v0.0.4
github.com/onsi/ginkgo v1.16.2
github.com/onsi/gomega v1.12.0
github.com/onsi/ginkgo v1.16.5
github.com/onsi/gomega v1.19.0
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/image-spec v1.0.2
github.com/opencontainers/image-spec v1.0.3-0.20211202193544-a5463b7f9c84
github.com/pkg/errors v0.9.1
github.com/pkg/sftp v1.13.0
github.com/rifflock/lfshook v0.0.0-20180920164130-b9218ef580f5
github.com/sealyun/lvscare v1.1.2-alpha.2
github.com/shirou/gopsutil v3.21.11+incompatible
github.com/sirupsen/logrus v1.8.1
github.com/spf13/cobra v1.1.3
github.com/spf13/viper v1.7.0
github.com/tonistiigi/fsutil v0.0.0-20211208191308-f95797418e48
github.com/vbatts/tar-split v0.11.1
github.com/vishvananda/netlink v1.1.1-0.20201029203352-d40f9887b852
github.com/spf13/cobra v1.4.0
github.com/spf13/pflag v1.0.5
github.com/spf13/viper v1.10.0
github.com/tonistiigi/fsutil v0.0.0-20210609172227-d72af97c0eaf
github.com/vbatts/tar-split v0.11.2
github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5
github.com/vishvananda/netns v0.0.0-20211101163701-50045581ed74 // indirect
github.com/yusufpapurcu/wmi v1.2.2 // indirect
go.etcd.io/etcd/client/v3 v3.5.0
go.uber.org/zap v1.17.0
golang.org/x/crypto v0.0.0-20210817164053-32db794688a5
golang.org/x/net v0.0.0-20210510120150-4163338589ed
golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4
golang.org/x/net v0.0.0-20220225172249-27dd8689420f
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
golang.org/x/sys v0.0.0-20220114195835-da31bd327af9
golang.org/x/sys v0.0.0-20220422013727-9388b58f7150
gopkg.in/yaml.v2 v2.4.0
gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
gotest.tools v2.2.0+incompatible
Expand All @@ -54,9 +59,7 @@ require (
k8s.io/client-go v0.21.0
k8s.io/kube-proxy v0.21.0
k8s.io/kubelet v0.21.0
k8s.io/utils v0.0.0-20210111153108-fddb29f9d009
k8s.io/utils v0.0.0-20210930125809-cb0fa318a74b
sigs.k8s.io/controller-runtime v0.8.1
sigs.k8s.io/yaml v1.2.0
)

replace github.com/docker/docker => github.com/docker/docker v20.10.3-0.20211208011758-87521affb077+incompatible
Loading

0 comments on commit b89013c

Please sign in to comment.