Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(crate): introduce crate concept. #138

Merged
merged 16 commits into from
Mar 7, 2022
Merged
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
feat(oci): support template archive.
Zenithar committed Mar 7, 2022
commit 9342a36dce47a701f909ad55ec86a66b13cd9fb1
169 changes: 168 additions & 1 deletion pkg/crate/build.go
Original file line number Diff line number Diff line change
@@ -18,15 +18,24 @@
package crate

import (
"archive/tar"
"bytes"
"compress/gzip"
"crypto/rand"
"errors"
"fmt"
"io"
"os"
"path/filepath"

"github.com/gobwas/glob"
"go.uber.org/zap"

"github.com/elastic/harp/pkg/container"
"github.com/elastic/harp/pkg/crate/cratefile"
schemav1 "github.com/elastic/harp/pkg/crate/schema/v1"
"github.com/elastic/harp/pkg/sdk/log"
"github.com/elastic/harp/pkg/sdk/types"
)

const (
@@ -64,17 +73,175 @@ func Build(spec *cratefile.Config) (*Image, error) {
c = sc
}

// Prepare config manifest
cfg := schemav1.NewConfig()
cfg.SetContainers([]string{spec.Container.Name})

// Preapre archives
templateArchives := []*TemplateArchive{}

// Create archives
archiveNames := []string{}
for i, archive := range spec.Archives {
// Create tar.gz archive
var buf bytes.Buffer
if err := createArchive(&spec.Archives[i], &buf); err != nil {
return nil, fmt.Errorf("unable to create archive '%s': %w", archive.Name, err)
}

// Add archive format suffix
name := fmt.Sprintf("%s.tar.gz", archive.Name)

// Add to config manifest
archiveNames = append(archiveNames, name)

// Add layer.
templateArchives = append(templateArchives, &TemplateArchive{
Name: name,
Archive: buf.Bytes(),
})
}

// Update config manifest
cfg.SetTemplates(archiveNames)

// Create
res := &Image{
Config: schemav1.NewConfig(),
Config: cfg,
Containers: []*SealedContainer{
{
Name: spec.Container.Name,
Container: c,
},
},
TemplateArchives: templateArchives,
}

// No error
return res, nil
}

// -----------------------------------------------------------------------------

//nolint:gocyclo,funlen // to refactor
func createArchive(archive *cratefile.Archive, w io.Writer) error {
// Check arguments
if types.IsNil(w) {
return errors.New("output writer is nil")
}
if archive == nil {
return errors.New("archive is nil")
}

// Ensure the root actually exists before trying to tar it
if _, err := os.Stat(archive.RootPath); err != nil {
return fmt.Errorf("unable to tar files: %w", err)
}

// Create writer chain.
zr := gzip.NewWriter(w)
tw := tar.NewWriter(zr)

// Compile inclusion filters
var includes []glob.Glob
for _, f := range archive.IncludeGlob {
// Try to compile glob filter.
filter, err := glob.Compile(f)
if err != nil {
return fmt.Errorf("unable to compile glob filter '%s' for inclusion: %w", f, err)
}

// Add to explusion filters.
includes = append(includes, filter)
}

// Compile explusion filters
var excludes []glob.Glob
for _, f := range archive.ExcludeGlob {
// Try to compile glob filter.
filter, err := glob.Compile(f)
if err != nil {
return fmt.Errorf("unable to compile glob filter '%s' for exclusion: %w", f, err)
}

// Add to explusion filters.
excludes = append(excludes, filter)
}

// walk through every file in the folder
if errWalk := filepath.Walk(archive.RootPath, func(file string, fi os.FileInfo, errIn error) error {
// return on any error
if errIn != nil {
return errIn
}
// Ignore non regular files
if !fi.Mode().IsRegular() {
return nil
}

// Process inclusions
keep := false
for _, f := range includes {
if f.Match(file) {
keep = true
}
}
for _, f := range excludes {
if f.Match(file) {
keep = false
}
}
if !keep {
// Ignore this file.
log.Bg().Debug("ignoreing file ...", zap.String("file", file))
return nil
}

// generate tar header
header, err := tar.FileInfoHeader(fi, file)
if err != nil {
return err
}

// must provide real name
header.Name, _ = filepath.Rel(archive.RootPath, file)

// write header
if err := tw.WriteHeader(header); err != nil {
return err
}

// if not a dir, write file content
if !fi.IsDir() {
data, err := os.Open(file)
if err != nil {
return err
}
if _, err := io.Copy(tw, data); err != nil {
return err
}

// Manual close to prevent to wait all files to be processed
// to close all.
log.SafeClose(data, "unble to close file", zap.String("file", file))
}

// No error
return nil
}); errWalk != nil {
return fmt.Errorf("fail to walk folders for archive compression: %w", errWalk)
}

// produce tar
if err := tw.Close(); err != nil {
return err
}

// produce gzip
if err := zr.Close(); err != nil {
return err
}

// No error
return nil
}
2 changes: 1 addition & 1 deletion pkg/crate/oci.go
Original file line number Diff line number Diff line change
@@ -179,7 +179,7 @@ func addTemplateArchive(store StoreSetter, ta *TemplateArchive) (*ocispec.Descri

// Prepare a layer
containerDesc := ocispec.Descriptor{
MediaType: ocispec.MediaTypeImageLayerGzip,
MediaType: harpDataMediaType,
Digest: digest.FromBytes(ta.Archive),
Size: int64(len(ta.Archive)),
Annotations: map[string]string{
7 changes: 6 additions & 1 deletion samples/oci-crate/Cratefile
Original file line number Diff line number Diff line change
@@ -3,4 +3,9 @@ container "postgres" {
identities = [
"v1.ipk.Ky60amL-6AYYQomvxivXDVS5d50h6BJkpDA0Hrqny7s"
]
}
}

archive "postgres" {
root = "./templates"
includes = ["**"]
}
14 changes: 14 additions & 0 deletions samples/oci-crate/templates/ca.crt
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
// Copyright 2022 Thibault NORMAND
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

Empty file.