Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: enhance manifest fetch #541

Merged
merged 16 commits into from
Sep 16, 2022
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 4 additions & 11 deletions cmd/oras/blob/fetch.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,18 +24,16 @@ import (
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/spf13/cobra"
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content/oci"
"oras.land/oras/cmd/oras/internal/option"
"oras.land/oras/internal/cache"
)

type fetchBlobOptions struct {
option.Cache
option.Common
option.Descriptor
option.Pretty
option.Remote

cacheRoot string
outputPath string
targetRef string
}
Expand Down Expand Up @@ -74,7 +72,6 @@ Example - Fetch blob from the insecure registry:
return errors.New("`--output -` cannot be used with `--descriptor` at the same time")
}

opts.cacheRoot = os.Getenv("ORAS_CACHE")
return opts.ReadPassword()
},
Aliases: []string{"get"},
Expand All @@ -101,13 +98,9 @@ func fetchBlob(opts fetchBlobOptions) (fetchErr error) {
return fmt.Errorf("%s: blob reference must be of the form <name@digest>", opts.targetRef)
}

var src oras.ReadOnlyTarget = repo.Blobs()
if opts.cacheRoot != "" {
ociStore, err := oci.New(opts.cacheRoot)
if err != nil {
return err
}
src = cache.New(src, ociStore)
src, err := opts.CachedTarget(repo.Blobs())
if err != nil {
return err
}

var desc ocispec.Descriptor
Expand Down
41 changes: 41 additions & 0 deletions cmd/oras/internal/option/cache.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/*
Copyright The ORAS Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package option

import (
"os"

"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content/oci"
"oras.land/oras/internal/cache"
)

type Cache struct {
Root string
}

// CachedTarget gets the target storage with caching if cache root is specified.
func (opts *Cache) CachedTarget(src oras.ReadOnlyTarget) (oras.ReadOnlyTarget, error) {
opts.Root = os.Getenv("ORAS_CACHE")
if opts.Root != "" {
ociStore, err := oci.New(opts.Root)
if err != nil {
return nil, err
}
return cache.New(src, ociStore), nil
}
return src, nil
}
63 changes: 63 additions & 0 deletions cmd/oras/internal/option/cache_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,63 @@
/*
Copyright The ORAS Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package option

import (
"os"
"reflect"
"testing"

"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content/memory"
"oras.land/oras-go/v2/content/oci"
"oras.land/oras/internal/cache"
)

var mockTarget oras.ReadOnlyTarget = memory.New()

func TestCache_CachedTarget(t *testing.T) {
tempDir := t.TempDir()
os.Setenv("ORAS_CACHE", tempDir)
yuehaoliang marked this conversation as resolved.
Show resolved Hide resolved
defer os.Unsetenv("ORAS_CACHE")
opts := Cache{}

ociStore, err := oci.New(tempDir)
if err != nil {
t.Fatal("error calling oci.New(), error =", err)
}
want := cache.New(mockTarget, ociStore)
Copy link
Contributor Author

@yuehaoliang yuehaoliang Sep 15, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I was trying to write a better test here, but I didn't figure out a good way unless exporting the cache.target struct.

want := struct {
	oras.ReadOnlyTarget
	cache content.Storage
}{
	mockTarget,
	ociStore,
}

Building a struct like above doesn't work as want and got are not equal.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

It's not doable even if cache.target struct is exported because target.cache is not exported. I think we can keep it this way.


got, err := opts.CachedTarget(mockTarget)
if err != nil {
t.Fatal("Cache.CachedTarget() error=", err)
}
if !reflect.DeepEqual(got, want) {
t.Fatalf("Cache.CachedTarget() got %v, want %v", got, want)
}
}

func TestCache_CachedTarget_emptyRoot(t *testing.T) {
os.Setenv("ORAS_CACHE", "")
opts := Cache{}

got, err := opts.CachedTarget(mockTarget)
if err != nil {
t.Fatal("Cache.CachedTarget() error=", err)
}
if !reflect.DeepEqual(got, mockTarget) {
t.Fatalf("Cache.CachedTarget() got %v, want %v", got, mockTarget)
}
}
88 changes: 60 additions & 28 deletions cmd/oras/manifest/fetch.go
Original file line number Diff line number Diff line change
Expand Up @@ -16,26 +16,28 @@ limitations under the License.
package manifest

import (
"bytes"
"encoding/json"
"fmt"
"errors"
"os"

ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/spf13/cobra"
"oras.land/oras-go/v2"
oerrors "oras.land/oras/cmd/oras/internal/errors"
"oras.land/oras/cmd/oras/internal/option"
"oras.land/oras/internal/cas"
)

type fetchOptions struct {
option.Cache
option.Common
option.Descriptor
option.Remote
option.Platform
option.Pretty

targetRef string
pretty bool
mediaTypes []string
fetchDescriptor bool
mediaTypes []string
outputPath string
targetRef string
}

func fetchCmd() *cobra.Command {
Expand All @@ -44,6 +46,7 @@ func fetchCmd() *cobra.Command {
Use: "fetch [flags] <name:tag|name@digest>",
Short: "[Preview] Fetch manifest of the target artifact",
Long: `[Preview] Fetch manifest of the target artifact

** This command is in preview and under development. **

Example - Fetch raw manifest:
Expand All @@ -63,6 +66,10 @@ Example - Fetch manifest with prettified json result:
`,
Args: cobra.ExactArgs(1),
PreRunE: func(cmd *cobra.Command, args []string) error {
if opts.outputPath == "-" && opts.OutputDescriptor {
return errors.New("`--output -` cannot be used with `--descriptor` at the same time")
}

return opts.ReadPassword()
},
Aliases: []string{"get"},
Expand All @@ -72,46 +79,71 @@ Example - Fetch manifest with prettified json result:
},
}

cmd.Flags().BoolVarP(&opts.pretty, "pretty", "", false, "output prettified manifest")
cmd.Flags().BoolVarP(&opts.fetchDescriptor, "descriptor", "", false, "fetch a descriptor of the manifest")
cmd.Flags().StringSliceVarP(&opts.mediaTypes, "media-type", "", nil, "accepted media types")
cmd.Flags().StringVarP(&opts.outputPath, "output", "o", "", "output file path")
option.ApplyFlags(&opts, cmd.Flags())
return cmd
}

func fetchManifest(opts fetchOptions) error {
func fetchManifest(opts fetchOptions) (fetchErr error) {
ctx, _ := opts.SetLoggerLevel()
targetPlatform, err := opts.Parse()
if err != nil {
return err
}

repo, err := opts.NewRepository(opts.targetRef, opts.Common)
if err != nil {
return err
}

if repo.Reference.Reference == "" {
return oerrors.NewErrInvalidReference(repo.Reference)
}
repo.ManifestMediaTypes = opts.mediaTypes

// Fetch and output
var content []byte
if opts.fetchDescriptor {
content, err = cas.FetchDescriptor(ctx, repo, opts.targetRef, targetPlatform)
} else {
content, err = cas.FetchManifest(ctx, repo, opts.targetRef, targetPlatform)
targetPlatform, err := opts.Parse()
if err != nil {
return err
}

manifests, err := opts.CachedTarget(repo.Manifests())
if err != nil {
return err
}
if opts.pretty {
buf := bytes.NewBuffer(nil)
if err = json.Indent(buf, content, "", " "); err != nil {
return fmt.Errorf("failed to prettify: %w", err)

var desc ocispec.Descriptor
if opts.OutputDescriptor && opts.outputPath == "" {
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This if block can be merged with the if block at the end:

  1. If !opts.OutputDescriptor || opt.outputPath != "" => fetch manifest content
  2. If opts.OutputDescriptor => resolve and output to stdout

Copy link
Contributor Author

@yuehaoliang yuehaoliang Sep 16, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I couldn't really follow your idea here.

In the case of fetching manifest content, there still might be a need to output descriptor, eg. --output manifest.json --descriptor. Since oras.FetchBytes returns both the content and the descriptor, so if content has already been fetched, we don't need to call oras.Resolve again.

Current logic:

if opts.OutputDescriptor && opts.outputPath == "" {
	// oras.Resolve   fetch manifest descriptor only
} else {
	// oras.FetchBytes   fetch manifest descriptor content

	if opts.outputPath == "" || opts.outputPath == "-" {
		// output manifest content

		return
	}

	// save manifest content into the local file
}

if opts.OutputDescriptor {
	// output manifest descriptor
}

In the logic below, for --output manifest.json --descriptor, there's a redundant call of oras.Resolve.

if !opts.OutputDescriptor || opt.outputPath != "" {
	// oras.FetchBytes   fetch manifest descriptor content

	if opts.outputPath == "" || opts.outputPath == "-" {
		// output manifest content

		return
	}

	// save manifest content into the local file
} 

if opts.OutputDescriptor {
	// oras.Resolve   fetch manifest descriptor only

	// output manifest descriptor
}

Copy link
Contributor

@qweeah qweeah Sep 19, 2022

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Since oras.FetchBytes returns both the content and the descriptor, so if content has already been fetched, we don't need to call oras.Resolve again

This can be avoided by checking if the desc is an empty struct.

// fetch manifest descriptor only
desc, err = oras.Resolve(ctx, manifests, opts.targetRef, oras.DefaultResolveOptions)
if err != nil {
return err
}
} else {
// fetch manifest content
var content []byte
fetchOpts := oras.DefaultFetchBytesOptions
fetchOpts.TargetPlatform = targetPlatform
desc, content, err = oras.FetchBytes(ctx, manifests, opts.targetRef, fetchOpts)
if err != nil {
return err
}

if opts.outputPath == "" || opts.outputPath == "-" {
// output manifest content
return opts.Output(os.Stdout, content)
}

// save manifest content into the local file if the output path is provided
if err = os.WriteFile(opts.outputPath, content, 0666); err != nil {
return err
}
}

// output manifest's descriptor if `--descriptor` is used
if opts.OutputDescriptor {
descBytes, err := json.Marshal(desc)
if err != nil {
return err
}
buf.WriteByte('\n')
content = buf.Bytes()
return opts.Output(os.Stdout, descBytes)
}
_, err = os.Stdout.Write(content)
return err

return nil
}
16 changes: 4 additions & 12 deletions cmd/oras/pull.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,28 +18,25 @@ package main
import (
"context"
"fmt"
"os"
"sync"

ocispec "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/spf13/cobra"
"oras.land/oras-go/v2"
"oras.land/oras-go/v2/content"
"oras.land/oras-go/v2/content/file"
"oras.land/oras-go/v2/content/oci"
"oras.land/oras/cmd/oras/internal/display"
"oras.land/oras/cmd/oras/internal/errors"
"oras.land/oras/cmd/oras/internal/option"
"oras.land/oras/internal/cache"
"oras.land/oras/internal/docker"
)

type pullOptions struct {
option.Cache
option.Common
option.Remote

targetRef string
cacheRoot string
KeepOldFiles bool
PathTraversal bool
Output string
Expand Down Expand Up @@ -68,7 +65,6 @@ Example - Pull files with local cache:
`,
Args: cobra.ExactArgs(1),
PreRunE: func(cmd *cobra.Command, args []string) error {
opts.cacheRoot = os.Getenv("ORAS_CACHE")
return opts.ReadPassword()
},
RunE: func(cmd *cobra.Command, args []string) error {
Expand All @@ -94,13 +90,9 @@ func runPull(opts pullOptions) error {
if repo.Reference.Reference == "" {
return errors.NewErrInvalidReference(repo.Reference)
}
var src oras.ReadOnlyTarget = repo
if opts.cacheRoot != "" {
ociStore, err := oci.New(opts.cacheRoot)
if err != nil {
return err
}
src = cache.New(repo, ociStore)
src, err := opts.CachedTarget(repo)
if err != nil {
return err
}

// Copy Options
Expand Down
Loading