Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Store time of last modified in cache dir #2368

Merged
merged 4 commits into from
Jul 17, 2024
Merged

Store time of last modified in cache dir #2368

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
5524c02
Store time of Last-Modified in cache dir
afbjorklund May 27, 2024
e8841f6
Store type of Content-Type in cache dir
afbjorklund May 27, 2024
cf20b24
Return contents and not path in result
afbjorklund May 27, 2024
d6bddec
Use time.Time not string for LastModified
afbjorklund Jun 2, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
65 changes: 62 additions & 3 deletions pkg/downloader/downloader.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import (
"path"
"path/filepath"
"strings"
"time"

"github.com/cheggaaa/pb/v3"
"github.com/containerd/continuity/fs"
Expand Down Expand Up @@ -44,6 +45,8 @@ const (
type Result struct {
Status Status
CachePath string // "/Users/foo/Library/Caches/lima/download/by-url-sha256/<SHA256_OF_URL>/data"
LastModified time.Time
ContentType string
ValidatedDigest bool
}

Expand Down Expand Up @@ -118,6 +121,38 @@ func WithExpectedDigest(expectedDigest digest.Digest) Opt {
}
}

func readFile(path string) string {
afbjorklund marked this conversation as resolved.
Show resolved Hide resolved
if path == "" {
return ""
}
if _, err := os.Stat(path); err != nil {
return ""
}
b, err := os.ReadFile(path)
if err != nil {
return ""
}
return string(b)
}

func readTime(path string) time.Time {
if path == "" {
return time.Time{}
}
if _, err := os.Stat(path); err != nil {
return time.Time{}
}
b, err := os.ReadFile(path)
if err != nil {
return time.Time{}
}
t, err := time.Parse(http.TimeFormat, string(b))
if err != nil {
return time.Time{}
}
return t
}

// Download downloads the remote resource into the local path.
//
// Download caches the remote resource if WithCache or WithCacheDir option is specified.
Expand Down Expand Up @@ -175,7 +210,7 @@ func Download(ctx context.Context, local, remote string, opts ...Opt) (*Result,
}

if o.cacheDir == "" {
if err := downloadHTTP(ctx, localPath, remote, o.description, o.expectedDigest); err != nil {
if err := downloadHTTP(ctx, localPath, "", "", remote, o.description, o.expectedDigest); err != nil {
return nil, err
}
res := &Result{
Expand All @@ -187,6 +222,8 @@ func Download(ctx context.Context, local, remote string, opts ...Opt) (*Result,

shad := cacheDirectoryPath(o.cacheDir, remote)
shadData := filepath.Join(shad, "data")
shadTime := filepath.Join(shad, "time")
shadType := filepath.Join(shad, "type")
shadDigest, err := cacheDigestPath(shad, o.expectedDigest)
if err != nil {
return nil, err
Expand All @@ -210,6 +247,8 @@ func Download(ctx context.Context, local, remote string, opts ...Opt) (*Result,
res := &Result{
Status: StatusUsedCache,
CachePath: shadData,
LastModified: readTime(shadTime),
ContentType: readFile(shadType),
ValidatedDigest: o.expectedDigest != "",
}
return res, nil
Expand All @@ -224,7 +263,7 @@ func Download(ctx context.Context, local, remote string, opts ...Opt) (*Result,
if err := os.WriteFile(shadURL, []byte(remote), 0o644); err != nil {
return nil, err
}
if err := downloadHTTP(ctx, shadData, remote, o.description, o.expectedDigest); err != nil {
if err := downloadHTTP(ctx, shadData, shadTime, shadType, remote, o.description, o.expectedDigest); err != nil {
return nil, err
}
// no need to pass the digest to copyLocal(), as we already verified the digest
Expand All @@ -239,6 +278,8 @@ func Download(ctx context.Context, local, remote string, opts ...Opt) (*Result,
res := &Result{
Status: StatusDownloaded,
CachePath: shadData,
LastModified: readTime(shadTime),
ContentType: readFile(shadType),
ValidatedDigest: o.expectedDigest != "",
}
return res, nil
Expand Down Expand Up @@ -266,6 +307,8 @@ func Cached(remote string, opts ...Opt) (*Result, error) {

shad := cacheDirectoryPath(o.cacheDir, remote)
shadData := filepath.Join(shad, "data")
shadTime := filepath.Join(shad, "time")
shadType := filepath.Join(shad, "type")
shadDigest, err := cacheDigestPath(shad, o.expectedDigest)
if err != nil {
return nil, err
Expand All @@ -285,6 +328,8 @@ func Cached(remote string, opts ...Opt) (*Result, error) {
res := &Result{
Status: StatusUsedCache,
CachePath: shadData,
LastModified: readTime(shadTime),
ContentType: readFile(shadType),
ValidatedDigest: o.expectedDigest != "",
}
return res, nil
Expand All @@ -293,6 +338,8 @@ func Cached(remote string, opts ...Opt) (*Result, error) {
// cacheDirectoryPath returns the cache subdirectory path.
// - "url" file contains the url
// - "data" file contains the data
// - "time" file contains the time (Last-Modified header)
// - "type" file contains the type (Content-Type header)
func cacheDirectoryPath(cacheDir, remote string) string {
return filepath.Join(cacheDir, "download", "by-url-sha256", fmt.Sprintf("%x", sha256.Sum256([]byte(remote))))
}
Expand Down Expand Up @@ -470,7 +517,7 @@ func validateLocalFileDigest(localPath string, expectedDigest digest.Digest) err
return nil
}

func downloadHTTP(ctx context.Context, localPath, url, description string, expectedDigest digest.Digest) error {
func downloadHTTP(ctx context.Context, localPath, lastModified, contentType, url, description string, expectedDigest digest.Digest) error {
if localPath == "" {
return fmt.Errorf("downloadHTTP: got empty localPath")
}
Expand All @@ -489,6 +536,18 @@ func downloadHTTP(ctx context.Context, localPath, url, description string, expec
if err != nil {
return err
}
if lastModified != "" {
lm := resp.Header.Get("Last-Modified")
if err := os.WriteFile(lastModified, []byte(lm), 0o644); err != nil {
return err
}
}
if contentType != "" {
ct := resp.Header.Get("Content-Type")
if err := os.WriteFile(contentType, []byte(ct), 0o644); err != nil {
return err
}
}
defer resp.Body.Close()
bar, err := progressbar.New(resp.ContentLength)
if err != nil {
Expand Down
14 changes: 14 additions & 0 deletions pkg/downloader/downloader_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"runtime"
"strings"
"testing"
"time"

"github.com/opencontainers/go-digest"
"gotest.tools/v3/assert"
Expand All @@ -25,6 +26,8 @@ func TestDownloadRemote(t *testing.T) {
t.Cleanup(ts.Close)
dummyRemoteFileURL := ts.URL + "/downloader.txt"
const dummyRemoteFileDigest = "sha256:380481d26f897403368be7cb86ca03a4bc14b125bfaf2b93bff809a5a2ad717e"
dummyRemoteFileStat, err := os.Stat(filepath.Join("testdata", "downloader.txt"))
assert.NilError(t, err)

t.Run("without cache", func(t *testing.T) {
t.Run("without digest", func(t *testing.T) {
Expand Down Expand Up @@ -105,6 +108,17 @@ func TestDownloadRemote(t *testing.T) {
_, err = Cached(dummyRemoteFileURL, WithExpectedDigest(wrongDigest), WithCacheDir(cacheDir))
assert.ErrorContains(t, err, "expected digest")
})
t.Run("metadata", func(t *testing.T) {
_, err := Cached(dummyRemoteFileURL, WithExpectedDigest(dummyRemoteFileDigest))
assert.ErrorContains(t, err, "cache directory to be specified")

cacheDir := filepath.Join(t.TempDir(), "cache")
r, err := Download(context.Background(), "", dummyRemoteFileURL, WithExpectedDigest(dummyRemoteFileDigest), WithCacheDir(cacheDir))
assert.NilError(t, err)
assert.Equal(t, StatusDownloaded, r.Status)
assert.Equal(t, dummyRemoteFileStat.ModTime().Truncate(time.Second).UTC(), r.LastModified)
assert.Equal(t, "text/plain; charset=utf-8", r.ContentType)
})
}

func TestDownloadLocal(t *testing.T) {
Expand Down
Loading