Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Add Filesystem based remote store to support airgap. #397

Merged
merged 6 commits into from
Sep 30, 2022
Merged

feat: Add Filesystem based remote store to support airgap. #397

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
9fc21e2
Add Filesystem based remote store to support airgap.
vaikas Sep 24, 2022
b54f045
Try with T.Cleanup too. Wonder if this goes to 11 at some point?
vaikas Sep 28, 2022
6c98b79
close the file.
vaikas Sep 28, 2022
6a32d8c
t.Cleanup vs. defer.
vaikas Sep 28, 2022
dabde9f
omit one initializing test for windows.
vaikas Sep 29, 2022
f8b8151
Merge branch 'master' into air-gap
joshuagl Sep 30, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions client/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -807,6 +807,8 @@ type Destination interface {
// - The target does not exist in any targets
// - Metadata cannot be generated for the downloaded data
// - Generated metadata does not match local metadata for the given file
// - Size of the download does not match if the reported size is known and
// incorrect
func (c *Client) Download(name string, dest Destination) (err error) {
// delete dest if there is an error
defer func() {
Expand Down
163 changes: 118 additions & 45 deletions client/client_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,15 +31,19 @@ import (
func Test(t *testing.T) { TestingT(t) }

type ClientSuite struct {
store tuf.LocalStore
repo *tuf.Repo
local LocalStore
remote *fakeRemoteStore
expiredTime time.Time
keyIDs map[string][]string
store tuf.LocalStore
repo *tuf.Repo
local LocalStore
remote RemoteStore
expiredTime time.Time
keyIDs map[string][]string
useFileStore bool
// Only used with FileStore
tmpDir string
}

var _ = Suite(&ClientSuite{})
var _ = Suite(&ClientSuite{useFileStore: false})
var _ = Suite(&ClientSuite{useFileStore: true})

func newFakeRemoteStore() *fakeRemoteStore {
return &fakeRemoteStore{
Expand Down Expand Up @@ -69,6 +73,66 @@ func (f *fakeRemoteStore) get(name string, store map[string]*fakeFile) (io.ReadC
return file, file.size, nil
}

// These are helper methods for manipulating the internals of the Stores
// because the set/delete methods are not part of the Interface, we need to
// switch on the underlying implementation.
// Also readMeta method is convenience for ease of testing.
func (s *ClientSuite) setRemoteMeta(path string, data []byte) error {
switch impl := s.remote.(type) {
case *fakeRemoteStore:
impl.meta[path] = newFakeFile(data)
return nil
case *FileRemoteStore:
return impl.addMeta(path, data)
default:
return fmt.Errorf("non-supoprted RemoteStore, got %+v", impl)
vaikas marked this conversation as resolved.
Show resolved Hide resolved
}
}

func (s *ClientSuite) setRemoteTarget(path string, data []byte) error {
switch impl := s.remote.(type) {
case *fakeRemoteStore:
impl.targets[path] = newFakeFile(data)
return nil
case *FileRemoteStore:
return impl.addTarget(path, data)
default:
return fmt.Errorf("non-supoprted RemoteStore, got %+v", impl)
}
}

func (s *ClientSuite) deleteMeta(path string) error {
switch impl := s.remote.(type) {
case *fakeRemoteStore:
delete(impl.meta, path)
return nil
case *FileRemoteStore:
return impl.deleteMeta(path)
default:
return fmt.Errorf("non-supported RemoteStore, got %+v", impl)
}
}

func (s *ClientSuite) deleteTarget(path string) error {
switch impl := s.remote.(type) {
case *fakeRemoteStore:
delete(impl.targets, path)
return nil
case *FileRemoteStore:
return impl.deleteTarget(path)
default:
return fmt.Errorf("non-supported RemoteStore, got %+v", impl)
}
}

func (s *ClientSuite) readMeta(name string) ([]byte, error) {
stream, _, err := s.remote.GetMeta(name)
if err != nil {
return nil, err
}
return io.ReadAll(stream)
}

func newFakeFile(b []byte) *fakeFile {
return &fakeFile{buf: bytes.NewReader(b), size: int64(len(b))}
}
Expand Down Expand Up @@ -118,15 +182,28 @@ func (s *ClientSuite) SetUpTest(c *C) {
c.Assert(s.repo.Commit(), IsNil)

// create a remote store containing valid repo files
s.remote = newFakeRemoteStore()
if s.useFileStore {
s.remote, s.tmpDir, err = newTestFileStoreFS()
if err != nil {
c.Fatalf("failed to create new FileStore: %v", err)
}
} else {
s.remote = newFakeRemoteStore()
}
s.syncRemote(c)
for path, data := range targetFiles {
s.remote.targets[path] = newFakeFile(data)
s.setRemoteTarget(path, data)
}

s.expiredTime = time.Now().Add(time.Hour)
}

func (s *ClientSuite) TearDownTest(c *C) {
if s.tmpDir != "" {
rmrf(s.tmpDir, c.Logf)
}
}

func (s *ClientSuite) genKey(c *C, role string) []string {
ids, err := s.repo.GenKey(role)
c.Assert(err, IsNil)
Expand Down Expand Up @@ -163,7 +240,9 @@ func (s *ClientSuite) syncRemote(c *C) {
meta, err := s.store.GetMeta()
c.Assert(err, IsNil)
for name, data := range meta {
s.remote.meta[name] = newFakeFile(data)
if err := s.setRemoteMeta(name, data); err != nil {
panic(fmt.Sprintf("setMetadata failed: %v", err))
}
}
}

Expand Down Expand Up @@ -252,7 +331,7 @@ func (s *ClientSuite) TestInitAllowsExpired(c *C) {
c.Assert(s.repo.Commit(), IsNil)
s.syncRemote(c)
client := NewClient(MemoryLocalStore(), s.remote)
bytes, err := io.ReadAll(s.remote.meta["root.json"])
bytes, err := s.readMeta("root.json")
c.Assert(err, IsNil)
s.withMetaExpired(func() {
c.Assert(client.Init(bytes), IsNil)
Expand All @@ -261,7 +340,7 @@ func (s *ClientSuite) TestInitAllowsExpired(c *C) {

func (s *ClientSuite) TestInit(c *C) {
client := NewClient(MemoryLocalStore(), s.remote)
bytes, err := io.ReadAll(s.remote.meta["root.json"])
bytes, err := s.readMeta("root.json")
c.Assert(err, IsNil)
dataSigned := &data.Signed{}
c.Assert(json.Unmarshal(bytes, dataSigned), IsNil)
Expand Down Expand Up @@ -299,11 +378,11 @@ func (s *ClientSuite) TestFirstUpdate(c *C) {
func (s *ClientSuite) TestMissingRemoteMetadata(c *C) {
client := s.newClient(c)

delete(s.remote.meta, "targets.json")
s.deleteMeta("targets.json")
_, err := client.Update()
c.Assert(err, Equals, ErrMissingRemoteMetadata{"targets.json"})

delete(s.remote.meta, "timestamp.json")
s.deleteMeta("timestamp.json")
_, err = client.Update()
c.Assert(err, Equals, ErrMissingRemoteMetadata{"timestamp.json"})
}
Expand Down Expand Up @@ -774,7 +853,7 @@ func (s *ClientSuite) TestLocalExpired(c *C) {
}

func (s *ClientSuite) TestTimestampTooLarge(c *C) {
s.remote.meta["timestamp.json"] = newFakeFile(make([]byte, defaultTimestampDownloadLimit+1))
s.setRemoteMeta("timestamp.json", make([]byte, defaultTimestampDownloadLimit+1))
_, err := s.newClient(c).Update()
c.Assert(err, Equals, ErrMetaTooLarge{"timestamp.json", defaultTimestampDownloadLimit + 1, defaultTimestampDownloadLimit})
}
Expand Down Expand Up @@ -901,8 +980,8 @@ func (s *ClientSuite) TestUpdateMixAndMatchAttack(c *C) {
client := s.updatedClient(c)

// grab the remote targets.json
oldTargets, ok := s.remote.meta["targets.json"]
if !ok {
oldTargets, err := s.readMeta("targets.json")
if err != nil {
c.Fatal("missing remote targets.json")
}

Expand All @@ -912,22 +991,22 @@ func (s *ClientSuite) TestUpdateMixAndMatchAttack(c *C) {
c.Assert(s.repo.Timestamp(), IsNil)
c.Assert(s.repo.Commit(), IsNil)
s.syncRemote(c)
newTargets, ok := s.remote.meta["targets.json"]
if !ok {
newTargets, err := s.readMeta("targets.json")
if err != nil {
c.Fatal("missing remote targets.json")
}
s.remote.meta["targets.json"] = oldTargets
s.setRemoteMeta("targets.json", oldTargets)

// check update returns ErrWrongSize for targets.json
_, err := client.Update()
c.Assert(err, DeepEquals, ErrWrongSize{"targets.json", oldTargets.size, newTargets.size})
_, err = client.Update()
c.Assert(err, DeepEquals, ErrWrongSize{"targets.json", int64(len(oldTargets)), int64(len(newTargets))})

// do the same but keep the size the same
c.Assert(s.repo.RemoveTargetWithExpires("foo.txt", expires), IsNil)
c.Assert(s.repo.Snapshot(), IsNil)
c.Assert(s.repo.Timestamp(), IsNil)
s.syncRemote(c)
s.remote.meta["targets.json"] = oldTargets
s.setRemoteMeta("targets.json", oldTargets)

// check update returns ErrWrongHash
_, err = client.Update()
Expand All @@ -938,8 +1017,8 @@ func (s *ClientSuite) TestUpdateReplayAttack(c *C) {
client := s.updatedClient(c)

// grab the remote timestamp.json
oldTimestamp, ok := s.remote.meta["timestamp.json"]
if !ok {
oldTimestamp, err := s.readMeta("timestamp.json")
if err != nil {
c.Fatal("missing remote timestamp.json")
}

Expand All @@ -948,12 +1027,12 @@ func (s *ClientSuite) TestUpdateReplayAttack(c *C) {
c.Assert(version > 0, Equals, true)
c.Assert(s.repo.Timestamp(), IsNil)
s.syncRemote(c)
_, err := client.Update()
_, err = client.Update()
c.Assert(err, IsNil)
c.Assert(client.timestampVer > version, Equals, true)

// replace remote timestamp.json with the old one
s.remote.meta["timestamp.json"] = oldTimestamp
s.setRemoteMeta("timestamp.json", oldTimestamp)

// check update returns ErrLowVersion
_, err = client.Update()
Expand All @@ -970,8 +1049,8 @@ func (s *ClientSuite) TestUpdateForkTimestamp(c *C) {
client := s.updatedClient(c)

// grab the remote timestamp.json
oldTimestamp, ok := s.remote.meta["timestamp.json"]
if !ok {
oldTimestamp, err := s.readMeta("timestamp.json")
if err != nil {
c.Fatal("missing remote timestamp.json")
}

Expand All @@ -980,13 +1059,13 @@ func (s *ClientSuite) TestUpdateForkTimestamp(c *C) {
c.Assert(version > 0, Equals, true)
c.Assert(s.repo.Timestamp(), IsNil)
s.syncRemote(c)
_, err := client.Update()
_, err = client.Update()
c.Assert(err, IsNil)
newVersion := client.timestampVer
c.Assert(newVersion > version, Equals, true)

// generate a new, different timestamp with the *same version*
s.remote.meta["timestamp.json"] = oldTimestamp
s.setRemoteMeta("timestamp.json", oldTimestamp)
c.Assert(s.repo.Timestamp(), IsNil)
c.Assert(client.timestampVer, Equals, newVersion) // double-check: same version?
s.syncRemote(c)
Expand Down Expand Up @@ -1098,7 +1177,7 @@ func (s *ClientSuite) TestDownloadUnknownTarget(c *C) {

func (s *ClientSuite) TestDownloadNoExist(c *C) {
client := s.updatedClient(c)
delete(s.remote.targets, "foo.txt")
s.deleteTarget("foo.txt")
var dest testDestination
c.Assert(client.Download("foo.txt", &dest), Equals, ErrNotFound{"foo.txt"})
c.Assert(dest.deleted, Equals, true)
Expand All @@ -1117,38 +1196,32 @@ func (s *ClientSuite) TestDownloadOK(c *C) {

func (s *ClientSuite) TestDownloadWrongSize(c *C) {
client := s.updatedClient(c)
remoteFile := &fakeFile{buf: bytes.NewReader([]byte("wrong-size")), size: 10}
s.remote.targets["foo.txt"] = remoteFile
// Update with a file that's incorrect size.
s.setRemoteTarget("foo.txt", []byte("wrong-size"))
var dest testDestination
c.Assert(client.Download("foo.txt", &dest), DeepEquals, ErrWrongSize{"foo.txt", 10, 3})
c.Assert(remoteFile.bytesRead, Equals, 0)
c.Assert(dest.deleted, Equals, true)
}

func (s *ClientSuite) TestDownloadTargetTooLong(c *C) {
client := s.updatedClient(c)
remoteFile := s.remote.targets["foo.txt"]
remoteFile.buf = bytes.NewReader([]byte("foo-ooo"))
s.setRemoteTarget("foo.txt", []byte("foo-ooo"))
var dest testDestination
c.Assert(client.Download("foo.txt", &dest), IsNil)
c.Assert(remoteFile.bytesRead, Equals, 3)
c.Assert(dest.deleted, Equals, false)
c.Assert(dest.String(), Equals, "foo")
c.Assert(client.Download("foo.txt", &dest), DeepEquals, ErrWrongSize{"foo.txt", 7, 3})
c.Assert(dest.deleted, Equals, true)
}

func (s *ClientSuite) TestDownloadTargetTooShort(c *C) {
client := s.updatedClient(c)
remoteFile := s.remote.targets["foo.txt"]
remoteFile.buf = bytes.NewReader([]byte("fo"))
s.setRemoteTarget("foo.txt", []byte("fo"))
var dest testDestination
c.Assert(client.Download("foo.txt", &dest), DeepEquals, ErrWrongSize{"foo.txt", 2, 3})
c.Assert(dest.deleted, Equals, true)
}

func (s *ClientSuite) TestDownloadTargetCorruptData(c *C) {
client := s.updatedClient(c)
remoteFile := s.remote.targets["foo.txt"]
remoteFile.buf = bytes.NewReader([]byte("corrupt"))
s.setRemoteTarget("foo.txt", []byte("ooo"))
var dest testDestination
assertWrongHash(c, client.Download("foo.txt", &dest))
c.Assert(dest.deleted, Equals, true)
Expand Down
Loading