From 6d7713e643b788395b604198ce390769005a4bad Mon Sep 17 00:00:00 2001 From: James Carnegie Date: Mon, 11 Dec 2023 13:08:34 +0200 Subject: [PATCH] Fixes for windows & enable in CI Signed-off-by: James Carnegie --- .github/workflows/tests.yml | 5 +- examples/cli/tuf-client/cmd/get.go | 2 +- examples/cli/tuf-client/cmd/init.go | 2 +- metadata/metadata_api_test.go | 131 +++++++++--------- metadata/metadata_test.go | 6 +- .../trustedmetadata/trustedmetadata_test.go | 45 +++--- metadata/updater/updater.go | 63 ++++++++- .../updater_consistent_snapshot_test.go | 18 ++- .../updater/updater_top_level_update_test.go | 19 +-- testutils/simulator/repository_simulator.go | 44 +++++- .../simulator/repository_simulator_setup.go | 9 +- testutils/testutils/setup.go | 10 +- 12 files changed, 238 insertions(+), 116 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 03d26dd7..5767ae60 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -32,11 +32,14 @@ jobs: strategy: fail-fast: false # Keep running if one leg fails. matrix: - os: [ubuntu-latest] # , macos-latest, windows-latest] Enable later so we don't waste github actions resources + os: [ubuntu-latest, windows-latest] # , ] Enable later so we don't waste github actions resources go-version: ${{ fromJSON(needs.get-go-versions.outputs.matrix) }} runs-on: ${{ matrix.os }} needs: get-go-versions steps: + - name: Set git to use LF + run: git config --global core.autocrlf false + - name: Checkout code uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 diff --git a/examples/cli/tuf-client/cmd/get.go b/examples/cli/tuf-client/cmd/get.go index ec8b7209..9b695d96 100644 --- a/examples/cli/tuf-client/cmd/get.go +++ b/examples/cli/tuf-client/cmd/get.go @@ -157,7 +157,7 @@ func verifyEnv() (*localConfig, error) { return nil, fmt.Errorf("no local download folder: %w", err) } // verify there's a local root.json available for bootstrapping trust - _, err = os.Stat(fmt.Sprintf("%s/%s.json", env.MetadataDir, metadata.ROOT)) + _, err = os.Stat(filepath.Join(env.MetadataDir, fmt.Sprintf("%s.json", metadata.ROOT))) if err != nil { return nil, fmt.Errorf("no local download folder: %w", err) } diff --git a/examples/cli/tuf-client/cmd/init.go b/examples/cli/tuf-client/cmd/init.go index 8e4be0fc..af908dbe 100644 --- a/examples/cli/tuf-client/cmd/init.go +++ b/examples/cli/tuf-client/cmd/init.go @@ -75,7 +75,7 @@ func InitializeCmd() error { if err != nil { return err } - rootPath = fmt.Sprintf("%s/%s.json", rootPath, metadata.ROOT) + rootPath = filepath.Join(rootPath, fmt.Sprintf("%s.json", metadata.ROOT)) // no need to copy root.json to the metadata folder as we already download it in the expected location copyTrusted = false } diff --git a/metadata/metadata_api_test.go b/metadata/metadata_api_test.go index e70985c7..38950a81 100644 --- a/metadata/metadata_api_test.go +++ b/metadata/metadata_api_test.go @@ -25,6 +25,7 @@ import ( "fmt" "io/fs" "os" + "path/filepath" "strconv" "strings" "testing" @@ -34,7 +35,6 @@ import ( "github.com/sigstore/sigstore/pkg/signature" "github.com/stretchr/testify/assert" "github.com/theupdateframework/go-tuf/v2/testutils/testutils" - "golang.org/x/sys/unix" ) func TestMain(m *testing.M) { @@ -64,7 +64,7 @@ func TestGenericRead(t *testing.T) { _, err = Timestamp().FromBytes([]byte(badMetadata)) assert.ErrorIs(t, err, ErrValue{"expected metadata type timestamp, got - bad-metadata"}) - badMetadataPath := fmt.Sprintf("%s/bad-metadata.json", testutils.RepoDir) + badMetadataPath := filepath.Join(testutils.RepoDir, "bad-metadata.json") err = os.WriteFile(badMetadataPath, []byte(badMetadata), 0644) assert.NoError(t, err) assert.FileExists(t, badMetadataPath) @@ -85,46 +85,46 @@ func TestGenericRead(t *testing.T) { func TestGenericReadFromMismatchingRoles(t *testing.T) { // Test failing to load other roles from root metadata - _, err := Snapshot().FromFile(fmt.Sprintf("%s/root.json", testutils.RepoDir)) + _, err := Snapshot().FromFile(filepath.Join(testutils.RepoDir, "root.json")) assert.ErrorIs(t, err, ErrValue{"expected metadata type snapshot, got - root"}) - _, err = Timestamp().FromFile(fmt.Sprintf("%s/root.json", testutils.RepoDir)) + _, err = Timestamp().FromFile(filepath.Join(testutils.RepoDir, "root.json")) assert.ErrorIs(t, err, ErrValue{"expected metadata type timestamp, got - root"}) - _, err = Targets().FromFile(fmt.Sprintf("%s/root.json", testutils.RepoDir)) + _, err = Targets().FromFile(filepath.Join(testutils.RepoDir, "root.json")) assert.ErrorIs(t, err, ErrValue{"expected metadata type targets, got - root"}) // Test failing to load other roles from targets metadata - _, err = Snapshot().FromFile(fmt.Sprintf("%s/targets.json", testutils.RepoDir)) + _, err = Snapshot().FromFile(filepath.Join(testutils.RepoDir, "targets.json")) assert.ErrorIs(t, err, ErrValue{"expected metadata type snapshot, got - targets"}) - _, err = Timestamp().FromFile(fmt.Sprintf("%s/targets.json", testutils.RepoDir)) + _, err = Timestamp().FromFile(filepath.Join(testutils.RepoDir, "targets.json")) assert.ErrorIs(t, err, ErrValue{"expected metadata type timestamp, got - targets"}) - _, err = Root().FromFile(fmt.Sprintf("%s/targets.json", testutils.RepoDir)) + _, err = Root().FromFile(filepath.Join(testutils.RepoDir, "targets.json")) assert.ErrorIs(t, err, ErrValue{"expected metadata type root, got - targets"}) // Test failing to load other roles from timestamp metadata - _, err = Snapshot().FromFile(fmt.Sprintf("%s/timestamp.json", testutils.RepoDir)) + _, err = Snapshot().FromFile(filepath.Join(testutils.RepoDir, "timestamp.json")) assert.ErrorIs(t, err, ErrValue{"expected metadata type snapshot, got - timestamp"}) - _, err = Targets().FromFile(fmt.Sprintf("%s/timestamp.json", testutils.RepoDir)) + _, err = Targets().FromFile(filepath.Join(testutils.RepoDir, "timestamp.json")) assert.ErrorIs(t, err, ErrValue{"expected metadata type targets, got - timestamp"}) - _, err = Root().FromFile(fmt.Sprintf("%s/timestamp.json", testutils.RepoDir)) + _, err = Root().FromFile(filepath.Join(testutils.RepoDir, "timestamp.json")) assert.ErrorIs(t, err, ErrValue{"expected metadata type root, got - timestamp"}) // Test failing to load other roles from snapshot metadata - _, err = Targets().FromFile(fmt.Sprintf("%s/snapshot.json", testutils.RepoDir)) + _, err = Targets().FromFile(filepath.Join(testutils.RepoDir, "snapshot.json")) assert.ErrorIs(t, err, ErrValue{"expected metadata type targets, got - snapshot"}) - _, err = Timestamp().FromFile(fmt.Sprintf("%s/snapshot.json", testutils.RepoDir)) + _, err = Timestamp().FromFile(filepath.Join(testutils.RepoDir, "snapshot.json")) assert.ErrorIs(t, err, ErrValue{"expected metadata type timestamp, got - snapshot"}) - _, err = Root().FromFile(fmt.Sprintf("%s/snapshot.json", testutils.RepoDir)) + _, err = Root().FromFile(filepath.Join(testutils.RepoDir, "snapshot.json")) assert.ErrorIs(t, err, ErrValue{"expected metadata type root, got - snapshot"}) } func TestMDReadWriteFileExceptions(t *testing.T) { // Test writing to a file with bad filename - badMetadataPath := fmt.Sprintf("%s/bad-metadata.json", testutils.RepoDir) + badMetadataPath := filepath.Join(testutils.RepoDir, "bad-metadata.json") _, err := Root().FromFile(badMetadataPath) expectedErr := fs.PathError{ Op: "open", Path: badMetadataPath, - Err: unix.ENOENT, + Err: fs.ErrNotExist, } assert.ErrorIs(t, err, expectedErr.Err) @@ -134,39 +134,43 @@ func TestMDReadWriteFileExceptions(t *testing.T) { expectedErr = fs.PathError{ Op: "open", Path: "", - Err: unix.ENOENT, + Err: fs.ErrNotExist, } assert.ErrorIs(t, err, expectedErr.Err) } func TestCompareFromBytesFromFileToBytes(t *testing.T) { - rootBytesWant, err := os.ReadFile(fmt.Sprintf("%s/root.json", testutils.RepoDir)) + rootPath := filepath.Join(testutils.RepoDir, "root.json") + rootBytesWant, err := os.ReadFile(rootPath) assert.NoError(t, err) - root, err := Root().FromFile(fmt.Sprintf("%s/root.json", testutils.RepoDir)) + root, err := Root().FromFile(rootPath) assert.NoError(t, err) rootBytesActual, err := root.ToBytes(true) assert.NoError(t, err) assert.Equal(t, rootBytesWant, rootBytesActual) - targetsBytesWant, err := os.ReadFile(fmt.Sprintf("%s/targets.json", testutils.RepoDir)) + targetsPath := filepath.Join(testutils.RepoDir, "targets.json") + targetsBytesWant, err := os.ReadFile(targetsPath) assert.NoError(t, err) - targets, err := Targets().FromFile(fmt.Sprintf("%s/targets.json", testutils.RepoDir)) + targets, err := Targets().FromFile(targetsPath) assert.NoError(t, err) targetsBytesActual, err := targets.ToBytes(true) assert.NoError(t, err) assert.Equal(t, targetsBytesWant, targetsBytesActual) - snapshotBytesWant, err := os.ReadFile(fmt.Sprintf("%s/snapshot.json", testutils.RepoDir)) + snapshotPath := filepath.Join(testutils.RepoDir, "snapshot.json") + snapshotBytesWant, err := os.ReadFile(snapshotPath) assert.NoError(t, err) - snapshot, err := Snapshot().FromFile(fmt.Sprintf("%s/snapshot.json", testutils.RepoDir)) + snapshot, err := Snapshot().FromFile(snapshotPath) assert.NoError(t, err) snapshotBytesActual, err := snapshot.ToBytes(true) assert.NoError(t, err) assert.Equal(t, snapshotBytesWant, snapshotBytesActual) - timestampBytesWant, err := os.ReadFile(fmt.Sprintf("%s/timestamp.json", testutils.RepoDir)) + timestampPath := filepath.Join(testutils.RepoDir, "timestamp.json") + timestampBytesWant, err := os.ReadFile(timestampPath) assert.NoError(t, err) - timestamp, err := Timestamp().FromFile(fmt.Sprintf("%s/timestamp.json", testutils.RepoDir)) + timestamp, err := Timestamp().FromFile(timestampPath) assert.NoError(t, err) timestampBytesActual, err := timestamp.ToBytes(true) assert.NoError(t, err) @@ -174,7 +178,7 @@ func TestCompareFromBytesFromFileToBytes(t *testing.T) { } func TestRootReadWriteReadCompare(t *testing.T) { - src := testutils.RepoDir + "/root.json" + src := filepath.Join(testutils.RepoDir, "root.json") srcRoot, err := Root().FromFile(src) assert.NoError(t, err) @@ -196,7 +200,7 @@ func TestRootReadWriteReadCompare(t *testing.T) { } func TestSnapshotReadWriteReadCompare(t *testing.T) { - path1 := testutils.RepoDir + "/snapshot.json" + path1 := filepath.Join(testutils.RepoDir, "snapshot.json") snaphot1, err := Snapshot().FromFile(path1) assert.NoError(t, err) @@ -218,7 +222,7 @@ func TestSnapshotReadWriteReadCompare(t *testing.T) { } func TestTargetsReadWriteReadCompare(t *testing.T) { - path1 := testutils.RepoDir + "/targets.json" + path1 := filepath.Join(testutils.RepoDir, "targets.json") targets1, err := Targets().FromFile(path1) assert.NoError(t, err) @@ -240,7 +244,7 @@ func TestTargetsReadWriteReadCompare(t *testing.T) { } func TestTimestampReadWriteReadCompare(t *testing.T) { - path1 := testutils.RepoDir + "/timestamp.json" + path1 := filepath.Join(testutils.RepoDir, "timestamp.json") timestamp1, err := Timestamp().FromFile(path1) assert.NoError(t, err) @@ -263,7 +267,8 @@ func TestTimestampReadWriteReadCompare(t *testing.T) { func TestToFromBytes(t *testing.T) { // ROOT - data, err := os.ReadFile(testutils.RepoDir + "/root.json") + rootPath := filepath.Join(testutils.RepoDir, "root.json") + data, err := os.ReadFile(rootPath) assert.NoError(t, err) root, err := Root().FromBytes(data) assert.NoError(t, err) @@ -284,7 +289,7 @@ func TestToFromBytes(t *testing.T) { assert.Equal(t, rootBytesWant, rootBytesActual) // SNAPSHOT - data, err = os.ReadFile(testutils.RepoDir + "/snapshot.json") + data, err = os.ReadFile(filepath.Join(testutils.RepoDir, "snapshot.json")) assert.NoError(t, err) snapshot, err := Snapshot().FromBytes(data) assert.NoError(t, err) @@ -302,7 +307,7 @@ func TestToFromBytes(t *testing.T) { assert.Equal(t, snapshotBytesWant, snapshotBytesActual) // TARGETS - data, err = os.ReadFile(testutils.RepoDir + "/targets.json") + data, err = os.ReadFile(filepath.Join(testutils.RepoDir, "targets.json")) assert.NoError(t, err) targets, err := Targets().FromBytes(data) assert.NoError(t, err) @@ -320,7 +325,7 @@ func TestToFromBytes(t *testing.T) { assert.Equal(t, targetsBytesWant, targetsBytesActual) // TIMESTAMP - data, err = os.ReadFile(testutils.RepoDir + "/timestamp.json") + data, err = os.ReadFile(filepath.Join(testutils.RepoDir, "timestamp.json")) assert.NoError(t, err) timestamp, err := Timestamp().FromBytes(data) assert.NoError(t, err) @@ -340,7 +345,7 @@ func TestToFromBytes(t *testing.T) { } func TestSignVerify(t *testing.T) { - root, err := Root().FromFile(testutils.RepoDir + "/root.json") + root, err := Root().FromFile(filepath.Join(testutils.RepoDir, "root.json")) assert.NoError(t, err) // Locate the public keys we need from root @@ -352,7 +357,7 @@ func TestSignVerify(t *testing.T) { timestampKeyID := root.Signed.Roles[TIMESTAMP].KeyIDs[0] // Load sample metadata (targets) and assert ... - targets, err := Targets().FromFile(testutils.RepoDir + "/targets.json") + targets, err := Targets().FromFile(filepath.Join(testutils.RepoDir, "targets.json")) assert.NoError(t, err) sig, _ := getSignatureByKeyID(targets.Signatures, targetsKeyID) data, err := targets.Signed.MarshalJSON() @@ -382,7 +387,7 @@ func TestSignVerify(t *testing.T) { assert.ErrorContains(t, err, "crypto/rsa: verification error") // Append a new signature with the unrelated key and assert that ... - signer, err := signature.LoadSignerFromPEMFile(testutils.KeystoreDir+"/snapshot_key", crypto.SHA256, cryptoutils.SkipPassword) + signer, err := signature.LoadSignerFromPEMFile(filepath.Join(testutils.KeystoreDir, "snapshot_key"), crypto.SHA256, cryptoutils.SkipPassword) assert.NoError(t, err) snapshotSig, err := targets.Sign(signer) assert.NoError(t, err) @@ -397,7 +402,7 @@ func TestSignVerify(t *testing.T) { assert.Equal(t, snapshotSig.KeyID, snapshotKeyID) // Clear all signatures and add a new signature with the unrelated key and assert that ... - signer, err = signature.LoadSignerFromPEMFile(testutils.KeystoreDir+"/timestamp_key", crypto.SHA256, cryptoutils.SkipPassword) + signer, err = signature.LoadSignerFromPEMFile(filepath.Join(testutils.KeystoreDir, "timestamp_key"), crypto.SHA256, cryptoutils.SkipPassword) assert.NoError(t, err) targets.ClearSignatures() assert.Equal(t, 0, len(targets.Signatures)) @@ -420,7 +425,7 @@ func TestSignVerify(t *testing.T) { } func TestKeyVerifyFailures(t *testing.T) { - root, err := Root().FromFile(testutils.RepoDir + "/root.json") + root, err := Root().FromFile(filepath.Join(testutils.RepoDir, "root.json")) assert.NoError(t, err) // Locate the timestamp public key we need from root @@ -428,7 +433,7 @@ func TestKeyVerifyFailures(t *testing.T) { timestampKeyID := root.Signed.Roles[TIMESTAMP].KeyIDs[0] // Load sample metadata (timestamp) - timestamp, err := Timestamp().FromFile(testutils.RepoDir + "/timestamp.json") + timestamp, err := Timestamp().FromFile(filepath.Join(testutils.RepoDir, "timestamp.json")) assert.NoError(t, err) timestampSig, _ := getSignatureByKeyID(timestamp.Signatures, timestampKeyID) @@ -489,7 +494,7 @@ func TestKeyVerifyFailures(t *testing.T) { func TestMetadataSignedIsExpired(t *testing.T) { // Use of Snapshot is arbitrary, we're just testing the base class // features with real data - snapshot, err := Snapshot().FromFile(testutils.RepoDir + "/snapshot.json") + snapshot, err := Snapshot().FromFile(filepath.Join(testutils.RepoDir, "snapshot.json")) assert.NoError(t, err) assert.Equal(t, time.Date(2030, 8, 15, 14, 30, 45, 100, time.UTC), snapshot.Signed.Expires) @@ -505,17 +510,16 @@ func TestMetadataSignedIsExpired(t *testing.T) { func TestMetadataVerifyDelegate(t *testing.T) { - root, err := Root().FromFile(fmt.Sprintf("%s/root.json", testutils.RepoDir)) + root, err := Root().FromFile(filepath.Join(testutils.RepoDir, "root.json")) assert.NoError(t, err) - snapshot, err := Snapshot().FromFile(fmt.Sprintf("%s/snapshot.json", testutils.RepoDir)) + snapshot, err := Snapshot().FromFile(filepath.Join(testutils.RepoDir, "snapshot.json")) assert.NoError(t, err) - targets, err := Targets().FromFile(fmt.Sprintf("%s/targets.json", testutils.RepoDir)) + targets, err := Targets().FromFile(filepath.Join(testutils.RepoDir, "targets.json")) assert.NoError(t, err) - role1, err := Targets().FromFile(fmt.Sprintf("%s/role1.json", testutils.RepoDir)) + role1, err := Targets().FromFile(filepath.Join(testutils.RepoDir, "role1.json")) assert.NoError(t, err) - role2, err := Targets().FromFile(fmt.Sprintf("%s/role2.json", testutils.RepoDir)) + role2, err := Targets().FromFile(filepath.Join(testutils.RepoDir, "role2.json")) assert.NoError(t, err) - // Test the expected delegation tree err = root.VerifyDelegate(ROOT, root) assert.NoError(t, err) @@ -583,7 +587,7 @@ func TestMetadataVerifyDelegate(t *testing.T) { // Verify succeeds when we correct the new signature and reach the // threshold of 2 keys - signer, err := signature.LoadSignerFromPEMFile(testutils.KeystoreDir+"/timestamp_key", crypto.SHA256, cryptoutils.SkipPassword) + signer, err := signature.LoadSignerFromPEMFile(filepath.Join(testutils.KeystoreDir, "timestamp_key"), crypto.SHA256, cryptoutils.SkipPassword) assert.NoError(t, err) _, err = snapshot.Sign(signer) assert.NoError(t, err) @@ -592,11 +596,11 @@ func TestMetadataVerifyDelegate(t *testing.T) { } func TestRootAddKeyAndRevokeKey(t *testing.T) { - root, err := Root().FromFile(fmt.Sprintf("%s/root.json", testutils.RepoDir)) + root, err := Root().FromFile(filepath.Join(testutils.RepoDir, "root.json")) assert.NoError(t, err) // Create a new key - signer, err := signature.LoadSignerFromPEMFile(testutils.KeystoreDir+"/root_key2", crypto.SHA256, cryptoutils.SkipPassword) + signer, err := signature.LoadSignerFromPEMFile(filepath.Join(testutils.KeystoreDir, "root_key2"), crypto.SHA256, cryptoutils.SkipPassword) assert.NoError(t, err) key, err := signer.PublicKey() assert.NoError(t, err) @@ -654,7 +658,7 @@ func TestRootAddKeyAndRevokeKey(t *testing.T) { } func TestTargetsKeyAPI(t *testing.T) { - targets, err := Targets().FromFile(fmt.Sprintf("%s/targets.json", testutils.RepoDir)) + targets, err := Targets().FromFile(filepath.Join(testutils.RepoDir, "targets.json")) assert.NoError(t, err) delegatedRole := DelegatedRole{ @@ -732,7 +736,7 @@ func TestTargetsKeyAPI(t *testing.T) { } func TestTargetsKeyAPIWithSuccinctRoles(t *testing.T) { - targets, err := Targets().FromFile(testutils.RepoDir + "/targets.json") + targets, err := Targets().FromFile(filepath.Join(testutils.RepoDir, "targets.json")) assert.NoError(t, err) // Remove delegated roles @@ -785,13 +789,13 @@ func TestLengthAndHashValidation(t *testing.T) { // Use timestamp to get a MetaFile object and snapshot // for untrusted metadata file to verify. - timestamp, err := Timestamp().FromFile(testutils.RepoDir + "/timestamp.json") + timestamp, err := Timestamp().FromFile(filepath.Join(testutils.RepoDir, "timestamp.json")) assert.NoError(t, err) snapshotMetafile := timestamp.Signed.Meta["snapshot.json"] assert.NotNil(t, snapshotMetafile) - snapshotData, err := os.ReadFile(testutils.RepoDir + "/snapshot.json") + snapshotData, err := os.ReadFile(filepath.Join(testutils.RepoDir, "snapshot.json")) assert.NoError(t, err) h32 := sha256.Sum256(snapshotData) h := h32[:] @@ -800,7 +804,7 @@ func TestLengthAndHashValidation(t *testing.T) { } snapshotMetafile.Length = 652 - data, err := os.ReadFile(testutils.RepoDir + "/snapshot.json") + data, err := os.ReadFile(filepath.Join(testutils.RepoDir, "snapshot.json")) assert.NoError(t, err) err = snapshotMetafile.VerifyLengthHashes(data) assert.NoError(t, err) @@ -829,10 +833,10 @@ func TestLengthAndHashValidation(t *testing.T) { assert.NoError(t, err) // Test target files' hash and length verification - targets, err := Targets().FromFile(testutils.RepoDir + "/targets.json") + targets, err := Targets().FromFile(filepath.Join(testutils.RepoDir, "targets.json")) assert.NoError(t, err) targetFile := targets.Signed.Targets["file1.txt"] - targetFileData, err := os.ReadFile(testutils.TargetsDir + "/" + targetFile.Path) + targetFileData, err := os.ReadFile(filepath.Join(testutils.TargetsDir, targetFile.Path)) assert.NoError(t, err) // test exceptions @@ -849,21 +853,23 @@ func TestLengthAndHashValidation(t *testing.T) { func TestTargetFileFromFile(t *testing.T) { // Test with an existing file and valid hash algorithm - targetFileFromFile, err := TargetFile().FromFile(testutils.TargetsDir+"/file1.txt", "sha256") + targetFilePath := filepath.Join(testutils.TargetsDir, "file1.txt") + targetFileFromFile, err := TargetFile().FromFile(targetFilePath, "sha256") assert.NoError(t, err) - targetFileData, err := os.ReadFile(testutils.TargetsDir + "/file1.txt") + targetFileData, err := os.ReadFile(targetFilePath) assert.NoError(t, err) err = targetFileFromFile.VerifyLengthHashes(targetFileData) assert.NoError(t, err) // Test with mismatching target file data - mismatchingTargetFileData, err := os.ReadFile(testutils.TargetsDir + "/file2.txt") + mismatchingTargetFilePath := filepath.Join(testutils.TargetsDir, "file2.txt") + mismatchingTargetFileData, err := os.ReadFile(mismatchingTargetFilePath) assert.NoError(t, err) err = targetFileFromFile.VerifyLengthHashes(mismatchingTargetFileData) assert.ErrorIs(t, err, ErrLengthOrHashMismatch{"hash verification failed - mismatch for algorithm sha256"}) // Test with an unsupported algorithm - _, err = TargetFile().FromFile(testutils.TargetsDir+"/file1.txt", "123") + _, err = TargetFile().FromFile(targetFilePath, "123") assert.ErrorIs(t, err, ErrValue{"failed generating TargetFile - unsupported hashing algorithm - 123"}) } @@ -879,15 +885,16 @@ func TestTargetFileCustom(t *testing.T) { func TestTargetFileFromBytes(t *testing.T) { data := []byte("Inline test content") + path := filepath.Join(testutils.TargetsDir, "file1.txt") // Test with a valid hash algorithm - targetFileFromData, err := TargetFile().FromBytes(testutils.TargetsDir+"/file1.txt", data, "sha256") + targetFileFromData, err := TargetFile().FromBytes(path, data, "sha256") assert.NoError(t, err) err = targetFileFromData.VerifyLengthHashes(data) assert.NoError(t, err) // Test with no algorithms specified - targetFileFromDataWithNoAlg, err := TargetFile().FromBytes(testutils.TargetsDir+"/file1.txt", data) + targetFileFromDataWithNoAlg, err := TargetFile().FromBytes(path, data) assert.NoError(t, err) err = targetFileFromDataWithNoAlg.VerifyLengthHashes(data) assert.NoError(t, err) diff --git a/metadata/metadata_test.go b/metadata/metadata_test.go index b128666f..cebf888b 100644 --- a/metadata/metadata_test.go +++ b/metadata/metadata_test.go @@ -21,8 +21,8 @@ import ( "crypto/ed25519" "crypto/sha256" "encoding/json" - "fmt" "os" + "path/filepath" "testing" "time" @@ -506,7 +506,7 @@ func TestToByte(t *testing.T) { func TestFromFile(t *testing.T) { root := Root(fixedExpire) - _, err := root.FromFile(fmt.Sprintf("%s/1.root.json", TEST_REPOSITORY_DATA)) + _, err := root.FromFile(filepath.Join(TEST_REPOSITORY_DATA, "1.root.json")) assert.NoError(t, err) assert.Equal(t, fixedExpire, root.Signed.Expires) @@ -554,7 +554,7 @@ func TestToFile(t *testing.T) { tmpDir, err := os.MkdirTemp(tmp, "0750") assert.NoError(t, err) - fileName := fmt.Sprintf("%s/1.root.json", tmpDir) + fileName := filepath.Join(tmpDir, "1.root.json") assert.NoFileExists(t, fileName) root, err := Root().FromBytes(testRootBytes) assert.NoError(t, err) diff --git a/metadata/trustedmetadata/trustedmetadata_test.go b/metadata/trustedmetadata/trustedmetadata_test.go index 8113a76b..6baf0602 100644 --- a/metadata/trustedmetadata/trustedmetadata_test.go +++ b/metadata/trustedmetadata/trustedmetadata_test.go @@ -19,8 +19,8 @@ package trustedmetadata import ( "crypto" - "fmt" "os" + "path/filepath" "testing" "time" @@ -38,39 +38,50 @@ func setAllRolesBytes(path string) { log := metadata.GetLogger() allRoles = make(map[string][]byte) - root, err := os.ReadFile(fmt.Sprintf("%s/root.json", path)) + rootPath := filepath.Join(path, "root.json") + root, err := os.ReadFile(rootPath) if err != nil { - log.Error(err, "failed to root bytes") + log.Error(err, "failed to read root bytes") os.Exit(1) } allRoles[metadata.ROOT] = root - targets, err := os.ReadFile(fmt.Sprintf("%s/targets.json", path)) + + targetsPath := filepath.Join(path, "targets.json") + targets, err := os.ReadFile(targetsPath) if err != nil { - log.Error(err, "failed to targets bytes") + log.Error(err, "failed to read targets bytes") os.Exit(1) } allRoles[metadata.TARGETS] = targets - snapshot, err := os.ReadFile(fmt.Sprintf("%s/snapshot.json", path)) + + snapshotPath := filepath.Join(path, "snapshot.json") + snapshot, err := os.ReadFile(snapshotPath) if err != nil { - log.Error(err, "failed to snapshot bytes") + log.Error(err, "failed to read snapshot bytes") os.Exit(1) } allRoles[metadata.SNAPSHOT] = snapshot - timestamp, err := os.ReadFile(fmt.Sprintf("%s/timestamp.json", path)) + + timestampPath := filepath.Join(path, "timestamp.json") + timestamp, err := os.ReadFile(timestampPath) if err != nil { - log.Error(err, "failed to timestamp bytes") + log.Error(err, "failed to read timestamp bytes") os.Exit(1) } allRoles[metadata.TIMESTAMP] = timestamp - role1, err := os.ReadFile(fmt.Sprintf("%s/role1.json", path)) + + role1Path := filepath.Join(path, "role1.json") + role1, err := os.ReadFile(role1Path) if err != nil { - log.Error(err, "failed to role1 bytes") + log.Error(err, "failed to read role1 bytes") os.Exit(1) } allRoles["role1"] = role1 - role2, err := os.ReadFile(fmt.Sprintf("%s/role2.json", path)) + + role2Path := filepath.Join(path, "role2.json") + role2, err := os.ReadFile(role2Path) if err != nil { - log.Error(err, "failed to role2 bytes") + log.Error(err, "failed to read role2 bytes") os.Exit(1) } allRoles["role2"] = role2 @@ -104,7 +115,7 @@ func modifyRootMetadata(fn modifyRoot) ([]byte, error) { } fn(root) - signer, err := signature.LoadSignerFromPEMFile(testutils.KeystoreDir+"/root_key", crypto.SHA256, cryptoutils.SkipPassword) + signer, err := signature.LoadSignerFromPEMFile(filepath.Join(testutils.KeystoreDir, "root_key"), crypto.SHA256, cryptoutils.SkipPassword) if err != nil { log.Error(err, "failed to load signer from pem file") } @@ -127,7 +138,7 @@ func modifyTimestamptMetadata(fn modifyTimestamp) ([]byte, error) { } fn(timestamp) - signer, err := signature.LoadSignerFromPEMFile(testutils.KeystoreDir+"/timestamp_key", crypto.SHA256, cryptoutils.SkipPassword) + signer, err := signature.LoadSignerFromPEMFile(filepath.Join(testutils.KeystoreDir, "timestamp_key"), crypto.SHA256, cryptoutils.SkipPassword) if err != nil { log.Error(err, "failed to load signer from pem file") } @@ -150,7 +161,7 @@ func modifySnapshotMetadata(fn modifySnapshot) ([]byte, error) { } fn(snapshot) - signer, err := signature.LoadSignerFromPEMFile(testutils.KeystoreDir+"/snapshot_key", crypto.SHA256, cryptoutils.SkipPassword) + signer, err := signature.LoadSignerFromPEMFile(filepath.Join(testutils.KeystoreDir, "snapshot_key"), crypto.SHA256, cryptoutils.SkipPassword) if err != nil { log.Error(err, "failed to load signer from pem file") } @@ -173,7 +184,7 @@ func modifyTargetsMetadata(fn modifyTargets) ([]byte, error) { } fn(targets) - signer, err := signature.LoadSignerFromPEMFile(testutils.KeystoreDir+"/targets_key", crypto.SHA256, cryptoutils.SkipPassword) + signer, err := signature.LoadSignerFromPEMFile(filepath.Join(testutils.KeystoreDir, "targets_key"), crypto.SHA256, cryptoutils.SkipPassword) if err != nil { log.Error(err, "failed to load signer from pem file") } diff --git a/metadata/updater/updater.go b/metadata/updater/updater.go index 1e767be7..63169af5 100644 --- a/metadata/updater/updater.go +++ b/metadata/updater/updater.go @@ -26,6 +26,8 @@ import ( "net/url" "os" "path/filepath" + "regexp" + "runtime" "strconv" "strings" "time" @@ -237,7 +239,7 @@ func (update *Updater) DownloadTarget(targetFile *metadata.TargetFiles, filePath targetFilePath = fmt.Sprintf("%s.%s", hashes, dirName) } else { // /. - targetFilePath = fmt.Sprintf("%s/%s.%s", dirName, hashes, baseName) + targetFilePath = filepath.Join(dirName, fmt.Sprintf("%s.%s", hashes, baseName)) } } fullURL := fmt.Sprintf("%s%s", targetBaseURL, targetFilePath) @@ -575,6 +577,40 @@ func (update *Updater) preOrderDepthFirstWalk(targetFilePath string) (*metadata. return nil, fmt.Errorf("target %s not found", targetFilePath) } +// on windows, you can't rename a file across drives, so let's move instead +func MoveFile(source, destination string) (err error) { + if runtime.GOOS == "windows" { + inputFile, err := os.Open(source) + if err != nil { + return fmt.Errorf("Couldn't open source file: %s", err) + } + defer inputFile.Close() + outputFile, err := os.Create(destination) + if err != nil { + inputFile.Close() + return fmt.Errorf("Couldn't open dest file: %s", err) + } + defer outputFile.Close() + c, err := io.Copy(outputFile, inputFile) + if err != nil { + return fmt.Errorf("Writing to output file failed: %s", err) + } + if c <= 0 { + return fmt.Errorf("Nothing copied to output file") + } + inputFile.Close() + // The copy was successful, so now delete the original file + err = os.Remove(source) + if err != nil { + return fmt.Errorf("Failed removing original file: %s", err) + } + return nil + } else { + return os.Rename(source, destination) + } + +} + // persistMetadata writes metadata to disk atomically to avoid data loss func (update *Updater) persistMetadata(roleName string, data []byte) error { log := metadata.GetLogger() @@ -593,6 +629,7 @@ func (update *Updater) persistMetadata(roleName string, data []byte) error { if err != nil { return err } + defer file.Close() // write the data content to the temporary file err = os.WriteFile(file.Name(), data, 0644) if err != nil { @@ -603,11 +640,21 @@ func (update *Updater) persistMetadata(roleName string, data []byte) error { } return err } + + // can't move/rename an open file on windows, so close it first + file.Close() // if all okay, rename the temporary file to the desired one - err = os.Rename(file.Name(), fileName) + err = MoveFile(file.Name(), fileName) + if err != nil { + return err + } + read, err := os.ReadFile(fileName) if err != nil { return err } + if string(read) != string(data) { + return fmt.Errorf("failed to persist metadata: %w", err) + } return nil } @@ -648,8 +695,20 @@ func (update *Updater) GetTrustedMetadataSet() trustedmetadata.TrustedMetadata { return *update.trusted } +func IsWindowsPath(path string) bool { + match, _ := regexp.MatchString(`^[a-zA-Z]:\\`, path) + return match +} + // ensureTrailingSlash ensures url ends with a slash func ensureTrailingSlash(url string) string { + if IsWindowsPath(url) { + slash := string(filepath.Separator) + if strings.HasSuffix(url, slash) { + return url + } + return url + slash + } if strings.HasSuffix(url, "/") { return url } diff --git a/metadata/updater/updater_consistent_snapshot_test.go b/metadata/updater/updater_consistent_snapshot_test.go index e649240d..60eac3e0 100644 --- a/metadata/updater/updater_consistent_snapshot_test.go +++ b/metadata/updater/updater_consistent_snapshot_test.go @@ -39,10 +39,12 @@ func TestTopLevelRolesUpdateWithConsistentSnapshotDisabled(t *testing.T) { updaterConfig, err := loadUpdaterConfig() assert.NoError(t, err) updater := initUpdater(updaterConfig) - // cleanup fetch tracker metadata simulator.Sim.FetchTracker.Metadata = []simulator.FTMetadata{} err = updater.Refresh() + if err != nil { + t.Fatal(err) + } assert.NoError(t, err) // metadata files are fetched with the expected version (or None) @@ -71,7 +73,9 @@ func TestTopLevelRolesUpdateWithConsistentSnapshotEnabled(t *testing.T) { updaterConfig, err := loadUpdaterConfig() assert.NoError(t, err) updater := initUpdater(updaterConfig) - + if updater == nil { + t.Fatal("updater is nil") + } // cleanup fetch tracker metadata simulator.Sim.FetchTracker.Metadata = []simulator.FTMetadata{} err = updater.Refresh() @@ -133,7 +137,9 @@ func TestDelegatesRolesUpdateWithConsistentSnapshotDisabled(t *testing.T) { updaterConfig, err := loadUpdaterConfig() assert.NoError(t, err) updater := initUpdater(updaterConfig) - + if updater == nil { + t.Fatal("updater is nil") + } err = updater.Refresh() assert.NoError(t, err) @@ -149,7 +155,7 @@ func TestDelegatesRolesUpdateWithConsistentSnapshotDisabled(t *testing.T) { {Name: "..", Value: -1}, {Name: ".", Value: -1}, } - assert.EqualValues(t, expectedsnapshotEnabled, simulator.Sim.FetchTracker.Metadata) + assert.ElementsMatch(t, expectedsnapshotEnabled, simulator.Sim.FetchTracker.Metadata) // metadata files are always persisted without a version prefix assertFilesExist(t, metadata.TOP_LEVEL_ROLE_NAMES[:]) } @@ -197,7 +203,9 @@ func TestDelegatesRolesUpdateWithConsistentSnapshotEnabled(t *testing.T) { updaterConfig, err := loadUpdaterConfig() assert.NoError(t, err) updater := initUpdater(updaterConfig) - + if updater == nil { + t.Fatal("updater is nil") + } err = updater.Refresh() assert.NoError(t, err) diff --git a/metadata/updater/updater_top_level_update_test.go b/metadata/updater/updater_top_level_update_test.go index c849f39f..598630ba 100644 --- a/metadata/updater/updater_top_level_update_test.go +++ b/metadata/updater/updater_top_level_update_test.go @@ -20,6 +20,7 @@ package updater import ( "fmt" "os" + "path/filepath" "testing" "time" @@ -99,7 +100,7 @@ func runRefresh(updaterConfig *config.UpdaterConfig, moveInTime time.Time) (Upda return *updater, err } -func initUpdater(updaterConfig *config.UpdaterConfig) Updater { +func initUpdater(updaterConfig *config.UpdaterConfig) *Updater { if len(simulator.Sim.DumpDir) > 0 { simulator.Sim.Write() } @@ -108,7 +109,7 @@ func initUpdater(updaterConfig *config.UpdaterConfig) Updater { if err != nil { log.Debugf("failed to create new updater config: %v", err) } - return *updater + return updater } // Asserts that local metadata files exist for 'roles' @@ -153,13 +154,13 @@ func assertContentEquals(t *testing.T, role string, version *int) { expectedContent, err := simulator.Sim.FetchMetadata(role, version) assert.NoError(t, err) - content, err := os.ReadFile(fmt.Sprintf("%s/%s.json", simulator.MetadataDir, role)) + content, err := os.ReadFile(filepath.Join(simulator.MetadataDir, fmt.Sprintf("%s.json", role))) assert.NoError(t, err) assert.Equal(t, string(expectedContent), string(content)) } func assertVersionEquals(t *testing.T, role string, expectedVersion int64) { - path := fmt.Sprintf("%s/%s.json", simulator.MetadataDir, role) + path := filepath.Join(simulator.MetadataDir, fmt.Sprintf("%s.json", role)) switch role { case metadata.ROOT: md, err := simulator.Sim.MDRoot.FromFile(path) @@ -353,7 +354,7 @@ func TestTrustedRootUnsigned(t *testing.T) { err := loadOrResetTrustedRootMetadata() assert.NoError(t, err) - rootPath := fmt.Sprintf("%s/%s.json", simulator.MetadataDir, metadata.ROOT) + rootPath := filepath.Join(simulator.MetadataDir, fmt.Sprintf("%s.json", metadata.ROOT)) mdRoot, err := simulator.Sim.MDRoot.FromFile(rootPath) assert.NoError(t, err) @@ -394,7 +395,7 @@ func TestMaxRootRotations(t *testing.T) { simulator.Sim.PublishRoot() } - rootPath := fmt.Sprintf("%s/%s.json", simulator.MetadataDir, metadata.ROOT) + rootPath := filepath.Join(simulator.MetadataDir, fmt.Sprintf("%s.json", metadata.ROOT)) mdRoot, err := simulator.Sim.MDRoot.FromFile(rootPath) assert.NoError(t, err) initialRootVersion := mdRoot.Signed.Version @@ -997,15 +998,15 @@ func TestExpiredMetadata(t *testing.T) { // which means a successful refresh is performed // with expired local metadata - mdTimestamp, err := metadata.Timestamp().FromFile(simulator.MetadataDir + "/timestamp.json") + mdTimestamp, err := metadata.Timestamp().FromFile(filepath.Join(simulator.MetadataDir, "timestamp.json")) assert.NoError(t, err) assert.Equal(t, int64(2), mdTimestamp.Signed.Version) - mdSnapshot, err := metadata.Snapshot().FromFile(simulator.MetadataDir + "/snapshot.json") + mdSnapshot, err := metadata.Snapshot().FromFile(filepath.Join(simulator.MetadataDir, "snapshot.json")) assert.NoError(t, err) assert.Equal(t, int64(2), mdSnapshot.Signed.Version) - mdTargets, err := metadata.Targets().FromFile(simulator.MetadataDir + "/targets.json") + mdTargets, err := metadata.Targets().FromFile(filepath.Join(simulator.MetadataDir, "targets.json")) assert.NoError(t, err) assert.Equal(t, int64(2), mdTargets.Signed.Version) } diff --git a/testutils/simulator/repository_simulator.go b/testutils/simulator/repository_simulator.go index 94d4f701..926b8741 100644 --- a/testutils/simulator/repository_simulator.go +++ b/testutils/simulator/repository_simulator.go @@ -58,6 +58,7 @@ import ( "net/url" "os" "path/filepath" + "regexp" "strconv" "strings" "time" @@ -314,10 +315,41 @@ func (rs *RepositorySimulator) DownloadFile(urlPath string, maxLength int64, tim return data, err } +func IsWindowsPath(path string) bool { + match, _ := regexp.MatchString(`^[a-zA-Z]:\\`, path) + return match +} + +func trimPrefix(path string, prefix string) (string, error) { + var toTrim string + if IsWindowsPath(path) { + toTrim = path + } else { + parsedURL, e := url.Parse(path) + if e != nil { + return "", e + } + toTrim = parsedURL.Path + } + + return strings.TrimPrefix(toTrim, prefix), nil +} + +func hasPrefix(path, prefix string) bool { + return strings.HasPrefix(filepath.ToSlash(path), prefix) +} + +func hasSuffix(path, prefix string) bool { + return strings.HasSuffix(filepath.ToSlash(path), prefix) +} + func (rs *RepositorySimulator) fetch(urlPath string) ([]byte, error) { - parsedURL, _ := url.Parse(urlPath) - path := strings.TrimPrefix(parsedURL.Path, rs.LocalDir) - if strings.HasPrefix(path, "/metadata/") && strings.HasSuffix(path, ".json") { + + path, err := trimPrefix(urlPath, rs.LocalDir) + if err != nil { + return nil, err + } + if hasPrefix(path, "/metadata/") && hasSuffix(path, ".json") { fileName := path[len("/metadata/"):] verAndName := fileName[:len(path)-len("/metadata/")-len(".json")] versionStr, role := partition(verAndName, ".") @@ -333,16 +365,16 @@ func (rs *RepositorySimulator) fetch(urlPath string) ([]byte, error) { version = -1 } return rs.FetchMetadata(role, &version) - } else if strings.HasPrefix(path, "/targets/") { + } else if hasPrefix(path, "/targets/") { targetPath := path[len("/targets/"):] - dirParts, sep, prefixedFilename := lastIndex(targetPath, "/") + dirParts, sep, prefixedFilename := lastIndex(targetPath, string(filepath.Separator)) var filename string prefix := "" filename = prefixedFilename if rs.MDRoot.Signed.ConsistentSnapshot && rs.PrefixTargetsWithHash { prefix, filename = partition(prefixedFilename, ".") } - targetPath = fmt.Sprintf("%s%s%s", dirParts, sep, filename) + targetPath = filepath.Join(dirParts, sep, filename) target, err := rs.FetchTarget(targetPath, prefix) if err != nil { log.Printf("failed to fetch target: %v", err) diff --git a/testutils/simulator/repository_simulator_setup.go b/testutils/simulator/repository_simulator_setup.go index df789c8d..38260951 100644 --- a/testutils/simulator/repository_simulator_setup.go +++ b/testutils/simulator/repository_simulator_setup.go @@ -19,6 +19,7 @@ package simulator import ( "os" + "path/filepath" "time" log "github.com/sirupsen/logrus" @@ -65,11 +66,11 @@ func InitMetadataDir() (*RepositorySimulator, string, string, error) { if err != nil { log.Fatal("failed to initialize environment: ", err) } - metadataDir := LocalDir + metadataPath + metadataDir := filepath.Join(LocalDir, metadataPath) sim := NewRepository() - f, err := os.Create(metadataDir + "/root.json") + f, err := os.Create(filepath.Join(metadataDir, "root.json")) if err != nil { log.Fatalf("failed to create root: %v", err) } @@ -78,13 +79,13 @@ func InitMetadataDir() (*RepositorySimulator, string, string, error) { if err != nil { log.Debugf("repository simulator setup: failed to write signed roots: %v", err) } - targetsDir := LocalDir + targetsPath + targetsDir := filepath.Join(LocalDir, targetsPath) sim.LocalDir = LocalDir return sim, metadataDir, targetsDir, err } func GetRootBytes(localMetadataDir string) ([]byte, error) { - return os.ReadFile(localMetadataDir + "/root.json") + return os.ReadFile(filepath.Join(localMetadataDir, "root.json")) } func RepositoryCleanup(tmpDir string) { diff --git a/testutils/testutils/setup.go b/testutils/testutils/setup.go index 1558ac0c..ac409444 100644 --- a/testutils/testutils/setup.go +++ b/testutils/testutils/setup.go @@ -39,7 +39,7 @@ func SetupTestDirs(repoPath string, targetsPath string, keystorePath string) err return fmt.Errorf("failed to create temporary directory: %w", err) } - RepoDir = fmt.Sprintf("%s/repository_data/repository", TempDir) + RepoDir = filepath.Join(TempDir, "repository_data", "repository") absPath, err := filepath.Abs(repoPath) if err != nil { return fmt.Errorf("failed to get absolute path: %w", err) @@ -49,7 +49,7 @@ func SetupTestDirs(repoPath string, targetsPath string, keystorePath string) err return fmt.Errorf("failed to copy metadata to %s: %w", RepoDir, err) } - TargetsDir = fmt.Sprintf("%s/repository_data/repository/targets", TempDir) + TargetsDir = filepath.Join(TempDir, "repository_data", "repository", "targets") targetsAbsPath, err := filepath.Abs(targetsPath) if err != nil { return fmt.Errorf("failed to get absolute targets path: %w", err) @@ -59,7 +59,7 @@ func SetupTestDirs(repoPath string, targetsPath string, keystorePath string) err return fmt.Errorf("failed to copy metadata to %s: %w", RepoDir, err) } - KeystoreDir = fmt.Sprintf("%s/keystore", TempDir) + KeystoreDir = filepath.Join(TempDir, "keystore") err = os.Mkdir(KeystoreDir, 0750) if err != nil { return fmt.Errorf("failed to create keystore dir %s: %w", KeystoreDir, err) @@ -86,11 +86,11 @@ func Copy(fromPath string, toPath string) error { return fmt.Errorf("failed to read path %s: %w", fromPath, err) } for _, file := range files { - data, err := os.ReadFile(fmt.Sprintf("%s/%s", fromPath, file.Name())) + data, err := os.ReadFile(filepath.Join(fromPath, file.Name())) if err != nil { return fmt.Errorf("failed to read file %s: %w", file.Name(), err) } - filePath := fmt.Sprintf("%s/%s", toPath, file.Name()) + filePath := filepath.Join(toPath, file.Name()) err = os.WriteFile(filePath, data, 0750) if err != nil { return fmt.Errorf("failed to write file %s: %w", filePath, err)