Skip to content

Commit

Permalink
issue 7211: support concatenate objects
Browse files Browse the repository at this point in the history
Signed-off-by: Lyndon-Li <[email protected]>
  • Loading branch information
Lyndon-Li committed Feb 21, 2024
1 parent 3b8370e commit 2075574
Show file tree
Hide file tree
Showing 12 changed files with 387 additions and 39 deletions.
34 changes: 34 additions & 0 deletions pkg/repository/udmrepo/kopialib/lib_repo.go
Original file line number Diff line number Diff line change
Expand Up @@ -366,6 +366,40 @@ func (kr *kopiaRepository) Flush(ctx context.Context) error {
return nil
}

func (kr *kopiaRepository) GetAdvancedFeatures() udmrepo.AdvancedFeatureInfo {
return udmrepo.AdvancedFeatureInfo{
MultiPartBackup: true,
}

Check warning on line 372 in pkg/repository/udmrepo/kopialib/lib_repo.go

View check run for this annotation

Codecov / codecov/patch

pkg/repository/udmrepo/kopialib/lib_repo.go#L369-L372

Added lines #L369 - L372 were not covered by tests
}

func (kr *kopiaRepository) ConcatenateObjects(ctx context.Context, objectIDs []udmrepo.ID) (udmrepo.ID, error) {
if kr.rawWriter == nil {
return "", errors.New("repo writer is closed or not open")
}

Check warning on line 378 in pkg/repository/udmrepo/kopialib/lib_repo.go

View check run for this annotation

Codecov / codecov/patch

pkg/repository/udmrepo/kopialib/lib_repo.go#L377-L378

Added lines #L377 - L378 were not covered by tests

if len(objectIDs) == 0 {
return udmrepo.ID(""), errors.New("object list is empty")
}

rawIDs := []object.ID{}
for _, id := range objectIDs {
rawID, err := object.ParseID(string(id))
if err != nil {
return udmrepo.ID(""), errors.Wrapf(err, "error to parse object ID from %v", id)
}

rawIDs = append(rawIDs, rawID)
}

result, err := kr.rawWriter.ConcatenateObjects(ctx, rawIDs)
if err != nil {
return udmrepo.ID(""), errors.Wrap(err, "error to concatenate objects")
}

return udmrepo.ID(result.String()), nil

}

// updateProgress is called when the repository writes a piece of blob data to the storage during data write
func (kr *kopiaRepository) updateProgress(uploaded int64) {
total := atomic.AddInt64(&kr.uploaded, uploaded)
Expand Down
59 changes: 59 additions & 0 deletions pkg/repository/udmrepo/kopialib/lib_repo_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -716,6 +716,65 @@ func TestFlush(t *testing.T) {
}
}

func TestConcatenateObjects(t *testing.T) {
testCases := []struct {
name string
rawWriter *repomocks.DirectRepositoryWriter
rawWriterRetErr error
objectIDs []udmrepo.ID
expectedErr string
}{
{
name: "empty object list",
expectedErr: "object list is empty",
},
{
name: "invalid object id",
objectIDs: []udmrepo.ID{
"I123456",
"fake-id",
"I678901",
},
expectedErr: "error to parse object ID from fake-id: malformed content ID: \"fake-id\": invalid content prefix",
},
{
name: "concatenate error",
rawWriter: repomocks.NewDirectRepositoryWriter(t),
rawWriterRetErr: errors.New("fake-concatenate-error"),
objectIDs: []udmrepo.ID{
"I123456",
},
expectedErr: "error to concatenate objects: fake-concatenate-error",
},
{
name: "succeed",
rawWriter: repomocks.NewDirectRepositoryWriter(t),
objectIDs: []udmrepo.ID{
"I123456",
},
},
}

for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
kr := &kopiaRepository{}
if tc.rawWriter != nil {
tc.rawWriter.On("ConcatenateObjects", mock.Anything, mock.Anything).Return(object.ID{}, tc.rawWriterRetErr)
}

kr.rawWriter = tc.rawWriter

_, err := kr.ConcatenateObjects(context.Background(), tc.objectIDs)

if tc.expectedErr == "" {
assert.NoError(t, err)
} else {
assert.EqualError(t, err, tc.expectedErr)
}
})
}
}

func TestNewObjectWriter(t *testing.T) {
rawObjWriter := repomocks.NewWriter(t)
testCases := []struct {
Expand Down
108 changes: 99 additions & 9 deletions pkg/repository/udmrepo/mocks/BackupRepo.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

Loading

0 comments on commit 2075574

Please sign in to comment.