diff --git a/lib/backend/gcsbackend/client.go b/lib/backend/gcsbackend/client.go index dee62c228..539afaca6 100644 --- a/lib/backend/gcsbackend/client.go +++ b/lib/backend/gcsbackend/client.go @@ -186,14 +186,28 @@ func (c *Client) List(prefix string, opts ...backend.ListOption) (*backend.ListR } pager := iterator.NewPager(pageIterator, maxKeys, paginationToken) - result, err := c.gcs.NextPage(pager) + blobs, continuationToken, err := c.gcs.NextPage(pager) if err != nil { return nil, err } + + var names []string + for _, b := range blobs { + name, err := c.pather.NameFromBlobPath(b) + if err != nil { + log.With("blob", b).Errorf("Error converting blob path into name: %s", err) + continue + } + names = append(names, name) + } + result := &backend.ListResult{ + Names: names, + ContinuationToken: continuationToken, + } + if !options.Paginated { result.ContinuationToken = "" } - return result, nil } @@ -261,21 +275,18 @@ func (g *GCSImpl) GetObjectIterator(prefix string) iterator.Pageable { return g.bucket.Objects(g.ctx, &query) } -func (g *GCSImpl) NextPage(pager *iterator.Pager) (*backend.ListResult, +func (g *GCSImpl) NextPage(pager *iterator.Pager) ([]string, string, error) { var objectAttrs []*storage.ObjectAttrs continuationToken, err := pager.NextPage(&objectAttrs) if err != nil { - return nil, err + return nil, "", err } names := make([]string, len(objectAttrs)) for idx, objectAttr := range objectAttrs { names[idx] = objectAttr.Name } - return &backend.ListResult{ - Names: names, - ContinuationToken: continuationToken, - }, nil + return names, continuationToken, nil } diff --git a/lib/backend/gcsbackend/client_test.go b/lib/backend/gcsbackend/client_test.go index 85ddc22b9..74b302423 100644 --- a/lib/backend/gcsbackend/client_test.go +++ b/lib/backend/gcsbackend/client_test.go @@ -199,21 +199,19 @@ func TestClientList(t *testing.T) { for i := 0; i < maxIterate; { count := (rand.Int() % 10) + 1 var expected []string + var ret []string for j := i; j < (i+count) && j < maxIterate; j++ { expected = append(expected, "test/"+strconv.Itoa(j)) + ret = append(ret, "/root/test/"+strconv.Itoa(j)) } continuationToken := "" if (i + count) < maxIterate { strconv.Itoa(i + count) } - result := &backend.ListResult{ - Names: expected, - ContinuationToken: continuationToken, - } mocks.gcs.EXPECT().NextPage( gomock.Any(), - ).Return(result, nil) + ).Return(ret, continuationToken, nil) result, err := client.List("test", backend.ListWithPagination(), backend.ListWithMaxKeys(count), diff --git a/lib/backend/gcsbackend/gcs.go b/lib/backend/gcsbackend/gcs.go index d8f9742e7..f6733c692 100644 --- a/lib/backend/gcsbackend/gcs.go +++ b/lib/backend/gcsbackend/gcs.go @@ -17,7 +17,6 @@ import ( "io" "cloud.google.com/go/storage" - "github.com/uber/kraken/lib/backend" "google.golang.org/api/iterator" ) @@ -27,5 +26,5 @@ type GCS interface { Download(objectName string, w io.Writer) (int64, error) Upload(objectName string, r io.Reader) (int64, error) GetObjectIterator(prefix string) iterator.Pageable - NextPage(pager *iterator.Pager) (*backend.ListResult, error) + NextPage(pager *iterator.Pager) ([]string, string, error) } diff --git a/mocks/lib/backend/gcsbackend/gcs.go b/mocks/lib/backend/gcsbackend/gcs.go index 8af1879b3..736bb0ac6 100644 --- a/mocks/lib/backend/gcsbackend/gcs.go +++ b/mocks/lib/backend/gcsbackend/gcs.go @@ -7,7 +7,6 @@ package mockgcsbackend import ( storage "cloud.google.com/go/storage" gomock "github.com/golang/mock/gomock" - backend "github.com/uber/kraken/lib/backend" iterator "google.golang.org/api/iterator" io "io" reflect "reflect" @@ -66,12 +65,13 @@ func (mr *MockGCSMockRecorder) GetObjectIterator(arg0 interface{}) *gomock.Call } // NextPage mocks base method -func (m *MockGCS) NextPage(arg0 *iterator.Pager) (*backend.ListResult, error) { +func (m *MockGCS) NextPage(arg0 *iterator.Pager) ([]string, string, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "NextPage", arg0) - ret0, _ := ret[0].(*backend.ListResult) - ret1, _ := ret[1].(error) - return ret0, ret1 + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(string) + ret2, _ := ret[2].(error) + return ret0, ret1, ret2 } // NextPage indicates an expected call of NextPage