diff --git a/src/internal/connector/discovery/api/beta_service.go b/src/internal/connector/discovery/api/beta_service.go index 5ff65ac773..0208ace696 100644 --- a/src/internal/connector/discovery/api/beta_service.go +++ b/src/internal/connector/discovery/api/beta_service.go @@ -11,23 +11,23 @@ import ( // Service wraps BetaClient's functionality. // Abstraction created to comply loosely with graph.Servicer // methods for ease of switching between v1.0 and beta connnectors -type Service struct { +type BetaService struct { client *betasdk.BetaClient } -func (s Service) Client() *betasdk.BetaClient { +func (s BetaService) Client() *betasdk.BetaClient { return s.client } -func NewBetaService(adpt *msgraphsdk.GraphRequestAdapter) *Service { - return &Service{ +func NewBetaService(adpt *msgraphsdk.GraphRequestAdapter) *BetaService { + return &BetaService{ client: betasdk.NewBetaClient(adpt), } } // Seraialize writes an M365 parsable object into a byte array using the built-in // application/json writer within the adapter. -func (s Service) Serialize(object absser.Parsable) ([]byte, error) { +func (s BetaService) Serialize(object absser.Parsable) ([]byte, error) { writer, err := s.client.Adapter(). GetSerializationWriterFactory(). GetSerializationWriter("application/json") diff --git a/src/internal/connector/sharepoint/api/api.go b/src/internal/connector/sharepoint/api/api.go new file mode 100644 index 0000000000..c05eaad6b5 --- /dev/null +++ b/src/internal/connector/sharepoint/api/api.go @@ -0,0 +1,6 @@ +package api + +type Tuple struct { + Name string + ID string +} diff --git a/src/internal/connector/sharepoint/api/helper_test.go b/src/internal/connector/sharepoint/api/helper_test.go new file mode 100644 index 0000000000..631dd7b3b2 --- /dev/null +++ b/src/internal/connector/sharepoint/api/helper_test.go @@ -0,0 +1,21 @@ +package api + +import ( + "testing" + + "github.com/alcionai/corso/src/internal/connector/discovery/api" + "github.com/alcionai/corso/src/internal/connector/graph" + "github.com/alcionai/corso/src/pkg/account" + "github.com/stretchr/testify/require" +) + +func createTestBetaService(t *testing.T, credentials account.M365Config) *api.BetaService { + adapter, err := graph.CreateAdapter( + credentials.AzureTenantID, + credentials.AzureClientID, + credentials.AzureClientSecret, + ) + require.NoError(t, err) + + return api.NewBetaService(adapter) +} diff --git a/src/internal/connector/sharepoint/api/pages.go b/src/internal/connector/sharepoint/api/pages.go new file mode 100644 index 0000000000..a2232140c3 --- /dev/null +++ b/src/internal/connector/sharepoint/api/pages.go @@ -0,0 +1,93 @@ +package api + +import ( + "context" + + "github.com/alcionai/corso/src/internal/connector/discovery/api" + "github.com/alcionai/corso/src/internal/connector/graph/betasdk/models" + "github.com/alcionai/corso/src/internal/connector/graph/betasdk/sites" + "github.com/alcionai/corso/src/internal/connector/support" +) + +// GetSitePages retrieves a collection of Pages related to the give Site. +// Returns error if error experienced during the call +func GetSitePage( + ctx context.Context, + serv *api.BetaService, + siteID string, + pages []string, +) ([]models.SitePageable, error) { + col := make([]models.SitePageable, 0) + opts := retrieveSitePageOptions() + + for _, entry := range pages { + page, err := serv.Client().SitesById(siteID).PagesById(entry).Get(ctx, opts) + if err != nil { + return nil, support.ConnectorStackErrorTraceWrap(err, "fetching page: "+entry) + } + + col = append(col, page) + } + + return col, nil +} + +// fetchPages utility function to return the tuple of item +func FetchPages(ctx context.Context, bs *api.BetaService, siteID string) ([]Tuple, error) { + var ( + builder = bs.Client().SitesById(siteID).Pages() + opts = fetchPageOptions() + pageTuples = make([]Tuple, 0) + ) + + for { + resp, err := builder.Get(ctx, opts) + if err != nil { + return nil, support.ConnectorStackErrorTraceWrap(err, "failed fetching site page") + } + + for _, entry := range resp.GetValue() { + pid := *entry.GetId() + temp := Tuple{pid, pid} + + if entry.GetName() != nil { + temp.Name = *entry.GetName() + } + + pageTuples = append(pageTuples, temp) + } + + if resp.GetOdataNextLink() == nil { + break + } + + builder = sites.NewItemPagesRequestBuilder(*resp.GetOdataNextLink(), bs.Client().Adapter()) + } + + return pageTuples, nil +} + +// fetchPageOptions is used to return minimal information reltating to Site Pages +// Pages API: https://learn.microsoft.com/en-us/graph/api/resources/sitepage?view=graph-rest-beta +func fetchPageOptions() *sites.ItemPagesRequestBuilderGetRequestConfiguration { + fields := []string{"id", "name"} + options := &sites.ItemPagesRequestBuilderGetRequestConfiguration{ + QueryParameters: &sites.ItemPagesRequestBuilderGetQueryParameters{ + Select: fields, + }, + } + + return options +} + +// retrievePageOptions returns options to expand +func retrieveSitePageOptions() *sites.ItemPagesSitePageItemRequestBuilderGetRequestConfiguration { + fields := []string{"canvasLayout"} + options := &sites.ItemPagesSitePageItemRequestBuilderGetRequestConfiguration{ + QueryParameters: &sites.ItemPagesSitePageItemRequestBuilderGetQueryParameters{ + Expand: fields, + }, + } + + return options +} diff --git a/src/internal/connector/sharepoint/api/pages_test.go b/src/internal/connector/sharepoint/api/pages_test.go new file mode 100644 index 0000000000..ecc2cf18d5 --- /dev/null +++ b/src/internal/connector/sharepoint/api/pages_test.go @@ -0,0 +1,71 @@ +package api + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" + + "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/account" +) + +type SharePointPageSuite struct { + suite.Suite + siteID string + creds account.M365Config +} + +func (suite *SharePointPageSuite) SetupSuite() { + t := suite.T() + tester.MustGetEnvSets(t, tester.M365AcctCredEnvs) + + suite.siteID = tester.M365SiteID(t) + a := tester.NewM365Account(t) + m365, err := a.M365Config() + require.NoError(t, err) + + suite.creds = m365 +} + +func TestSharePointPageSuite(t *testing.T) { + tester.RunOnAny( + t, + tester.CorsoCITests, + tester.CorsoGraphConnectorSharePointTests) + suite.Run(t, new(SharePointPageSuite)) +} + +func (suite *SharePointPageSuite) TestFetchPages() { + ctx, flush := tester.NewContext() + defer flush() + + t := suite.T() + service := createTestBetaService(t, suite.creds) + + pgs, err := FetchPages(ctx, service, suite.siteID) + assert.NoError(t, err) + require.NotNil(t, pgs) + assert.NotZero(t, len(pgs)) + + for _, entry := range pgs { + t.Logf("id: %s\t name: %s\n", entry.ID, entry.Name) + } +} + +func (suite *SharePointPageSuite) TestGetSitePage() { + ctx, flush := tester.NewContext() + defer flush() + + t := suite.T() + service := createTestBetaService(t, suite.creds) + tuples, err := FetchPages(ctx, service, suite.siteID) + require.NoError(t, err) + require.NotNil(t, tuples) + + jobs := []string{tuples[0].ID} + pages, err := GetSitePage(ctx, service, suite.siteID, jobs) + assert.NoError(t, err) + assert.NotEmpty(t, pages) +} diff --git a/src/internal/connector/sharepoint/collection.go b/src/internal/connector/sharepoint/collection.go index c34d2a2d14..c540af4e60 100644 --- a/src/internal/connector/sharepoint/collection.go +++ b/src/internal/connector/sharepoint/collection.go @@ -9,6 +9,7 @@ import ( kw "github.com/microsoft/kiota-serialization-json-go" "github.com/microsoftgraph/msgraph-sdk-go/models" + "github.com/alcionai/corso/src/internal/connector/discovery/api" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" @@ -46,6 +47,7 @@ type Collection struct { jobs []string // M365 IDs of the items of this collection service graph.Servicer + betaService *api.BetaService statusUpdater support.StatusUpdater } diff --git a/src/internal/connector/sharepoint/collection_test.go b/src/internal/connector/sharepoint/collection_test.go index f049ab26ff..c2b1ac830a 100644 --- a/src/internal/connector/sharepoint/collection_test.go +++ b/src/internal/connector/sharepoint/collection_test.go @@ -17,11 +17,27 @@ import ( "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/tester" + "github.com/alcionai/corso/src/pkg/account" + "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/path" ) type SharePointCollectionSuite struct { suite.Suite + siteID string + creds account.M365Config +} + +func (suite *SharePointCollectionSuite) SetupSuite() { + t := suite.T() + tester.MustGetEnvSets(t, tester.M365AcctCredEnvs) + + suite.siteID = tester.M365SiteID(t) + a := tester.NewM365Account(t) + m365, err := a.M365Config() + require.NoError(t, err) + + suite.creds = m365 } func TestSharePointCollectionSuite(t *testing.T) { @@ -95,20 +111,33 @@ func (suite *SharePointCollectionSuite) TestSharePointListCollection() { assert.Equal(t, testName, shareInfo.Info().SharePoint.ItemName) } +func (suite *SharePointCollectionSuite) TestCollectPages() { + ctx, flush := tester.NewContext() + defer flush() + + t := suite.T() + col, err := collectPages( + ctx, + suite.creds, + nil, + account.AzureTenantID, + suite.siteID, + nil, + &MockGraphService{}, + control.Defaults(), + ) + assert.NoError(t, err) + assert.NotEmpty(t, col) +} + // TestRestoreListCollection verifies Graph Restore API for the List Collection func (suite *SharePointCollectionSuite) TestRestoreListCollection() { ctx, flush := tester.NewContext() defer flush() t := suite.T() - siteID := tester.M365SiteID(t) - a := tester.NewM365Account(t) - account, err := a.M365Config() - require.NoError(t, err) - - service, err := createTestService(account) - require.NoError(t, err) + service := createTestService(t, suite.creds) listing := mockconnector.GetMockListDefault("Mock List") testName := "MockListing" listing.SetDisplayName(&testName) @@ -123,13 +152,13 @@ func (suite *SharePointCollectionSuite) TestRestoreListCollection() { destName := "Corso_Restore_" + common.FormatNow(common.SimpleTimeTesting) - deets, err := restoreListItem(ctx, service, listData, siteID, destName) + deets, err := restoreListItem(ctx, service, listData, suite.siteID, destName) assert.NoError(t, err) t.Logf("List created: %s\n", deets.SharePoint.ItemName) // Clean-Up var ( - builder = service.Client().SitesById(siteID).Lists() + builder = service.Client().SitesById(suite.siteID).Lists() isFound bool deleteID string ) @@ -156,7 +185,7 @@ func (suite *SharePointCollectionSuite) TestRestoreListCollection() { } if isFound { - err := DeleteList(ctx, service, siteID, deleteID) + err := DeleteList(ctx, service, suite.siteID, deleteID) assert.NoError(t, err) } } @@ -168,25 +197,18 @@ func (suite *SharePointCollectionSuite) TestRestoreLocation() { defer flush() t := suite.T() - a := tester.NewM365Account(t) - account, err := a.M365Config() - require.NoError(t, err) - - service, err := createTestService(account) - require.NoError(t, err) + service := createTestService(t, suite.creds) rootFolder := "General_" + common.FormatNow(common.SimpleTimeTesting) - siteID := tester.M365SiteID(t) - - folderID, err := createRestoreFolders(ctx, service, siteID, []string{rootFolder}) + folderID, err := createRestoreFolders(ctx, service, suite.siteID, []string{rootFolder}) assert.NoError(t, err) t.Log("FolderID: " + folderID) - _, err = createRestoreFolders(ctx, service, siteID, []string{rootFolder, "Tsao"}) + _, err = createRestoreFolders(ctx, service, suite.siteID, []string{rootFolder, "Tsao"}) assert.NoError(t, err) // CleanUp - siteDrive, err := service.Client().SitesById(siteID).Drive().Get(ctx, nil) + siteDrive, err := service.Client().SitesById(suite.siteID).Drive().Get(ctx, nil) require.NoError(t, err) driveID := *siteDrive.GetId() diff --git a/src/internal/connector/sharepoint/data_collections.go b/src/internal/connector/sharepoint/data_collections.go index 1fd2f786dc..88e16882c8 100644 --- a/src/internal/connector/sharepoint/data_collections.go +++ b/src/internal/connector/sharepoint/data_collections.go @@ -6,11 +6,14 @@ import ( "github.com/pkg/errors" + "github.com/alcionai/corso/src/internal/connector/discovery/api" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/onedrive" + sapi "github.com/alcionai/corso/src/internal/connector/sharepoint/api" "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/internal/data" "github.com/alcionai/corso/src/internal/observe" + "github.com/alcionai/corso/src/pkg/account" "github.com/alcionai/corso/src/pkg/control" "github.com/alcionai/corso/src/pkg/logger" "github.com/alcionai/corso/src/pkg/path" @@ -162,6 +165,55 @@ func collectLibraries( return append(collections, odcs...), excludes, errs } +// collectPages constructs a sharepoint Collections struct and Get()s the associated +// M365 IDs for the associated Pages +func collectPages( + ctx context.Context, + creds account.M365Config, + serv graph.Servicer, + tenantID, siteID string, + scope selectors.SharePointScope, + updater statusUpdater, + ctrlOpts control.Options, +) ([]data.Collection, error) { + logger.Ctx(ctx).With("site", siteID).Debug("Creating SharePoint Pages collections") + + spcs := make([]data.Collection, 0) + + // make the betaClient + adpt, err := graph.CreateAdapter(creds.AzureTenantID, creds.AzureClientID, creds.AzureClientSecret) + if err != nil { + return nil, errors.Wrap(err, "adapter for betaservice not created") + } + + betaService := api.NewBetaService(adpt) + + tuples, err := sapi.FetchPages(ctx, betaService, siteID) + if err != nil { + return nil, err + } + + for _, tuple := range tuples { + dir, err := path.Builder{}.Append(tuple.Name). + ToDataLayerSharePointPath( + tenantID, + siteID, + path.PagesCategory, + false) + if err != nil { + return nil, errors.Wrapf(err, "failed to create collection path for site: %s", siteID) + } + + collection := NewCollection(dir, serv, updater.UpdateStatus) + collection.betaService = betaService + collection.AddJob(tuple.ID) + + spcs = append(spcs, collection) + } + + return spcs, nil +} + type folderMatcher struct { scope selectors.SharePointScope } diff --git a/src/internal/connector/sharepoint/helper_test.go b/src/internal/connector/sharepoint/helper_test.go index e716a5bae8..30d589389f 100644 --- a/src/internal/connector/sharepoint/helper_test.go +++ b/src/internal/connector/sharepoint/helper_test.go @@ -4,11 +4,11 @@ import ( "testing" msgraphsdk "github.com/microsoftgraph/msgraph-sdk-go" - "github.com/pkg/errors" "github.com/stretchr/testify/require" "github.com/alcionai/corso/src/internal/connector/graph" "github.com/alcionai/corso/src/internal/connector/onedrive" + "github.com/alcionai/corso/src/internal/connector/support" "github.com/alcionai/corso/src/pkg/account" ) @@ -29,21 +29,22 @@ func (ms *MockGraphService) Adapter() *msgraphsdk.GraphRequestAdapter { return nil } +func (ms *MockGraphService) UpdateStatus(*support.ConnectorOperationStatus) { +} + // --------------------------------------------------------------------------- // Helper Functions // --------------------------------------------------------------------------- -func createTestService(credentials account.M365Config) (*graph.Service, error) { +func createTestService(t *testing.T, credentials account.M365Config) *graph.Service { adapter, err := graph.CreateAdapter( credentials.AzureTenantID, credentials.AzureClientID, credentials.AzureClientSecret, ) - if err != nil { - return nil, errors.Wrap(err, "creating microsoft graph service for exchange") - } + require.NoError(t, err, "creating microsoft graph service for exchange") - return graph.NewService(adapter), nil + return graph.NewService(adapter) } func expectedPathAsSlice(t *testing.T, tenant, user string, rest ...string) []string { diff --git a/src/internal/connector/sharepoint/list_test.go b/src/internal/connector/sharepoint/list_test.go index c798be368a..2571c91838 100644 --- a/src/internal/connector/sharepoint/list_test.go +++ b/src/internal/connector/sharepoint/list_test.go @@ -49,9 +49,7 @@ func (suite *SharePointSuite) TestLoadList() { defer flush() t := suite.T() - service, err := createTestService(suite.creds) - require.NoError(t, err) - + service := createTestService(t, suite.creds) tuples, err := preFetchLists(ctx, service, "root") require.NoError(t, err)