Merge branch 'main' into beta-page-connector

src/internal/connector/onedrive/collections.go

@@ -24,6 +24,7 @@ const (
 	OneDriveSource
 	SharePointSource
 )
+const restrictedDirectory = "Site Pages"
 
 func (ds driveSource) toPathServiceCat() (path.ServiceType, path.CategoryType) {
 	switch ds {
@@ -102,8 +103,9 @@ func (c *Collections) Get(ctx context.Context) ([]data.Collection, error) {
 	// Update the collection map with items from each drive
 	for _, d := range drives {
 		driveID := *d.GetId()
+		driveName := *d.GetName()
 
-		delta, paths, err := collectItems(ctx, c.service, driveID, c.UpdateCollections)
+		delta, paths, err := collectItems(ctx, c.service, driveID, driveName, c.UpdateCollections)
 		if err != nil {
 			return nil, err
 		}
@@ -162,7 +164,7 @@ func (c *Collections) Get(ctx context.Context) ([]data.Collection, error) {
 // A new collection is created for every drive folder (or package)
 func (c *Collections) UpdateCollections(
 	ctx context.Context,
-	driveID string,
+	driveID, driveName string,
 	items []models.DriveItemable,
 	paths map[string]string,
 ) error {
@@ -188,7 +190,7 @@ func (c *Collections) UpdateCollections(
 		}
 
 		// Skip items that don't match the folder selectors we were given.
-		if !includePath(ctx, c.matcher, collectionPath) {
+		if shouldSkipDrive(ctx, collectionPath, c.matcher, driveName) {
 			logger.Ctx(ctx).Infof("Skipping path %s", collectionPath.String())
 			continue
 		}
@@ -239,6 +241,11 @@ func (c *Collections) UpdateCollections(
 	return nil
 }
 
+func shouldSkipDrive(ctx context.Context, drivePath path.Path, m folderMatcher, driveName string) bool {
+	return !includePath(ctx, m, drivePath) ||
+		(drivePath.Category() == path.LibrariesCategory && restrictedDirectory == driveName)
+}
+
 // GetCanonicalPath constructs the standard path for the given source.
 func GetCanonicalPath(p, tenant, resourceOwner string, source driveSource) (path.Path, error) {
 	var (

src/internal/connector/onedrive/collections_test.go

@@ -326,7 +326,7 @@ func (suite *OneDriveCollectionsSuite) TestUpdateCollections() {
 				nil,
 				control.Options{})
 
-			err := c.UpdateCollections(ctx, "driveID", tt.items, paths)
+			err := c.UpdateCollections(ctx, "driveID", "General", tt.items, paths)
 			tt.expect(t, err)
 			assert.Equal(t, len(tt.expectedCollectionPaths), len(c.CollectionMap), "collection paths")
 			assert.Equal(t, tt.expectedItemCount, c.NumItems, "item count")

src/internal/connector/onedrive/drive.go

@@ -163,7 +163,7 @@ func userDrives(ctx context.Context, service graph.Servicer, user string) ([]mod
 // itemCollector functions collect the items found in a drive
 type itemCollector func(
 	ctx context.Context,
-	driveID string,
+	driveID, driveName string,
 	driveItems []models.DriveItemable,
 	paths map[string]string,
 ) error
@@ -173,7 +173,7 @@ type itemCollector func(
 func collectItems(
 	ctx context.Context,
 	service graph.Servicer,
-	driveID string,
+	driveID, driveName string,
 	collector itemCollector,
 ) (string, map[string]string, error) {
 	var (
@@ -219,7 +219,7 @@ func collectItems(
 			)
 		}
 
-		err = collector(ctx, driveID, r.GetValue(), paths)
+		err = collector(ctx, driveID, driveName, r.GetValue(), paths)
 		if err != nil {
 			return "", nil, err
 		}
@@ -349,9 +349,10 @@ func GetAllFolders(
 			ctx,
 			gs,
 			*d.GetId(),
+			*d.GetName(),
 			func(
 				innerCtx context.Context,
-				driveID string,
+				driveID, driveName string,
 				items []models.DriveItemable,
 				paths map[string]string,
 			) error {

src/internal/connector/onedrive/item_test.go

@@ -97,7 +97,7 @@ func (suite *ItemIntegrationSuite) TestItemReader_oneDrive() {
 	// This item collector tries to find "a" drive item that is a file to test the reader function
 	itemCollector := func(
 		ctx context.Context,
-		driveID string,
+		driveID, driveName string,
 		items []models.DriveItemable,
 		paths map[string]string,
 	) error {
@@ -110,7 +110,7 @@ func (suite *ItemIntegrationSuite) TestItemReader_oneDrive() {
 
 		return nil
 	}
-	_, _, err := collectItems(ctx, suite, suite.userDriveID, itemCollector)
+	_, _, err := collectItems(ctx, suite, suite.userDriveID, "General", itemCollector)
 	require.NoError(suite.T(), err)
 
 	// Test Requirement 2: Need a file

src/internal/connector/sharepoint/data_collections_test.go

@@ -96,7 +96,7 @@ func (suite *SharePointLibrariesSuite) TestUpdateCollections() {
 				&MockGraphService{},
 				nil,
 				control.Options{})
-			err := c.UpdateCollections(ctx, "driveID", test.items, paths)
+			err := c.UpdateCollections(ctx, "driveID", "General", test.items, paths)
 			test.expect(t, err)
 			assert.Equal(t, len(test.expectedCollectionPaths), len(c.CollectionMap), "collection paths")
 			assert.Equal(t, test.expectedItemCount, c.NumItems, "item count")

src/internal/connector/sharepoint/listInfo_test.go

@@ -20,33 +20,33 @@ func TestSharePointInfoSuite(t *testing.T) {
 
 func (suite *SharePointInfoSuite) TestSharePointInfo() {
 	tests := []struct {
-		name      string
-		listAndRP func() (models.Listable, *details.SharePointInfo)
+		name         string
+		listAndDeets func() (models.Listable, *details.SharePointInfo)
 	}{
 		{
 			name: "Empty List",
-			listAndRP: func() (models.Listable, *details.SharePointInfo) {
+			listAndDeets: func() (models.Listable, *details.SharePointInfo) {
 				i := &details.SharePointInfo{ItemType: details.SharePointItem}
 				return models.NewList(), i
 			},
 		}, {
 			name: "Only Name",
-			listAndRP: func() (models.Listable, *details.SharePointInfo) {
+			listAndDeets: func() (models.Listable, *details.SharePointInfo) {
 				aTitle := "Whole List"
 				listing := models.NewList()
 				listing.SetDisplayName(&aTitle)
 				i := &details.SharePointInfo{
 					ItemType: details.SharePointItem,
 					ItemName: aTitle,
-					Size:     10,
 				}
+
 				return listing, i
 			},
 		},
 	}
 	for _, test := range tests {
 		suite.T().Run(test.name, func(t *testing.T) {
-			list, expected := test.listAndRP()
+			list, expected := test.listAndDeets()
 			info := sharePointListInfo(list, 10)
 			assert.Equal(t, expected.ItemType, info.ItemType)
 			assert.Equal(t, expected.ItemName, info.ItemName)

src/internal/connector/sharepoint/pageInfo.go

@@ -0,0 +1,42 @@
+package sharepoint
+
+import (
+	"time"
+
+	"github.com/alcionai/corso/src/pkg/backup/details"
+)
+
+// sharePointPageInfo propagates metadata from  the SharePoint Page data type
+// into searchable content.
+// Page Details: https://learn.microsoft.com/en-us/graph/api/resources/sitepage?view=graph-rest-beta
+func sharePointPageInfo(page SitePageable, size int64) *details.SharePointInfo {
+	var (
+		name, webURL      string
+		created, modified time.Time
+	)
+
+	if page.GetTitle() != nil {
+		name = *page.GetTitle()
+	}
+
+	if page.GetWebUrl() != nil {
+		webURL = *page.GetWebUrl()
+	}
+
+	if page.GetCreatedDateTime() != nil {
+		created = *page.GetCreatedDateTime()
+	}
+
+	if page.GetLastModifiedDateTime() != nil {
+		modified = *page.GetLastModifiedDateTime()
+	}
+
+	return &details.SharePointInfo{
+		ItemType: details.SharePointItem,
+		ItemName: name,
+		Created:  created,
+		Modified: modified,
+		WebURL:   webURL,
+		Size:     size,
+	}
+}

src/internal/connector/sharepoint/pageInfo_test.go

@@ -0,0 +1,47 @@
+package sharepoint
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/alcionai/corso/src/pkg/backup/details"
+)
+
+func (suite *SharePointInfoSuite) TestSharePointInfo_Pages() {
+	tests := []struct {
+		name         string
+		pageAndDeets func() (SitePageable, *details.SharePointInfo)
+	}{
+		{
+			name: "Empty Page",
+			pageAndDeets: func() (SitePageable, *details.SharePointInfo) {
+				deets := &details.SharePointInfo{ItemType: details.SharePointItem}
+				return NewSitePage(), deets
+			},
+		},
+		{
+			name: "Only Name",
+			pageAndDeets: func() (SitePageable, *details.SharePointInfo) {
+				title := "Blank Page"
+				sPage := NewSitePage()
+				sPage.SetTitle(&title)
+				deets := &details.SharePointInfo{
+					ItemType: details.SharePointItem,
+					ItemName: title,
+				}
+
+				return sPage, deets
+			},
+		},
+	}
+	for _, test := range tests {
+		suite.T().Run(test.name, func(t *testing.T) {
+			paged, expected := test.pageAndDeets()
+			info := sharePointPageInfo(paged, 0)
+			assert.Equal(t, expected.ItemType, info.ItemType)
+			assert.Equal(t, expected.ItemName, info.ItemName)
+			assert.Equal(t, expected.WebURL, info.WebURL)
+		})
+	}
+}

src/internal/connector/sharepoint/restore.go

@@ -76,6 +76,12 @@ func RestoreCollections(
 				deets,
 				errUpdater,
 			)
+		case path.PagesCategory:
+			errorMessage := fmt.Sprintf("restore of %s not supported", dc.FullPath().Category())
+			logger.Ctx(ctx).Error(errorMessage)
+
+			return nil, errors.New(errorMessage)
+
 		default:
 			return nil, errors.Errorf("category %s not supported", dc.FullPath().Category())
 		}