bug: fixes issues exporting more than 1000 products #611 (#612)

apigee · Dec 29, 2024 · 3d0d7c2 · 3d0d7c2
1 parent 5a06ccc
commit 3d0d7c2
Show file tree

Hide file tree

Showing 4 changed files with 95 additions and 23 deletions.
diff --git a/internal/client/apps/apps.go b/internal/client/apps/apps.go
@@ -294,16 +294,7 @@ func Export(conn int) (payload [][]byte, err error) {
 	var mu sync.Mutex
 	const entityType = "apps"
 
-	u, _ := url.Parse(apiclient.GetApigeeBaseURL())
-	u.Path = path.Join(u.Path, apiclient.GetApigeeOrg(), entityType)
-
-	respBody, err := apiclient.HttpClient(u.String())
-	if err != nil {
-		return apiclient.GetEntityPayloadList(), err
-	}
-
-	entities := apps{}
-	err = json.Unmarshal(respBody, &entities)
+	entities, err := listAllApps()
 	if err != nil {
 		return apiclient.GetEntityPayloadList(), err
 	}
@@ -574,3 +565,48 @@ func getNewDeveloperId(oldDeveloperId string, developerEntities developers.Appde
 	}
 	return "", "", fmt.Errorf("developer not imported into Apigee")
 }
+
+func listAllApps() (appList apps, err error) {
+	var startKey string
+	appList = apps{}
+
+	u, _ := url.Parse(apiclient.GetApigeeBaseURL())
+	u.Path = path.Join(u.Path, apiclient.GetApigeeOrg(), "apps")
+
+	// don't print to sysout
+	apiclient.ClientPrintHttpResponse.Set(false)
+
+	for {
+
+		a := apps{}
+
+		if startKey != "" {
+			q := u.Query()
+			q.Set("startKey", startKey)
+			q.Set("rows", "10000")
+			u.RawQuery = q.Encode()
+		}
+
+		respBody, err := apiclient.HttpClient(u.String())
+		startKey = ""
+		if err != nil {
+			return appList, err
+		}
+
+		err = json.Unmarshal(respBody, &a)
+		if err != nil {
+			return appList, err
+		}
+
+		appList.Apps = append(appList.Apps, a.Apps...)
+
+		if len(a.Apps) == 10000 {
+			startKey = a.Apps[len(a.Apps)-1].AppID
+		} else if len(a.Apps) < 10000 {
+			break
+		}
+	}
+
+	apiclient.ClientPrintHttpResponse.Set(apiclient.GetCmdPrintHttpResponseSetting())
+	return appList, nil
+}
diff --git a/internal/client/products/products.go b/internal/client/products/products.go
@@ -309,20 +309,9 @@ func Export(conn int) (payload [][]byte, err error) {
 	// parent workgroup
 	var pwg sync.WaitGroup
 	var mu sync.Mutex
-	const entityType = "apiproducts"
 
-	u, _ := url.Parse(apiclient.GetApigeeBaseURL())
-	u.Path = path.Join(u.Path, apiclient.GetApigeeOrg(), entityType)
-	// don't print to sysout
-	apiclient.ClientPrintHttpResponse.Set(false)
-	respBody, err := apiclient.HttpClient(u.String())
-	apiclient.ClientPrintHttpResponse.Set(apiclient.GetCmdPrintHttpResponseSetting())
-	if err != nil {
-		return apiclient.GetEntityPayloadList(), err
-	}
-
-	products := apiProducts{}
-	err = json.Unmarshal(respBody, &products)
+	entityType := "apiproducts"
+	products, err := listAllProducts()
 	if err != nil {
 		return apiclient.GetEntityPayloadList(), err
 	}
@@ -471,3 +460,48 @@ func readProductsFile(filePath string) ([]APIProduct, error) {
 
 	return products, nil
 }
+
+func listAllProducts() (products apiProducts, err error) {
+	var startKey string
+	products = apiProducts{}
+
+	u, _ := url.Parse(apiclient.GetApigeeBaseURL())
+	u.Path = path.Join(u.Path, apiclient.GetApigeeOrg(), "apiproducts")
+
+	// don't print to sysout
+	apiclient.ClientPrintHttpResponse.Set(false)
+
+	for {
+
+		p := apiProducts{}
+
+		if startKey != "" {
+			q := u.Query()
+			q.Set("startKey", startKey)
+			q.Set("count", "1000")
+			u.RawQuery = q.Encode()
+		}
+
+		respBody, err := apiclient.HttpClient(u.String())
+		startKey = ""
+		if err != nil {
+			return products, err
+		}
+
+		err = json.Unmarshal(respBody, &p)
+		if err != nil {
+			return products, err
+		}
+
+		products.APIProduct = append(products.APIProduct, p.APIProduct...)
+
+		if len(p.APIProduct) == 1000 {
+			startKey = p.APIProduct[len(p.APIProduct)-1].Name
+		} else if len(p.APIProduct) < 1000 {
+			break
+		}
+	}
+
+	apiclient.ClientPrintHttpResponse.Set(apiclient.GetCmdPrintHttpResponseSetting())
+	return products, nil
+}
diff --git a/internal/cmd/apps/expapp.go b/internal/cmd/apps/expapp.go
@@ -34,6 +34,7 @@ var ExpCmd = &cobra.Command{
 		cmd.SilenceUsage = true
 
 		const exportFileName = "apps.json"
+		apiclient.DisableCmdPrintHttpResponse()
 		payload, err := apps.Export(conn)
 		if err != nil {
 			return err

diff --git a/internal/cmd/products/expprod.go b/internal/cmd/products/expprod.go
@@ -34,6 +34,7 @@ var ExpCmd = &cobra.Command{
 		cmd.SilenceUsage = true
 
 		const exportFileName = "products.json"
+		apiclient.DisableCmdPrintHttpResponse()
 		payload, err := products.Export(conn)
 		if err != nil {
 			return err