fix(ingestion/bigquery): user exceeded quota for concurrent project.lists requests

metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py

@@ -150,12 +150,22 @@ def get_query_result(self, query: str) -> RowIterator:
     def get_projects(self) -> List[BigqueryProject]:
         with self.report.list_projects:
             try:
-                projects = self.bq_client.list_projects()
-
-                return [
-                    BigqueryProject(id=p.project_id, name=p.friendly_name)
-                    for p in projects
-                ]
+                projects: List[BigqueryProject] = []
+                page_token: Optional[str] = None
+                while True:
+                    projects_iterator = self.bq_client.list_projects(
+                        max_results=5, page_token=page_token
+                    )
+                    projects.extend(
+                        [
+                            BigqueryProject(id=p.project_id, name=p.friendly_name)
+                            for p in projects_iterator
+                        ]
+                    )
+                    page_token = projects_iterator.next_page_token
+                    if page_token is None:
+                        break
+                return projects
             except Exception as e:
                 logger.error(f"Error getting projects. {e}", exc_info=True)
                 return []

metadata-ingestion/tests/unit/test_bigquery_source.py

@@ -271,18 +271,24 @@ def test_get_projects_with_single_project_id(get_bq_client_mock):
 
 @patch.object(BigQueryV2Config, "get_bigquery_client")
 def test_get_projects_by_list(get_bq_client_mock):
+    mock_projects = MagicMock()
+    mock_projects.__iter__.return_value = iter(
+        [
+            SimpleNamespace(
+                project_id="test-1",
+                friendly_name="one",
+            ),
+            SimpleNamespace(
+                project_id="test-2",
+                friendly_name="two",
+            ),
+        ]
+    )
+    mock_projects.next_page_token = None
+
     client_mock = MagicMock()
     get_bq_client_mock.return_value = client_mock
-    client_mock.list_projects.return_value = [
-        SimpleNamespace(
-            project_id="test-1",
-            friendly_name="one",
-        ),
-        SimpleNamespace(
-            project_id="test-2",
-            friendly_name="two",
-        ),
-    ]
+    client_mock.list_projects.return_value = mock_projects
 
     config = BigQueryV2Config.parse_obj({})
     source = BigqueryV2Source(config=config, ctx=PipelineContext(run_id="test1"))