fix(ingestion): looker - fix dashboard and chart-usage calculations (#…

…5791) Co-authored-by: Shirshanka Das <[email protected]> Co-authored-by: MohdSiddique Bagwan <[email protected]>
datahub-project · Aug 31, 2022 · dbcd309 · dbcd309
1 parent 3059d9d
commit dbcd309
Show file tree

Hide file tree

Showing 3 changed files with 102 additions and 175 deletions.
diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_usage.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_usage.py
@@ -245,9 +245,9 @@ def _fill_user_stat_aspect(
                 continue
 
             # Confirm for the user row (entity + time) the entity stat is present for the same time
-            entity_stat_aspect: Aspect = entity_usage_stat[
+            entity_stat_aspect: Optional[Aspect] = entity_usage_stat.get(
                 self.get_entity_stat_key(row)
-            ]
+            )
             if entity_stat_aspect is None:
                 logger.warning(
                     "entity stat is not found for the user stat key = {}".format(
@@ -256,7 +256,9 @@ def _fill_user_stat_aspect(
                 )
                 continue
             self.append_user_stat(entity_stat_aspect, user, row)
-            yield looker_object, entity_stat_aspect
+
+        for (id, _), aspect in entity_usage_stat.items():
+            yield self.id_vs_model[id], aspect
 
     def _execute_query(self, query: LookerQuery, query_name: str) -> List[Dict]:
         start_time = datetime.datetime.now()
@@ -390,7 +392,6 @@ def to_entity_absolute_stat_aspect(self, looker_object: model.Model) -> Aspect:
                 lastViewedAt=round(looker_dashboard.last_viewed_at.timestamp() * 1000)
                 if looker_dashboard.last_viewed_at
                 else None,
-                userCounts=[],
             ),
         )
 
@@ -413,7 +414,6 @@ def to_entity_timeseries_stat_aspect(self, row: dict) -> Aspect:
                 eventGranularity=TimeWindowSizeClass(unit=CalendarIntervalClass.DAY),
                 uniqueUserCount=row[HistoryViewField.HISTORY_DASHBOARD_USER],
                 executionsCount=row[HistoryViewField.HISTORY_DASHBOARD_RUN_COUNT],
-                userCounts=[],
             ),
         )
 
@@ -508,7 +508,6 @@ def to_entity_absolute_stat_aspect(self, looker_object: model.Model) -> Aspect:
             ChartUsageStatisticsClass(
                 timestampMillis=round(datetime.datetime.now().timestamp() * 1000),
                 viewsCount=looker_look.view_count,
-                userCounts=[],
             ),
         )
 
@@ -529,7 +528,6 @@ def to_entity_timeseries_stat_aspect(self, row: dict) -> Aspect:
                 ),
                 eventGranularity=TimeWindowSizeClass(unit=CalendarIntervalClass.DAY),
                 viewsCount=row[HistoryViewField.HISTORY_COUNT],
-                userCounts=[],
             ),
         )