Merge branch 'master' into remove-archived-docs

datahub-web-react/src/app/ingest/source/builder/RecipeForm/RecipeForm.tsx

@@ -5,10 +5,9 @@ import YAML from 'yamljs';
 import { ApiOutlined, FilterOutlined, QuestionCircleOutlined, SettingOutlined } from '@ant-design/icons';
 import styled from 'styled-components/macro';
 import { jsonToYaml } from '../../utils';
-import { RECIPE_FIELDS } from './constants';
+import { CONNECTORS_WITH_TEST_CONNECTION, RECIPE_FIELDS } from './constants';
 import FormField from './FormField';
 import TestConnectionButton from './TestConnection/TestConnectionButton';
-import { SNOWFLAKE } from '../../conf/snowflake/snowflake';
 import { useListSecretsQuery } from '../../../../../graphql/ingestion.generated';
 import { RecipeField, setFieldValueOnRecipe } from './common';
 
@@ -145,7 +144,7 @@ function RecipeForm(props: Props) {
                             removeMargin={i === fields.length - 1}
                         />
                     ))}
-                    {type === SNOWFLAKE && (
+                    {CONNECTORS_WITH_TEST_CONNECTION.has(type) && (
                         <TestConnectionWrapper>
                             <TestConnectionButton type={type} recipe={displayRecipe} />
                         </TestConnectionWrapper>

datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts

@@ -200,3 +200,5 @@ export const RECIPE_FIELDS: RecipeFields = {
 };
 
 export const CONNECTORS_WITH_FORM = new Set(Object.keys(RECIPE_FIELDS));
+
+export const CONNECTORS_WITH_TEST_CONNECTION = new Set([SNOWFLAKE, LOOKER]);

metadata-ingestion/src/datahub/configuration/common.py

@@ -105,7 +105,7 @@ class OauthConfiguration(ConfigModel):
     scopes: Optional[List[str]] = Field(
         description="scopes required to connect to snowflake"
     )
-    use_certificate: Optional[bool] = Field(
+    use_certificate: bool = Field(
         description="Do you want to use certificate and private key to authenticate using oauth",
         default=False,
     )

metadata-ingestion/src/datahub/ingestion/source/looker/looker_usage.py

@@ -245,9 +245,9 @@ def _fill_user_stat_aspect(
                 continue
 
             # Confirm for the user row (entity + time) the entity stat is present for the same time
-            entity_stat_aspect: Aspect = entity_usage_stat[
+            entity_stat_aspect: Optional[Aspect] = entity_usage_stat.get(
                 self.get_entity_stat_key(row)
-            ]
+            )
             if entity_stat_aspect is None:
                 logger.warning(
                     "entity stat is not found for the user stat key = {}".format(
@@ -256,7 +256,9 @@ def _fill_user_stat_aspect(
                 )
                 continue
             self.append_user_stat(entity_stat_aspect, user, row)
-            yield looker_object, entity_stat_aspect
+
+        for (id, _), aspect in entity_usage_stat.items():
+            yield self.id_vs_model[id], aspect
 
     def _execute_query(self, query: LookerQuery, query_name: str) -> List[Dict]:
         start_time = datetime.datetime.now()
@@ -390,7 +392,6 @@ def to_entity_absolute_stat_aspect(self, looker_object: model.Model) -> Aspect:
                 lastViewedAt=round(looker_dashboard.last_viewed_at.timestamp() * 1000)
                 if looker_dashboard.last_viewed_at
                 else None,
-                userCounts=[],
             ),
         )
 
@@ -413,7 +414,6 @@ def to_entity_timeseries_stat_aspect(self, row: dict) -> Aspect:
                 eventGranularity=TimeWindowSizeClass(unit=CalendarIntervalClass.DAY),
                 uniqueUserCount=row[HistoryViewField.HISTORY_DASHBOARD_USER],
                 executionsCount=row[HistoryViewField.HISTORY_DASHBOARD_RUN_COUNT],
-                userCounts=[],
             ),
         )
 
@@ -508,7 +508,6 @@ def to_entity_absolute_stat_aspect(self, looker_object: model.Model) -> Aspect:
             ChartUsageStatisticsClass(
                 timestampMillis=round(datetime.datetime.now().timestamp() * 1000),
                 viewsCount=looker_look.view_count,
-                userCounts=[],
             ),
         )
 
@@ -529,7 +528,6 @@ def to_entity_timeseries_stat_aspect(self, row: dict) -> Aspect:
                 ),
                 eventGranularity=TimeWindowSizeClass(unit=CalendarIntervalClass.DAY),
                 viewsCount=row[HistoryViewField.HISTORY_COUNT],
-                userCounts=[],
             ),
         )
 

metadata-ingestion/src/datahub/ingestion/source_config/sql/snowflake.py

@@ -213,12 +213,12 @@ def authenticator_type_is_valid(cls, v, values, field):
                     f"but should be set when using {v} authentication"
                 )
             if values.get("oauth_config").use_certificate is True:
-                if values.get("oauth_config").base64_encoded_oauth_private_key is None:
+                if values.get("oauth_config").encoded_oauth_private_key is None:
                     raise ValueError(
                         "'base64_encoded_oauth_private_key' was none "
                         "but should be set when using certificate for oauth_config"
                     )
-                if values.get("oauth").base64_encoded_oauth_public_key is None:
+                if values.get("oauth").encoded_oauth_public_key is None:
                     raise ValueError(
                         "'base64_encoded_oauth_public_key' was none"
                         "but should be set when using use_certificate true for oauth_config"