fix(ui) Fix bigquery and redshift forms for lineage fields (#6215)

datahub-project · Oct 17, 2022 · a7b1763 · a7b1763
1 parent 956bee9
commit a7b1763
Show file tree

Hide file tree

Showing 3 changed files with 20 additions and 5 deletions.
diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx
@@ -255,6 +255,15 @@ export const INCLUDE_LINEAGE: RecipeField = {
     },
 };
 
+export const INCLUDE_TABLE_LINEAGE: RecipeField = {
+    name: 'include_table_lineage',
+    label: 'Include Table Lineage',
+    tooltip: 'Whether or not table lineage should be ingested.',
+    type: FieldType.BOOLEAN,
+    fieldPath: 'source.config.include_table_lineage',
+    rules: null,
+};
+
 export const PROFILING_ENABLED: RecipeField = {
     name: 'profiling.enabled',
     label: 'Enable Profiling',

diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts
@@ -22,6 +22,7 @@ import {
     SKIP_PERSONAL_FOLDERS,
     RecipeField,
     START_TIME,
+    INCLUDE_TABLE_LINEAGE,
 } from './common';
 import {
     SNOWFLAKE_ACCOUNT_ID,
@@ -137,7 +138,12 @@ export const RECIPE_FIELDS: RecipeFields = {
             BIGQUERY_CLIENT_EMAIL,
             BIGQUERY_CLIENT_ID,
         ],
-        advancedFields: [INCLUDE_LINEAGE, PROFILING_ENABLED, STATEFUL_INGESTION_ENABLED, UPSTREAM_LINEAGE_IN_REPORT],
+        advancedFields: [
+            INCLUDE_TABLE_LINEAGE,
+            PROFILING_ENABLED,
+            STATEFUL_INGESTION_ENABLED,
+            UPSTREAM_LINEAGE_IN_REPORT,
+        ],
         filterFields: [
             BIGQUERY_SCHEMA_ALLOW,
             BIGQUERY_SCHEMA_DENY,
@@ -157,7 +163,7 @@ export const RECIPE_FIELDS: RecipeFields = {
             BIGQUERY_CLIENT_EMAIL,
             BIGQUERY_CLIENT_ID,
         ],
-        advancedFields: [INCLUDE_LINEAGE, PROFILING_ENABLED, STATEFUL_INGESTION_ENABLED, START_TIME],
+        advancedFields: [INCLUDE_TABLE_LINEAGE, PROFILING_ENABLED, STATEFUL_INGESTION_ENABLED, START_TIME],
         filterFields: [
             PROJECT_ALLOW,
             PROJECT_DENY,
@@ -173,7 +179,7 @@ export const RECIPE_FIELDS: RecipeFields = {
     },
     [REDSHIFT]: {
         fields: [REDSHIFT_HOST_PORT, REDSHIFT_DATABASE, REDSHIFT_USERNAME, REDSHIFT_PASSWORD],
-        advancedFields: [INCLUDE_LINEAGE, PROFILING_ENABLED, STATEFUL_INGESTION_ENABLED, TABLE_LINEAGE_MODE],
+        advancedFields: [INCLUDE_TABLE_LINEAGE, PROFILING_ENABLED, STATEFUL_INGESTION_ENABLED, TABLE_LINEAGE_MODE],
         filterFields: [
             REDSHIFT_SCHEMA_ALLOW,
             REDSHIFT_SCHEMA_DENY,

diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json
@@ -4,14 +4,14 @@
         "name": "bigquery",
         "displayName": "BigQuery",
         "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/bigquery/",
-        "recipe": "source:\n    type: bigquery\n    config:\n        # Coordinates\n        project_id: # Your BigQuery project id, e.g. sample_project_id\n        # Credentials\n        credential:\n            project_id: # Your BQ project id, e.g. sample_project_id\n            private_key_id: \"${BQ_PRIVATE_KEY_ID}\"\n            private_key: \"${BQ_PRIVATE_KEY}\"\n            client_email: # Your BQ client email, e.g. \"[email protected]\"\n            client_id: # Your BQ client id, e.g.\"123456678890\"\n            \n        include_table_lineage: true\n        include_view_lineage: true\n        profiling:\n            enabled: true\n        stateful_ingestion:\n            enabled: true"
+        "recipe": "source:\n    type: bigquery\n    config:\n        # Coordinates\n        project_id: # Your BigQuery project id, e.g. sample_project_id\n        # Credentials\n        credential:\n            project_id: # Your BQ project id, e.g. sample_project_id\n            private_key_id: \"${BQ_PRIVATE_KEY_ID}\"\n            private_key: \"${BQ_PRIVATE_KEY}\"\n            client_email: # Your BQ client email, e.g. \"[email protected]\"\n            client_id: # Your BQ client id, e.g.\"123456678890\"\n            \n        include_table_lineage: true\n        profiling:\n            enabled: true\n        stateful_ingestion:\n            enabled: true"
     },
     {
         "urn": "urn:li:dataPlatform:redshift",
         "name": "redshift",
         "displayName": "Redshift",
         "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/redshift/",
-        "recipe": "source: \n    type: redshift\n    config:\n        # Coordinates\n        host_port: # Your Redshift host and post, e.g. example.something.us-west-2.redshift.amazonaws.com:5439\n        database: # Your Redshift database, e.g. SampleDatabase\n\n        # Credentials\n        # Add secret in Secrets Tab with relevant names for each variable\n        username: \"${REDSHIFT_USERNAME}\" # Your Redshift username, e.g. admin\n        password: \"${REDSHIFT_PASSWORD}\" # Your Redshift password, e.g. password_01\n\n        table_lineage_mode: stl_scan_based\n        include_table_lineage: true\n        include_view_lineage: true\n        profiling:\n            enabled: true\n        stateful_ingestion:\n            enabled: true"
+        "recipe": "source: \n    type: redshift\n    config:\n        # Coordinates\n        host_port: # Your Redshift host and post, e.g. example.something.us-west-2.redshift.amazonaws.com:5439\n        database: # Your Redshift database, e.g. SampleDatabase\n\n        # Credentials\n        # Add secret in Secrets Tab with relevant names for each variable\n        username: \"${REDSHIFT_USERNAME}\" # Your Redshift username, e.g. admin\n        password: \"${REDSHIFT_PASSWORD}\" # Your Redshift password, e.g. password_01\n\n        table_lineage_mode: stl_scan_based\n        include_table_lineage: true\n        profiling:\n            enabled: true\n        stateful_ingestion:\n            enabled: true"
     },
     {
         "urn": "urn:li:dataPlatform:snowflake",