From 08c86e8b33968afb5974cdeeec2534cb9abea6e5 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Mon, 17 Oct 2022 10:34:48 -0400 Subject: [PATCH] fix(ui) Fix bigquery and redshift forms for lineage fields --- .../app/ingest/source/builder/RecipeForm/common.tsx | 9 +++++++++ .../ingest/source/builder/RecipeForm/constants.ts | 12 +++++++++--- .../src/app/ingest/source/builder/sources.json | 4 ++-- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx index 1136101da2409..2bc5fedcf52c5 100644 --- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx @@ -255,6 +255,15 @@ export const INCLUDE_LINEAGE: RecipeField = { }, }; +export const INCLUDE_TABLE_LINEAGE: RecipeField = { + name: 'include_table_lineage', + label: 'Include Table Lineage', + tooltip: 'Whether or not table lineage should be ingested.', + type: FieldType.BOOLEAN, + fieldPath: 'source.config.include_table_lineage', + rules: null, +}; + export const PROFILING_ENABLED: RecipeField = { name: 'profiling.enabled', label: 'Enable Profiling', diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts index 630416f3e6410..1a0e1e83ee71b 100644 --- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts @@ -22,6 +22,7 @@ import { SKIP_PERSONAL_FOLDERS, RecipeField, START_TIME, + INCLUDE_TABLE_LINEAGE, } from './common'; import { SNOWFLAKE_ACCOUNT_ID, @@ -137,7 +138,12 @@ export const RECIPE_FIELDS: RecipeFields = { BIGQUERY_CLIENT_EMAIL, BIGQUERY_CLIENT_ID, ], - advancedFields: [INCLUDE_LINEAGE, PROFILING_ENABLED, STATEFUL_INGESTION_ENABLED, UPSTREAM_LINEAGE_IN_REPORT], + advancedFields: [ + INCLUDE_TABLE_LINEAGE, + PROFILING_ENABLED, + STATEFUL_INGESTION_ENABLED, + UPSTREAM_LINEAGE_IN_REPORT, + ], filterFields: [ BIGQUERY_SCHEMA_ALLOW, BIGQUERY_SCHEMA_DENY, @@ -157,7 +163,7 @@ export const RECIPE_FIELDS: RecipeFields = { BIGQUERY_CLIENT_EMAIL, BIGQUERY_CLIENT_ID, ], - advancedFields: [INCLUDE_LINEAGE, PROFILING_ENABLED, STATEFUL_INGESTION_ENABLED, START_TIME], + advancedFields: [INCLUDE_TABLE_LINEAGE, PROFILING_ENABLED, STATEFUL_INGESTION_ENABLED, START_TIME], filterFields: [ PROJECT_ALLOW, PROJECT_DENY, @@ -173,7 +179,7 @@ export const RECIPE_FIELDS: RecipeFields = { }, [REDSHIFT]: { fields: [REDSHIFT_HOST_PORT, REDSHIFT_DATABASE, REDSHIFT_USERNAME, REDSHIFT_PASSWORD], - advancedFields: [INCLUDE_LINEAGE, PROFILING_ENABLED, STATEFUL_INGESTION_ENABLED, TABLE_LINEAGE_MODE], + advancedFields: [INCLUDE_TABLE_LINEAGE, PROFILING_ENABLED, STATEFUL_INGESTION_ENABLED, TABLE_LINEAGE_MODE], filterFields: [ REDSHIFT_SCHEMA_ALLOW, REDSHIFT_SCHEMA_DENY, diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json index ba228d69920e1..b1dc4b88e7896 100644 --- a/datahub-web-react/src/app/ingest/source/builder/sources.json +++ b/datahub-web-react/src/app/ingest/source/builder/sources.json @@ -4,14 +4,14 @@ "name": "bigquery", "displayName": "BigQuery", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/bigquery/", - "recipe": "source:\n type: bigquery\n config:\n # Coordinates\n project_id: # Your BigQuery project id, e.g. sample_project_id\n # Credentials\n credential:\n project_id: # Your BQ project id, e.g. sample_project_id\n private_key_id: \"${BQ_PRIVATE_KEY_ID}\"\n private_key: \"${BQ_PRIVATE_KEY}\"\n client_email: # Your BQ client email, e.g. \"test@suppproject-id-1234567.iam.gserviceaccount.com\"\n client_id: # Your BQ client id, e.g.\"123456678890\"\n \n include_table_lineage: true\n include_view_lineage: true\n profiling:\n enabled: true\n stateful_ingestion:\n enabled: true" + "recipe": "source:\n type: bigquery\n config:\n # Coordinates\n project_id: # Your BigQuery project id, e.g. sample_project_id\n # Credentials\n credential:\n project_id: # Your BQ project id, e.g. sample_project_id\n private_key_id: \"${BQ_PRIVATE_KEY_ID}\"\n private_key: \"${BQ_PRIVATE_KEY}\"\n client_email: # Your BQ client email, e.g. \"test@suppproject-id-1234567.iam.gserviceaccount.com\"\n client_id: # Your BQ client id, e.g.\"123456678890\"\n \n include_table_lineage: true\n profiling:\n enabled: true\n stateful_ingestion:\n enabled: true" }, { "urn": "urn:li:dataPlatform:redshift", "name": "redshift", "displayName": "Redshift", "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/redshift/", - "recipe": "source: \n type: redshift\n config:\n # Coordinates\n host_port: # Your Redshift host and post, e.g. example.something.us-west-2.redshift.amazonaws.com:5439\n database: # Your Redshift database, e.g. SampleDatabase\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n username: \"${REDSHIFT_USERNAME}\" # Your Redshift username, e.g. admin\n password: \"${REDSHIFT_PASSWORD}\" # Your Redshift password, e.g. password_01\n\n table_lineage_mode: stl_scan_based\n include_table_lineage: true\n include_view_lineage: true\n profiling:\n enabled: true\n stateful_ingestion:\n enabled: true" + "recipe": "source: \n type: redshift\n config:\n # Coordinates\n host_port: # Your Redshift host and post, e.g. example.something.us-west-2.redshift.amazonaws.com:5439\n database: # Your Redshift database, e.g. SampleDatabase\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n username: \"${REDSHIFT_USERNAME}\" # Your Redshift username, e.g. admin\n password: \"${REDSHIFT_PASSWORD}\" # Your Redshift password, e.g. password_01\n\n table_lineage_mode: stl_scan_based\n include_table_lineage: true\n profiling:\n enabled: true\n stateful_ingestion:\n enabled: true" }, { "urn": "urn:li:dataPlatform:snowflake",