Skip to content

Commit

Permalink
fix(ui) Fix bigquery and redshift forms for lineage fields (datahub-p…
Browse files Browse the repository at this point in the history 
…roject#6215)
  • Loading branch information
@chriscollins3456 @cccs-tom
chriscollins3456 authored and cccs-tom committed Nov 18, 2022
1 parent 372ceef commit 2b81a3d
Show file tree
Hide file tree
Showing 3 changed files with 20 additions and 5 deletions.
9 changes: 9 additions & 0 deletions datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -255,6 +255,15 @@ export const INCLUDE_LINEAGE: RecipeField = {
},
};

export const INCLUDE_TABLE_LINEAGE: RecipeField = {
name: 'include_table_lineage',
label: 'Include Table Lineage',
tooltip: 'Whether or not table lineage should be ingested.',
type: FieldType.BOOLEAN,
fieldPath: 'source.config.include_table_lineage',
rules: null,
};

export const PROFILING_ENABLED: RecipeField = {
name: 'profiling.enabled',
label: 'Enable Profiling',
Expand Down
12 changes: 9 additions & 3 deletions datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import {
SKIP_PERSONAL_FOLDERS,
RecipeField,
START_TIME,
INCLUDE_TABLE_LINEAGE,
} from './common';
import {
SNOWFLAKE_ACCOUNT_ID,
Expand Down Expand Up @@ -137,7 +138,12 @@ export const RECIPE_FIELDS: RecipeFields = {
BIGQUERY_CLIENT_EMAIL,
BIGQUERY_CLIENT_ID,
],
advancedFields: [INCLUDE_LINEAGE, PROFILING_ENABLED, STATEFUL_INGESTION_ENABLED, UPSTREAM_LINEAGE_IN_REPORT],
advancedFields: [
INCLUDE_TABLE_LINEAGE,
PROFILING_ENABLED,
STATEFUL_INGESTION_ENABLED,
UPSTREAM_LINEAGE_IN_REPORT,
],
filterFields: [
BIGQUERY_SCHEMA_ALLOW,
BIGQUERY_SCHEMA_DENY,
Expand All @@ -157,7 +163,7 @@ export const RECIPE_FIELDS: RecipeFields = {
BIGQUERY_CLIENT_EMAIL,
BIGQUERY_CLIENT_ID,
],
advancedFields: [INCLUDE_LINEAGE, PROFILING_ENABLED, STATEFUL_INGESTION_ENABLED, START_TIME],
advancedFields: [INCLUDE_TABLE_LINEAGE, PROFILING_ENABLED, STATEFUL_INGESTION_ENABLED, START_TIME],
filterFields: [
PROJECT_ALLOW,
PROJECT_DENY,
Expand All @@ -173,7 +179,7 @@ export const RECIPE_FIELDS: RecipeFields = {
},
[REDSHIFT]: {
fields: [REDSHIFT_HOST_PORT, REDSHIFT_DATABASE, REDSHIFT_USERNAME, REDSHIFT_PASSWORD],
advancedFields: [INCLUDE_LINEAGE, PROFILING_ENABLED, STATEFUL_INGESTION_ENABLED, TABLE_LINEAGE_MODE],
advancedFields: [INCLUDE_TABLE_LINEAGE, PROFILING_ENABLED, STATEFUL_INGESTION_ENABLED, TABLE_LINEAGE_MODE],
filterFields: [
REDSHIFT_SCHEMA_ALLOW,
REDSHIFT_SCHEMA_DENY,
Expand Down
4 changes: 2 additions & 2 deletions datahub-web-react/src/app/ingest/source/builder/sources.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,14 @@
"name": "bigquery",
"displayName": "BigQuery",
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/bigquery/",
"recipe": "source:\n type: bigquery\n config:\n # Coordinates\n project_id: # Your BigQuery project id, e.g. sample_project_id\n # Credentials\n credential:\n project_id: # Your BQ project id, e.g. sample_project_id\n private_key_id: \"${BQ_PRIVATE_KEY_ID}\"\n private_key: \"${BQ_PRIVATE_KEY}\"\n client_email: # Your BQ client email, e.g. \"[email protected]\"\n client_id: # Your BQ client id, e.g.\"123456678890\"\n \n include_table_lineage: true\n include_view_lineage: true\n profiling:\n enabled: true\n stateful_ingestion:\n enabled: true"
"recipe": "source:\n type: bigquery\n config:\n # Coordinates\n project_id: # Your BigQuery project id, e.g. sample_project_id\n # Credentials\n credential:\n project_id: # Your BQ project id, e.g. sample_project_id\n private_key_id: \"${BQ_PRIVATE_KEY_ID}\"\n private_key: \"${BQ_PRIVATE_KEY}\"\n client_email: # Your BQ client email, e.g. \"[email protected]\"\n client_id: # Your BQ client id, e.g.\"123456678890\"\n \n include_table_lineage: true\n profiling:\n enabled: true\n stateful_ingestion:\n enabled: true"
},
{
"urn": "urn:li:dataPlatform:redshift",
"name": "redshift",
"displayName": "Redshift",
"docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/redshift/",
"recipe": "source: \n type: redshift\n config:\n # Coordinates\n host_port: # Your Redshift host and post, e.g. example.something.us-west-2.redshift.amazonaws.com:5439\n database: # Your Redshift database, e.g. SampleDatabase\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n username: \"${REDSHIFT_USERNAME}\" # Your Redshift username, e.g. admin\n password: \"${REDSHIFT_PASSWORD}\" # Your Redshift password, e.g. password_01\n\n table_lineage_mode: stl_scan_based\n include_table_lineage: true\n include_view_lineage: true\n profiling:\n enabled: true\n stateful_ingestion:\n enabled: true"
"recipe": "source: \n type: redshift\n config:\n # Coordinates\n host_port: # Your Redshift host and post, e.g. example.something.us-west-2.redshift.amazonaws.com:5439\n database: # Your Redshift database, e.g. SampleDatabase\n\n # Credentials\n # Add secret in Secrets Tab with relevant names for each variable\n username: \"${REDSHIFT_USERNAME}\" # Your Redshift username, e.g. admin\n password: \"${REDSHIFT_PASSWORD}\" # Your Redshift password, e.g. password_01\n\n table_lineage_mode: stl_scan_based\n include_table_lineage: true\n profiling:\n enabled: true\n stateful_ingestion:\n enabled: true"
},
{
"urn": "urn:li:dataPlatform:snowflake",
Expand Down

0 comments on commit 2b81a3d

Please sign in to comment.