From 537b527f59063bec8734d629f19f02a9824b9502 Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Thu, 13 Oct 2022 14:46:55 -0400 Subject: [PATCH 1/2] feat(ingestion-ui) Add new form for the bigquery-beta connector --- .../source/builder/RecipeForm/FormField.tsx | 29 +++++--- .../source/builder/RecipeForm/bigquery.ts | 6 ++ .../source/builder/RecipeForm/bigqueryBeta.ts | 71 +++++++++++++++++++ .../source/builder/RecipeForm/common.tsx | 33 +++++++++ .../source/builder/RecipeForm/constants.ts | 25 +++++++ 5 files changed, 155 insertions(+), 9 deletions(-) create mode 100644 datahub-web-react/src/app/ingest/source/builder/RecipeForm/bigqueryBeta.ts diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/FormField.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/FormField.tsx index faa4ebd3fc65cd..2cf7ee70d19a72 100644 --- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/FormField.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/FormField.tsx @@ -1,5 +1,5 @@ import React from 'react'; -import { Checkbox, Form, Input, Select, Tooltip } from 'antd'; +import { Checkbox, DatePicker, Form, Input, Select, Tooltip } from 'antd'; import styled from 'styled-components/macro'; import Button from 'antd/lib/button'; import { MinusCircleOutlined, PlusOutlined } from '@ant-design/icons'; @@ -19,17 +19,12 @@ const StyledRemoveIcon = styled(MinusCircleOutlined)` margin-left: 10px; `; -interface ListFieldProps { +interface CommonFieldProps { field: RecipeField; removeMargin?: boolean; } -interface SelectFieldProps { - field: RecipeField; - removeMargin?: boolean; -} - -function ListField({ field, removeMargin }: ListFieldProps) { +function ListField({ field, removeMargin }: CommonFieldProps) { return ( {(fields, { add, remove }, { errors }) => ( @@ -58,7 +53,7 @@ function ListField({ field, removeMargin }: ListFieldProps) { ); } -function SelectField({ field, removeMargin }: SelectFieldProps) { +function SelectField({ field, removeMargin }: CommonFieldProps) { return ( + + + ); +} + interface Props { field: RecipeField; secrets: Secret[]; @@ -92,6 +101,8 @@ function FormField(props: Props) { if (field.type === FieldType.SELECT) return ; + if (field.type === FieldType.DATE) return ; + if (field.type === FieldType.SECRET) return ( diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/bigquery.ts b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/bigquery.ts index 6aaef70f4a71bb..19811034a1f522 100644 --- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/bigquery.ts +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/bigquery.ts @@ -4,6 +4,7 @@ export const BIGQUERY_PROJECT_ID: RecipeField = { name: 'project_id', label: 'BigQuery Project ID', tooltip: 'Project ID where you have rights to run queries and create tables.', + placeholder: 'my-project-123', type: FieldType.TEXT, fieldPath: 'source.config.project_id', rules: null, @@ -13,6 +14,7 @@ export const BIGQUERY_CREDENTIAL_PROJECT_ID: RecipeField = { name: 'credential.project_id', label: 'Credentials Project ID', tooltip: 'Project id to set the credentials.', + placeholder: 'my-project-123', type: FieldType.TEXT, fieldPath: 'source.config.credential.project_id', rules: null, @@ -22,6 +24,7 @@ export const BIGQUERY_PRIVATE_KEY_ID: RecipeField = { name: 'credential.private_key_id', label: 'Private Key Id', tooltip: 'Private key id.', + placeholder: 'BQ_PRIVATE_KEY_ID', type: FieldType.SECRET, fieldPath: 'source.config.credential.private_key_id', rules: null, @@ -30,6 +33,7 @@ export const BIGQUERY_PRIVATE_KEY_ID: RecipeField = { export const BIGQUERY_PRIVATE_KEY: RecipeField = { name: 'credential.private_key', label: 'Private Key', + placeholder: 'BQ_PRIVATE_KEY', tooltip: 'Private key in a form of "-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n".', type: FieldType.SECRET, fieldPath: 'source.config.credential.private_key', @@ -40,6 +44,7 @@ export const BIGQUERY_CLIENT_EMAIL: RecipeField = { name: 'credential.client_email', label: 'Client Email', tooltip: 'Client email.', + placeholder: 'client_email@gmail.com', type: FieldType.TEXT, fieldPath: 'source.config.credential.client_email', rules: null, @@ -49,6 +54,7 @@ export const BIGQUERY_CLIENT_ID: RecipeField = { name: 'credential.client_id', label: 'Client ID', tooltip: 'Client ID.', + placeholder: '123456789098765432101', type: FieldType.TEXT, fieldPath: 'source.config.credential.client_id', rules: null, diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/bigqueryBeta.ts b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/bigqueryBeta.ts new file mode 100644 index 00000000000000..24e2393830c37f --- /dev/null +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/bigqueryBeta.ts @@ -0,0 +1,71 @@ +import { FieldType, RecipeField, setListValuesOnRecipe } from './common'; + +export const BIGQUERY_BETA_PROJECT_ID: RecipeField = { + name: 'credential.project_id', + label: 'Project ID', + tooltip: 'Project id to set the credentials.', + placeholder: 'my-project-123', + type: FieldType.TEXT, + fieldPath: 'source.config.credential.project_id', + rules: null, +}; + +const projectIdAllowFieldPath = 'source.config.project_id_pattern.allow'; +export const PROJECT_ALLOW: RecipeField = { + name: 'project_id_pattern.allow', + label: 'Allow Patterns', + tooltip: 'Use regex here to filter for project IDs.', + placeholder: '^my_db$', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: projectIdAllowFieldPath, + rules: null, + section: 'Projects', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, projectIdAllowFieldPath), +}; + +const projectIdDenyFieldPath = 'source.config.project_id_pattern.deny'; +export const PROJECT_DENY: RecipeField = { + name: 'project_id_pattern.deny', + label: 'Deny Patterns', + tooltip: 'Use regex here to filter for project IDs.', + placeholder: '^my_db$', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: projectIdDenyFieldPath, + rules: null, + section: 'Projects', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, projectIdDenyFieldPath), +}; + +const datasetAllowFieldPath = 'source.config.dataset_pattern.allow'; +export const DATASET_ALLOW: RecipeField = { + name: 'dataset_pattern.allow', + label: 'Allow Patterns', + tooltip: 'Use regex here.', + placeholder: '^my_db$', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: datasetAllowFieldPath, + rules: null, + section: 'Datasets', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, datasetAllowFieldPath), +}; + +const datasetDenyFieldPath = 'source.config.dataset_pattern.deny'; +export const DATASET_DENY: RecipeField = { + name: 'dataset_pattern.deny', + label: 'Deny Patterns', + tooltip: 'Use regex here.', + placeholder: '^my_db$', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: datasetDenyFieldPath, + rules: null, + section: 'Datasets', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, datasetDenyFieldPath), +}; diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx index ca04b8fbc2c7a0..b8833f0a813686 100644 --- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx @@ -1,5 +1,6 @@ import React from 'react'; import { set, get } from 'lodash'; +import moment, { Moment } from 'moment-timezone'; export enum FieldType { TEXT, @@ -9,6 +10,7 @@ export enum FieldType { SECRET, DICT, TEXTAREA, + DATE, } interface Option { @@ -357,3 +359,34 @@ export const SKIP_PERSONAL_FOLDERS: RecipeField = { fieldPath: 'source.config.skip_personal_folders', rules: null, }; + +const NUM_CHARACTERS_TO_REMOVE_FROM_DATE = 5; + +const startTimeFieldPath = 'source.config.start_time'; +export const START_TIME: RecipeField = { + name: 'start_time', + label: 'Start Time', + tooltip: + 'Earliest date of audit logs to process for usage, lineage etc. Default: Last full day in UTC or last time DataHub ingested usage (if stateful ingestion is enabled). Tip: Set this to an older date (e.g. 1 month ago) for your first ingestion run, and then uncheck it for future runs.', + placeholder: 'Select date and time', + type: FieldType.DATE, + fieldPath: startTimeFieldPath, + rules: null, + getValueFromRecipeOverride: (recipe: any) => { + const isoDateString = get(recipe, startTimeFieldPath); + if (isoDateString) { + return moment(isoDateString); + } + return isoDateString; + }, + setValueOnRecipeOverride: (recipe: any, value?: Moment) => { + if (!value) { + return setFieldValueOnRecipe(recipe, null, startTimeFieldPath); + } + const isoDateString = value.toISOString(); + const formattedDateString = isoDateString + .substring(0, isoDateString.length - NUM_CHARACTERS_TO_REMOVE_FROM_DATE) + .concat('Z'); + return setFieldValueOnRecipe(recipe, formattedDateString, startTimeFieldPath); + }, +}; diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts index 3ab6dff12fbae6..630416f3e64108 100644 --- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts @@ -21,6 +21,7 @@ import { EXTRACT_OWNERS, SKIP_PERSONAL_FOLDERS, RecipeField, + START_TIME, } from './common'; import { SNOWFLAKE_ACCOUNT_ID, @@ -89,6 +90,8 @@ import { PARSE_TABLE_NAMES_FROM_SQL, PROJECT_NAME, } from './lookml'; +import { BIGQUERY_BETA } from '../constants'; +import { BIGQUERY_BETA_PROJECT_ID, DATASET_ALLOW, DATASET_DENY, PROJECT_ALLOW, PROJECT_DENY } from './bigqueryBeta'; export enum RecipeSections { Connection = 0, @@ -146,6 +149,28 @@ export const RECIPE_FIELDS: RecipeFields = { filterSectionTooltip: 'Filter out data assets based on allow/deny regex patterns we match against. Deny patterns take precedence over allow patterns.', }, + [BIGQUERY_BETA]: { + fields: [ + BIGQUERY_BETA_PROJECT_ID, + BIGQUERY_PRIVATE_KEY, + BIGQUERY_PRIVATE_KEY_ID, + BIGQUERY_CLIENT_EMAIL, + BIGQUERY_CLIENT_ID, + ], + advancedFields: [INCLUDE_LINEAGE, PROFILING_ENABLED, STATEFUL_INGESTION_ENABLED, START_TIME], + filterFields: [ + PROJECT_ALLOW, + PROJECT_DENY, + DATASET_ALLOW, + DATASET_DENY, + BIGQUERY_TABLE_ALLOW, + BIGQUERY_TABLE_DENY, + BIGQUERY_VIEW_ALLOW, + BIGQUERY_VIEW_DENY, + ], + filterSectionTooltip: + 'Filter out data assets based on allow/deny regex patterns we match against. Deny patterns take precedence over allow patterns.', + }, [REDSHIFT]: { fields: [REDSHIFT_HOST_PORT, REDSHIFT_DATABASE, REDSHIFT_USERNAME, REDSHIFT_PASSWORD], advancedFields: [INCLUDE_LINEAGE, PROFILING_ENABLED, STATEFUL_INGESTION_ENABLED, TABLE_LINEAGE_MODE], From 9186cdd956695c9c35a253525756522b7671acfe Mon Sep 17 00:00:00 2001 From: Chris Collins Date: Thu, 13 Oct 2022 16:00:26 -0400 Subject: [PATCH 2/2] move update date values function to be a shared function --- .../source/builder/RecipeForm/common.tsx | 30 +++++++++++-------- 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx index b8833f0a813686..1136101da2409c 100644 --- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx @@ -76,6 +76,23 @@ export function setListValuesOnRecipe(recipe: any, values: string[] | undefined, return updatedRecipe; } +const NUM_CHARACTERS_TO_REMOVE_FROM_DATE = 5; + +export function setDateValueOnRecipe(recipe: any, value: Moment | undefined, fieldPath: string) { + const updatedRecipe = { ...recipe }; + if (value !== undefined) { + if (!value) { + return setFieldValueOnRecipe(updatedRecipe, null, fieldPath); + } + const isoDateString = value.toISOString(); + const formattedDateString = isoDateString + .substring(0, isoDateString.length - NUM_CHARACTERS_TO_REMOVE_FROM_DATE) + .concat('Z'); + return setFieldValueOnRecipe(updatedRecipe, formattedDateString, fieldPath); + } + return updatedRecipe; +} + /* ---------------------------------------------------- Filter Section ---------------------------------------------------- */ const databaseAllowFieldPath = 'source.config.database_pattern.allow'; export const DATABASE_ALLOW: RecipeField = { @@ -360,8 +377,6 @@ export const SKIP_PERSONAL_FOLDERS: RecipeField = { rules: null, }; -const NUM_CHARACTERS_TO_REMOVE_FROM_DATE = 5; - const startTimeFieldPath = 'source.config.start_time'; export const START_TIME: RecipeField = { name: 'start_time', @@ -379,14 +394,5 @@ export const START_TIME: RecipeField = { } return isoDateString; }, - setValueOnRecipeOverride: (recipe: any, value?: Moment) => { - if (!value) { - return setFieldValueOnRecipe(recipe, null, startTimeFieldPath); - } - const isoDateString = value.toISOString(); - const formattedDateString = isoDateString - .substring(0, isoDateString.length - NUM_CHARACTERS_TO_REMOVE_FROM_DATE) - .concat('Z'); - return setFieldValueOnRecipe(recipe, formattedDateString, startTimeFieldPath); - }, + setValueOnRecipeOverride: (recipe: any, value?: Moment) => setDateValueOnRecipe(recipe, value, startTimeFieldPath), };