feat(ingestion-ui) Add new form for the bigquery-beta connector (data…

…hub-project#6200)
CybercentreCanada · Nov 18, 2022 · a808b2e · a808b2e
1 parent d1e3355
commit a808b2e
Show file tree

Hide file tree

Showing 5 changed files with 161 additions and 9 deletions.
diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/FormField.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/FormField.tsx
@@ -1,5 +1,5 @@
 import React from 'react';
-import { Checkbox, Form, Input, Select, Tooltip } from 'antd';
+import { Checkbox, DatePicker, Form, Input, Select, Tooltip } from 'antd';
 import styled from 'styled-components/macro';
 import Button from 'antd/lib/button';
 import { MinusCircleOutlined, PlusOutlined } from '@ant-design/icons';
@@ -19,17 +19,12 @@ const StyledRemoveIcon = styled(MinusCircleOutlined)`
     margin-left: 10px;
 `;
 
-interface ListFieldProps {
+interface CommonFieldProps {
     field: RecipeField;
     removeMargin?: boolean;
 }
 
-interface SelectFieldProps {
-    field: RecipeField;
-    removeMargin?: boolean;
-}
-
-function ListField({ field, removeMargin }: ListFieldProps) {
+function ListField({ field, removeMargin }: CommonFieldProps) {
     return (
         <Form.List name={field.name} rules={field.rules || undefined}>
             {(fields, { add, remove }, { errors }) => (
@@ -58,7 +53,7 @@ function ListField({ field, removeMargin }: ListFieldProps) {
     );
 }
 
-function SelectField({ field, removeMargin }: SelectFieldProps) {
+function SelectField({ field, removeMargin }: CommonFieldProps) {
     return (
         <StyledFormItem
             name={field.name}
@@ -78,6 +73,20 @@ function SelectField({ field, removeMargin }: SelectFieldProps) {
     );
 }
 
+function DateField({ field, removeMargin }: CommonFieldProps) {
+    return (
+        <StyledFormItem
+            name={field.name}
+            label={field.label}
+            tooltip={field.tooltip}
+            removeMargin={!!removeMargin}
+            rules={field.rules || undefined}
+        >
+            <DatePicker showTime />
+        </StyledFormItem>
+    );
+}
+
 interface Props {
     field: RecipeField;
     secrets: Secret[];
@@ -92,6 +101,8 @@ function FormField(props: Props) {
 
     if (field.type === FieldType.SELECT) return <SelectField field={field} removeMargin={removeMargin} />;
 
+    if (field.type === FieldType.DATE) return <DateField field={field} removeMargin={removeMargin} />;
+
     if (field.type === FieldType.SECRET)
         return (
             <SecretField field={field} secrets={secrets} removeMargin={removeMargin} refetchSecrets={refetchSecrets} />

diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/bigquery.ts b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/bigquery.ts
@@ -4,6 +4,7 @@ export const BIGQUERY_PROJECT_ID: RecipeField = {
     name: 'project_id',
     label: 'BigQuery Project ID',
     tooltip: 'Project ID where you have rights to run queries and create tables.',
+    placeholder: 'my-project-123',
     type: FieldType.TEXT,
     fieldPath: 'source.config.project_id',
     rules: null,
@@ -13,6 +14,7 @@ export const BIGQUERY_CREDENTIAL_PROJECT_ID: RecipeField = {
     name: 'credential.project_id',
     label: 'Credentials Project ID',
     tooltip: 'Project id to set the credentials.',
+    placeholder: 'my-project-123',
     type: FieldType.TEXT,
     fieldPath: 'source.config.credential.project_id',
     rules: null,
@@ -22,6 +24,7 @@ export const BIGQUERY_PRIVATE_KEY_ID: RecipeField = {
     name: 'credential.private_key_id',
     label: 'Private Key Id',
     tooltip: 'Private key id.',
+    placeholder: 'BQ_PRIVATE_KEY_ID',
     type: FieldType.SECRET,
     fieldPath: 'source.config.credential.private_key_id',
     rules: null,
@@ -30,6 +33,7 @@ export const BIGQUERY_PRIVATE_KEY_ID: RecipeField = {
 export const BIGQUERY_PRIVATE_KEY: RecipeField = {
     name: 'credential.private_key',
     label: 'Private Key',
+    placeholder: 'BQ_PRIVATE_KEY',
     tooltip: 'Private key in a form of "-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n".',
     type: FieldType.SECRET,
     fieldPath: 'source.config.credential.private_key',
@@ -40,6 +44,7 @@ export const BIGQUERY_CLIENT_EMAIL: RecipeField = {
     name: 'credential.client_email',
     label: 'Client Email',
     tooltip: 'Client email.',
+    placeholder: '[email protected]',
     type: FieldType.TEXT,
     fieldPath: 'source.config.credential.client_email',
     rules: null,
@@ -49,6 +54,7 @@ export const BIGQUERY_CLIENT_ID: RecipeField = {
     name: 'credential.client_id',
     label: 'Client ID',
     tooltip: 'Client ID.',
+    placeholder: '123456789098765432101',
     type: FieldType.TEXT,
     fieldPath: 'source.config.credential.client_id',
     rules: null,

diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/bigqueryBeta.ts b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/bigqueryBeta.ts
@@ -0,0 +1,71 @@
+import { FieldType, RecipeField, setListValuesOnRecipe } from './common';
+
+export const BIGQUERY_BETA_PROJECT_ID: RecipeField = {
+    name: 'credential.project_id',
+    label: 'Project ID',
+    tooltip: 'Project id to set the credentials.',
+    placeholder: 'my-project-123',
+    type: FieldType.TEXT,
+    fieldPath: 'source.config.credential.project_id',
+    rules: null,
+};
+
+const projectIdAllowFieldPath = 'source.config.project_id_pattern.allow';
+export const PROJECT_ALLOW: RecipeField = {
+    name: 'project_id_pattern.allow',
+    label: 'Allow Patterns',
+    tooltip: 'Use regex here to filter for project IDs.',
+    placeholder: '^my_db$',
+    type: FieldType.LIST,
+    buttonLabel: 'Add pattern',
+    fieldPath: projectIdAllowFieldPath,
+    rules: null,
+    section: 'Projects',
+    setValueOnRecipeOverride: (recipe: any, values: string[]) =>
+        setListValuesOnRecipe(recipe, values, projectIdAllowFieldPath),
+};
+
+const projectIdDenyFieldPath = 'source.config.project_id_pattern.deny';
+export const PROJECT_DENY: RecipeField = {
+    name: 'project_id_pattern.deny',
+    label: 'Deny Patterns',
+    tooltip: 'Use regex here to filter for project IDs.',
+    placeholder: '^my_db$',
+    type: FieldType.LIST,
+    buttonLabel: 'Add pattern',
+    fieldPath: projectIdDenyFieldPath,
+    rules: null,
+    section: 'Projects',
+    setValueOnRecipeOverride: (recipe: any, values: string[]) =>
+        setListValuesOnRecipe(recipe, values, projectIdDenyFieldPath),
+};
+
+const datasetAllowFieldPath = 'source.config.dataset_pattern.allow';
+export const DATASET_ALLOW: RecipeField = {
+    name: 'dataset_pattern.allow',
+    label: 'Allow Patterns',
+    tooltip: 'Use regex here.',
+    placeholder: '^my_db$',
+    type: FieldType.LIST,
+    buttonLabel: 'Add pattern',
+    fieldPath: datasetAllowFieldPath,
+    rules: null,
+    section: 'Datasets',
+    setValueOnRecipeOverride: (recipe: any, values: string[]) =>
+        setListValuesOnRecipe(recipe, values, datasetAllowFieldPath),
+};
+
+const datasetDenyFieldPath = 'source.config.dataset_pattern.deny';
+export const DATASET_DENY: RecipeField = {
+    name: 'dataset_pattern.deny',
+    label: 'Deny Patterns',
+    tooltip: 'Use regex here.',
+    placeholder: '^my_db$',
+    type: FieldType.LIST,
+    buttonLabel: 'Add pattern',
+    fieldPath: datasetDenyFieldPath,
+    rules: null,
+    section: 'Datasets',
+    setValueOnRecipeOverride: (recipe: any, values: string[]) =>
+        setListValuesOnRecipe(recipe, values, datasetDenyFieldPath),
+};
diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/common.tsx
@@ -1,5 +1,6 @@
 import React from 'react';
 import { set, get } from 'lodash';
+import moment, { Moment } from 'moment-timezone';
 
 export enum FieldType {
     TEXT,
@@ -9,6 +10,7 @@ export enum FieldType {
     SECRET,
     DICT,
     TEXTAREA,
+    DATE,
 }
 
 interface Option {
@@ -74,6 +76,23 @@ export function setListValuesOnRecipe(recipe: any, values: string[] | undefined,
     return updatedRecipe;
 }
 
+const NUM_CHARACTERS_TO_REMOVE_FROM_DATE = 5;
+
+export function setDateValueOnRecipe(recipe: any, value: Moment | undefined, fieldPath: string) {
+    const updatedRecipe = { ...recipe };
+    if (value !== undefined) {
+        if (!value) {
+            return setFieldValueOnRecipe(updatedRecipe, null, fieldPath);
+        }
+        const isoDateString = value.toISOString();
+        const formattedDateString = isoDateString
+            .substring(0, isoDateString.length - NUM_CHARACTERS_TO_REMOVE_FROM_DATE)
+            .concat('Z');
+        return setFieldValueOnRecipe(updatedRecipe, formattedDateString, fieldPath);
+    }
+    return updatedRecipe;
+}
+
 /* ---------------------------------------------------- Filter Section ---------------------------------------------------- */
 const databaseAllowFieldPath = 'source.config.database_pattern.allow';
 export const DATABASE_ALLOW: RecipeField = {
@@ -357,3 +376,23 @@ export const SKIP_PERSONAL_FOLDERS: RecipeField = {
     fieldPath: 'source.config.skip_personal_folders',
     rules: null,
 };
+
+const startTimeFieldPath = 'source.config.start_time';
+export const START_TIME: RecipeField = {
+    name: 'start_time',
+    label: 'Start Time',
+    tooltip:
+        'Earliest date of audit logs to process for usage, lineage etc. Default: Last full day in UTC or last time DataHub ingested usage (if stateful ingestion is enabled). Tip: Set this to an older date (e.g. 1 month ago) for your first ingestion run, and then uncheck it for future runs.',
+    placeholder: 'Select date and time',
+    type: FieldType.DATE,
+    fieldPath: startTimeFieldPath,
+    rules: null,
+    getValueFromRecipeOverride: (recipe: any) => {
+        const isoDateString = get(recipe, startTimeFieldPath);
+        if (isoDateString) {
+            return moment(isoDateString);
+        }
+        return isoDateString;
+    },
+    setValueOnRecipeOverride: (recipe: any, value?: Moment) => setDateValueOnRecipe(recipe, value, startTimeFieldPath),
+};
diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts
@@ -21,6 +21,7 @@ import {
     EXTRACT_OWNERS,
     SKIP_PERSONAL_FOLDERS,
     RecipeField,
+    START_TIME,
 } from './common';
 import {
     SNOWFLAKE_ACCOUNT_ID,
@@ -89,6 +90,8 @@ import {
     PARSE_TABLE_NAMES_FROM_SQL,
     PROJECT_NAME,
 } from './lookml';
+import { BIGQUERY_BETA } from '../constants';
+import { BIGQUERY_BETA_PROJECT_ID, DATASET_ALLOW, DATASET_DENY, PROJECT_ALLOW, PROJECT_DENY } from './bigqueryBeta';
 
 export enum RecipeSections {
     Connection = 0,
@@ -146,6 +149,28 @@ export const RECIPE_FIELDS: RecipeFields = {
         filterSectionTooltip:
             'Filter out data assets based on allow/deny regex patterns we match against. Deny patterns take precedence over allow patterns.',
     },
+    [BIGQUERY_BETA]: {
+        fields: [
+            BIGQUERY_BETA_PROJECT_ID,
+            BIGQUERY_PRIVATE_KEY,
+            BIGQUERY_PRIVATE_KEY_ID,
+            BIGQUERY_CLIENT_EMAIL,
+            BIGQUERY_CLIENT_ID,
+        ],
+        advancedFields: [INCLUDE_LINEAGE, PROFILING_ENABLED, STATEFUL_INGESTION_ENABLED, START_TIME],
+        filterFields: [
+            PROJECT_ALLOW,
+            PROJECT_DENY,
+            DATASET_ALLOW,
+            DATASET_DENY,
+            BIGQUERY_TABLE_ALLOW,
+            BIGQUERY_TABLE_DENY,
+            BIGQUERY_VIEW_ALLOW,
+            BIGQUERY_VIEW_DENY,
+        ],
+        filterSectionTooltip:
+            'Filter out data assets based on allow/deny regex patterns we match against. Deny patterns take precedence over allow patterns.',
+    },
     [REDSHIFT]: {
         fields: [REDSHIFT_HOST_PORT, REDSHIFT_DATABASE, REDSHIFT_USERNAME, REDSHIFT_PASSWORD],
         advancedFields: [INCLUDE_LINEAGE, PROFILING_ENABLED, STATEFUL_INGESTION_ENABLED, TABLE_LINEAGE_MODE],