Skip to content
This repository has been archived by the owner on Jan 27, 2025. It is now read-only.

Commit

Permalink
refactor(ui): Making improvements to UI ingestion forms, adding MySQL…
Browse files Browse the repository at this point in the history
…, Trino, Presto, MSSQL, MariaDB forms (datahub-project#6607)
  • Loading branch information
jjoyce0510 authored and cccs-Dustin committed Feb 1, 2023
1 parent 9fb2145 commit 65bdc36
Show file tree
Hide file tree
Showing 23 changed files with 798 additions and 568 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -14,13 +14,13 @@ export const LookerWarning = ({ type }: Props) => {
if (type === LOOKER) {
link = (
<a href={LOOKML_DOC_LINK} target="_blank" rel="noopener noreferrer">
DataHub lookml module
DataHub LookML Ingestion Source
</a>
);
} else if (type === LOOK_ML) {
link = (
<a href={LOOKER_DOC_LINK} target="_blank" rel="noopener noreferrer">
DataHub looker module
DataHub Looker Ingestion Source
</a>
);
}
Expand All @@ -32,8 +32,8 @@ export const LookerWarning = ({ type }: Props) => {
banner
message={
<>
To get complete Looker metadata integration (including Looker views and lineage to the underlying
warehouse tables), you must <b>also</b> use the {link}.
To complete the Looker integration (including Looker views and lineage to the underlying warehouse
tables), you must <b>also</b> use the {link}.
</>
}
/>
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import React from 'react';
import { Checkbox, DatePicker, Form, Input, Select, Tooltip, FormInstance } from 'antd';
import { Checkbox, DatePicker, Form, Input, Select, Tooltip } from 'antd';
import styled from 'styled-components/macro';
import Button from 'antd/lib/button';
import { MinusCircleOutlined, PlusOutlined } from '@ant-design/icons';
Expand Down Expand Up @@ -94,11 +94,11 @@ interface Props {
secrets: Secret[];
refetchSecrets: () => void;
removeMargin?: boolean;
form: FormInstance<any>;
updateFormValue: (field, value) => void;
}

function FormField(props: Props) {
const { field, secrets, refetchSecrets, removeMargin, form } = props;
const { field, secrets, refetchSecrets, removeMargin, updateFormValue } = props;

if (field.type === FieldType.LIST) return <ListField field={field} removeMargin={removeMargin} />;

Expand All @@ -113,7 +113,7 @@ function FormField(props: Props) {
secrets={secrets}
removeMargin={removeMargin}
refetchSecrets={refetchSecrets}
form={form}
updateFormValue={updateFormValue}
/>
);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,8 +60,9 @@ function getInitialValues(displayRecipe: string, allFields: any[]) {
}
if (recipeObj) {
allFields.forEach((field) => {
initialValues[field.name] =
field.getValueFromRecipeOverride?.(recipeObj) || get(recipeObj, field.fieldPath);
initialValues[field.name] = field.getValueFromRecipeOverride
? field.getValueFromRecipeOverride(recipeObj)
: get(recipeObj, field.fieldPath);
});
}

Expand Down Expand Up @@ -123,16 +124,21 @@ function RecipeForm(props: Props) {
Object.keys(changedValues).forEach((fieldName) => {
const recipeField = allFields.find((f) => f.name === fieldName);
if (recipeField) {
updatedValues =
recipeField.setValueOnRecipeOverride?.(updatedValues, allValues[fieldName]) ||
setFieldValueOnRecipe(updatedValues, allValues[fieldName], recipeField.fieldPath);
updatedValues = recipeField.setValueOnRecipeOverride
? recipeField.setValueOnRecipeOverride(updatedValues, allValues[fieldName])
: setFieldValueOnRecipe(updatedValues, allValues[fieldName], recipeField.fieldPath);
}
});

const stagedRecipe = jsonToYaml(JSON.stringify(updatedValues));
setStagedRecipe(stagedRecipe);
}

function updateFormValue(fieldName, fieldValue) {
updateFormValues({ [fieldName]: fieldValue }, { [fieldName]: fieldValue });
form.setFieldsValue({ [fieldName]: fieldValue });
}

return (
<Form
layout="vertical"
Expand All @@ -149,7 +155,7 @@ function RecipeForm(props: Props) {
secrets={secrets}
refetchSecrets={refetchSecrets}
removeMargin={i === fields.length - 1}
form={form}
updateFormValue={updateFormValue}
/>
))}
{CONNECTORS_WITH_TEST_CONNECTION.has(type as string) && (
Expand Down Expand Up @@ -187,7 +193,7 @@ function RecipeForm(props: Props) {
secrets={secrets}
refetchSecrets={refetchSecrets}
removeMargin={i === filterFields.length - 1}
form={form}
updateFormValue={updateFormValue}
/>
</MarginWrapper>
</>
Expand All @@ -213,7 +219,7 @@ function RecipeForm(props: Props) {
secrets={secrets}
refetchSecrets={refetchSecrets}
removeMargin={i === advancedFields.length - 1}
form={form}
updateFormValue={updateFormValue}
/>
))}
</Collapse.Panel>
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,11 +24,12 @@ const CreateButton = styled(Button)`
`;

interface Props {
initialState?: SecretBuilderState;
onSubmit?: (state: SecretBuilderState) => void;
refetchSecrets: () => void;
}

function CreateSecretButton({ onSubmit, refetchSecrets }: Props) {
function CreateSecretButton({ initialState, onSubmit, refetchSecrets }: Props) {
const [isCreateModalVisible, setIsCreateModalVisible] = useState(false);
const [createSecretMutation] = useCreateSecretMutation();

Expand Down Expand Up @@ -62,6 +63,7 @@ function CreateSecretButton({ onSubmit, refetchSecrets }: Props) {
</CreateButton>
{isCreateModalVisible && (
<SecretBuilderModal
initialState={initialState}
visible={isCreateModalVisible}
onCancel={() => setIsCreateModalVisible(false)}
onSubmit={createSecret}
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import React, { ReactNode } from 'react';
import { AutoComplete, Divider, Form, FormInstance } from 'antd';
import { AutoComplete, Divider, Form } from 'antd';
import styled from 'styled-components/macro';
import { Secret } from '../../../../../../types.generated';
import CreateSecretButton from './CreateSecretButton';
Expand Down Expand Up @@ -52,7 +52,7 @@ interface SecretFieldProps {
secrets: Secret[];
removeMargin?: boolean;
refetchSecrets: () => void;
form: FormInstance<any>;
updateFormValue: (field, value) => void;
}

function SecretFieldTooltip({ tooltipLabel }: { tooltipLabel?: string | ReactNode }) {
Expand Down Expand Up @@ -84,7 +84,7 @@ const encodeSecret = (secretName: string) => {
return `\${${secretName}}`;
};

function SecretField({ field, secrets, removeMargin, form, refetchSecrets }: SecretFieldProps) {
function SecretField({ field, secrets, removeMargin, updateFormValue, refetchSecrets }: SecretFieldProps) {
const options = secrets.map((secret) => ({ value: encodeSecret(secret.name), label: secret.name }));

return (
Expand All @@ -108,9 +108,7 @@ function SecretField({ field, secrets, removeMargin, form, refetchSecrets }: Sec
{menu}
<StyledDivider />
<CreateSecretButton
onSubmit={(state) =>
form.setFields([{ name: field.name, value: encodeSecret(state.name as string) }])
}
onSubmit={(state) => updateFormValue(field.name, encodeSecret(state.name as string))}
refetchSecrets={refetchSecrets}
/>
</>
Expand Down
113 changes: 14 additions & 99 deletions datahub-web-react/src/app/ingest/source/builder/RecipeForm/bigquery.ts
Original file line number Diff line number Diff line change
@@ -1,19 +1,20 @@
import { RecipeField, FieldType, setListValuesOnRecipe } from './common';
import { RecipeField, FieldType } from './common';

export const BIGQUERY_PROJECT_ID: RecipeField = {
name: 'project_id',
label: 'BigQuery Project ID',
label: 'Project ID',
tooltip: 'Project ID where you have rights to run queries and create tables.',
placeholder: 'my-project-123',
type: FieldType.TEXT,
fieldPath: 'source.config.project_id',
rules: null,
required: true,
};

export const BIGQUERY_CREDENTIAL_PROJECT_ID: RecipeField = {
name: 'credential.project_id',
label: 'Credentials Project ID',
tooltip: 'Project id to set the credentials.',
tooltip: "The Project ID, which can be found in your service account's JSON Key (project_id)",
placeholder: 'my-project-123',
type: FieldType.TEXT,
fieldPath: 'source.config.credential.project_id',
Expand All @@ -23,129 +24,43 @@ export const BIGQUERY_CREDENTIAL_PROJECT_ID: RecipeField = {
export const BIGQUERY_PRIVATE_KEY_ID: RecipeField = {
name: 'credential.private_key_id',
label: 'Private Key Id',
tooltip: 'Private key id.',
placeholder: 'BQ_PRIVATE_KEY_ID',
tooltip: "The Private Key id, which can be found in your service account's JSON Key (private_key_id)",
type: FieldType.SECRET,
fieldPath: 'source.config.credential.private_key_id',
placeholder: 'd0121d0000882411234e11166c6aaa23ed5d74e0',
rules: null,
required: true,
};

export const BIGQUERY_PRIVATE_KEY: RecipeField = {
name: 'credential.private_key',
label: 'Private Key',
placeholder: 'BQ_PRIVATE_KEY',
tooltip: 'Private key in a form of "-----BEGIN PRIVATE KEY-----\nprivate-key\n-----END PRIVATE KEY-----\n".',
tooltip: "The Private key, which can be found in your service account's JSON Key (private_key).",
placeholder: '-----BEGIN PRIVATE KEY-----....\n-----END PRIVATE KEY-----',
type: FieldType.SECRET,
fieldPath: 'source.config.credential.private_key',
rules: null,
required: true,
};

export const BIGQUERY_CLIENT_EMAIL: RecipeField = {
name: 'credential.client_email',
label: 'Client Email',
tooltip: 'Client email.',
tooltip: "The Client Email, which can be found in your service account's JSON Key (client_email).",
placeholder: '[email protected]',
type: FieldType.TEXT,
fieldPath: 'source.config.credential.client_email',
rules: null,
required: true,
};

export const BIGQUERY_CLIENT_ID: RecipeField = {
name: 'credential.client_id',
label: 'Client ID',
tooltip: 'Client ID.',
tooltip: "The Client ID, which can be found in your service account's JSON Key (client_id).",
placeholder: '123456789098765432101',
type: FieldType.TEXT,
fieldPath: 'source.config.credential.client_id',
rules: null,
};

const schemaAllowFieldPath = 'source.config.schema_pattern.allow';
export const BIGQUERY_SCHEMA_ALLOW: RecipeField = {
name: 'schema_pattern.allow',
label: 'Allow Patterns',
tooltip: 'Use regex here.',
placeholder: '^my_schema$',
type: FieldType.LIST,
buttonLabel: 'Add pattern',
fieldPath: schemaAllowFieldPath,
rules: null,
section: 'Schemas',
setValueOnRecipeOverride: (recipe: any, values: string[]) =>
setListValuesOnRecipe(recipe, values, schemaAllowFieldPath),
};

const schemaDenyFieldPath = 'source.config.schema_pattern.deny';
export const BIGQUERY_SCHEMA_DENY: RecipeField = {
name: 'schema_pattern.deny',
label: 'Deny Patterns',
tooltip: 'Use regex here.',
placeholder: '^my_schema$',
type: FieldType.LIST,
buttonLabel: 'Add pattern',
fieldPath: schemaDenyFieldPath,
rules: null,
section: 'Schemas',
setValueOnRecipeOverride: (recipe: any, values: string[]) =>
setListValuesOnRecipe(recipe, values, schemaDenyFieldPath),
};

const tableAllowFieldPath = 'source.config.table_pattern.allow';
export const BIGQUERY_TABLE_ALLOW: RecipeField = {
name: 'table_pattern.allow',
label: 'Allow Patterns',
tooltip: 'Use regex here.',
placeholder: '^my_schema\\.table_name$',
type: FieldType.LIST,
buttonLabel: 'Add pattern',
fieldPath: tableAllowFieldPath,
rules: null,
section: 'Tables',
setValueOnRecipeOverride: (recipe: any, values: string[]) =>
setListValuesOnRecipe(recipe, values, tableAllowFieldPath),
};

const tableDenyFieldPath = 'source.config.table_pattern.deny';
export const BIGQUERY_TABLE_DENY: RecipeField = {
name: 'table_pattern.deny',
label: 'Deny Patterns',
tooltip: 'Use regex here.',
placeholder: '^my_schema\\.table_name$',
type: FieldType.LIST,
buttonLabel: 'Add pattern',
fieldPath: tableDenyFieldPath,
rules: null,
section: 'Tables',
setValueOnRecipeOverride: (recipe: any, values: string[]) =>
setListValuesOnRecipe(recipe, values, tableDenyFieldPath),
};

const viewAllowFieldPath = 'source.config.view_pattern.allow';
export const BIGQUERY_VIEW_ALLOW: RecipeField = {
name: 'view_pattern.allow',
label: 'Allow Patterns',
tooltip: 'Use regex here.',
placeholder: '^my_schema\\.view_name$',
type: FieldType.LIST,
buttonLabel: 'Add pattern',
fieldPath: viewAllowFieldPath,
rules: null,
section: 'Views',
setValueOnRecipeOverride: (recipe: any, values: string[]) =>
setListValuesOnRecipe(recipe, values, viewAllowFieldPath),
};

const viewDenyFieldPath = 'source.config.view_pattern.deny';
export const BIGQUERY_VIEW_DENY: RecipeField = {
name: 'view_pattern.deny',
label: 'Deny Patterns',
tooltip: 'Use regex here.',
placeholder: '^my_schema\\.view_name$',
type: FieldType.LIST,
buttonLabel: 'Add pattern',
fieldPath: viewDenyFieldPath,
rules: null,
section: 'Views',
setValueOnRecipeOverride: (recipe: any, values: string[]) =>
setListValuesOnRecipe(recipe, values, viewDenyFieldPath),
required: true,
};
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,12 @@ import { FieldType, RecipeField, setListValuesOnRecipe } from './common';
export const BIGQUERY_BETA_PROJECT_ID: RecipeField = {
name: 'credential.project_id',
label: 'Project ID',
tooltip: 'Project id to set the credentials.',
tooltip: "The Project ID, which can be found in your service account's JSON Key (project_id)",
placeholder: 'my-project-123',
type: FieldType.TEXT,
fieldPath: 'source.config.credential.project_id',
rules: null,
required: true,
};

const projectIdAllowFieldPath = 'source.config.project_id_pattern.allow';
Expand Down
Loading

0 comments on commit 65bdc36

Please sign in to comment.