feat(ingest/vertica): Adding Vertica as source in Datahub UI (#7010)

Co-authored-by: Vishal <[email protected]> Co-authored-by: VISHAL KUMAR <[email protected]> Co-authored-by: John Joyce <[email protected]>
datahub-project · Jan 13, 2023 · cd9fc26 · cd9fc26
1 parent b238272
commit cd9fc26
Show file tree

Hide file tree

Showing 7 changed files with 39 additions and 4 deletions.
diff --git a/datahub-web-react/src/app/ingest/source/builder/constants.ts b/datahub-web-react/src/app/ingest/source/builder/constants.ts
@@ -26,6 +26,7 @@ import metabaseLogo from '../../../../images/metabaselogo.png';
 import powerbiLogo from '../../../../images/powerbilogo.png';
 import modeLogo from '../../../../images/modelogo.png';
 import databricksLogo from '../../../../images/databrickslogo.png';
+import verticaLogo from '../../../../images/verticalogo.png';
 
 export const ATHENA = 'athena';
 export const ATHENA_URN = `urn:li:dataPlatform:${ATHENA}`;
@@ -96,6 +97,8 @@ export const UNITY_CATALOG = 'unity-catalog';
 export const UNITY_CATALOG_URN = `urn:li:dataPlatform:${UNITY_CATALOG}`;
 export const DBT_CLOUD = 'dbt-cloud';
 export const DBT_CLOUD_URN = `urn:li:dataPlatform:dbt`;
+export const VERTICA = 'vertica';
+export const VERTICA_URN = `urn:li:dataPlatform:${VERTICA}`;
 
 export const PLATFORM_URN_TO_LOGO = {
     [ATHENA_URN]: athenaLogo,
@@ -126,6 +129,7 @@ export const PLATFORM_URN_TO_LOGO = {
     [TRINO_URN]: trinoLogo,
     [SUPERSET_URN]: supersetLogo,
     [UNITY_CATALOG_URN]: databricksLogo,
+    [VERTICA_URN]: verticaLogo,
 };
 
 export const SOURCE_TO_PLATFORM_URN = {
@@ -139,4 +143,5 @@ export const SOURCE_TO_PLATFORM_URN = {
     [SNOWFLAKE_USAGE]: SNOWFLAKE_URN,
     [STARBURST_TRINO_USAGE]: TRINO_URN,
     [DBT_CLOUD]: DBT_URN,
+    [VERTICA]: VERTICA_URN,
 };
diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json
@@ -195,11 +195,18 @@
         "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/okta/",
         "recipe": "source:\n    type: okta\n    config:\n        # Coordinates\n        okta_domain: # Your Okta Domain, e.g. \"dev-35531955.okta.com\"\n\n        # Credentials\n        # Add secret in Secrets Tab with relevant names for each variable\n        okta_api_token: \"${OKTA_API_TOKEN}\" # Your Okta API Token, e.g. \"11be4R_M2MzDqXawbTHfKGpKee0kuEOfX1RCQSRx99\"\n\n        # Optional flags to ingest users, groups, or both\n        ingest_users: True\n        ingest_groups: True\n\n        # Optional: Customize the mapping to DataHub Username from an attribute appearing in the Okta User\n        # profile. Reference: https://developer.okta.com/docs/reference/api/users/\n        # okta_profile_to_username_attr: str = \"login\"\n        # okta_profile_to_username_regex: str = \"([^@]+)\"\n    \n        # Optional: Customize the mapping to DataHub Group from an attribute appearing in the Okta Group\n        # profile. Reference: https://developer.okta.com/docs/reference/api/groups/\n        # okta_profile_to_group_name_attr: str = \"name\"\n        # okta_profile_to_group_name_regex: str = \"(.*)\"\n        \n        # Optional: Include deprovisioned or suspended Okta users in the ingestion.\n        # include_deprovisioned_users = False\n        # include_suspended_users = False"
     },
+    {
+        "urn": "urn:li:dataPlatform:vertica",
+        "name": "vertica",
+        "displayName": "Vertica",
+        "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/vertica/",
+        "recipe": "source:\n    type: vertica\n    config:\n        # Coordinates\n        host_port: localhost:5433\n        # The name of the vertica database\n        database: Vmart\n        # Credentials\n        username: dbadmin\n    password:null\n    include_tables: true\n        include_views: true\n    include_projections: true\n    include_oauth: true\n   include_models: true\n        include_view_lineage: true\n      include_projection_lineage: true\n    profiling:\n            enabled: true\n        stateful_ingestion:\n            enabled: true  "
+    },
     {
         "urn": "urn:li:dataPlatform:custom",
         "name": "custom",
         "displayName": "Other",
         "docsUrl": "https://datahubproject.io/docs/metadata-ingestion/",
         "recipe": "source:\n  type: <source-type>\n  config:\n    # Source-type specifics config\n    <source-configs>"
     }
-]
+]
diff --git a/datahub-web-react/src/images/verticalogo.png b/datahub-web-react/src/images/verticalogo.png
diff --git a/metadata-ingestion/docs/sources/vertica/README.md b/metadata-ingestion/docs/sources/vertica/README.md
@@ -3,9 +3,13 @@
 <!-- Plain-language description of what this integration is meant to do.  -->
 <!-- Include details about where metadata is extracted from (ie. logs, source API, manifest, etc.)   -->
 
-This plugin extracts metadata for Tables and Views on Vertica.
+The DataHub Vertica Plugin extracts the following: 
+
+* Metadata for databases, schemas, views, tables, and projections
+* Table level lineage
+* Metadata for ML Models
+* Metadata for Vertica OAuth
 
-This plugin is in beta and has only been tested on sample data on the Vertica database.
 
 ### Concept Mapping
 
@@ -21,6 +25,7 @@ This ingestion source maps the following Source System Concepts to DataHub Conce
 | `Vertica`      | [Data Platform](../../metamodel/entities/dataPlatform.md) |       |
 | Table          | [Dataset](../../metamodel/entities/dataset.md)            |       |
 | View           | [Dataset](../../metamodel/entities/dataset.md)            |       |
+| Projections    | [Dataset](../../metamodel/entities/dataset.md)            |       |
 
 ## Metadata Ingestion Quickstart
 

diff --git a/metadata-ingestion/docs/sources/vertica/vertica_recipe.yml b/metadata-ingestion/docs/sources/vertica/vertica_recipe.yml
@@ -8,6 +8,14 @@ source:
     # Credentials
     username: "${VERTICA_USER}"
     password: "${VERTICA_PASSWORD}"
+
+    include_tables: true
+    include_views: true
+    include_projections: true
+    include_oauth: true
+    include_models: true
+    include_view_lineage: true
+    include_projection_lineage: true
 
 sink:
   # sink configs
diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py
@@ -343,7 +343,7 @@ def get_long_description():
     "nifi": {"requests", "packaging"},
     "powerbi": microsoft_common | {"lark[regex]==1.1.4", "sqlparse"},
     "powerbi-report-server": powerbi_report_server,
-    "vertica": sql_common | {"sqlalchemy-vertica-dialect[vertica-python]==0.1.4"},
+    "vertica": sql_common | {"vertica-sqlalchemy-dialect[vertica-python]==0.0.1"},
     "unity-catalog": databricks_cli | {"requests"},
 }
 

diff --git a/metadata-service/war/src/main/resources/boot/data_platforms.json b/metadata-service/war/src/main/resources/boot/data_platforms.json
@@ -524,5 +524,15 @@
       "type": "OTHERS",
       "logoUrl": "/assets/platforms/databrickslogo.png"
     }
+  },
+  {
+    "urn": "urn:li:dataPlatform:vertica",
+    "aspect": {
+      "datasetNameDelimiter": ".",
+      "name": "vertica",
+      "displayName": "Vertica",
+      "type": "OLAP_DATASTORE",
+      "logoUrl": "/assets/platforms/verticalogo.png"
+    }
   }
 ]