feat(datahub-lite): introduces a new experimental lightweight impleme…

docs-website/sidebars.js

@@ -324,6 +324,7 @@ module.exports = {
       // concrete tutorials for how to work with the DataHub CLI & APIs
       // "docs/wip/developer-guides",
       "docs/cli",
+      "docs/datahub_lite",
       {
         "GraphQL API": [
           {

metadata-ingestion/setup.py

@@ -55,6 +55,7 @@ def get_long_description():
     "ijson",
     "click-spinner",
     "requests_file",
+    "duckdb",
 }
 
 rest_common = {
@@ -240,6 +241,7 @@ def get_long_description():
     # Sink plugins.
     "datahub-kafka": kafka_common,
     "datahub-rest": rest_common,
+    "datahub-lite": set(),
     # Integrations.
     "airflow": {
         "apache-airflow >= 2.0.2",
@@ -426,6 +428,7 @@ def get_long_description():
             "sagemaker",
             "kafka",
             "datahub-rest",
+            "datahub-lite",
             "presto",
             "redash",
             "redshift",
@@ -564,6 +567,7 @@ def get_long_description():
         "console = datahub.ingestion.sink.console:ConsoleSink",
         "datahub-kafka = datahub.ingestion.sink.datahub_kafka:DatahubKafkaSink",
         "datahub-rest = datahub.ingestion.sink.datahub_rest:DatahubRestSink",
+        "datahub-lite = datahub.ingestion.sink.datahub_lite:DataHubLiteSink",
     ],
     "datahub.ingestion.checkpointing_provider.plugins": [
         "datahub = datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider:DatahubIngestionCheckpointingProvider",

metadata-ingestion/sink_docs/datahub.md

@@ -116,6 +116,69 @@ The options in the producer config and schema registry config are passed to the
 
 For a full example with a number of security options, see this [example recipe](../examples/recipes/secured_kafka.dhub.yaml).
 
+## DataHub Lite (experimental)
+
+A sink that provides integration with [DataHub Lite](../../docs/datahub_lite.md) for local metadata exploration and serving.
+
+### Setup
+
+To install this plugin, run `pip install 'acryl-datahub[datahub-lite]'`.
+
+### Capabilities
+
+Pushes metadata to a local DataHub Lite instance.
+
+### Quickstart recipe
+
+Check out the following recipe to get started with ingestion! See [below](#config-details) for full configuration options.
+
+For general pointers on writing and running a recipe, see our [main recipe guide](../README.md#recipes).
+
+```yml
+source:
+  # source configs
+sink:
+  type: "datahub-lite"
+```
+
+By default, `datahub-lite` uses a **DuckDB** database and will write to a database file located under **~/.datahub/lite/**.
+
+To configure the location, you can specify it directly in the config:
+
+```yml
+source:
+  # source configs
+sink:
+  type: "datahub-lite"
+  config:
+    type: "duckdb"
+    config:
+      file: "<path_to_duckdb_file>"
+```
+
+:::note
+
+DataHub Lite currently doesn't support stateful ingestion, so you'll have to turn off stateful ingestion in your recipe to use it. This will be fixed shortly.
+
+:::
+
+### Config details
+
+Note that a `.` is used to denote nested fields in the YAML recipe.
+
+| Field                      | Required | Default              | Description                                                                                        |
+|----------------------------|----------|----------------------|----------------------------------------------------------------------------------------------------|
+| `type`                   |       |      duckdb                | Type of DataHub Lite implementation to use |
+| `config`              |          | `{"file": "~/.datahub/lite/datahub.duckdb"}`                   | Config dictionary to pass through to the DataHub Lite implementation. See below for fields accepted by the DuckDB implementation |
+
+#### DuckDB Config Details
+
+| Field                      | Required | Default              | Description                                                                                        |
+|----------------------------|----------|----------------------|----------------------------------------------------------------------------------------------------|
+| `file`                   |       |   `"~/.datahub/lite/datahub.duckdb"`     | File to use for DuckDB storage |
+| `options`                |          | `{}`                   | Options dictionary to pass through to DuckDB library. See [the official spec](https://duckdb.org/docs/sql/configuration.html) for the options supported by DuckDB. |
+
+
 ## Questions
 
 If you've got any questions on configuring this sink, feel free to ping us on [our Slack](https://slack.datahubproject.io/)!

metadata-ingestion/src/datahub/cli/cli_utils.py

@@ -70,18 +70,33 @@ def set_env_variables_override_config(url: str, token: Optional[str]) -> None:
         config_override[ENV_METADATA_TOKEN] = token
 
 
-def write_datahub_config(host: str, token: Optional[str]) -> None:
-    config = {
-        "gms": {
-            "server": host,
-            "token": token,
-        }
-    }
+def persist_datahub_config(config: dict) -> None:
     with open(DATAHUB_CONFIG_PATH, "w+") as outfile:
         yaml.dump(config, outfile, default_flow_style=False)
     return None
 
 
+def write_gms_config(
+    host: str, token: Optional[str], merge_with_previous: bool = True
+) -> None:
+    config = DatahubConfig(gms=GmsConfig(server=host, token=token))
+    if merge_with_previous:
+        try:
+            previous_config = get_client_config(as_dict=True)
+            assert isinstance(previous_config, dict)
+        except Exception as e:
+            # ok to fail on this
+            previous_config = {}
+            log.debug(
+                f"Failed to retrieve config from file {DATAHUB_CONFIG_PATH}. This isn't fatal.",
+                e,
+            )
+        config_dict = {**previous_config, **config.dict()}
+    else:
+        config_dict = config.dict()
+    persist_datahub_config(config_dict)
+
+
 def should_skip_config() -> bool:
     return get_boolean_env_variable(ENV_SKIP_CONFIG, False)
 
@@ -92,30 +107,40 @@ def ensure_datahub_config() -> None:
             f"No {CONDENSED_DATAHUB_CONFIG_PATH} file found, generating one for you...",
             bold=True,
         )
-        write_datahub_config(DEFAULT_GMS_HOST, None)
+        write_gms_config(DEFAULT_GMS_HOST, None)
 
 
-def get_details_from_config():
+def get_client_config(as_dict: bool = False) -> Union[Optional[DatahubConfig], dict]:
     with open(DATAHUB_CONFIG_PATH, "r") as stream:
         try:
             config_json = yaml.safe_load(stream)
+            if as_dict:
+                return config_json
             try:
-                datahub_config = DatahubConfig(**config_json)
+                datahub_config = DatahubConfig.parse_obj(config_json)
+                return datahub_config
             except ValidationError as e:
                 click.echo(
                     f"Received error, please check your {CONDENSED_DATAHUB_CONFIG_PATH}"
                 )
                 click.echo(e, err=True)
                 sys.exit(1)
-
-            gms_config = datahub_config.gms
-
-            gms_host = gms_config.server
-            gms_token = gms_config.token
-            return gms_host, gms_token
         except yaml.YAMLError as exc:
             click.secho(f"{DATAHUB_CONFIG_PATH} malformed, error: {exc}", bold=True)
-    return None, None
+            return None
+
+
+def get_details_from_config():
+    datahub_config = get_client_config(as_dict=False)
+    assert isinstance(datahub_config, DatahubConfig)
+    if datahub_config is not None:
+        gms_config = datahub_config.gms
+
+        gms_host = gms_config.server
+        gms_token = gms_config.token
+        return gms_host, gms_token
+    else:
+        return None, None
 
 
 def get_details_from_env() -> Tuple[Optional[str], Optional[str]]: