From 5c99f20b7ded38aefb7f6641f462d534a1c56912 Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Mon, 12 Dec 2022 15:18:01 +0530 Subject: [PATCH 01/19] fix(ingest): mysql - fix mysql ingestion issue with non-lowercase database (#6713) --- .../source/sql/two_tier_sql_source.py | 8 +++++ .../mysql/mysql_mces_no_db_golden.json | 34 +++++++++---------- .../tests/integration/mysql/setup/setup.sql | 10 +++--- 3 files changed, 30 insertions(+), 22 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/sql/two_tier_sql_source.py b/metadata-ingestion/src/datahub/ingestion/source/sql/two_tier_sql_source.py index c62c9c88f88543..f7e18dc066647e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/sql/two_tier_sql_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/sql/two_tier_sql_source.py @@ -94,3 +94,11 @@ def gen_schema_containers( self, schema: str, db_name: str ) -> typing.Iterable[MetadataWorkUnit]: return [] + + def get_db_name(self, inspector: Inspector) -> str: + engine = inspector.engine + + if engine and hasattr(engine, "url") and hasattr(engine.url, "database"): + return str(engine.url.database).strip('"') + else: + raise Exception("Unable to get database name from Sqlalchemy inspector") diff --git a/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json b/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json index ee49fbbf2ccebe..59f6e127eff0cf 100644 --- a/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json +++ b/metadata-ingestion/tests/integration/mysql/mysql_mces_no_db_golden.json @@ -1,11 +1,11 @@ [ { "entityType": "container", - "entityUrn": "urn:li:container:9191fea5add3487ba6b8266d4c74a7d1", + "entityUrn": "urn:li:container:0f72a1bc79da282eb614cc089c0ba302", "changeType": "UPSERT", "aspectName": "containerProperties", "aspect": { - "value": "{\"customProperties\": {\"platform\": \"mysql\", \"instance\": \"PROD\", \"database\": \"datacharmer\"}, \"name\": \"datacharmer\"}", + "value": "{\"customProperties\": {\"platform\": \"mysql\", \"instance\": \"PROD\", \"database\": \"dataCharmer\"}, \"name\": \"dataCharmer\"}", "contentType": "application/json" }, "systemMetadata": { @@ -15,7 +15,7 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:9191fea5add3487ba6b8266d4c74a7d1", + "entityUrn": "urn:li:container:0f72a1bc79da282eb614cc089c0ba302", "changeType": "UPSERT", "aspectName": "status", "aspect": { @@ -29,7 +29,7 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:9191fea5add3487ba6b8266d4c74a7d1", + "entityUrn": "urn:li:container:0f72a1bc79da282eb614cc089c0ba302", "changeType": "UPSERT", "aspectName": "dataPlatformInstance", "aspect": { @@ -43,7 +43,7 @@ }, { "entityType": "container", - "entityUrn": "urn:li:container:9191fea5add3487ba6b8266d4c74a7d1", + "entityUrn": "urn:li:container:0f72a1bc79da282eb614cc089c0ba302", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -57,11 +57,11 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.employees,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,dataCharmer.employees,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:9191fea5add3487ba6b8266d4c74a7d1\"}", + "value": "{\"container\": \"urn:li:container:0f72a1bc79da282eb614cc089c0ba302\"}", "contentType": "application/json" }, "systemMetadata": { @@ -72,7 +72,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.employees,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,dataCharmer.employees,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -88,7 +88,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "datacharmer.employees", + "schemaName": "dataCharmer.employees", "platform": "urn:li:dataPlatform:mysql", "version": 0, "created": { @@ -191,7 +191,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.employees,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,dataCharmer.employees,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -205,11 +205,11 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.salaries,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,dataCharmer.salaries,PROD)", "changeType": "UPSERT", "aspectName": "container", "aspect": { - "value": "{\"container\": \"urn:li:container:9191fea5add3487ba6b8266d4c74a7d1\"}", + "value": "{\"container\": \"urn:li:container:0f72a1bc79da282eb614cc089c0ba302\"}", "contentType": "application/json" }, "systemMetadata": { @@ -220,7 +220,7 @@ { "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { - "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.salaries,PROD)", + "urn": "urn:li:dataset:(urn:li:dataPlatform:mysql,dataCharmer.salaries,PROD)", "aspects": [ { "com.linkedin.pegasus2avro.common.Status": { @@ -236,7 +236,7 @@ }, { "com.linkedin.pegasus2avro.schema.SchemaMetadata": { - "schemaName": "datacharmer.salaries", + "schemaName": "dataCharmer.salaries", "platform": "urn:li:dataPlatform:mysql", "version": 0, "created": { @@ -315,7 +315,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.salaries,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,dataCharmer.salaries,PROD)", "changeType": "UPSERT", "aspectName": "subTypes", "aspect": { @@ -329,7 +329,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.employees,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,dataCharmer.employees,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -343,7 +343,7 @@ }, { "entityType": "dataset", - "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,datacharmer.salaries,PROD)", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:mysql,dataCharmer.salaries,PROD)", "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { diff --git a/metadata-ingestion/tests/integration/mysql/setup/setup.sql b/metadata-ingestion/tests/integration/mysql/setup/setup.sql index ed6df6ac17dd8d..c8a88aff0f2533 100644 --- a/metadata-ingestion/tests/integration/mysql/setup/setup.sql +++ b/metadata-ingestion/tests/integration/mysql/setup/setup.sql @@ -48,13 +48,13 @@ CREATE TABLE metadata_index ( CREATE VIEW metadata_index_view AS SELECT id, urn, path, doubleVal FROM metadata_index; -- ----------------------------------------------------- --- Some sample data, from https://github.com/datacharmer/test_db. +-- Some sample data, from https://github.com/dataCharmer/test_db. -- ----------------------------------------------------- -CREATE SCHEMA IF NOT EXISTS `datacharmer` ; -USE `datacharmer` ; +CREATE SCHEMA IF NOT EXISTS `dataCharmer` ; +USE `dataCharmer` ; -CREATE TABLE `datacharmer`.`employees` ( +CREATE TABLE `dataCharmer`.`employees` ( emp_no INT NOT NULL, birth_date DATE NOT NULL, first_name VARCHAR(14) NOT NULL, @@ -64,7 +64,7 @@ CREATE TABLE `datacharmer`.`employees` ( PRIMARY KEY (emp_no) ); -CREATE TABLE `datacharmer`.`salaries` ( +CREATE TABLE `dataCharmer`.`salaries` ( emp_no INT NOT NULL, salary INT NOT NULL, from_date DATE NOT NULL, From fd911c9820b442202cbd3f0b655dce210f5d6575 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Mon, 12 Dec 2022 04:48:26 -0500 Subject: [PATCH 02/19] feat(ingest): redact configs reported in ingestion_run_summary (#6696) --- .../src/datahub/configuration/common.py | 52 +++++++++++++++++++ .../datahub_ingestion_run_summary_provider.py | 9 +++- .../tests/unit/config/test_config_model.py | 25 ++++++++- 3 files changed, 83 insertions(+), 3 deletions(-) diff --git a/metadata-ingestion/src/datahub/configuration/common.py b/metadata-ingestion/src/datahub/configuration/common.py index 6aa7e992b77135..f57a0262649cdf 100644 --- a/metadata-ingestion/src/datahub/configuration/common.py +++ b/metadata-ingestion/src/datahub/configuration/common.py @@ -15,6 +15,58 @@ _ConfigSelf = TypeVar("_ConfigSelf", bound="ConfigModel") +REDACT_KEYS = { + "password", + "token", + "secret", + "options", + "sqlalchemy_uri", +} +REDACT_SUFFIXES = { + "_password", + "_secret", + "_token", + "_key", + "_key_id", +} + + +def _should_redact_key(key: str) -> bool: + return key in REDACT_KEYS or any(key.endswith(suffix) for suffix in REDACT_SUFFIXES) + + +def _redact_value(value: Any) -> Any: + if isinstance(value, str): + # If it's just a variable reference, it's ok to show as-is. + if value.startswith("$"): + return value + return "********" + elif value is None: + return None + elif isinstance(value, bool): + # We don't have any sensitive boolean fields. + return value + elif isinstance(value, list) and not value: + # Empty states are fine. + return [] + elif isinstance(value, dict) and not value: + return {} + else: + return "********" + + +def redact_raw_config(obj: Any) -> Any: + if isinstance(obj, dict): + return { + k: _redact_value(v) if _should_redact_key(k) else redact_raw_config(v) + for k, v in obj.items() + } + elif isinstance(obj, list): + return [redact_raw_config(v) for v in obj] + else: + return obj + + class ConfigModel(BaseModel): class Config: extra = Extra.forbid diff --git a/metadata-ingestion/src/datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py b/metadata-ingestion/src/datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py index 8afe34b93749aa..55f6f1a81b7687 100644 --- a/metadata-ingestion/src/datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py +++ b/metadata-ingestion/src/datahub/ingestion/reporting/datahub_ingestion_run_summary_provider.py @@ -4,7 +4,12 @@ from typing import Any, Dict, Optional from datahub import nice_version_name -from datahub.configuration.common import ConfigModel, DynamicTypedConfig, IgnorableError +from datahub.configuration.common import ( + ConfigModel, + DynamicTypedConfig, + IgnorableError, + redact_raw_config, +) from datahub.emitter.mce_builder import datahub_guid from datahub.emitter.mcp import MetadataChangeProposalWrapper from datahub.emitter.mcp_builder import make_data_platform_urn @@ -149,7 +154,7 @@ def _get_recipe_to_report(self, ctx: PipelineContext) -> str: if not self.report_recipe or not ctx.pipeline_config._raw_dict: return "" else: - return json.dumps(ctx.pipeline_config._raw_dict) + return json.dumps(redact_raw_config(ctx.pipeline_config._raw_dict)) def _emit_aspect( self, entity_urn: Urn, aspect_name: str, aspect_value: _Aspect diff --git a/metadata-ingestion/tests/unit/config/test_config_model.py b/metadata-ingestion/tests/unit/config/test_config_model.py index 3a92810356b53d..4b6e17df67497f 100644 --- a/metadata-ingestion/tests/unit/config/test_config_model.py +++ b/metadata-ingestion/tests/unit/config/test_config_model.py @@ -3,7 +3,7 @@ import pydantic import pytest -from datahub.configuration.common import ConfigModel +from datahub.configuration.common import ConfigModel, redact_raw_config def test_extras_not_allowed(): @@ -49,3 +49,26 @@ class MyConfig(ConfigModel): assert config_2.items == [] assert config_2.items_field == [] + + +def test_config_redaction(): + obj = { + "config": { + "password": "this_is_sensitive", + "aws_key_id": "${AWS_KEY_ID}", + "projects": ["default"], + "options": {}, + }, + "options": {"foo": "bar"}, + } + + redacted = redact_raw_config(obj) + assert redacted == { + "config": { + "password": "********", + "aws_key_id": "${AWS_KEY_ID}", + "projects": ["default"], + "options": {}, + }, + "options": "********", + } From d3fca44e1636b0a667843fb84fd61fb743477e71 Mon Sep 17 00:00:00 2001 From: Jan Hicken Date: Mon, 12 Dec 2022 10:58:23 +0100 Subject: [PATCH 03/19] fix(ingest): bigquery - rectify filter for BigQuery external tables (#6691) --- .../datahub/ingestion/source/bigquery_v2/bigquery_schema.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py index 3302c873bd56ce..c46bea3788a784 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_schema.py @@ -119,7 +119,7 @@ class BigqueryQuery: table_name) as p on t.table_name = p.table_name WHERE - table_type in ('BASE TABLE', 'EXTERNAL TABLE') + table_type in ('BASE TABLE', 'EXTERNAL') {table_filter} order by table_schema ASC, @@ -146,7 +146,7 @@ class BigqueryQuery: and t.TABLE_NAME = tos.TABLE_NAME and tos.OPTION_NAME = "description" WHERE - table_type in ('BASE TABLE', 'EXTERNAL TABLE') + table_type in ('BASE TABLE', 'EXTERNAL') {table_filter} order by table_schema ASC, From 65ba13d9aa91ce50eeefd542f8463d000c8bbc20 Mon Sep 17 00:00:00 2001 From: Mayuri Nehate <33225191+mayurinehate@users.noreply.github.com> Date: Mon, 12 Dec 2022 19:53:12 +0530 Subject: [PATCH 04/19] feat(ingest): snowflake - add separate config for include_column_lineage in snowflake (#6712) --- .../source/snowflake/snowflake_config.py | 15 +- .../source/snowflake/snowflake_lineage.py | 15 +- .../source/snowflake/snowflake_query.py | 16 +- .../source/snowflake/snowflake_report.py | 1 + .../source/snowflake/snowflake_v2.py | 3 +- .../snowflake_privatelink_beta_golden.json | 140 ++++++++++++++++++ .../snowflake-beta/test_snowflake_beta.py | 14 +- .../tests/unit/test_snowflake_beta_source.py | 33 +++++ 8 files changed, 221 insertions(+), 16 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py index 643ba4f1db579c..9e066b41fbef2d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_config.py @@ -1,7 +1,7 @@ import logging from typing import Dict, Optional, cast -from pydantic import Field, SecretStr, root_validator +from pydantic import Field, SecretStr, root_validator, validator from datahub.configuration.common import AllowDenyPattern from datahub.ingestion.glossary.classifier import ClassificationConfig @@ -30,6 +30,11 @@ class SnowflakeV2Config(SnowflakeConfig, SnowflakeUsageConfig): description="If enabled, populates the snowflake technical schema and descriptions.", ) + include_column_lineage: bool = Field( + default=True, + description="If enabled, populates the column lineage. Supported only for snowflake table-to-table and view-to-table lineage edge (not supported in table-to-view or view-to-view lineage edge yet). Requires appropriate grants given to the role.", + ) + check_role_grants: bool = Field( default=False, description="Not supported", @@ -54,6 +59,14 @@ class SnowflakeV2Config(SnowflakeConfig, SnowflakeUsageConfig): description="Whether `schema_pattern` is matched against fully qualified schema name `.`.", ) + @validator("include_column_lineage") + def validate_include_column_lineage(cls, v, values): + if not values.get("include_table_lineage") and v: + raise ValueError( + "include_table_lineage must be True for include_column_lineage to be set." + ) + return v + @root_validator(pre=False) def validate_unsupported_configs(cls, values: Dict) -> Dict: diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage.py index 933431b014e394..6e0fa30575ccfc 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_lineage.py @@ -138,11 +138,13 @@ class SnowflakeTableLineage: default_factory=lambda: defaultdict(SnowflakeColumnUpstreams), init=False ) - def update_lineage(self, table: SnowflakeUpstreamTable) -> None: + def update_lineage( + self, table: SnowflakeUpstreamTable, include_column_lineage: bool = True + ) -> None: if table.upstreamDataset not in self.upstreamTables.keys(): self.upstreamTables[table.upstreamDataset] = table - if table.downstreamColumns: + if include_column_lineage and table.downstreamColumns: for col in table.downstreamColumns: if col.directSourceColumns: @@ -380,6 +382,7 @@ def _populate_lineage(self, conn: SnowflakeConnection) -> None: if not self.config.ignore_start_time_lineage else 0, end_time_millis=int(self.config.end_time.timestamp() * 1000), + include_column_lineage=self.config.include_column_lineage, ) num_edges: int = 0 self._lineage_map = defaultdict(SnowflakeTableLineage) @@ -404,6 +407,7 @@ def _populate_lineage(self, conn: SnowflakeConnection) -> None: db_row["UPSTREAM_TABLE_COLUMNS"], db_row["DOWNSTREAM_TABLE_COLUMNS"], ), + self.config.include_column_lineage, ) num_edges += 1 logger.debug( @@ -452,7 +456,8 @@ def _populate_view_upstream_lineage(self, conn: SnowflakeConnection) -> None: # key is the downstream view name self._lineage_map[view_name].update_lineage( # (, , ) - SnowflakeUpstreamTable.from_dict(view_upstream, None, None) + SnowflakeUpstreamTable.from_dict(view_upstream, None, None), + self.config.include_column_lineage, ) num_edges += 1 logger.debug( @@ -477,6 +482,7 @@ def _populate_view_downstream_lineage(self, conn: SnowflakeConnection) -> None: if not self.config.ignore_start_time_lineage else 0, end_time_millis=int(self.config.end_time.timestamp() * 1000), + include_column_lineage=self.config.include_column_lineage, ) assert self._lineage_map is not None @@ -512,7 +518,8 @@ def _populate_view_downstream_lineage(self, conn: SnowflakeConnection) -> None: view_name, db_row["VIEW_COLUMNS"], db_row["DOWNSTREAM_TABLE_COLUMNS"], - ) + ), + self.config.include_column_lineage, ) self.report.num_view_to_table_edges_scanned += 1 diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py index 1c2d780afa8fef..c1957f46d25bfb 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_query.py @@ -232,7 +232,9 @@ def operational_data_for_time_window( @staticmethod def table_to_table_lineage_history( - start_time_millis: int, end_time_millis: int + start_time_millis: int, + end_time_millis: int, + include_column_lineage: bool = True, ) -> str: return f""" WITH table_lineage_history AS ( @@ -263,8 +265,7 @@ def table_to_table_lineage_history( WHERE upstream_table_domain in ('Table', 'External table') and downstream_table_domain = 'Table' QUALIFY ROW_NUMBER() OVER ( PARTITION BY downstream_table_name, - upstream_table_name, - downstream_table_columns + upstream_table_name{", downstream_table_columns" if include_column_lineage else ""} ORDER BY query_start_time DESC ) = 1""" @@ -289,7 +290,11 @@ def view_dependencies() -> str: """ @staticmethod - def view_lineage_history(start_time_millis: int, end_time_millis: int) -> str: + def view_lineage_history( + start_time_millis: int, + end_time_millis: int, + include_column_lineage: bool = True, + ) -> str: return f""" WITH view_lineage_history AS ( SELECT @@ -330,8 +335,7 @@ def view_lineage_history(start_time_millis: int, end_time_millis: int) -> str: view_domain in ('View', 'Materialized view') QUALIFY ROW_NUMBER() OVER ( PARTITION BY view_name, - downstream_table_name, - downstream_table_columns + downstream_table_name {", downstream_table_columns" if include_column_lineage else ""} ORDER BY query_start_time DESC ) = 1 diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py index e70c48771c49bc..db605f1d629f87 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_report.py @@ -11,6 +11,7 @@ class SnowflakeV2Report(SnowflakeReport, SnowflakeUsageReport, ProfilingSqlRepor include_usage_stats: bool = False include_operational_stats: bool = False include_technical_schema: bool = False + include_column_lineage: bool = False usage_aggregation_query_secs: float = -1 table_lineage_query_secs: float = -1 diff --git a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py index be02c92a49beaf..62a53b002ff3ca 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py +++ b/metadata-ingestion/src/datahub/ingestion/source/snowflake/snowflake_v2.py @@ -169,7 +169,7 @@ ) @capability( SourceCapability.LINEAGE_FINE, - "Enabled by default, can be disabled via configuration `include_table_lineage` and `include_view_lineage`", + "Enabled by default, can be disabled via configuration `include_column_lineage`", ) @capability( SourceCapability.USAGE_STATS, @@ -1101,6 +1101,7 @@ def add_config_to_report(self): self.report.check_role_grants = self.config.check_role_grants self.report.include_usage_stats = self.config.include_usage_stats self.report.include_operational_stats = self.config.include_operational_stats + self.report.include_column_lineage = self.config.include_column_lineage if self.report.include_usage_stats or self.config.include_operational_stats: self.report.window_start_time = self.config.start_time self.report.window_end_time = self.config.end_time diff --git a/metadata-ingestion/tests/integration/snowflake-beta/snowflake_privatelink_beta_golden.json b/metadata-ingestion/tests/integration/snowflake-beta/snowflake_privatelink_beta_golden.json index ae406291240e15..2f3ec2f32a2fb2 100644 --- a/metadata-ingestion/tests/integration/snowflake-beta/snowflake_privatelink_beta_golden.json +++ b/metadata-ingestion/tests/integration/snowflake-beta/snowflake_privatelink_beta_golden.json @@ -195,6 +195,20 @@ "runId": "snowflake-beta-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_1,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", @@ -265,6 +279,20 @@ "runId": "snowflake-beta-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", @@ -335,6 +363,20 @@ "runId": "snowflake-beta-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_3,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", @@ -405,6 +447,20 @@ "runId": "snowflake-beta-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_4,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", @@ -475,6 +531,20 @@ "runId": "snowflake-beta-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_5,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", @@ -545,6 +615,20 @@ "runId": "snowflake-beta-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_6,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", @@ -615,6 +699,20 @@ "runId": "snowflake-beta-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_7,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", @@ -685,6 +783,20 @@ "runId": "snowflake-beta-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_8,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", @@ -755,6 +867,20 @@ "runId": "snowflake-beta-2022_06_07-17_00_00" } }, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_9,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } +}, { "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", @@ -824,5 +950,19 @@ "lastObserved": 1654621200000, "runId": "snowflake-beta-2022_06_07-17_00_00" } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_10,PROD)", + "changeType": "UPSERT", + "aspectName": "upstreamLineage", + "aspect": { + "value": "{\"upstreams\": [{\"auditStamp\": {\"time\": 0, \"actor\": \"urn:li:corpuser:unknown\"}, \"dataset\": \"urn:li:dataset:(urn:li:dataPlatform:snowflake,test_db.test_schema.table_2,PROD)\", \"type\": \"TRANSFORMED\"}]}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1654621200000, + "runId": "snowflake-beta-2022_06_07-17_00_00" + } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/snowflake-beta/test_snowflake_beta.py b/metadata-ingestion/tests/integration/snowflake-beta/test_snowflake_beta.py index 95b5f63a0d95d8..4b51ad71119953 100644 --- a/metadata-ingestion/tests/integration/snowflake-beta/test_snowflake_beta.py +++ b/metadata-ingestion/tests/integration/snowflake-beta/test_snowflake_beta.py @@ -258,9 +258,14 @@ def default_query_results(query): } for op_idx in range(1, NUM_OPS + 1) ] - elif query == snowflake_query.SnowflakeQuery.table_to_table_lineage_history( - 1654499820000, - 1654586220000, + elif query in ( + snowflake_query.SnowflakeQuery.table_to_table_lineage_history( + 1654499820000, + 1654586220000, + ), + snowflake_query.SnowflakeQuery.table_to_table_lineage_history( + 1654499820000, 1654586220000, False + ), ): return [ { @@ -426,7 +431,8 @@ def test_snowflake_private_link(pytestconfig, tmp_path, mock_time, mock_datahub_ include_views=False, schema_pattern=AllowDenyPattern(allow=["test_schema"]), include_technical_schema=True, - include_table_lineage=False, + include_table_lineage=True, + include_column_lineage=False, include_view_lineage=False, include_usage_stats=False, include_operational_stats=False, diff --git a/metadata-ingestion/tests/unit/test_snowflake_beta_source.py b/metadata-ingestion/tests/unit/test_snowflake_beta_source.py index ed58e4aeb1a507..9e22a723bf9b83 100644 --- a/metadata-ingestion/tests/unit/test_snowflake_beta_source.py +++ b/metadata-ingestion/tests/unit/test_snowflake_beta_source.py @@ -1,6 +1,7 @@ from unittest.mock import MagicMock, patch import pytest +from pydantic import ValidationError from datahub.configuration.common import ConfigurationError, OauthConfiguration from datahub.ingestion.api.source import SourceCapability @@ -182,6 +183,38 @@ def test_options_contain_connect_args(): assert connect_args is not None +def test_snowflake_config_with_view_lineage_no_table_lineage_throws_error(): + with pytest.raises(ValidationError): + SnowflakeV2Config.parse_obj( + { + "username": "user", + "password": "password", + "host_port": "acctname", + "database_pattern": {"allow": {"^demo$"}}, + "warehouse": "COMPUTE_WH", + "role": "sysadmin", + "include_view_lineage": True, + "include_table_lineage": False, + } + ) + + +def test_snowflake_config_with_column_lineage_no_table_lineage_throws_error(): + with pytest.raises(ValidationError): + SnowflakeV2Config.parse_obj( + { + "username": "user", + "password": "password", + "host_port": "acctname", + "database_pattern": {"allow": {"^demo$"}}, + "warehouse": "COMPUTE_WH", + "role": "sysadmin", + "include_column_lineage": True, + "include_table_lineage": False, + } + ) + + @patch("snowflake.connector.connect") def test_test_connection_failure(mock_connect): mock_connect.side_effect = Exception("Failed to connect to snowflake") From 3099bd148675e4577c50addf480f33c734b120f3 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Mon, 12 Dec 2022 22:02:19 +0530 Subject: [PATCH 05/19] fix(ci): flakiness due to onboarding tour in add user test (#6734) --- .../src/app/onboarding/OnboardingTour.tsx | 4 ++++ smoke-test/.gitignore | 3 ++- .../cypress/integration/mutations/add_users.js | 13 +++++++++---- .../tests/cypress/cypress/support/commands.js | 4 ++++ 4 files changed, 19 insertions(+), 5 deletions(-) diff --git a/datahub-web-react/src/app/onboarding/OnboardingTour.tsx b/datahub-web-react/src/app/onboarding/OnboardingTour.tsx index 3205f837eb2313..5a046971fe53fe 100644 --- a/datahub-web-react/src/app/onboarding/OnboardingTour.tsx +++ b/datahub-web-react/src/app/onboarding/OnboardingTour.tsx @@ -25,6 +25,10 @@ export const OnboardingTour = ({ stepIds }: Props) => { setReshow(true); setIsOpen(true); } + if (e.metaKey && e.ctrlKey && e.key === 'h') { + setReshow(false); + setIsOpen(false); + } } document.addEventListener('keydown', handleKeyDown); }, []); diff --git a/smoke-test/.gitignore b/smoke-test/.gitignore index 55142a4a3630f3..44d3f620a19372 100644 --- a/smoke-test/.gitignore +++ b/smoke-test/.gitignore @@ -130,4 +130,5 @@ dmypy.json # Pyre type checker .pyre/ -junit* \ No newline at end of file +junit* +tests/cypress/onboarding.json \ No newline at end of file diff --git a/smoke-test/tests/cypress/cypress/integration/mutations/add_users.js b/smoke-test/tests/cypress/cypress/integration/mutations/add_users.js index 3151638a477f54..7dea5ba01d5d59 100644 --- a/smoke-test/tests/cypress/cypress/integration/mutations/add_users.js +++ b/smoke-test/tests/cypress/cypress/integration/mutations/add_users.js @@ -1,6 +1,8 @@ const tryToSignUp = () => { - cy.enterTextInTestId("email", "example@example.com") - cy.enterTextInTestId("name", "Example Name") + let number = Math.floor(Math.random() * 100000); + let name = `Example Name ${number}`; + cy.enterTextInTestId("email", `example${number}@example.com`) + cy.enterTextInTestId("name", name) cy.enterTextInTestId("password", "Example password") cy.enterTextInTestId("confirmPassword", "Example password") @@ -8,6 +10,7 @@ const tryToSignUp = () => { cy.waitTextVisible("Other").click() cy.get("[type=submit]").click() + return name; }; describe("add_user", () => { @@ -23,8 +26,10 @@ describe("add_user", () => { const inviteLink = $elem.text(); cy.logout(); cy.visit(inviteLink); - tryToSignUp(); - cy.waitTextVisible("Accepted invite!") + let name = tryToSignUp(); + cy.waitTextVisible("Welcome to DataHub"); + cy.hideOnboardingTour(); + cy.waitTextVisible(name); }).then(() => { cy.logout(); cy.visit("/signup?invite_token=bad_token"); diff --git a/smoke-test/tests/cypress/cypress/support/commands.js b/smoke-test/tests/cypress/cypress/support/commands.js index c9174825bff84e..2abea209a4ec76 100644 --- a/smoke-test/tests/cypress/cypress/support/commands.js +++ b/smoke-test/tests/cypress/cypress/support/commands.js @@ -136,6 +136,10 @@ Cypress.Commands.add("clickOptionWithTestId", (id) => { }); }) +Cypress.Commands.add("hideOnboardingTour", () => { + cy.get('body').type("{ctrl} {meta} h"); +}); + Cypress.Commands.add('addTermToDataset', (urn, dataset_name, term) => { cy.goToDataset(urn, dataset_name); cy.clickOptionWithText("Add Term"); From 621974e559083ffb4ad67be86e0165d08da4e20b Mon Sep 17 00:00:00 2001 From: John Joyce Date: Mon, 12 Dec 2022 09:14:06 -0800 Subject: [PATCH 06/19] feat(ui): Support DataBricks Unity Catalog Source in Ui Ingestion (#6707) --- .../source/builder/RecipeForm/constants.ts | 30 +++- .../builder/RecipeForm/unity_catalog.tsx | 153 ++++++++++++++++++ .../app/ingest/source/builder/constants.ts | 4 + .../app/ingest/source/builder/sources.json | 7 + 4 files changed, 192 insertions(+), 2 deletions(-) create mode 100644 datahub-web-react/src/app/ingest/source/builder/RecipeForm/unity_catalog.tsx diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts index fa4cf551ba2eb9..1b0963f1f9db2f 100644 --- a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/constants.ts @@ -83,12 +83,23 @@ import { PROJECT_NAME, } from './lookml'; import { PRESTO, PRESTO_HOST_PORT, PRESTO_DATABASE, PRESTO_USERNAME, PRESTO_PASSWORD } from './presto'; -import { BIGQUERY_BETA, MYSQL } from '../constants'; +import { BIGQUERY_BETA, MYSQL, UNITY_CATALOG } from '../constants'; import { BIGQUERY_BETA_PROJECT_ID, DATASET_ALLOW, DATASET_DENY, PROJECT_ALLOW, PROJECT_DENY } from './bigqueryBeta'; import { MYSQL_HOST_PORT, MYSQL_PASSWORD, MYSQL_USERNAME } from './mysql'; import { MSSQL, MSSQL_DATABASE, MSSQL_HOST_PORT, MSSQL_PASSWORD, MSSQL_USERNAME } from './mssql'; import { TRINO, TRINO_DATABASE, TRINO_HOST_PORT, TRINO_PASSWORD, TRINO_USERNAME } from './trino'; import { MARIADB, MARIADB_DATABASE, MARIADB_HOST_PORT, MARIADB_PASSWORD, MARIADB_USERNAME } from './mariadb'; +import { + INCLUDE_COLUMN_LINEAGE, + TOKEN, + UNITY_CATALOG_ALLOW, + UNITY_CATALOG_DENY, + UNITY_METASTORE_ID_ALLOW, + UNITY_METASTORE_ID_DENY, + UNITY_TABLE_ALLOW, + UNITY_TABLE_DENY, + WORKSPACE_URL, +} from './unity_catalog'; export enum RecipeSections { Connection = 0, @@ -338,8 +349,23 @@ export const RECIPE_FIELDS: RecipeFields = { ], filterSectionTooltip: 'Include or exclude specific Schemas, Tables and Views from ingestion.', }, + [UNITY_CATALOG]: { + fields: [WORKSPACE_URL, TOKEN], + filterFields: [ + UNITY_METASTORE_ID_ALLOW, + UNITY_METASTORE_ID_DENY, + UNITY_CATALOG_ALLOW, + UNITY_CATALOG_DENY, + SCHEMA_ALLOW, + SCHEMA_DENY, + UNITY_TABLE_ALLOW, + UNITY_TABLE_DENY, + ], + advancedFields: [INCLUDE_TABLE_LINEAGE, INCLUDE_COLUMN_LINEAGE, STATEFUL_INGESTION_ENABLED], + filterSectionTooltip: 'Include or exclude specific Metastores, Catalogs, Schemas, and Tables from ingestion.', + }, }; export const CONNECTORS_WITH_FORM = new Set(Object.keys(RECIPE_FIELDS)); -export const CONNECTORS_WITH_TEST_CONNECTION = new Set([SNOWFLAKE, LOOKER, BIGQUERY_BETA, BIGQUERY]); +export const CONNECTORS_WITH_TEST_CONNECTION = new Set([SNOWFLAKE, LOOKER, BIGQUERY_BETA, BIGQUERY, UNITY_CATALOG]); diff --git a/datahub-web-react/src/app/ingest/source/builder/RecipeForm/unity_catalog.tsx b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/unity_catalog.tsx new file mode 100644 index 00000000000000..3b5565ab1abd43 --- /dev/null +++ b/datahub-web-react/src/app/ingest/source/builder/RecipeForm/unity_catalog.tsx @@ -0,0 +1,153 @@ +import React from 'react'; +import { RecipeField, FieldType, setListValuesOnRecipe } from './common'; + +export const UNITY_CATALOG = 'unity-catalog'; + +export const TOKEN: RecipeField = { + name: 'token', + label: 'Token', + tooltip: 'A personal access token associated with the Databricks account used to extract metadata.', + type: FieldType.SECRET, + fieldPath: 'source.config.token', + placeholder: 'dapi1a2b3c45d67890e1f234567a8bc9012d', + required: true, + rules: null, +}; + +export const WORKSPACE_URL: RecipeField = { + name: 'workspace_url', + label: 'Workspace URL', + tooltip: 'The URL for the Databricks workspace from which to extract metadata.', + type: FieldType.TEXT, + fieldPath: 'source.config.workspace_url', + placeholder: 'https://abcsales.cloud.databricks.com', + required: true, + rules: null, +}; + +export const INCLUDE_TABLE_LINEAGE: RecipeField = { + name: 'include_table_lineage', + label: 'Include Table Lineage', + tooltip: ( +
+ Extract Table Lineage from Unity Catalog. Note that this requires that your Databricks accounts meets + certain requirements. View them{' '} + here +
+ ), + type: FieldType.BOOLEAN, + fieldPath: 'source.config.include_table_lineage', + rules: null, +}; + +export const INCLUDE_COLUMN_LINEAGE: RecipeField = { + name: 'include_column_lineage', + label: 'Include Column Lineage', + tooltip: ( +
+ Extract Column Lineage from Unity Catalog. Note that this requires that your Databricks accounts meets + certain requirements. View them{' '} + here. + Enabling this feature may increase the duration of ingestion. +
+ ), + type: FieldType.BOOLEAN, + fieldPath: 'source.config.include_column_lineage', + rules: null, +}; + +const metastoreIdAllowFieldPath = 'source.config.metastore_id_pattern.allow'; +export const UNITY_METASTORE_ID_ALLOW: RecipeField = { + name: 'metastore_id_pattern.allow', + label: 'Allow Patterns', + tooltip: + 'Only include specific Metastores by providing the id of a Metastore, or a Regular Expression (REGEX) to include specific Metastores. If not provided, all Metastores will be included.', + placeholder: '11111-2222-33333-44-555555', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: metastoreIdAllowFieldPath, + rules: null, + section: 'Metastores', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, metastoreIdAllowFieldPath), +}; + +const metastoreIdDenyFieldPath = 'source.config.metastore_id_pattern.deny'; +export const UNITY_METASTORE_ID_DENY: RecipeField = { + name: 'metastore_id_pattern.deny', + label: 'Deny Patterns', + tooltip: + 'Exclude specific Metastores by providing the id of a Metastores, or a Regular Expression (REGEX). If not provided, all Metastores will be included. Deny patterns always take precedence over Allow patterns.', + placeholder: '11111-2222-33333-44-555555', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: metastoreIdDenyFieldPath, + rules: null, + section: 'Metastores', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, metastoreIdDenyFieldPath), +}; + +const catalogAllowFieldPath = 'source.config.catalog_pattern.allow'; +export const UNITY_CATALOG_ALLOW: RecipeField = { + name: 'catalog_pattern.allow', + label: 'Allow Patterns', + tooltip: + 'Only include specific Catalogs by providing the name of a Catalog, or a Regular Expression (REGEX) to include specific Catalogs. If not provided, all Catalogs will be included.', + placeholder: 'my_catalog', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: catalogAllowFieldPath, + rules: null, + section: 'Catalogs', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, catalogAllowFieldPath), +}; + +const catalogDenyFieldPath = 'source.config.catalog_pattern.deny'; +export const UNITY_CATALOG_DENY: RecipeField = { + name: 'catalog_pattern.allow', + label: 'Allow Patterns', + tooltip: + 'Exclude specific Catalogs by providing the name of a Catalog, or a Regular Expression (REGEX) to exclude specific Catalogs. If not provided, all Catalogs will be included. Deny patterns always take precedence over Allow patterns.', + placeholder: 'my_catalog', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: catalogDenyFieldPath, + rules: null, + section: 'Catalogs', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, catalogDenyFieldPath), +}; + +const tableAllowFieldPath = 'source.config.metastore_id_pattern.allow'; +export const UNITY_TABLE_ALLOW: RecipeField = { + name: 'catalog_pattern.allow', + label: 'Allow Patterns', + tooltip: + 'Only include specific Tables by providing the fully-qualified name of a Table, or a Regular Expression (REGEX) to include specific Tables. If not provided, all Tables will be included.', + placeholder: 'catalog.schema.table', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: tableAllowFieldPath, + rules: null, + section: 'Tables', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, tableAllowFieldPath), +}; + +const tableDenyFieldPath = 'source.config.metastore_id_pattern.deny'; +export const UNITY_TABLE_DENY: RecipeField = { + name: 'catalog_pattern.allow', + label: 'Allow Patterns', + tooltip: + 'Exclude specific Tables by providing the fully-qualified name of a Table, or a Regular Expression (REGEX) to exclude specific Tables. If not provided, all Tables will be included. Deny patterns always take precedence over Allow patterns.', + placeholder: 'catalog.schema.table', + type: FieldType.LIST, + buttonLabel: 'Add pattern', + fieldPath: tableDenyFieldPath, + rules: null, + section: 'Tables', + setValueOnRecipeOverride: (recipe: any, values: string[]) => + setListValuesOnRecipe(recipe, values, tableDenyFieldPath), +}; diff --git a/datahub-web-react/src/app/ingest/source/builder/constants.ts b/datahub-web-react/src/app/ingest/source/builder/constants.ts index 282be15e132268..ca925aa3a7821e 100644 --- a/datahub-web-react/src/app/ingest/source/builder/constants.ts +++ b/datahub-web-react/src/app/ingest/source/builder/constants.ts @@ -25,6 +25,7 @@ import mariadbLogo from '../../../../images/mariadblogo.png'; import metabaseLogo from '../../../../images/metabaselogo.png'; import powerbiLogo from '../../../../images/powerbilogo.png'; import modeLogo from '../../../../images/modelogo.png'; +import databricksLogo from '../../../../images/databrickslogo.png'; export const ATHENA = 'athena'; export const ATHENA_URN = `urn:li:dataPlatform:${ATHENA}`; @@ -91,6 +92,8 @@ export const TRINO = 'trino'; export const TRINO_URN = `urn:li:dataPlatform:${TRINO}`; export const CUSTOM = 'custom'; export const CUSTOM_URN = `urn:li:dataPlatform:${CUSTOM}`; +export const UNITY_CATALOG = 'unity-catalog'; +export const UNITY_CATALOG_URN = `urn:li:dataPlatform:${UNITY_CATALOG}`; export const PLATFORM_URN_TO_LOGO = { [ATHENA_URN]: athenaLogo, @@ -120,6 +123,7 @@ export const PLATFORM_URN_TO_LOGO = { [TABLEAU_URN]: tableauLogo, [TRINO_URN]: trinoLogo, [SUPERSET_URN]: supersetLogo, + [UNITY_CATALOG_URN]: databricksLogo, }; export const SOURCE_TO_PLATFORM_URN = { diff --git a/datahub-web-react/src/app/ingest/source/builder/sources.json b/datahub-web-react/src/app/ingest/source/builder/sources.json index 6f6c63672fb8a1..939c0dda1c64b3 100644 --- a/datahub-web-react/src/app/ingest/source/builder/sources.json +++ b/datahub-web-react/src/app/ingest/source/builder/sources.json @@ -97,6 +97,13 @@ "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/mariadb/", "recipe": "source:\n type: mariadb\n config:\n # Coordinates\n host_port: null\n # The name\n database: null\n # Credentials\n username: null\n include_views: true\n include_tables: true\n profiling:\n enabled: true\n profile_table_level_only: true\n stateful_ingestion:\n enabled: true" }, + { + "urn": "urn:li:dataPlatform:unity-catalog", + "name": "unity-catalog", + "displayName": "Databricks Unity Catalog", + "docsUrl": "https://datahubproject.io/docs/generated/ingestion/sources/databricks/#module-unity-catalog", + "recipe": "source:\n type: unity-catalog\n config:\n # Coordinates\n workspace_url: null\n include_table_lineage: true\n include_column_lineage: false\n stateful_ingestion:\n enabled: true" + }, { "urn": "urn:li:dataPlatform:mongodb", "name": "mongodb", From 2cc64742e05812ead78042f0d227343107ca6f67 Mon Sep 17 00:00:00 2001 From: cccs-Dustin <96579982+cccs-Dustin@users.noreply.github.com> Date: Mon, 12 Dec 2022 13:06:03 -0500 Subject: [PATCH 07/19] feat(ingest/iceberg): add stateful ingestion (#6344) --- .../ingestion/source/iceberg/iceberg.py | 49 +++++- .../source/iceberg/iceberg_common.py | 31 +++- .../ingestion/source/state/iceberg_state.py | 45 +++++ .../ingest_test}/iceberg_mces_golden.json | 56 ++---- .../iceberg_test/metadata/v1.metadata.json | 0 .../iceberg_test/metadata/v2.metadata.json | 0 .../iceberg_test}/metadata/version-hint.text | 0 ...-bb5c-4ffd-8ead-08f33fa2675d-00001.parquet | Bin ...-498a-4ce9-b525-8242758d18f8-00001.parquet | Bin ...-191f-4a11-9953-09435ffce01d-00001.parquet | Bin ...-b547-40b9-89ca-caf4fcfe6685-00001.parquet | Bin ...-e02b-44b1-8ec8-4dfa287c3bd5.metadata.json | 0 ...-5f25-4180-99e2-065ef0b9791b.metadata.json | 0 ...-4c12-46d0-9a75-ce3578ec03d4.metadata.json | 0 ...acaffc-9bed-4d97-8ddd-0ea1ea15a2b8-m0.avro | Bin ...0bd970-e5ef-4843-abcb-e96a35a8f14d-m0.avro | Bin ...-ec0bd970-e5ef-4843-abcb-e96a35a8f14d.avro | Bin ...-23acaffc-9bed-4d97-8ddd-0ea1ea15a2b8.avro | Bin .../profiling}/metadata/version-hint.text | 0 .../iceberg_mces_golden.json | 59 ++----- .../iceberg_deleted_table_mces_golden.json | 159 ++++++++++++++++++ .../iceberg_test/metadata/v1.metadata.json | 105 ++++++++++++ .../iceberg_test/metadata/v2.metadata.json | 118 +++++++++++++ .../iceberg_test/metadata/version-hint.text | 1 + .../iceberg_test_2/metadata/v1.metadata.json | 105 ++++++++++++ .../iceberg_test_2/metadata/v2.metadata.json | 118 +++++++++++++ .../iceberg_test_2/metadata/version-hint.text | 1 + .../iceberg_test/metadata/v1.metadata.json | 105 ++++++++++++ .../iceberg_test/metadata/v2.metadata.json | 118 +++++++++++++ .../iceberg_test/metadata/version-hint.text | 1 + .../tests/integration/iceberg/test_iceberg.py | 145 +++++++++++++++- 31 files changed, 1107 insertions(+), 109 deletions(-) create mode 100644 metadata-ingestion/src/datahub/ingestion/source/state/iceberg_state.py rename metadata-ingestion/tests/integration/iceberg/{ => test_data/ingest_test}/iceberg_mces_golden.json (73%) rename metadata-ingestion/tests/integration/iceberg/test_data/{ => ingest_test}/namespace/iceberg_test/metadata/v1.metadata.json (100%) rename metadata-ingestion/tests/integration/iceberg/test_data/{ => ingest_test}/namespace/iceberg_test/metadata/v2.metadata.json (100%) rename metadata-ingestion/tests/integration/iceberg/test_data/{datahub/integration/profiling => ingest_test/namespace/iceberg_test}/metadata/version-hint.text (100%) mode change 100755 => 100644 rename metadata-ingestion/tests/integration/iceberg/test_data/{ => profiling_test}/datahub/integration/profiling/data/00000-0-72133c37-bb5c-4ffd-8ead-08f33fa2675d-00001.parquet (100%) rename metadata-ingestion/tests/integration/iceberg/test_data/{ => profiling_test}/datahub/integration/profiling/data/00000-3-c638dd0f-498a-4ce9-b525-8242758d18f8-00001.parquet (100%) rename metadata-ingestion/tests/integration/iceberg/test_data/{ => profiling_test}/datahub/integration/profiling/data/00001-1-5f69f6ed-191f-4a11-9953-09435ffce01d-00001.parquet (100%) rename metadata-ingestion/tests/integration/iceberg/test_data/{ => profiling_test}/datahub/integration/profiling/data/00001-4-b21a5375-b547-40b9-89ca-caf4fcfe6685-00001.parquet (100%) rename metadata-ingestion/tests/integration/iceberg/test_data/{ => profiling_test}/datahub/integration/profiling/metadata/00000-331b9f67-e02b-44b1-8ec8-4dfa287c3bd5.metadata.json (100%) rename metadata-ingestion/tests/integration/iceberg/test_data/{ => profiling_test}/datahub/integration/profiling/metadata/00001-fb50681e-5f25-4180-99e2-065ef0b9791b.metadata.json (100%) rename metadata-ingestion/tests/integration/iceberg/test_data/{ => profiling_test}/datahub/integration/profiling/metadata/00002-cc241948-4c12-46d0-9a75-ce3578ec03d4.metadata.json (100%) rename metadata-ingestion/tests/integration/iceberg/test_data/{ => profiling_test}/datahub/integration/profiling/metadata/23acaffc-9bed-4d97-8ddd-0ea1ea15a2b8-m0.avro (100%) rename metadata-ingestion/tests/integration/iceberg/test_data/{ => profiling_test}/datahub/integration/profiling/metadata/ec0bd970-e5ef-4843-abcb-e96a35a8f14d-m0.avro (100%) rename metadata-ingestion/tests/integration/iceberg/test_data/{ => profiling_test}/datahub/integration/profiling/metadata/snap-4220723025353071767-1-ec0bd970-e5ef-4843-abcb-e96a35a8f14d.avro (100%) rename metadata-ingestion/tests/integration/iceberg/test_data/{ => profiling_test}/datahub/integration/profiling/metadata/snap-4437197002876030991-1-23acaffc-9bed-4d97-8ddd-0ea1ea15a2b8.avro (100%) rename metadata-ingestion/tests/integration/iceberg/test_data/{namespace/iceberg_test => profiling_test/datahub/integration/profiling}/metadata/version-hint.text (100%) mode change 100644 => 100755 rename metadata-ingestion/tests/integration/iceberg/test_data/{datahub/integration/profiling => profiling_test}/iceberg_mces_golden.json (74%) create mode 100644 metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/iceberg_deleted_table_mces_golden.json create mode 100644 metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run1/namespace/iceberg_test/metadata/v1.metadata.json create mode 100644 metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run1/namespace/iceberg_test/metadata/v2.metadata.json create mode 100644 metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run1/namespace/iceberg_test/metadata/version-hint.text create mode 100644 metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run1/namespace/iceberg_test_2/metadata/v1.metadata.json create mode 100644 metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run1/namespace/iceberg_test_2/metadata/v2.metadata.json create mode 100644 metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run1/namespace/iceberg_test_2/metadata/version-hint.text create mode 100644 metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run2/namespace/iceberg_test/metadata/v1.metadata.json create mode 100644 metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run2/namespace/iceberg_test/metadata/v2.metadata.json create mode 100644 metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run2/namespace/iceberg_test/metadata/version-hint.text diff --git a/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py b/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py index e4d4d8f5f2a5f0..2e8d379f12ab92 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py +++ b/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg.py @@ -27,7 +27,7 @@ platform_name, support_status, ) -from datahub.ingestion.api.source import Source, SourceReport +from datahub.ingestion.api.source import SourceReport from datahub.ingestion.api.workunit import MetadataWorkUnit from datahub.ingestion.extractor import schema_util from datahub.ingestion.source.iceberg.iceberg_common import ( @@ -35,6 +35,14 @@ IcebergSourceReport, ) from datahub.ingestion.source.iceberg.iceberg_profiler import IcebergProfiler +from datahub.ingestion.source.state.iceberg_state import IcebergCheckpointState +from datahub.ingestion.source.state.stale_entity_removal_handler import ( + StaleEntityRemovalHandler, +) +from datahub.ingestion.source.state.stateful_ingestion_base import ( + StatefulIngestionSourceBase, +) +from datahub.metadata.com.linkedin.pegasus2avro.common import Status from datahub.metadata.com.linkedin.pegasus2avro.metadata.snapshot import DatasetSnapshot from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent from datahub.metadata.com.linkedin.pegasus2avro.schema import ( @@ -50,6 +58,10 @@ OwnershipClass, OwnershipTypeClass, ) +from datahub.utilities.source_helpers import ( + auto_stale_entity_removal, + auto_status_aspect, +) LOGGER = logging.getLogger(__name__) @@ -81,7 +93,8 @@ SourceCapability.OWNERSHIP, "Optionally enabled via configuration by specifying which Iceberg table property holds user or group ownership.", ) -class IcebergSource(Source): +@capability(SourceCapability.DELETION_DETECTION, "Enabled via stateful ingestion") +class IcebergSource(StatefulIngestionSourceBase): """ ## Integration Details @@ -101,18 +114,32 @@ class IcebergSource(Source): """ def __init__(self, config: IcebergSourceConfig, ctx: PipelineContext) -> None: - super().__init__(ctx) - self.PLATFORM: str = "iceberg" + super().__init__(config, ctx) + self.platform: str = "iceberg" self.report: IcebergSourceReport = IcebergSourceReport() self.config: IcebergSourceConfig = config self.iceberg_client: FilesystemTables = config.filesystem_tables + self.stale_entity_removal_handler = StaleEntityRemovalHandler( + source=self, + config=self.config, + state_type_class=IcebergCheckpointState, + pipeline_name=self.ctx.pipeline_name, + run_id=self.ctx.run_id, + ) + @classmethod def create(cls, config_dict: Dict, ctx: PipelineContext) -> "IcebergSource": config = IcebergSourceConfig.parse_obj(config_dict) return cls(config, ctx) def get_workunits(self) -> Iterable[MetadataWorkUnit]: + return auto_stale_entity_removal( + self.stale_entity_removal_handler, + auto_status_aspect(self.get_workunits_internal()), + ) + + def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: for dataset_path, dataset_name in self.config.get_paths(): # Tuple[str, str] try: if not self.config.table_pattern.allowed(dataset_name): @@ -140,14 +167,14 @@ def _create_iceberg_workunit( ) -> Iterable[MetadataWorkUnit]: self.report.report_table_scanned(dataset_name) dataset_urn: str = make_dataset_urn_with_platform_instance( - self.PLATFORM, + self.platform, dataset_name, self.config.platform_instance, self.config.env, ) dataset_snapshot = DatasetSnapshot( urn=dataset_urn, - aspects=[], + aspects=[Status(removed=False)], ) custom_properties: Dict = dict(table.properties()) @@ -227,9 +254,9 @@ def _get_dataplatform_instance_aspect( entityUrn=dataset_urn, aspectName="dataPlatformInstance", aspect=DataPlatformInstanceClass( - platform=make_data_platform_urn(self.PLATFORM), + platform=make_data_platform_urn(self.platform), instance=make_dataplatform_instance_urn( - self.PLATFORM, self.config.platform_instance + self.platform, self.config.platform_instance ), ), ) @@ -247,7 +274,7 @@ def _create_schema_metadata( ) schema_metadata = SchemaMetadata( schemaName=dataset_name, - platform=make_data_platform_urn(self.PLATFORM), + platform=make_data_platform_urn(self.platform), version=0, hash="", platformSchema=OtherSchema(rawSchema=repr(table.schema())), @@ -295,6 +322,10 @@ def _get_avro_schema_from_data_type(self, column: NestedField) -> Dict[str, Any] ], } + def get_platform_instance_id(self) -> str: + assert self.config.platform_instance is not None + return self.config.platform_instance + def get_report(self) -> SourceReport: return self.report diff --git a/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg_common.py b/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg_common.py index 746f639cb9a526..1c9c2d0436beef 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/iceberg/iceberg_common.py @@ -2,6 +2,7 @@ from dataclasses import dataclass, field from typing import Dict, Iterable, List, Optional, Tuple +import pydantic from azure.storage.filedatalake import FileSystemClient, PathProperties from iceberg.core.filesystem.abfss_filesystem import AbfssFileSystem from iceberg.core.filesystem.filesystem_tables import FilesystemTables @@ -12,9 +13,14 @@ ConfigModel, ConfigurationError, ) -from datahub.configuration.source_common import DatasetSourceConfigBase -from datahub.ingestion.api.source import SourceReport from datahub.ingestion.source.azure.azure_common import AdlsSourceConfig +from datahub.ingestion.source.state.stale_entity_removal_handler import ( + StaleEntityRemovalSourceReport, + StatefulStaleMetadataRemovalConfig, +) +from datahub.ingestion.source.state.stateful_ingestion_base import ( + StatefulIngestionConfigBase, +) class IcebergProfilingConfig(ConfigModel): @@ -44,7 +50,11 @@ class IcebergProfilingConfig(ConfigModel): # include_field_sample_values: bool = True -class IcebergSourceConfig(DatasetSourceConfigBase): +class IcebergSourceConfig(StatefulIngestionConfigBase): + # Override the stateful_ingestion config param with the Iceberg custom stateful ingestion config in the IcebergSourceConfig + stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = pydantic.Field( + default=None, description="Iceberg Stateful Ingestion Config." + ) adls: Optional[AdlsSourceConfig] = Field( default=None, description="[Azure Data Lake Storage](https://docs.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-introduction) to crawl for Iceberg tables. This is one filesystem type supported by this source and **only one can be configured**.", @@ -71,6 +81,19 @@ class IcebergSourceConfig(DatasetSourceConfigBase): ) profiling: IcebergProfilingConfig = IcebergProfilingConfig() + @pydantic.root_validator + def validate_platform_instance(cls: "IcebergSourceConfig", values: Dict) -> Dict: + stateful_ingestion = values.get("stateful_ingestion") + if ( + stateful_ingestion + and stateful_ingestion.enabled + and not values.get("platform_instance") + ): + raise ConfigurationError( + "Enabling Iceberg stateful ingestion requires to specify a platform instance." + ) + return values + @root_validator() def _ensure_one_filesystem_is_configured( cls: "IcebergSourceConfig", values: Dict @@ -160,7 +183,7 @@ def get_paths(self) -> Iterable[Tuple[str, str]]: @dataclass -class IcebergSourceReport(SourceReport): +class IcebergSourceReport(StaleEntityRemovalSourceReport): tables_scanned: int = 0 entities_profiled: int = 0 filtered: List[str] = field(default_factory=list) diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/iceberg_state.py b/metadata-ingestion/src/datahub/ingestion/source/state/iceberg_state.py new file mode 100644 index 00000000000000..a5983eaad269ea --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/state/iceberg_state.py @@ -0,0 +1,45 @@ +import logging +from typing import Iterable, List + +import pydantic + +from datahub.ingestion.source.state.stale_entity_removal_handler import ( + StaleEntityCheckpointStateBase, +) +from datahub.utilities.urns.urn import guess_entity_type + +logger = logging.getLogger(__name__) + + +class IcebergCheckpointState(StaleEntityCheckpointStateBase["IcebergCheckpointState"]): + """ + This Class represents the checkpoint state for Iceberg based sources. + Stores all the tables being ingested and is used to remove any stale entities. + """ + + urns: List[str] = pydantic.Field(default_factory=list) + + @classmethod + def get_supported_types(cls) -> List[str]: + return ["*"] + + def add_checkpoint_urn(self, type: str, urn: str) -> None: + self.urns.append(urn) + + def get_urns_not_in( + self, type: str, other_checkpoint_state: "IcebergCheckpointState" + ) -> Iterable[str]: + diff = set(self.urns) - set(other_checkpoint_state.urns) + + # To maintain backwards compatibility, we provide this filtering mechanism. + if type == "*": + yield from diff + else: + yield from (urn for urn in diff if guess_entity_type(urn) == type) + + def get_percent_entities_changed( + self, old_checkpoint_state: "IcebergCheckpointState" + ) -> float: + return StaleEntityCheckpointStateBase.compute_percent_entities_changed( + [(self.urns, old_checkpoint_state.urns)] + ) diff --git a/metadata-ingestion/tests/integration/iceberg/iceberg_mces_golden.json b/metadata-ingestion/tests/integration/iceberg/test_data/ingest_test/iceberg_mces_golden.json similarity index 73% rename from metadata-ingestion/tests/integration/iceberg/iceberg_mces_golden.json rename to metadata-ingestion/tests/integration/iceberg/test_data/ingest_test/iceberg_mces_golden.json index ecbb1c7570b33f..b106b91275835c 100644 --- a/metadata-ingestion/tests/integration/iceberg/iceberg_mces_golden.json +++ b/metadata-ingestion/tests/integration/iceberg/test_data/ingest_test/iceberg_mces_golden.json @@ -1,10 +1,14 @@ [ { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:iceberg,namespace.iceberg_test,PROD)", "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { @@ -12,11 +16,6 @@ "provider": "ICEBERG", "location": "/namespace/iceberg_test" }, - "externalUrl": null, - "name": null, - "qualifiedName": null, - "description": null, - "uri": null, "tags": [] } }, @@ -25,19 +24,16 @@ "owners": [ { "owner": "urn:li:corpuser:new_owner", - "type": "TECHNICAL_OWNER", - "source": null + "type": "TECHNICAL_OWNER" }, { "owner": "urn:li:corpGroup:new_owner", - "type": "TECHNICAL_OWNER", - "source": null + "type": "TECHNICAL_OWNER" } ], "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" } } }, @@ -48,17 +44,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.OtherSchema": { @@ -68,7 +59,6 @@ "fields": [ { "fieldPath": "[version=2.0].[type=struct].[type=string].level", - "jsonPath": null, "nullable": false, "description": "level documentation", "type": { @@ -78,15 +68,11 @@ }, "nativeDataType": "string", "recursive": false, - "globalTags": null, - "glossaryTerms": null, "isPartOfKey": false, - "isPartitioningKey": null, "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": false}" }, { "fieldPath": "[version=2.0].[type=struct].[type=long].event_time", - "jsonPath": null, "nullable": false, "description": "event_time documentation", "type": { @@ -96,15 +82,11 @@ }, "nativeDataType": "timestamptz", "recursive": false, - "globalTags": null, - "glossaryTerms": null, "isPartOfKey": false, - "isPartitioningKey": null, "jsonProps": "{\"logicalType\": \"timestamp-micros\", \"native_data_type\": \"timestamptz\", \"_nullable\": false}" }, { "fieldPath": "[version=2.0].[type=struct].[type=string].message", - "jsonPath": null, "nullable": false, "description": "message documentation", "type": { @@ -114,15 +96,11 @@ }, "nativeDataType": "string", "recursive": false, - "globalTags": null, - "glossaryTerms": null, "isPartOfKey": false, - "isPartitioningKey": null, "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": false}" }, { "fieldPath": "[version=2.0].[type=struct].[type=array].[type=string].call_stack", - "jsonPath": null, "nullable": true, "description": "call_stack documentation", "type": { @@ -136,28 +114,18 @@ }, "nativeDataType": "list", "recursive": false, - "globalTags": null, - "glossaryTerms": null, "isPartOfKey": false, - "isPartitioningKey": null, "jsonProps": "{\"native_data_type\": \"list\", \"_nullable\": true}" } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "iceberg-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "iceberg-test" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/iceberg/test_data/namespace/iceberg_test/metadata/v1.metadata.json b/metadata-ingestion/tests/integration/iceberg/test_data/ingest_test/namespace/iceberg_test/metadata/v1.metadata.json similarity index 100% rename from metadata-ingestion/tests/integration/iceberg/test_data/namespace/iceberg_test/metadata/v1.metadata.json rename to metadata-ingestion/tests/integration/iceberg/test_data/ingest_test/namespace/iceberg_test/metadata/v1.metadata.json diff --git a/metadata-ingestion/tests/integration/iceberg/test_data/namespace/iceberg_test/metadata/v2.metadata.json b/metadata-ingestion/tests/integration/iceberg/test_data/ingest_test/namespace/iceberg_test/metadata/v2.metadata.json similarity index 100% rename from metadata-ingestion/tests/integration/iceberg/test_data/namespace/iceberg_test/metadata/v2.metadata.json rename to metadata-ingestion/tests/integration/iceberg/test_data/ingest_test/namespace/iceberg_test/metadata/v2.metadata.json diff --git a/metadata-ingestion/tests/integration/iceberg/test_data/datahub/integration/profiling/metadata/version-hint.text b/metadata-ingestion/tests/integration/iceberg/test_data/ingest_test/namespace/iceberg_test/metadata/version-hint.text old mode 100755 new mode 100644 similarity index 100% rename from metadata-ingestion/tests/integration/iceberg/test_data/datahub/integration/profiling/metadata/version-hint.text rename to metadata-ingestion/tests/integration/iceberg/test_data/ingest_test/namespace/iceberg_test/metadata/version-hint.text diff --git a/metadata-ingestion/tests/integration/iceberg/test_data/datahub/integration/profiling/data/00000-0-72133c37-bb5c-4ffd-8ead-08f33fa2675d-00001.parquet b/metadata-ingestion/tests/integration/iceberg/test_data/profiling_test/datahub/integration/profiling/data/00000-0-72133c37-bb5c-4ffd-8ead-08f33fa2675d-00001.parquet similarity index 100% rename from metadata-ingestion/tests/integration/iceberg/test_data/datahub/integration/profiling/data/00000-0-72133c37-bb5c-4ffd-8ead-08f33fa2675d-00001.parquet rename to metadata-ingestion/tests/integration/iceberg/test_data/profiling_test/datahub/integration/profiling/data/00000-0-72133c37-bb5c-4ffd-8ead-08f33fa2675d-00001.parquet diff --git a/metadata-ingestion/tests/integration/iceberg/test_data/datahub/integration/profiling/data/00000-3-c638dd0f-498a-4ce9-b525-8242758d18f8-00001.parquet b/metadata-ingestion/tests/integration/iceberg/test_data/profiling_test/datahub/integration/profiling/data/00000-3-c638dd0f-498a-4ce9-b525-8242758d18f8-00001.parquet similarity index 100% rename from metadata-ingestion/tests/integration/iceberg/test_data/datahub/integration/profiling/data/00000-3-c638dd0f-498a-4ce9-b525-8242758d18f8-00001.parquet rename to metadata-ingestion/tests/integration/iceberg/test_data/profiling_test/datahub/integration/profiling/data/00000-3-c638dd0f-498a-4ce9-b525-8242758d18f8-00001.parquet diff --git a/metadata-ingestion/tests/integration/iceberg/test_data/datahub/integration/profiling/data/00001-1-5f69f6ed-191f-4a11-9953-09435ffce01d-00001.parquet b/metadata-ingestion/tests/integration/iceberg/test_data/profiling_test/datahub/integration/profiling/data/00001-1-5f69f6ed-191f-4a11-9953-09435ffce01d-00001.parquet similarity index 100% rename from metadata-ingestion/tests/integration/iceberg/test_data/datahub/integration/profiling/data/00001-1-5f69f6ed-191f-4a11-9953-09435ffce01d-00001.parquet rename to metadata-ingestion/tests/integration/iceberg/test_data/profiling_test/datahub/integration/profiling/data/00001-1-5f69f6ed-191f-4a11-9953-09435ffce01d-00001.parquet diff --git a/metadata-ingestion/tests/integration/iceberg/test_data/datahub/integration/profiling/data/00001-4-b21a5375-b547-40b9-89ca-caf4fcfe6685-00001.parquet b/metadata-ingestion/tests/integration/iceberg/test_data/profiling_test/datahub/integration/profiling/data/00001-4-b21a5375-b547-40b9-89ca-caf4fcfe6685-00001.parquet similarity index 100% rename from metadata-ingestion/tests/integration/iceberg/test_data/datahub/integration/profiling/data/00001-4-b21a5375-b547-40b9-89ca-caf4fcfe6685-00001.parquet rename to metadata-ingestion/tests/integration/iceberg/test_data/profiling_test/datahub/integration/profiling/data/00001-4-b21a5375-b547-40b9-89ca-caf4fcfe6685-00001.parquet diff --git a/metadata-ingestion/tests/integration/iceberg/test_data/datahub/integration/profiling/metadata/00000-331b9f67-e02b-44b1-8ec8-4dfa287c3bd5.metadata.json b/metadata-ingestion/tests/integration/iceberg/test_data/profiling_test/datahub/integration/profiling/metadata/00000-331b9f67-e02b-44b1-8ec8-4dfa287c3bd5.metadata.json similarity index 100% rename from metadata-ingestion/tests/integration/iceberg/test_data/datahub/integration/profiling/metadata/00000-331b9f67-e02b-44b1-8ec8-4dfa287c3bd5.metadata.json rename to metadata-ingestion/tests/integration/iceberg/test_data/profiling_test/datahub/integration/profiling/metadata/00000-331b9f67-e02b-44b1-8ec8-4dfa287c3bd5.metadata.json diff --git a/metadata-ingestion/tests/integration/iceberg/test_data/datahub/integration/profiling/metadata/00001-fb50681e-5f25-4180-99e2-065ef0b9791b.metadata.json b/metadata-ingestion/tests/integration/iceberg/test_data/profiling_test/datahub/integration/profiling/metadata/00001-fb50681e-5f25-4180-99e2-065ef0b9791b.metadata.json similarity index 100% rename from metadata-ingestion/tests/integration/iceberg/test_data/datahub/integration/profiling/metadata/00001-fb50681e-5f25-4180-99e2-065ef0b9791b.metadata.json rename to metadata-ingestion/tests/integration/iceberg/test_data/profiling_test/datahub/integration/profiling/metadata/00001-fb50681e-5f25-4180-99e2-065ef0b9791b.metadata.json diff --git a/metadata-ingestion/tests/integration/iceberg/test_data/datahub/integration/profiling/metadata/00002-cc241948-4c12-46d0-9a75-ce3578ec03d4.metadata.json b/metadata-ingestion/tests/integration/iceberg/test_data/profiling_test/datahub/integration/profiling/metadata/00002-cc241948-4c12-46d0-9a75-ce3578ec03d4.metadata.json similarity index 100% rename from metadata-ingestion/tests/integration/iceberg/test_data/datahub/integration/profiling/metadata/00002-cc241948-4c12-46d0-9a75-ce3578ec03d4.metadata.json rename to metadata-ingestion/tests/integration/iceberg/test_data/profiling_test/datahub/integration/profiling/metadata/00002-cc241948-4c12-46d0-9a75-ce3578ec03d4.metadata.json diff --git a/metadata-ingestion/tests/integration/iceberg/test_data/datahub/integration/profiling/metadata/23acaffc-9bed-4d97-8ddd-0ea1ea15a2b8-m0.avro b/metadata-ingestion/tests/integration/iceberg/test_data/profiling_test/datahub/integration/profiling/metadata/23acaffc-9bed-4d97-8ddd-0ea1ea15a2b8-m0.avro similarity index 100% rename from metadata-ingestion/tests/integration/iceberg/test_data/datahub/integration/profiling/metadata/23acaffc-9bed-4d97-8ddd-0ea1ea15a2b8-m0.avro rename to metadata-ingestion/tests/integration/iceberg/test_data/profiling_test/datahub/integration/profiling/metadata/23acaffc-9bed-4d97-8ddd-0ea1ea15a2b8-m0.avro diff --git a/metadata-ingestion/tests/integration/iceberg/test_data/datahub/integration/profiling/metadata/ec0bd970-e5ef-4843-abcb-e96a35a8f14d-m0.avro b/metadata-ingestion/tests/integration/iceberg/test_data/profiling_test/datahub/integration/profiling/metadata/ec0bd970-e5ef-4843-abcb-e96a35a8f14d-m0.avro similarity index 100% rename from metadata-ingestion/tests/integration/iceberg/test_data/datahub/integration/profiling/metadata/ec0bd970-e5ef-4843-abcb-e96a35a8f14d-m0.avro rename to metadata-ingestion/tests/integration/iceberg/test_data/profiling_test/datahub/integration/profiling/metadata/ec0bd970-e5ef-4843-abcb-e96a35a8f14d-m0.avro diff --git a/metadata-ingestion/tests/integration/iceberg/test_data/datahub/integration/profiling/metadata/snap-4220723025353071767-1-ec0bd970-e5ef-4843-abcb-e96a35a8f14d.avro b/metadata-ingestion/tests/integration/iceberg/test_data/profiling_test/datahub/integration/profiling/metadata/snap-4220723025353071767-1-ec0bd970-e5ef-4843-abcb-e96a35a8f14d.avro similarity index 100% rename from metadata-ingestion/tests/integration/iceberg/test_data/datahub/integration/profiling/metadata/snap-4220723025353071767-1-ec0bd970-e5ef-4843-abcb-e96a35a8f14d.avro rename to metadata-ingestion/tests/integration/iceberg/test_data/profiling_test/datahub/integration/profiling/metadata/snap-4220723025353071767-1-ec0bd970-e5ef-4843-abcb-e96a35a8f14d.avro diff --git a/metadata-ingestion/tests/integration/iceberg/test_data/datahub/integration/profiling/metadata/snap-4437197002876030991-1-23acaffc-9bed-4d97-8ddd-0ea1ea15a2b8.avro b/metadata-ingestion/tests/integration/iceberg/test_data/profiling_test/datahub/integration/profiling/metadata/snap-4437197002876030991-1-23acaffc-9bed-4d97-8ddd-0ea1ea15a2b8.avro similarity index 100% rename from metadata-ingestion/tests/integration/iceberg/test_data/datahub/integration/profiling/metadata/snap-4437197002876030991-1-23acaffc-9bed-4d97-8ddd-0ea1ea15a2b8.avro rename to metadata-ingestion/tests/integration/iceberg/test_data/profiling_test/datahub/integration/profiling/metadata/snap-4437197002876030991-1-23acaffc-9bed-4d97-8ddd-0ea1ea15a2b8.avro diff --git a/metadata-ingestion/tests/integration/iceberg/test_data/namespace/iceberg_test/metadata/version-hint.text b/metadata-ingestion/tests/integration/iceberg/test_data/profiling_test/datahub/integration/profiling/metadata/version-hint.text old mode 100644 new mode 100755 similarity index 100% rename from metadata-ingestion/tests/integration/iceberg/test_data/namespace/iceberg_test/metadata/version-hint.text rename to metadata-ingestion/tests/integration/iceberg/test_data/profiling_test/datahub/integration/profiling/metadata/version-hint.text diff --git a/metadata-ingestion/tests/integration/iceberg/test_data/datahub/integration/profiling/iceberg_mces_golden.json b/metadata-ingestion/tests/integration/iceberg/test_data/profiling_test/iceberg_mces_golden.json similarity index 74% rename from metadata-ingestion/tests/integration/iceberg/test_data/datahub/integration/profiling/iceberg_mces_golden.json rename to metadata-ingestion/tests/integration/iceberg/test_data/profiling_test/iceberg_mces_golden.json index c695567bd4e347..edfa8f80670cfa 100644 --- a/metadata-ingestion/tests/integration/iceberg/test_data/datahub/integration/profiling/iceberg_mces_golden.json +++ b/metadata-ingestion/tests/integration/iceberg/test_data/profiling_test/iceberg_mces_golden.json @@ -1,10 +1,14 @@ [ { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { "urn": "urn:li:dataset:(urn:li:dataPlatform:iceberg,datahub.integration.profiling,PROD)", "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, { "com.linkedin.pegasus2avro.dataset.DatasetProperties": { "customProperties": { @@ -14,11 +18,6 @@ "snapshot-id": "4220723025353071767", "manifest-list": "/home/iceberg/warehouse/datahub/integration/profiling/metadata/snap-4220723025353071767-1-ec0bd970-e5ef-4843-abcb-e96a35a8f14d.avro" }, - "externalUrl": null, - "name": null, - "qualifiedName": null, - "description": null, - "uri": null, "tags": [] } }, @@ -27,19 +26,16 @@ "owners": [ { "owner": "urn:li:corpuser:root", - "type": "TECHNICAL_OWNER", - "source": null + "type": "TECHNICAL_OWNER" }, { "owner": "urn:li:corpGroup:root", - "type": "TECHNICAL_OWNER", - "source": null + "type": "TECHNICAL_OWNER" } ], "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" } } }, @@ -50,17 +46,12 @@ "version": 0, "created": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, "lastModified": { "time": 0, - "actor": "urn:li:corpuser:unknown", - "impersonator": null + "actor": "urn:li:corpuser:unknown" }, - "deleted": null, - "dataset": null, - "cluster": null, "hash": "", "platformSchema": { "com.linkedin.pegasus2avro.schema.OtherSchema": { @@ -70,7 +61,6 @@ "fields": [ { "fieldPath": "[version=2.0].[type=struct].[type=long].field_int", - "jsonPath": null, "nullable": true, "description": "An integer field", "type": { @@ -80,15 +70,11 @@ }, "nativeDataType": "long", "recursive": false, - "globalTags": null, - "glossaryTerms": null, "isPartOfKey": false, - "isPartitioningKey": null, "jsonProps": "{\"native_data_type\": \"long\", \"_nullable\": true}" }, { "fieldPath": "[version=2.0].[type=struct].[type=string].field_str", - "jsonPath": null, "nullable": true, "description": "A string field", "type": { @@ -98,15 +84,11 @@ }, "nativeDataType": "string", "recursive": false, - "globalTags": null, - "glossaryTerms": null, "isPartOfKey": false, - "isPartitioningKey": null, "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": true}" }, { "fieldPath": "[version=2.0].[type=struct].[type=long].field_timestamp", - "jsonPath": null, "nullable": true, "description": "A timestamp field", "type": { @@ -116,35 +98,23 @@ }, "nativeDataType": "timestamptz", "recursive": false, - "globalTags": null, - "glossaryTerms": null, "isPartOfKey": false, - "isPartitioningKey": null, "jsonProps": "{\"logicalType\": \"timestamp-micros\", \"native_data_type\": \"timestamptz\", \"_nullable\": true}" } - ], - "primaryKeys": null, - "foreignKeysSpecs": null, - "foreignKeys": null + ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "iceberg-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "iceberg-test" } }, { - "auditHeader": null, "entityType": "dataset", "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:iceberg,datahub.integration.profiling,PROD)", - "entityKeyAspect": null, "changeType": "UPSERT", "aspectName": "datasetProfile", "aspect": { @@ -153,10 +123,7 @@ }, "systemMetadata": { "lastObserved": 1586847600000, - "runId": "iceberg-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "iceberg-test" } } ] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/iceberg_deleted_table_mces_golden.json b/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/iceberg_deleted_table_mces_golden.json new file mode 100644 index 00000000000000..d376d8b645d664 --- /dev/null +++ b/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/iceberg_deleted_table_mces_golden.json @@ -0,0 +1,159 @@ +[ +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.DatasetSnapshot": { + "urn": "urn:li:dataset:(urn:li:dataPlatform:iceberg,test_platform_instance.namespace.iceberg_test,PROD)", + "aspects": [ + { + "com.linkedin.pegasus2avro.common.Status": { + "removed": false + } + }, + { + "com.linkedin.pegasus2avro.dataset.DatasetProperties": { + "customProperties": { + "owner": "new_owner", + "provider": "ICEBERG", + "location": "/namespace/iceberg_test" + }, + "tags": [] + } + }, + { + "com.linkedin.pegasus2avro.common.Ownership": { + "owners": [ + { + "owner": "urn:li:corpuser:new_owner", + "type": "TECHNICAL_OWNER" + }, + { + "owner": "urn:li:corpGroup:new_owner", + "type": "TECHNICAL_OWNER" + } + ], + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + } + } + }, + { + "com.linkedin.pegasus2avro.schema.SchemaMetadata": { + "schemaName": "namespace.iceberg_test", + "platform": "urn:li:dataPlatform:iceberg", + "version": 0, + "created": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "lastModified": { + "time": 0, + "actor": "urn:li:corpuser:unknown" + }, + "hash": "", + "platformSchema": { + "com.linkedin.pegasus2avro.schema.OtherSchema": { + "rawSchema": "Schema(1: level: required string(level documentation),2: event_time: required timestamptz(event_time documentation),3: message: required string(message documentation),4: call_stack: optional list(call_stack documentation))" + } + }, + "fields": [ + { + "fieldPath": "[version=2.0].[type=struct].[type=string].level", + "nullable": false, + "description": "level documentation", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": false}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=long].event_time", + "nullable": false, + "description": "event_time documentation", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.TimeType": {} + } + }, + "nativeDataType": "timestamptz", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"logicalType\": \"timestamp-micros\", \"native_data_type\": \"timestamptz\", \"_nullable\": false}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=string].message", + "nullable": false, + "description": "message documentation", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.StringType": {} + } + }, + "nativeDataType": "string", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"string\", \"_nullable\": false}" + }, + { + "fieldPath": "[version=2.0].[type=struct].[type=array].[type=string].call_stack", + "nullable": true, + "description": "call_stack documentation", + "type": { + "type": { + "com.linkedin.pegasus2avro.schema.ArrayType": { + "nestedType": [ + "string" + ] + } + } + }, + "nativeDataType": "list", + "recursive": false, + "isPartOfKey": false, + "jsonProps": "{\"native_data_type\": \"list\", \"_nullable\": true}" + } + ] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "iceberg-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:iceberg,test_platform_instance.namespace.iceberg_test,PROD)", + "changeType": "UPSERT", + "aspectName": "dataPlatformInstance", + "aspect": { + "value": "{\"platform\": \"urn:li:dataPlatform:iceberg\", \"instance\": \"urn:li:dataPlatformInstance:(urn:li:dataPlatform:iceberg,test_platform_instance)\"}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "iceberg-2020_04_14-07_00_00" + } +}, +{ + "entityType": "dataset", + "entityUrn": "urn:li:dataset:(urn:li:dataPlatform:iceberg,test_platform_instance.namespace.iceberg_test_2,PROD)", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": true}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1586847600000, + "runId": "iceberg-2020_04_14-07_00_00" + } +} +] \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run1/namespace/iceberg_test/metadata/v1.metadata.json b/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run1/namespace/iceberg_test/metadata/v1.metadata.json new file mode 100644 index 00000000000000..e4ac0b9685ddc4 --- /dev/null +++ b/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run1/namespace/iceberg_test/metadata/v1.metadata.json @@ -0,0 +1,105 @@ +{ + "format-version" : 1, + "table-uuid" : "11bbe5de-5ef6-4074-80db-f041065f9862", + "location" : "/namespace/iceberg_test", + "last-updated-ms" : 1648729616724, + "last-column-id" : 5, + "schema" : { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "level", + "required" : true, + "type" : "string" + }, { + "id" : 2, + "name" : "event_time", + "required" : true, + "type" : "timestamptz" + }, { + "id" : 3, + "name" : "message", + "required" : true, + "type" : "string" + }, { + "id" : 4, + "name" : "call_stack", + "required" : false, + "type" : { + "type" : "list", + "element-id" : 5, + "element" : "string", + "element-required" : true + } + } ] + }, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "level", + "required" : true, + "type" : "string" + }, { + "id" : 2, + "name" : "event_time", + "required" : true, + "type" : "timestamptz" + }, { + "id" : 3, + "name" : "message", + "required" : true, + "type" : "string" + }, { + "id" : 4, + "name" : "call_stack", + "required" : false, + "type" : { + "type" : "list", + "element-id" : 5, + "element" : "string", + "element-required" : true + } + } ] + } ], + "partition-spec" : [ { + "name" : "event_time_hour", + "transform" : "hour", + "source-id" : 2, + "field-id" : 1000 + }, { + "name" : "level", + "transform" : "identity", + "source-id" : 1, + "field-id" : 1001 + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ { + "name" : "event_time_hour", + "transform" : "hour", + "source-id" : 2, + "field-id" : 1000 + }, { + "name" : "level", + "transform" : "identity", + "source-id" : 1, + "field-id" : 1001 + } ] + } ], + "last-partition-id" : 1001, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { }, + "current-snapshot-id" : -1, + "snapshots" : [ ], + "snapshot-log" : [ ], + "metadata-log" : [ ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run1/namespace/iceberg_test/metadata/v2.metadata.json b/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run1/namespace/iceberg_test/metadata/v2.metadata.json new file mode 100644 index 00000000000000..02221330b06654 --- /dev/null +++ b/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run1/namespace/iceberg_test/metadata/v2.metadata.json @@ -0,0 +1,118 @@ +{ + "format-version" : 1, + "table-uuid" : "16e6ecee-cd5d-470f-a7a6-a197944fa4db", + "location" : "/namespace/iceberg_test", + "last-updated-ms" : 1649086837695, + "last-column-id" : 5, + "schema" : { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "level", + "required" : true, + "type" : "string", + "doc" : "level documentation" + }, { + "id" : 2, + "name" : "event_time", + "required" : true, + "type" : "timestamptz", + "doc" : "event_time documentation" + }, { + "id" : 3, + "name" : "message", + "required" : true, + "type" : "string", + "doc" : "message documentation" + }, { + "id" : 4, + "name" : "call_stack", + "required" : false, + "type" : { + "type" : "list", + "element-id" : 5, + "element" : "string", + "element-required" : true + }, + "doc" : "call_stack documentation" + } ] + }, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "level", + "required" : true, + "type" : "string", + "doc" : "level documentation" + }, { + "id" : 2, + "name" : "event_time", + "required" : true, + "type" : "timestamptz", + "doc" : "event_time documentation" + }, { + "id" : 3, + "name" : "message", + "required" : true, + "type" : "string", + "doc" : "message documentation" + }, { + "id" : 4, + "name" : "call_stack", + "required" : false, + "type" : { + "type" : "list", + "element-id" : 5, + "element" : "string", + "element-required" : true + }, + "doc" : "call_stack documentation" + } ] + } ], + "partition-spec" : [ { + "name" : "event_time_hour", + "transform" : "hour", + "source-id" : 2, + "field-id" : 1000 + }, { + "name" : "level", + "transform" : "identity", + "source-id" : 1, + "field-id" : 1001 + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ { + "name" : "event_time_hour", + "transform" : "hour", + "source-id" : 2, + "field-id" : 1000 + }, { + "name" : "level", + "transform" : "identity", + "source-id" : 1, + "field-id" : 1001 + } ] + } ], + "last-partition-id" : 1001, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "new_owner" + }, + "current-snapshot-id" : -1, + "snapshots" : [ ], + "snapshot-log" : [ ], + "metadata-log" : [ { + "timestamp-ms" : 1649086837511, + "metadata-file" : "/namespace/iceberg_test/metadata/v1.metadata.json" + } ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run1/namespace/iceberg_test/metadata/version-hint.text b/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run1/namespace/iceberg_test/metadata/version-hint.text new file mode 100644 index 00000000000000..d8263ee9860594 --- /dev/null +++ b/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run1/namespace/iceberg_test/metadata/version-hint.text @@ -0,0 +1 @@ +2 \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run1/namespace/iceberg_test_2/metadata/v1.metadata.json b/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run1/namespace/iceberg_test_2/metadata/v1.metadata.json new file mode 100644 index 00000000000000..e4ac0b9685ddc4 --- /dev/null +++ b/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run1/namespace/iceberg_test_2/metadata/v1.metadata.json @@ -0,0 +1,105 @@ +{ + "format-version" : 1, + "table-uuid" : "11bbe5de-5ef6-4074-80db-f041065f9862", + "location" : "/namespace/iceberg_test", + "last-updated-ms" : 1648729616724, + "last-column-id" : 5, + "schema" : { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "level", + "required" : true, + "type" : "string" + }, { + "id" : 2, + "name" : "event_time", + "required" : true, + "type" : "timestamptz" + }, { + "id" : 3, + "name" : "message", + "required" : true, + "type" : "string" + }, { + "id" : 4, + "name" : "call_stack", + "required" : false, + "type" : { + "type" : "list", + "element-id" : 5, + "element" : "string", + "element-required" : true + } + } ] + }, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "level", + "required" : true, + "type" : "string" + }, { + "id" : 2, + "name" : "event_time", + "required" : true, + "type" : "timestamptz" + }, { + "id" : 3, + "name" : "message", + "required" : true, + "type" : "string" + }, { + "id" : 4, + "name" : "call_stack", + "required" : false, + "type" : { + "type" : "list", + "element-id" : 5, + "element" : "string", + "element-required" : true + } + } ] + } ], + "partition-spec" : [ { + "name" : "event_time_hour", + "transform" : "hour", + "source-id" : 2, + "field-id" : 1000 + }, { + "name" : "level", + "transform" : "identity", + "source-id" : 1, + "field-id" : 1001 + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ { + "name" : "event_time_hour", + "transform" : "hour", + "source-id" : 2, + "field-id" : 1000 + }, { + "name" : "level", + "transform" : "identity", + "source-id" : 1, + "field-id" : 1001 + } ] + } ], + "last-partition-id" : 1001, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { }, + "current-snapshot-id" : -1, + "snapshots" : [ ], + "snapshot-log" : [ ], + "metadata-log" : [ ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run1/namespace/iceberg_test_2/metadata/v2.metadata.json b/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run1/namespace/iceberg_test_2/metadata/v2.metadata.json new file mode 100644 index 00000000000000..02221330b06654 --- /dev/null +++ b/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run1/namespace/iceberg_test_2/metadata/v2.metadata.json @@ -0,0 +1,118 @@ +{ + "format-version" : 1, + "table-uuid" : "16e6ecee-cd5d-470f-a7a6-a197944fa4db", + "location" : "/namespace/iceberg_test", + "last-updated-ms" : 1649086837695, + "last-column-id" : 5, + "schema" : { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "level", + "required" : true, + "type" : "string", + "doc" : "level documentation" + }, { + "id" : 2, + "name" : "event_time", + "required" : true, + "type" : "timestamptz", + "doc" : "event_time documentation" + }, { + "id" : 3, + "name" : "message", + "required" : true, + "type" : "string", + "doc" : "message documentation" + }, { + "id" : 4, + "name" : "call_stack", + "required" : false, + "type" : { + "type" : "list", + "element-id" : 5, + "element" : "string", + "element-required" : true + }, + "doc" : "call_stack documentation" + } ] + }, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "level", + "required" : true, + "type" : "string", + "doc" : "level documentation" + }, { + "id" : 2, + "name" : "event_time", + "required" : true, + "type" : "timestamptz", + "doc" : "event_time documentation" + }, { + "id" : 3, + "name" : "message", + "required" : true, + "type" : "string", + "doc" : "message documentation" + }, { + "id" : 4, + "name" : "call_stack", + "required" : false, + "type" : { + "type" : "list", + "element-id" : 5, + "element" : "string", + "element-required" : true + }, + "doc" : "call_stack documentation" + } ] + } ], + "partition-spec" : [ { + "name" : "event_time_hour", + "transform" : "hour", + "source-id" : 2, + "field-id" : 1000 + }, { + "name" : "level", + "transform" : "identity", + "source-id" : 1, + "field-id" : 1001 + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ { + "name" : "event_time_hour", + "transform" : "hour", + "source-id" : 2, + "field-id" : 1000 + }, { + "name" : "level", + "transform" : "identity", + "source-id" : 1, + "field-id" : 1001 + } ] + } ], + "last-partition-id" : 1001, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "new_owner" + }, + "current-snapshot-id" : -1, + "snapshots" : [ ], + "snapshot-log" : [ ], + "metadata-log" : [ { + "timestamp-ms" : 1649086837511, + "metadata-file" : "/namespace/iceberg_test/metadata/v1.metadata.json" + } ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run1/namespace/iceberg_test_2/metadata/version-hint.text b/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run1/namespace/iceberg_test_2/metadata/version-hint.text new file mode 100644 index 00000000000000..d8263ee9860594 --- /dev/null +++ b/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run1/namespace/iceberg_test_2/metadata/version-hint.text @@ -0,0 +1 @@ +2 \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run2/namespace/iceberg_test/metadata/v1.metadata.json b/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run2/namespace/iceberg_test/metadata/v1.metadata.json new file mode 100644 index 00000000000000..e4ac0b9685ddc4 --- /dev/null +++ b/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run2/namespace/iceberg_test/metadata/v1.metadata.json @@ -0,0 +1,105 @@ +{ + "format-version" : 1, + "table-uuid" : "11bbe5de-5ef6-4074-80db-f041065f9862", + "location" : "/namespace/iceberg_test", + "last-updated-ms" : 1648729616724, + "last-column-id" : 5, + "schema" : { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "level", + "required" : true, + "type" : "string" + }, { + "id" : 2, + "name" : "event_time", + "required" : true, + "type" : "timestamptz" + }, { + "id" : 3, + "name" : "message", + "required" : true, + "type" : "string" + }, { + "id" : 4, + "name" : "call_stack", + "required" : false, + "type" : { + "type" : "list", + "element-id" : 5, + "element" : "string", + "element-required" : true + } + } ] + }, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "level", + "required" : true, + "type" : "string" + }, { + "id" : 2, + "name" : "event_time", + "required" : true, + "type" : "timestamptz" + }, { + "id" : 3, + "name" : "message", + "required" : true, + "type" : "string" + }, { + "id" : 4, + "name" : "call_stack", + "required" : false, + "type" : { + "type" : "list", + "element-id" : 5, + "element" : "string", + "element-required" : true + } + } ] + } ], + "partition-spec" : [ { + "name" : "event_time_hour", + "transform" : "hour", + "source-id" : 2, + "field-id" : 1000 + }, { + "name" : "level", + "transform" : "identity", + "source-id" : 1, + "field-id" : 1001 + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ { + "name" : "event_time_hour", + "transform" : "hour", + "source-id" : 2, + "field-id" : 1000 + }, { + "name" : "level", + "transform" : "identity", + "source-id" : 1, + "field-id" : 1001 + } ] + } ], + "last-partition-id" : 1001, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { }, + "current-snapshot-id" : -1, + "snapshots" : [ ], + "snapshot-log" : [ ], + "metadata-log" : [ ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run2/namespace/iceberg_test/metadata/v2.metadata.json b/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run2/namespace/iceberg_test/metadata/v2.metadata.json new file mode 100644 index 00000000000000..02221330b06654 --- /dev/null +++ b/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run2/namespace/iceberg_test/metadata/v2.metadata.json @@ -0,0 +1,118 @@ +{ + "format-version" : 1, + "table-uuid" : "16e6ecee-cd5d-470f-a7a6-a197944fa4db", + "location" : "/namespace/iceberg_test", + "last-updated-ms" : 1649086837695, + "last-column-id" : 5, + "schema" : { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "level", + "required" : true, + "type" : "string", + "doc" : "level documentation" + }, { + "id" : 2, + "name" : "event_time", + "required" : true, + "type" : "timestamptz", + "doc" : "event_time documentation" + }, { + "id" : 3, + "name" : "message", + "required" : true, + "type" : "string", + "doc" : "message documentation" + }, { + "id" : 4, + "name" : "call_stack", + "required" : false, + "type" : { + "type" : "list", + "element-id" : 5, + "element" : "string", + "element-required" : true + }, + "doc" : "call_stack documentation" + } ] + }, + "current-schema-id" : 0, + "schemas" : [ { + "type" : "struct", + "schema-id" : 0, + "fields" : [ { + "id" : 1, + "name" : "level", + "required" : true, + "type" : "string", + "doc" : "level documentation" + }, { + "id" : 2, + "name" : "event_time", + "required" : true, + "type" : "timestamptz", + "doc" : "event_time documentation" + }, { + "id" : 3, + "name" : "message", + "required" : true, + "type" : "string", + "doc" : "message documentation" + }, { + "id" : 4, + "name" : "call_stack", + "required" : false, + "type" : { + "type" : "list", + "element-id" : 5, + "element" : "string", + "element-required" : true + }, + "doc" : "call_stack documentation" + } ] + } ], + "partition-spec" : [ { + "name" : "event_time_hour", + "transform" : "hour", + "source-id" : 2, + "field-id" : 1000 + }, { + "name" : "level", + "transform" : "identity", + "source-id" : 1, + "field-id" : 1001 + } ], + "default-spec-id" : 0, + "partition-specs" : [ { + "spec-id" : 0, + "fields" : [ { + "name" : "event_time_hour", + "transform" : "hour", + "source-id" : 2, + "field-id" : 1000 + }, { + "name" : "level", + "transform" : "identity", + "source-id" : 1, + "field-id" : 1001 + } ] + } ], + "last-partition-id" : 1001, + "default-sort-order-id" : 0, + "sort-orders" : [ { + "order-id" : 0, + "fields" : [ ] + } ], + "properties" : { + "owner" : "new_owner" + }, + "current-snapshot-id" : -1, + "snapshots" : [ ], + "snapshot-log" : [ ], + "metadata-log" : [ { + "timestamp-ms" : 1649086837511, + "metadata-file" : "/namespace/iceberg_test/metadata/v1.metadata.json" + } ] +} \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run2/namespace/iceberg_test/metadata/version-hint.text b/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run2/namespace/iceberg_test/metadata/version-hint.text new file mode 100644 index 00000000000000..d8263ee9860594 --- /dev/null +++ b/metadata-ingestion/tests/integration/iceberg/test_data/stateful_test/run2/namespace/iceberg_test/metadata/version-hint.text @@ -0,0 +1 @@ +2 \ No newline at end of file diff --git a/metadata-ingestion/tests/integration/iceberg/test_iceberg.py b/metadata-ingestion/tests/integration/iceberg/test_iceberg.py index 97cb716314c1d0..7da50200cba5ff 100644 --- a/metadata-ingestion/tests/integration/iceberg/test_iceberg.py +++ b/metadata-ingestion/tests/integration/iceberg/test_iceberg.py @@ -1,5 +1,5 @@ from pathlib import PosixPath -from typing import Union +from typing import Any, Dict, Optional, Union, cast from unittest.mock import patch import pytest @@ -8,9 +8,27 @@ from iceberg.core.filesystem.local_filesystem import LocalFileSystem from datahub.ingestion.run.pipeline import Pipeline +from datahub.ingestion.source.iceberg.iceberg import IcebergSource +from datahub.ingestion.source.state.checkpoint import Checkpoint +from datahub.ingestion.source.state.iceberg_state import IcebergCheckpointState from tests.test_helpers import mce_helpers +from tests.test_helpers.state_helpers import ( + run_and_get_pipeline, + validate_all_providers_have_committed_successfully, +) FROZEN_TIME = "2020-04-14 07:00:00" +GMS_PORT = 8080 +GMS_SERVER = f"http://localhost:{GMS_PORT}" + + +def get_current_checkpoint_from_pipeline( + pipeline: Pipeline, +) -> Optional[Checkpoint]: + iceberg_source = cast(IcebergSource, pipeline.source) + return iceberg_source.get_current_checkpoint( + iceberg_source.stale_entity_removal_handler.job_id + ) @freeze_time(FROZEN_TIME) @@ -25,7 +43,7 @@ def test_iceberg_ingest(pytestconfig, tmp_path, mock_time): "source": { "type": "iceberg", "config": { - "localfs": str(test_resources_dir / "test_data"), + "localfs": str(test_resources_dir / "test_data/ingest_test"), "user_ownership_property": "owner", "group_ownership_property": "owner", }, @@ -45,10 +63,124 @@ def test_iceberg_ingest(pytestconfig, tmp_path, mock_time): mce_helpers.check_golden_file( pytestconfig, output_path=tmp_path / "iceberg_mces.json", - golden_path=test_resources_dir / "iceberg_mces_golden.json", + golden_path=test_resources_dir + / "test_data/ingest_test/iceberg_mces_golden.json", ) +@freeze_time(FROZEN_TIME) +@pytest.mark.integration +def test_iceberg_stateful_ingest(pytestconfig, tmp_path, mock_time, mock_datahub_graph): + test_resources_dir = ( + pytestconfig.rootpath / "tests/integration/iceberg/test_data/stateful_test" + ) + platform_instance = "test_platform_instance" + + scd_before_deletion: Dict[str, Any] = { + "localfs": str(test_resources_dir / "run1"), + "user_ownership_property": "owner", + "group_ownership_property": "owner", + "platform_instance": f"{platform_instance}", + # enable stateful ingestion + "stateful_ingestion": { + "enabled": True, + "remove_stale_metadata": True, + "fail_safe_threshold": 100.0, + "state_provider": { + "type": "datahub", + "config": {"datahub_api": {"server": GMS_SERVER}}, + }, + }, + } + + scd_after_deletion: Dict[str, Any] = { + "localfs": str(test_resources_dir / "run2"), + "user_ownership_property": "owner", + "group_ownership_property": "owner", + "platform_instance": f"{platform_instance}", + # enable stateful ingestion + "stateful_ingestion": { + "enabled": True, + "remove_stale_metadata": True, + "fail_safe_threshold": 100.0, + "state_provider": { + "type": "datahub", + "config": {"datahub_api": {"server": GMS_SERVER}}, + }, + }, + } + + pipeline_config_dict: Dict[str, Any] = { + "source": { + "type": "iceberg", + "config": scd_before_deletion, + }, + "sink": { + # we are not really interested in the resulting events for this test + "type": "console" + }, + "pipeline_name": "test_pipeline", + } + + with patch( + "datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider.DataHubGraph", + mock_datahub_graph, + ) as mock_checkpoint: + + # Both checkpoint and reporting will use the same mocked graph instance. + mock_checkpoint.return_value = mock_datahub_graph + + # Do the first run of the pipeline and get the default job's checkpoint. + pipeline_run1 = run_and_get_pipeline(pipeline_config_dict) + checkpoint1 = get_current_checkpoint_from_pipeline(pipeline_run1) + + assert checkpoint1 + assert checkpoint1.state + + # Set iceberg config where a table is deleted. + pipeline_config_dict["source"]["config"] = scd_after_deletion + # Capture MCEs of second run to validate Status(removed=true) + deleted_mces_path = f"{tmp_path}/iceberg_deleted_mces.json" + pipeline_config_dict["sink"]["type"] = "file" + pipeline_config_dict["sink"]["config"] = {"filename": deleted_mces_path} + + # Do the second run of the pipeline. + pipeline_run2 = run_and_get_pipeline(pipeline_config_dict) + checkpoint2 = get_current_checkpoint_from_pipeline(pipeline_run2) + + assert checkpoint2 + assert checkpoint2.state + + # Perform all assertions on the states. The deleted table should not be + # part of the second state + state1 = cast(IcebergCheckpointState, checkpoint1.state) + state2 = cast(IcebergCheckpointState, checkpoint2.state) + difference_urns = list( + state1.get_urns_not_in(type="dataset", other_checkpoint_state=state2) + ) + + assert len(difference_urns) == 1 + + urn1 = "urn:li:dataset:(urn:li:dataPlatform:iceberg,test_platform_instance.namespace.iceberg_test_2,PROD)" + + assert urn1 in difference_urns + + # Validate that all providers have committed successfully. + validate_all_providers_have_committed_successfully( + pipeline=pipeline_run1, expected_providers=1 + ) + validate_all_providers_have_committed_successfully( + pipeline=pipeline_run2, expected_providers=1 + ) + + # Verify the output. + mce_helpers.check_golden_file( + pytestconfig, + output_path=deleted_mces_path, + golden_path=test_resources_dir / "iceberg_deleted_table_mces_golden.json", + ) + + @freeze_time(FROZEN_TIME) @pytest.mark.integration def test_iceberg_profiling(pytestconfig, tmp_path, mock_time): @@ -69,7 +201,9 @@ def test_iceberg_profiling(pytestconfig, tmp_path, mock_time): When importing the metadata files into this test, we need to create a `version-hint.text` with a value that reflects the version of the table, and then change the code in `TestLocalFileSystem._replace_path()` accordingly. """ - test_resources_dir = pytestconfig.rootpath / "tests/integration/iceberg/test_data" + test_resources_dir = ( + pytestconfig.rootpath / "tests/integration/iceberg/test_data/profiling_test" + ) # Run the metadata ingestion pipeline. pipeline = Pipeline.create( @@ -161,6 +295,5 @@ def exists(self, path: str) -> bool: mce_helpers.check_golden_file( pytestconfig, output_path=tmp_path / "iceberg_mces.json", - golden_path=test_resources_dir - / "datahub/integration/profiling/iceberg_mces_golden.json", + golden_path=test_resources_dir / "iceberg_mces_golden.json", ) From 574bc85c734537e456c321dee1115cf8aba1931f Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Tue, 13 Dec 2022 00:15:11 +0530 Subject: [PATCH 08/19] doc(restore): document restore indices API endpoint (#6737) --- docs/how/restore-indices.md | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/docs/how/restore-indices.md b/docs/how/restore-indices.md index a9e2cd9799d19f..2885a16d691ea1 100644 --- a/docs/how/restore-indices.md +++ b/docs/how/restore-indices.md @@ -48,4 +48,17 @@ Once restore indices job template has been deployed, run the following command t kubectl create job --from=cronjob/datahub-datahub-restore-indices-job-template datahub-restore-indices-adhoc ``` -Once the job completes, your indices will have been restored. \ No newline at end of file +Once the job completes, your indices will have been restored. + +## Through API + +You can do a HTTP POST request to `/gms/aspects?action=restoreIndices` endpoint with the `urn` as part of JSON Payload to restore indices for the particular URN. + +``` +curl --location --request POST 'https://demo.datahubproject.io/api/gms/aspects?action=restoreIndices' \ +--header 'Authorization: Bearer TOKEN' \ +--header 'Content-Type: application/json' \ +--data-raw '{ + "urn": "YOUR_URN" +}' +``` \ No newline at end of file From 54230a8d81c9f11a8d640c2f33f36aa8986530ff Mon Sep 17 00:00:00 2001 From: John Joyce Date: Mon, 12 Dec 2022 12:30:09 -0800 Subject: [PATCH 09/19] feat(): Views Feature Milestone 1 (#6666) --- .../datahub/graphql/GmsGraphQLEngine.java | 74 ++- .../authorization/AuthorizationUtils.java | 4 + .../graphql/resolvers/EntityTypeMapper.java | 2 + .../datahub/graphql/resolvers/MeResolver.java | 1 + .../graphql/resolvers/ResolverUtils.java | 6 +- .../resolvers/config/AppConfigResolver.java | 11 +- .../mutate/UpdateUserSettingResolver.java | 7 +- .../ListRecommendationsResolver.java | 4 +- .../search/SearchAcrossEntitiesResolver.java | 51 +- .../graphql/resolvers/search/SearchUtils.java | 221 +++++++ .../UpdateCorpUserViewsSettingsResolver.java | 83 +++ .../view/GlobalViewsSettingsResolver.java | 51 ++ .../UpdateGlobalViewsSettingsResolver.java | 76 +++ .../resolvers/view/CreateViewResolver.java | 78 +++ .../resolvers/view/DeleteViewResolver.java | 48 ++ .../view/ListGlobalViewsResolver.java | 112 ++++ .../resolvers/view/ListMyViewsResolver.java | 125 ++++ .../resolvers/view/UpdateViewResolver.java | 71 ++ .../graphql/resolvers/view/ViewUtils.java | 146 +++++ .../common/mappers/UrnToEntityMapper.java | 6 + .../corpuser/mappers/CorpUserMapper.java | 34 +- .../graphql/types/view/DataHubViewMapper.java | 146 +++++ .../graphql/types/view/DataHubViewType.java | 76 +++ .../src/main/resources/app.graphql | 56 ++ .../src/main/resources/entity.graphql | 318 ++++++++- .../src/main/resources/recommendation.graphql | 27 +- .../src/main/resources/search.graphql | 40 +- .../linkedin/datahub/graphql/TestUtils.java | 23 +- .../SearchAcrossEntitiesResolverTest.java | 527 +++++++++++++++ .../resolvers/search/SearchUtilsTest.java | 352 ++++++++++ ...dateCorpUserViewsSettingsResolverTest.java | 164 +++++ .../view/GlobalViewsSettingsResolverTest.java | 121 ++++ ...UpdateGlobalViewsSettingsResolverTest.java | 168 +++++ .../view/CreateViewResolverTest.java | 161 +++++ .../view/DeleteViewResolverTest.java | 164 +++++ .../view/ListGlobalViewsResolverTest.java | 138 ++++ .../view/ListMyViewsResolverTest.java | 195 ++++++ .../view/UpdateViewResolverTest.java | 238 +++++++ .../graphql/resolvers/view/ViewUtilsTest.java | 188 ++++++ .../types/view/DataHubViewTypeTest.java | 243 +++++++ datahub-web-react/src/AppConfigProvider.tsx | 2 +- datahub-web-react/src/Mocks.tsx | 6 +- datahub-web-react/src/app/ProtectedRoutes.tsx | 21 +- datahub-web-react/src/app/analytics/event.ts | 46 +- .../src/app/context/UserContextProvider.tsx | 145 ++++ .../src/app/context/useUserContext.tsx | 9 + .../src/app/context/userContext.tsx | 62 ++ .../src/app/entity/dataset/DatasetEntity.tsx | 1 + .../styled/search/DownloadAsCsvModal.tsx | 4 +- .../search/EmbeddedListSearchHeader.tsx | 4 +- .../styled/search/SearchExtendedMenu.tsx | 4 +- .../src/app/entity/view/ManageViews.tsx | 43 ++ .../src/app/entity/view/ViewTypeLabel.tsx | 54 ++ .../src/app/entity/view/ViewsList.tsx | 144 ++++ .../src/app/entity/view/ViewsTable.tsx | 61 ++ .../app/entity/view/builder/ViewBuilder.tsx | 123 ++++ .../entity/view/builder/ViewBuilderForm.tsx | 115 ++++ .../entity/view/builder/ViewBuilderModal.tsx | 102 +++ .../view/builder/ViewDefinitionBuilder.tsx | 154 +++++ .../src/app/entity/view/builder/types.ts | 10 + .../src/app/entity/view/builder/utils.ts | 92 +++ .../src/app/entity/view/cacheUtils.ts | 250 +++++++ .../app/entity/view/menu/ViewDropdownMenu.tsx | 267 ++++++++ .../entity/view/menu/item/DeleteViewItem.tsx | 20 + .../entity/view/menu/item/EditViewItem.tsx | 20 + .../entity/view/menu/item/IconItemTitle.tsx | 38 ++ .../entity/view/menu/item/PreviewViewItem.tsx | 20 + .../menu/item/RemoveGlobalDefaultItem.tsx | 24 + .../view/menu/item/RemoveUserDefaultItem.tsx | 24 + .../view/menu/item/SetGlobalDefaultItem.tsx | 24 + .../view/menu/item/SetUserDefaultItem.tsx | 24 + .../src/app/entity/view/select/ViewOption.tsx | 60 ++ .../app/entity/view/select/ViewOptionName.tsx | 24 + .../view/select/ViewOptionTooltipTitle.tsx | 22 + .../src/app/entity/view/select/ViewSelect.tsx | 215 ++++++ .../entity/view/select/ViewSelectDropdown.tsx | 21 + .../entity/view/select/ViewSelectFooter.tsx | 46 ++ .../entity/view/select/ViewSelectHeader.tsx | 46 ++ .../entity/view/select/ViewSelectToolTip.tsx | 29 + .../entity/view/select/ViewsTableColumns.tsx | 92 +++ .../view/select/renderViewOptionGroup.tsx | 57 ++ .../entity/view/shared/DefaultViewIcon.tsx | 36 + .../view/shared/GlobalDefaultViewIcon.tsx | 11 + .../view/shared/UserDefaultViewIcon.tsx | 11 + .../src/app/entity/view/types.ts | 52 ++ .../src/app/entity/view/utils.ts | 83 +++ .../search/AdvancedFilterSelectValueModal.tsx | 49 +- .../search/AdvancedSearchAddFilterSelect.tsx | 30 +- .../src/app/search/AdvancedSearchFilter.tsx | 41 +- .../AdvancedSearchFilterConditionSelect.tsx | 36 +- ...ncedSearchFilterOverallUnionTypeSelect.tsx | 4 +- .../AdvancedSearchFilterValuesSection.tsx | 14 +- .../src/app/search/AdvancedSearchFilters.tsx | 109 ++-- .../src/app/search/ChooseEntityTypeModal.tsx | 30 +- .../src/app/search/SaveAsViewButton.tsx | 51 ++ .../src/app/search/SearchFilterLabel.tsx | 56 +- .../src/app/search/SearchFiltersSection.tsx | 79 ++- .../src/app/search/SearchHeader.tsx | 11 + .../src/app/search/SearchPage.tsx | 7 +- .../src/app/search/SimpleSearchFilter.tsx | 7 +- .../src/app/search/SimpleSearchFilters.tsx | 22 +- .../app/search/__tests__/constants.test.tsx | 7 + .../src/app/search/utils/constants.ts | 69 +- .../src/app/search/utils/filterUtils.ts | 21 +- .../utils/filtersToQueryStringParams.ts | 6 +- .../src/app/search/utils/generateOrFilters.ts | 4 +- .../app/search/utils/navigateToSearchUrl.ts | 1 - .../src/app/settings/SettingsPage.tsx | 19 +- .../src/app/shared/ManageAccount.tsx | 3 + datahub-web-react/src/appConfigContext.tsx | 3 + datahub-web-react/src/conf/Global.ts | 1 + datahub-web-react/src/graphql/app.graphql | 13 + datahub-web-react/src/graphql/entity.graphql | 7 + datahub-web-react/src/graphql/me.graphql | 6 + datahub-web-react/src/graphql/user.graphql | 6 +- datahub-web-react/src/graphql/view.graphql | 56 ++ docs/authorization/access-policies-guide.md | 1 + .../java/com/linkedin/metadata/Constants.java | 15 +- .../metadata/config/ViewsConfiguration.java | 14 + .../metadata/service/SettingsService.java | 145 ++++ .../metadata/service/ViewService.java | 225 +++++++ .../metadata/service/SettingsServiceTest.java | 317 +++++++++ .../metadata/service/ViewServiceTest.java | 617 ++++++++++++++++++ .../linkedin/identity/CorpUserSettings.pdl | 7 +- .../identity/CorpUserViewsSettings.pdl | 14 + .../linkedin/metadata/key/DataHubViewKey.pdl | 14 + .../metadata/key/GlobalSettingsKey.pdl | 17 + .../settings/global/GlobalSettingsInfo.pdl | 14 + .../settings/global/GlobalViewsSettings.pdl | 13 + .../linkedin/view/DataHubViewDefinition.pdl | 18 + .../com/linkedin/view/DataHubViewInfo.pdl | 73 +++ .../src/main/resources/entity-registry.yml | 11 + .../datahub/telemetry/TrackingService.java | 3 +- .../factory/config/ConfigurationProvider.java | 6 +- .../factory/graphql/GraphQLEngineFactory.java | 16 + .../search/views/ViewServiceFactory.java | 33 + .../settings/SettingsServiceFactory.java | 33 + .../factories/BootstrapManagerFactory.java | 4 +- .../IngestDefaultGlobalSettingsStep.java | 133 ++++ .../src/main/resources/application.yml | 3 + .../IngestDefaultGlobalSettingsStepTest.java | 122 ++++ .../test_global_settings_invalid_json.json | 4 + .../test_global_settings_invalid_model.json | 3 + .../boot/test_global_settings_valid.json | 5 + .../main/resources/boot/global_settings.json | 4 + .../war/src/main/resources/boot/policies.json | 6 +- .../authorization/PoliciesConfig.java | 9 +- .../cypress/integration/views/manage_views.js | 36 + .../cypress/integration/views/view_select.js | 102 +++ .../tests/cypress/cypress/support/commands.js | 11 + .../tests/cypress/cypress/support/index.js | 2 +- smoke-test/tests/views/__init__.py | 0 smoke-test/tests/views/views_test.py | 424 ++++++++++++ 153 files changed, 10200 insertions(+), 316 deletions(-) create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/settings/user/UpdateCorpUserViewsSettingsResolver.java create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/settings/view/GlobalViewsSettingsResolver.java create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/settings/view/UpdateGlobalViewsSettingsResolver.java create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/CreateViewResolver.java create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/DeleteViewResolver.java create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolver.java create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolver.java create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/UpdateViewResolver.java create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtils.java create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/view/DataHubViewMapper.java create mode 100644 datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/view/DataHubViewType.java create mode 100644 datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java create mode 100644 datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtilsTest.java create mode 100644 datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/settings/user/UpdateCorpUserViewsSettingsResolverTest.java create mode 100644 datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/settings/view/GlobalViewsSettingsResolverTest.java create mode 100644 datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/settings/view/UpdateGlobalViewsSettingsResolverTest.java create mode 100644 datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/CreateViewResolverTest.java create mode 100644 datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/DeleteViewResolverTest.java create mode 100644 datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolverTest.java create mode 100644 datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolverTest.java create mode 100644 datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/UpdateViewResolverTest.java create mode 100644 datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtilsTest.java create mode 100644 datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/view/DataHubViewTypeTest.java create mode 100644 datahub-web-react/src/app/context/UserContextProvider.tsx create mode 100644 datahub-web-react/src/app/context/useUserContext.tsx create mode 100644 datahub-web-react/src/app/context/userContext.tsx create mode 100644 datahub-web-react/src/app/entity/view/ManageViews.tsx create mode 100644 datahub-web-react/src/app/entity/view/ViewTypeLabel.tsx create mode 100644 datahub-web-react/src/app/entity/view/ViewsList.tsx create mode 100644 datahub-web-react/src/app/entity/view/ViewsTable.tsx create mode 100644 datahub-web-react/src/app/entity/view/builder/ViewBuilder.tsx create mode 100644 datahub-web-react/src/app/entity/view/builder/ViewBuilderForm.tsx create mode 100644 datahub-web-react/src/app/entity/view/builder/ViewBuilderModal.tsx create mode 100644 datahub-web-react/src/app/entity/view/builder/ViewDefinitionBuilder.tsx create mode 100644 datahub-web-react/src/app/entity/view/builder/types.ts create mode 100644 datahub-web-react/src/app/entity/view/builder/utils.ts create mode 100644 datahub-web-react/src/app/entity/view/cacheUtils.ts create mode 100644 datahub-web-react/src/app/entity/view/menu/ViewDropdownMenu.tsx create mode 100644 datahub-web-react/src/app/entity/view/menu/item/DeleteViewItem.tsx create mode 100644 datahub-web-react/src/app/entity/view/menu/item/EditViewItem.tsx create mode 100644 datahub-web-react/src/app/entity/view/menu/item/IconItemTitle.tsx create mode 100644 datahub-web-react/src/app/entity/view/menu/item/PreviewViewItem.tsx create mode 100644 datahub-web-react/src/app/entity/view/menu/item/RemoveGlobalDefaultItem.tsx create mode 100644 datahub-web-react/src/app/entity/view/menu/item/RemoveUserDefaultItem.tsx create mode 100644 datahub-web-react/src/app/entity/view/menu/item/SetGlobalDefaultItem.tsx create mode 100644 datahub-web-react/src/app/entity/view/menu/item/SetUserDefaultItem.tsx create mode 100644 datahub-web-react/src/app/entity/view/select/ViewOption.tsx create mode 100644 datahub-web-react/src/app/entity/view/select/ViewOptionName.tsx create mode 100644 datahub-web-react/src/app/entity/view/select/ViewOptionTooltipTitle.tsx create mode 100644 datahub-web-react/src/app/entity/view/select/ViewSelect.tsx create mode 100644 datahub-web-react/src/app/entity/view/select/ViewSelectDropdown.tsx create mode 100644 datahub-web-react/src/app/entity/view/select/ViewSelectFooter.tsx create mode 100644 datahub-web-react/src/app/entity/view/select/ViewSelectHeader.tsx create mode 100644 datahub-web-react/src/app/entity/view/select/ViewSelectToolTip.tsx create mode 100644 datahub-web-react/src/app/entity/view/select/ViewsTableColumns.tsx create mode 100644 datahub-web-react/src/app/entity/view/select/renderViewOptionGroup.tsx create mode 100644 datahub-web-react/src/app/entity/view/shared/DefaultViewIcon.tsx create mode 100644 datahub-web-react/src/app/entity/view/shared/GlobalDefaultViewIcon.tsx create mode 100644 datahub-web-react/src/app/entity/view/shared/UserDefaultViewIcon.tsx create mode 100644 datahub-web-react/src/app/entity/view/types.ts create mode 100644 datahub-web-react/src/app/entity/view/utils.ts create mode 100644 datahub-web-react/src/app/search/SaveAsViewButton.tsx create mode 100644 datahub-web-react/src/app/search/__tests__/constants.test.tsx create mode 100644 datahub-web-react/src/graphql/entity.graphql create mode 100644 datahub-web-react/src/graphql/view.graphql create mode 100644 metadata-io/src/main/java/com/linkedin/metadata/config/ViewsConfiguration.java create mode 100644 metadata-io/src/main/java/com/linkedin/metadata/service/SettingsService.java create mode 100644 metadata-io/src/main/java/com/linkedin/metadata/service/ViewService.java create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/service/SettingsServiceTest.java create mode 100644 metadata-io/src/test/java/com/linkedin/metadata/service/ViewServiceTest.java create mode 100644 metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserViewsSettings.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/metadata/key/DataHubViewKey.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/metadata/key/GlobalSettingsKey.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/settings/global/GlobalSettingsInfo.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/settings/global/GlobalViewsSettings.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/view/DataHubViewDefinition.pdl create mode 100644 metadata-models/src/main/pegasus/com/linkedin/view/DataHubViewInfo.pdl create mode 100644 metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/views/ViewServiceFactory.java create mode 100644 metadata-service/factories/src/main/java/com/linkedin/gms/factory/settings/SettingsServiceFactory.java create mode 100644 metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDefaultGlobalSettingsStep.java create mode 100644 metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestDefaultGlobalSettingsStepTest.java create mode 100644 metadata-service/factories/src/test/resources/boot/test_global_settings_invalid_json.json create mode 100644 metadata-service/factories/src/test/resources/boot/test_global_settings_invalid_model.json create mode 100644 metadata-service/factories/src/test/resources/boot/test_global_settings_valid.json create mode 100644 metadata-service/war/src/main/resources/boot/global_settings.json create mode 100644 smoke-test/tests/cypress/cypress/integration/views/manage_views.js create mode 100644 smoke-test/tests/cypress/cypress/integration/views/view_select.js create mode 100644 smoke-test/tests/views/__init__.py create mode 100644 smoke-test/tests/views/views_test.py diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java index ca57362c2ca917..c3b8ecb55241a7 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/GmsGraphQLEngine.java @@ -34,11 +34,13 @@ import com.linkedin.datahub.graphql.generated.CorpGroupInfo; import com.linkedin.datahub.graphql.generated.CorpUser; import com.linkedin.datahub.graphql.generated.CorpUserInfo; +import com.linkedin.datahub.graphql.generated.CorpUserViewsSettings; import com.linkedin.datahub.graphql.generated.Dashboard; import com.linkedin.datahub.graphql.generated.DashboardInfo; import com.linkedin.datahub.graphql.generated.DashboardStatsSummary; import com.linkedin.datahub.graphql.generated.DashboardUserUsageCounts; import com.linkedin.datahub.graphql.generated.DataFlow; +import com.linkedin.datahub.graphql.generated.DataHubView; import com.linkedin.datahub.graphql.generated.DataJob; import com.linkedin.datahub.graphql.generated.DataJobInputOutput; import com.linkedin.datahub.graphql.generated.DataPlatformInstance; @@ -61,6 +63,7 @@ import com.linkedin.datahub.graphql.generated.ListDomainsResult; import com.linkedin.datahub.graphql.generated.ListGroupsResult; import com.linkedin.datahub.graphql.generated.ListTestsResult; +import com.linkedin.datahub.graphql.generated.ListViewsResult; import com.linkedin.datahub.graphql.generated.MLFeature; import com.linkedin.datahub.graphql.generated.MLFeatureProperties; import com.linkedin.datahub.graphql.generated.MLFeatureTable; @@ -176,6 +179,7 @@ import com.linkedin.datahub.graphql.resolvers.mutate.UpdateNameResolver; import com.linkedin.datahub.graphql.resolvers.mutate.UpdateParentNodeResolver; import com.linkedin.datahub.graphql.resolvers.mutate.UpdateUserSettingResolver; +import com.linkedin.datahub.graphql.resolvers.settings.user.UpdateCorpUserViewsSettingsResolver; import com.linkedin.datahub.graphql.resolvers.operation.ReportOperationResolver; import com.linkedin.datahub.graphql.resolvers.policy.DeletePolicyResolver; import com.linkedin.datahub.graphql.resolvers.policy.GetGrantedPrivilegesResolver; @@ -196,6 +200,8 @@ import com.linkedin.datahub.graphql.resolvers.search.SearchResolver; import com.linkedin.datahub.graphql.resolvers.step.BatchGetStepStatesResolver; import com.linkedin.datahub.graphql.resolvers.step.BatchUpdateStepStatesResolver; +import com.linkedin.datahub.graphql.resolvers.settings.view.GlobalViewsSettingsResolver; +import com.linkedin.datahub.graphql.resolvers.settings.view.UpdateGlobalViewsSettingsResolver; import com.linkedin.datahub.graphql.resolvers.tag.CreateTagResolver; import com.linkedin.datahub.graphql.resolvers.tag.DeleteTagResolver; import com.linkedin.datahub.graphql.resolvers.tag.SetTagColorResolver; @@ -216,6 +222,11 @@ import com.linkedin.datahub.graphql.resolvers.user.ListUsersResolver; import com.linkedin.datahub.graphql.resolvers.user.RemoveUserResolver; import com.linkedin.datahub.graphql.resolvers.user.UpdateUserStatusResolver; +import com.linkedin.datahub.graphql.resolvers.view.CreateViewResolver; +import com.linkedin.datahub.graphql.resolvers.view.DeleteViewResolver; +import com.linkedin.datahub.graphql.resolvers.view.ListGlobalViewsResolver; +import com.linkedin.datahub.graphql.resolvers.view.ListMyViewsResolver; +import com.linkedin.datahub.graphql.resolvers.view.UpdateViewResolver; import com.linkedin.datahub.graphql.types.BrowsableEntityType; import com.linkedin.datahub.graphql.types.EntityType; import com.linkedin.datahub.graphql.types.LoadableType; @@ -252,10 +263,12 @@ import com.linkedin.datahub.graphql.types.schemafield.SchemaFieldType; import com.linkedin.datahub.graphql.types.tag.TagType; import com.linkedin.datahub.graphql.types.test.TestType; +import com.linkedin.datahub.graphql.types.view.DataHubViewType; import com.linkedin.entity.client.EntityClient; import com.linkedin.metadata.config.DataHubConfiguration; import com.linkedin.metadata.config.IngestionConfiguration; import com.linkedin.metadata.config.TestsConfiguration; +import com.linkedin.metadata.config.ViewsConfiguration; import com.linkedin.metadata.config.VisualConfiguration; import com.linkedin.metadata.entity.EntityService; import com.linkedin.metadata.graph.GraphClient; @@ -263,6 +276,8 @@ import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.recommendation.RecommendationsService; import com.linkedin.metadata.secret.SecretService; +import com.linkedin.metadata.service.SettingsService; +import com.linkedin.metadata.service.ViewService; import com.linkedin.metadata.telemetry.TelemetryConfiguration; import com.linkedin.metadata.timeline.TimelineService; import com.linkedin.metadata.timeseries.TimeseriesAspectService; @@ -322,6 +337,8 @@ public class GmsGraphQLEngine { private final RoleService roleService; private final InviteTokenService inviteTokenService; private final PostService postService; + private final SettingsService settingsService; + private final ViewService viewService; private final FeatureFlags featureFlags; @@ -332,6 +349,7 @@ public class GmsGraphQLEngine { private final TelemetryConfiguration telemetryConfiguration; private final TestsConfiguration testsConfiguration; private final DataHubConfiguration datahubConfiguration; + private final ViewsConfiguration viewsConfiguration; private final DatasetType datasetType; private final CorpUserType corpUserType; @@ -361,6 +379,7 @@ public class GmsGraphQLEngine { private final DataHubPolicyType dataHubPolicyType; private final DataHubRoleType dataHubRoleType; private final SchemaFieldType schemaFieldType; + private final DataHubViewType dataHubViewType; /** * Configures the graph objects that can be fetched primary key. @@ -398,8 +417,12 @@ public GmsGraphQLEngine(final EntityClient entityClient, final GraphClient graph final TimelineService timelineService, final boolean supportsImpactAnalysis, final VisualConfiguration visualConfiguration, final TelemetryConfiguration telemetryConfiguration, final TestsConfiguration testsConfiguration, final DataHubConfiguration datahubConfiguration, - final SiblingGraphService siblingGraphService, final GroupService groupService, final RoleService roleService, - final InviteTokenService inviteTokenService, final PostService postService, final FeatureFlags featureFlags) { + final ViewsConfiguration viewsConfiguration, final SiblingGraphService siblingGraphService, + final GroupService groupService, final RoleService roleService, + final InviteTokenService inviteTokenService, final PostService postService, + final ViewService viewService, + final SettingsService settingsService, + final FeatureFlags featureFlags) { this.entityClient = entityClient; this.graphClient = graphClient; @@ -421,6 +444,8 @@ public GmsGraphQLEngine(final EntityClient entityClient, final GraphClient graph this.roleService = roleService; this.inviteTokenService = inviteTokenService; this.postService = postService; + this.viewService = viewService; + this.settingsService = settingsService; this.ingestionConfiguration = Objects.requireNonNull(ingestionConfiguration); this.authenticationConfiguration = Objects.requireNonNull(authenticationConfiguration); @@ -429,6 +454,7 @@ public GmsGraphQLEngine(final EntityClient entityClient, final GraphClient graph this.telemetryConfiguration = telemetryConfiguration; this.testsConfiguration = testsConfiguration; this.datahubConfiguration = datahubConfiguration; + this.viewsConfiguration = viewsConfiguration; this.featureFlags = featureFlags; this.datasetType = new DatasetType(entityClient); @@ -459,6 +485,8 @@ public GmsGraphQLEngine(final EntityClient entityClient, final GraphClient graph this.dataHubPolicyType = new DataHubPolicyType(entityClient); this.dataHubRoleType = new DataHubRoleType(entityClient); this.schemaFieldType = new SchemaFieldType(); + this.dataHubViewType = new DataHubViewType(entityClient); + // Init Lists this.entityTypes = ImmutableList.of( datasetType, @@ -487,7 +515,8 @@ public GmsGraphQLEngine(final EntityClient entityClient, final GraphClient graph testType, dataHubPolicyType, dataHubRoleType, - schemaFieldType + schemaFieldType, + dataHubViewType ); this.loadableTypes = new ArrayList<>(entityTypes); this.ownerTypes = ImmutableList.of(corpUserType, corpGroupType); @@ -548,6 +577,7 @@ public void configureRuntimeWiring(final RuntimeWiring.Builder builder) { configureRoleResolvers(builder); configureSchemaFieldResolvers(builder); configureEntityPathResolvers(builder); + configureViewResolvers(builder); } public GraphQLEngine.Builder builder() { @@ -637,11 +667,12 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) { this.visualConfiguration, this.telemetryConfiguration, this.testsConfiguration, - this.datahubConfiguration + this.datahubConfiguration, + this.viewsConfiguration )) .dataFetcher("me", new MeResolver(this.entityClient, featureFlags)) .dataFetcher("search", new SearchResolver(this.entityClient)) - .dataFetcher("searchAcrossEntities", new SearchAcrossEntitiesResolver(this.entityClient)) + .dataFetcher("searchAcrossEntities", new SearchAcrossEntitiesResolver(this.entityClient, this.viewService)) .dataFetcher("searchAcrossLineage", new SearchAcrossLineageResolver(this.entityClient)) .dataFetcher("autoComplete", new AutoCompleteResolver(searchableTypes)) .dataFetcher("autoCompleteForMultiple", new AutoCompleteForMultipleResolver(searchableTypes)) @@ -697,6 +728,9 @@ private void configureQueryResolvers(final RuntimeWiring.Builder builder) { .dataFetcher("getInviteToken", new GetInviteTokenResolver(this.inviteTokenService)) .dataFetcher("listPosts", new ListPostsResolver(this.entityClient)) .dataFetcher("batchGetStepStates", new BatchGetStepStatesResolver(this.entityClient)) + .dataFetcher("listMyViews", new ListMyViewsResolver(this.entityClient)) + .dataFetcher("listGlobalViews", new ListGlobalViewsResolver(this.entityClient)) + .dataFetcher("globalViewsSettings", new GlobalViewsSettingsResolver(this.settingsService)) ); } @@ -821,6 +855,11 @@ private void configureMutationResolvers(final RuntimeWiring.Builder builder) { .dataFetcher("acceptRole", new AcceptRoleResolver(this.roleService, this.inviteTokenService)) .dataFetcher("createPost", new CreatePostResolver(this.postService)) .dataFetcher("batchUpdateStepStates", new BatchUpdateStepStatesResolver(this.entityClient)) + .dataFetcher("createView", new CreateViewResolver(this.viewService)) + .dataFetcher("updateView", new UpdateViewResolver(this.viewService)) + .dataFetcher("deleteView", new DeleteViewResolver(this.viewService)) + .dataFetcher("updateGlobalViewsSettings", new UpdateGlobalViewsSettingsResolver(this.settingsService)) + .dataFetcher("updateCorpUserViewsSettings", new UpdateCorpUserViewsSettingsResolver(this.settingsService)) ); } @@ -1489,6 +1528,31 @@ private void configureRoleResolvers(final RuntimeWiring.Builder builder) { typeWiring -> typeWiring.dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient))); } + private void configureViewResolvers(final RuntimeWiring.Builder builder) { + builder + .type("DataHubView", + typeWiring -> typeWiring.dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient))) + .type("ListViewsResult", typeWiring -> typeWiring + .dataFetcher("views", new LoadableTypeBatchResolver<>( + dataHubViewType, + (env) -> ((ListViewsResult) env.getSource()).getViews().stream() + .map(DataHubView::getUrn) + .collect(Collectors.toList()))) + ) + .type("CorpUserViewsSettings", typeWiring -> typeWiring + .dataFetcher("defaultView", new LoadableTypeResolver<>( + dataHubViewType, + (env) -> { + final CorpUserViewsSettings settings = env.getSource(); + if (settings.getDefaultView() != null) { + return settings.getDefaultView().getUrn(); + } + return null; + } + ) + )); + } + private void configureDataProcessInstanceResolvers(final RuntimeWiring.Builder builder) { builder.type("DataProcessInstance", typeWiring -> typeWiring.dataFetcher("relationships", new EntityRelationshipsResultResolver(graphClient)) diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java index bffd98f0d271ea..a65ad18723b82b 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/authorization/AuthorizationUtils.java @@ -108,6 +108,10 @@ public static boolean canCreateGlobalAnnouncements(@Nonnull QueryContext context return isAuthorized(context, Optional.empty(), PoliciesConfig.CREATE_GLOBAL_ANNOUNCEMENTS_PRIVILEGE); } + public static boolean canManageGlobalViews(@Nonnull QueryContext context) { + return isAuthorized(context, Optional.empty(), PoliciesConfig.MANAGE_GLOBAL_VIEWS); + } + public static boolean isAuthorized( @Nonnull QueryContext context, @Nonnull Optional resourceSpec, diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/EntityTypeMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/EntityTypeMapper.java index 150a51b5d9c37c..958f78a98b7fde 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/EntityTypeMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/EntityTypeMapper.java @@ -2,6 +2,7 @@ import com.google.common.collect.ImmutableMap; import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.metadata.Constants; import java.util.Map; import java.util.stream.Collectors; import javax.annotation.Nonnull; @@ -35,6 +36,7 @@ public class EntityTypeMapper { .put(EntityType.NOTEBOOK, "notebook") .put(EntityType.DATA_PLATFORM_INSTANCE, "dataPlatformInstance") .put(EntityType.TEST, "test") + .put(EntityType.DATAHUB_VIEW, Constants.DATAHUB_VIEW_ENTITY_NAME) .build(); private static final Map ENTITY_NAME_TO_TYPE = diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java index 01293f1cae2546..e22693c081b38b 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/MeResolver.java @@ -72,6 +72,7 @@ public CompletableFuture get(DataFetchingEnvironment environm platformPrivileges.setCreateDomains(AuthorizationUtils.canCreateDomains(context)); platformPrivileges.setCreateTags(AuthorizationUtils.canCreateTags(context)); platformPrivileges.setManageTags(AuthorizationUtils.canManageTags(context)); + platformPrivileges.setManageGlobalViews(AuthorizationUtils.canManageGlobalViews(context)); // Construct and return authenticated user object. final AuthenticatedUser authUser = new AuthenticatedUser(); diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java index d6bd87e0a74e09..a2a979dcb2d51c 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/ResolverUtils.java @@ -8,7 +8,7 @@ import com.linkedin.datahub.graphql.exception.ValidationException; import com.linkedin.datahub.graphql.generated.FacetFilterInput; -import com.linkedin.datahub.graphql.generated.OrFilter; +import com.linkedin.datahub.graphql.generated.AndFilterInput; import com.linkedin.metadata.query.filter.Condition; import com.linkedin.metadata.query.filter.Criterion; import com.linkedin.metadata.query.filter.CriterionArray; @@ -103,7 +103,7 @@ public static List criterionListFromAndFilter(List // In the case that user sends filters to be or-d together, we need to build a series of conjunctive criterion // arrays, rather than just one for the AND case. public static ConjunctiveCriterionArray buildConjunctiveCriterionArrayWithOr( - @Nonnull List orFilters + @Nonnull List orFilters ) { return new ConjunctiveCriterionArray(orFilters.stream().map(orFilter -> { CriterionArray andCriterionForOr = new CriterionArray(criterionListFromAndFilter(orFilter.getAnd())); @@ -115,7 +115,7 @@ public static ConjunctiveCriterionArray buildConjunctiveCriterionArrayWithOr( } @Nullable - public static Filter buildFilter(@Nullable List andFilters, @Nullable List orFilters) { + public static Filter buildFilter(@Nullable List andFilters, @Nullable List orFilters) { if ((andFilters == null || andFilters.isEmpty()) && (orFilters == null || orFilters.isEmpty())) { return null; } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java index 913f32eb83fea7..0272708c6461b5 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/config/AppConfigResolver.java @@ -15,10 +15,12 @@ import com.linkedin.datahub.graphql.generated.ResourcePrivileges; import com.linkedin.datahub.graphql.generated.TelemetryConfig; import com.linkedin.datahub.graphql.generated.TestsConfig; +import com.linkedin.datahub.graphql.generated.ViewsConfig; import com.linkedin.datahub.graphql.generated.VisualConfig; import com.linkedin.metadata.config.DataHubConfiguration; import com.linkedin.metadata.config.IngestionConfiguration; import com.linkedin.metadata.config.TestsConfiguration; +import com.linkedin.metadata.config.ViewsConfiguration; import com.linkedin.metadata.telemetry.TelemetryConfiguration; import com.linkedin.metadata.config.VisualConfiguration; import com.linkedin.metadata.version.GitVersion; @@ -43,6 +45,7 @@ public class AppConfigResolver implements DataFetcher get(final DataFetchingEnvironment environmen testsConfig.setEnabled(_testsConfiguration.isEnabled()); appConfig.setTestsConfig(testsConfig); + final ViewsConfig viewsConfig = new ViewsConfig(); + viewsConfig.setEnabled(_viewsConfiguration.isEnabled()); + appConfig.setViewsConfig(viewsConfig); + return CompletableFuture.completedFuture(appConfig); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateUserSettingResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateUserSettingResolver.java index 86a8415da3d39a..1531e0490ef040 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateUserSettingResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/mutate/UpdateUserSettingResolver.java @@ -5,6 +5,7 @@ import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.UpdateUserSettingInput; import com.linkedin.datahub.graphql.generated.UserSetting; +import com.linkedin.datahub.graphql.resolvers.settings.user.UpdateCorpUserViewsSettingsResolver; import com.linkedin.identity.CorpUserAppearanceSettings; import com.linkedin.identity.CorpUserSettings; import com.linkedin.metadata.entity.EntityService; @@ -20,6 +21,10 @@ import static com.linkedin.metadata.Constants.*; +/** + * Deprecated! Use {@link UpdateCorpUserViewsSettingsResolver} + * instead. + */ @Slf4j @RequiredArgsConstructor public class UpdateUserSettingResolver implements DataFetcher> { @@ -60,4 +65,4 @@ public CompletableFuture get(DataFetchingEnvironment environment) throw } }); } -} +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/recommendation/ListRecommendationsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/recommendation/ListRecommendationsResolver.java index f71a7143aa6eb2..df1a6d4d4b00dd 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/recommendation/ListRecommendationsResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/recommendation/ListRecommendationsResolver.java @@ -4,7 +4,7 @@ import com.linkedin.common.urn.Urn; import com.linkedin.datahub.graphql.generated.ContentParams; import com.linkedin.datahub.graphql.generated.EntityProfileParams; -import com.linkedin.datahub.graphql.generated.Filter; +import com.linkedin.datahub.graphql.generated.FacetFilter; import com.linkedin.datahub.graphql.generated.ListRecommendationsInput; import com.linkedin.datahub.graphql.generated.ListRecommendationsResult; import com.linkedin.datahub.graphql.generated.RecommendationContent; @@ -148,7 +148,7 @@ private RecommendationParams mapRecommendationParams( searchParams.setFilters(params.getSearchParams() .getFilters() .stream() - .map(criterion -> Filter.builder().setField(criterion.getField()).setValues( + .map(criterion -> FacetFilter.builder().setField(criterion.getField()).setValues( ImmutableList.of(criterion.getValue())).build()) .collect(Collectors.toList())); } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java index 20a6738c2abca5..5393dd42579151 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolver.java @@ -1,5 +1,9 @@ package com.linkedin.datahub.graphql.resolvers.search; +import com.datahub.authentication.Authentication; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.datahub.graphql.QueryContext; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.generated.SearchAcrossEntitiesInput; import com.linkedin.datahub.graphql.generated.SearchResults; @@ -7,11 +11,15 @@ import com.linkedin.datahub.graphql.resolvers.ResolverUtils; import com.linkedin.datahub.graphql.types.mappers.UrnSearchResultsMapper; import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.service.ViewService; +import com.linkedin.view.DataHubViewInfo; import graphql.schema.DataFetcher; import graphql.schema.DataFetchingEnvironment; import java.util.List; import java.util.concurrent.CompletableFuture; import java.util.stream.Collectors; +import javax.annotation.Nonnull; import lombok.RequiredArgsConstructor; import lombok.extern.slf4j.Slf4j; @@ -30,15 +38,17 @@ public class SearchAcrossEntitiesResolver implements DataFetcher get(DataFetchingEnvironment environment) { + final QueryContext context = environment.getContext(); final SearchAcrossEntitiesInput input = bindArgument(environment.getArgument("input"), SearchAcrossEntitiesInput.class); - List entityTypes = + final List entityTypes = (input.getTypes() == null || input.getTypes().isEmpty()) ? SEARCHABLE_ENTITY_TYPES : input.getTypes(); - List entityNames = entityTypes.stream().map(EntityTypeMapper::getName).collect(Collectors.toList()); + final List entityNames = entityTypes.stream().map(EntityTypeMapper::getName).collect(Collectors.toList()); // escape forward slash since it is a reserved character in Elasticsearch final String sanitizedQuery = ResolverUtils.escapeForwardSlash(input.getQuery()); @@ -47,12 +57,29 @@ public CompletableFuture get(DataFetchingEnvironment environment) final int count = input.getCount() != null ? input.getCount() : DEFAULT_COUNT; return CompletableFuture.supplyAsync(() -> { + + final DataHubViewInfo maybeResolvedView = (input.getViewUrn() != null) + ? resolveView(UrnUtils.getUrn(input.getViewUrn()), context.getAuthentication()) + : null; + + final Filter baseFilter = ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters()); + try { log.debug( "Executing search for multiple entities: entity types {}, query {}, filters: {}, start: {}, count: {}", input.getTypes(), input.getQuery(), input.getFilters(), start, count); - return UrnSearchResultsMapper.map(_entityClient.searchAcrossEntities(entityNames, sanitizedQuery, - ResolverUtils.buildFilter(input.getFilters(), input.getOrFilters()), start, count, ResolverUtils.getAuthentication(environment))); + + return UrnSearchResultsMapper.map(_entityClient.searchAcrossEntities( + maybeResolvedView != null + ? SearchUtils.intersectEntityTypes(entityNames, maybeResolvedView.getDefinition().getEntityTypes()) + : entityNames, + sanitizedQuery, + maybeResolvedView != null + ? SearchUtils.combineFilters(baseFilter, maybeResolvedView.getDefinition().getFilter()) + : baseFilter, + start, + count, + ResolverUtils.getAuthentication(environment))); } catch (Exception e) { log.error( "Failed to execute search for multiple entities: entity types {}, query {}, filters: {}, start: {}, count: {}", @@ -63,4 +90,20 @@ public CompletableFuture get(DataFetchingEnvironment environment) } }); } + + /** + * Attempts to resolve a View by urn. Throws {@link IllegalArgumentException} if a View with the specified + * urn cannot be found. + */ + private DataHubViewInfo resolveView(@Nonnull final Urn viewUrn, @Nonnull final Authentication authentication) { + try { + DataHubViewInfo maybeViewInfo = _viewService.getViewInfo(viewUrn, authentication); + if (maybeViewInfo == null) { + log.warn(String.format("Failed to resolve View with urn %s. View does not exist!", viewUrn)); + } + return maybeViewInfo; + } catch (Exception e) { + throw new RuntimeException(String.format("Caught exception while attempting to resolve View with URN %s", viewUrn), e); + } + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java index eae1713fb40d7e..c4d30071802459 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtils.java @@ -2,9 +2,21 @@ import com.google.common.collect.ImmutableList; import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; +import com.linkedin.metadata.query.filter.Filter; +import java.util.ArrayList; +import java.util.Collections; import java.util.List; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.extern.slf4j.Slf4j; +import org.codehaus.plexus.util.CollectionUtils; +@Slf4j public class SearchUtils { private SearchUtils() { } @@ -51,4 +63,213 @@ private SearchUtils() { EntityType.CORP_USER, EntityType.CORP_GROUP, EntityType.NOTEBOOK); + + + /** + * Combines two {@link Filter} instances in a conjunction and returns a new instance of {@link Filter} + * in disjunctive normal form. + * + * @param baseFilter the filter to apply the view to + * @param viewFilter the view filter, null if it doesn't exist + * + * @return a new instance of {@link Filter} representing the applied view. + */ + @Nonnull + public static Filter combineFilters(@Nullable final Filter baseFilter, @Nonnull final Filter viewFilter) { + final Filter finalBaseFilter = baseFilter == null + ? new Filter().setOr(new ConjunctiveCriterionArray(Collections.emptyList())) + : baseFilter; + + // Join the filter conditions in Disjunctive Normal Form. + return combineFiltersInConjunction(finalBaseFilter, viewFilter); + } + + /** + * Returns the intersection of two sets of entity types. (Really just string lists). + * If either is empty, consider the entity types list to mean "all" (take the other set). + * + * @param baseEntityTypes the entity types to apply the view to + * @param viewEntityTypes the view info, null if it doesn't exist + * + * @return the intersection of the two input sets + */ + @Nonnull + public static List intersectEntityTypes(@Nonnull final List baseEntityTypes, @Nonnull final List viewEntityTypes) { + if (baseEntityTypes.isEmpty()) { + return viewEntityTypes; + } + if (viewEntityTypes.isEmpty()) { + return baseEntityTypes; + } + // Join the entity types in intersection. + return new ArrayList<>(CollectionUtils.intersection(baseEntityTypes, viewEntityTypes)); + } + + /** + * Joins two filters in conjunction by reducing to Disjunctive Normal Form. + * + * @param filter1 the first filter in the pair + * @param filter2 the second filter in the pair + * + * This method supports either Filter format, where the "or" field is used, instead + * of criteria. If the criteria filter is used, then it will be converted into an "OR" before + * returning the new filter. + * + * @return the result of joining the 2 filters in a conjunction (AND) + * + * How does it work? It basically cross-products the conjunctions inside of each Filter clause. + * + * Example Inputs: + * filter1 -> + * { + * or: [ + * { + * and: [ + * { + * field: tags, + * condition: EQUAL, + * values: ["urn:li:tag:tag"] + * } + * ] + * }, + * { + * and: [ + * { + * field: glossaryTerms, + * condition: EQUAL, + * values: ["urn:li:glossaryTerm:term"] + * } + * ] + * } + * ] + * } + * filter2 -> + * { + * or: [ + * { + * and: [ + * { + * field: domain, + * condition: EQUAL, + * values: ["urn:li:domain:domain"] + * }, + * ] + * }, + * { + * and: [ + * { + * field: glossaryTerms, + * condition: EQUAL, + * values: ["urn:li:glossaryTerm:term2"] + * } + * ] + * } + * ] + * } + * Example Output: + * { + * or: [ + * { + * and: [ + * { + * field: tags, + * condition: EQUAL, + * values: ["urn:li:tag:tag"] + * }, + * { + * field: domain, + * condition: EQUAL, + * values: ["urn:li:domain:domain"] + * } + * ] + * }, + * { + * and: [ + * { + * field: tags, + * condition: EQUAL, + * values: ["urn:li:tag:tag"] + * }, + * { + * field: glossaryTerms, + * condition: EQUAL, + * values: ["urn:li:glosaryTerm:term2"] + * } + * ] + * }, + * { + * and: [ + * { + * field: glossaryTerm, + * condition: EQUAL, + * values: ["urn:li:glossaryTerm:term"] + * }, + * { + * field: domain, + * condition: EQUAL, + * values: ["urn:li:domain:domain"] + * } + * ] + * }, + * { + * and: [ + * { + * field: glossaryTerm, + * condition: EQUAL, + * values: ["urn:li:glossaryTerm:term"] + * }, + * { + * field: glossaryTerms, + * condition: EQUAL, + * values: ["urn:li:glosaryTerm:term2"] + * } + * ] + * }, + * ] + * } + */ + @Nonnull + private static Filter combineFiltersInConjunction(@Nonnull final Filter filter1, @Nonnull final Filter filter2) { + + final Filter finalFilter1 = convertToV2Filter(filter1); + final Filter finalFilter2 = convertToV2Filter(filter2); + + // If either filter is empty, simply return the other filter. + if (!finalFilter1.hasOr() || finalFilter1.getOr().size() == 0) { + return finalFilter2; + } + if (!finalFilter2.hasOr() || finalFilter2.getOr().size() == 0) { + return finalFilter1; + } + + // Iterate through the base filter, then cross-product with filter 2 conditions. + final Filter result = new Filter(); + final List newDisjunction = new ArrayList<>(); + for (ConjunctiveCriterion conjunction1 : finalFilter1.getOr()) { + for (ConjunctiveCriterion conjunction2 : finalFilter2.getOr()) { + final List joinedCriterion = new ArrayList<>(conjunction1.getAnd()); + joinedCriterion.addAll(conjunction2.getAnd()); + ConjunctiveCriterion newConjunction = new ConjunctiveCriterion().setAnd(new CriterionArray(joinedCriterion)); + newDisjunction.add(newConjunction); + } + } + result.setOr(new ConjunctiveCriterionArray(newDisjunction)); + return result; + } + + @Nonnull + private static Filter convertToV2Filter(@Nonnull Filter filter) { + if (filter.hasOr()) { + return filter; + } else if (filter.hasCriteria()) { + // Convert criteria to an OR + return new Filter() + .setOr(new ConjunctiveCriterionArray(ImmutableList.of( + new ConjunctiveCriterion() + .setAnd(filter.getCriteria()) + ))); + } + throw new IllegalArgumentException( + String.format("Illegal filter provided! Neither 'or' nor 'criteria' fields were populated for filter %s", filter)); + } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/settings/user/UpdateCorpUserViewsSettingsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/settings/user/UpdateCorpUserViewsSettingsResolver.java new file mode 100644 index 00000000000000..8c21277b66a69f --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/settings/user/UpdateCorpUserViewsSettingsResolver.java @@ -0,0 +1,83 @@ +package com.linkedin.datahub.graphql.resolvers.settings.user; + +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.SetMode; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.UpdateCorpUserViewsSettingsInput; +import com.linkedin.identity.CorpUserAppearanceSettings; +import com.linkedin.identity.CorpUserSettings; +import com.linkedin.identity.CorpUserViewsSettings; +import com.linkedin.metadata.service.SettingsService; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; +import lombok.RequiredArgsConstructor; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; + +/** + * Resolver responsible for updating the authenticated user's View-specific settings. + */ +@Slf4j +@RequiredArgsConstructor +public class UpdateCorpUserViewsSettingsResolver implements DataFetcher> { + + private final SettingsService _settingsService; + + @Override + public CompletableFuture get(DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + final UpdateCorpUserViewsSettingsInput input = bindArgument(environment.getArgument("input"), UpdateCorpUserViewsSettingsInput.class); + + return CompletableFuture.supplyAsync(() -> { + try { + + final Urn userUrn = UrnUtils.getUrn(context.getActorUrn()); + + final CorpUserSettings maybeSettings = _settingsService.getCorpUserSettings( + userUrn, + context.getAuthentication() + ); + + final CorpUserSettings newSettings = maybeSettings == null + ? new CorpUserSettings().setAppearance(new CorpUserAppearanceSettings().setShowSimplifiedHomepage(false)) + : maybeSettings; + + // Patch the new corp user settings. This does a R-M-F. + updateCorpUserSettings(newSettings, input); + + _settingsService.updateCorpUserSettings( + userUrn, + newSettings, + context.getAuthentication() + ); + return true; + } catch (Exception e) { + log.error("Failed to perform user view settings update against input {}, {}", input.toString(), e.getMessage()); + throw new RuntimeException(String.format("Failed to perform update to user view settings against input %s", input.toString()), e); + } + }); + } + + private static void updateCorpUserSettings( + @Nonnull final CorpUserSettings settings, + @Nonnull final UpdateCorpUserViewsSettingsInput input) { + final CorpUserViewsSettings newViewSettings = settings.hasViews() + ? settings.getViews() + : new CorpUserViewsSettings(); + updateCorpUserViewsSettings(newViewSettings, input); + settings.setViews(newViewSettings); + } + + private static void updateCorpUserViewsSettings( + @Nonnull final CorpUserViewsSettings settings, + @Nonnull final UpdateCorpUserViewsSettingsInput input) { + settings.setDefaultView(input.getDefaultView() != null + ? UrnUtils.getUrn(input.getDefaultView()) + : null, + SetMode.REMOVE_IF_NULL); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/settings/view/GlobalViewsSettingsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/settings/view/GlobalViewsSettingsResolver.java new file mode 100644 index 00000000000000..f1aba3d9247c58 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/settings/view/GlobalViewsSettingsResolver.java @@ -0,0 +1,51 @@ +package com.linkedin.datahub.graphql.resolvers.settings.view; + +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.GlobalViewsSettings; +import com.linkedin.metadata.service.SettingsService; +import com.linkedin.settings.global.GlobalSettingsInfo; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; + +/** + * Retrieves the Global Settings related to the Views feature. + * + * This capability requires the 'MANAGE_GLOBAL_VIEWS' Platform Privilege. + */ +@Slf4j +public class GlobalViewsSettingsResolver implements + DataFetcher> { + + private final SettingsService _settingsService; + + public GlobalViewsSettingsResolver(final SettingsService settingsService) { + _settingsService = Objects.requireNonNull(settingsService, "settingsService must not be null"); + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + return CompletableFuture.supplyAsync(() -> { + try { + final GlobalSettingsInfo globalSettings = _settingsService.getGlobalSettings(context.getAuthentication()); + return globalSettings != null && globalSettings.hasViews() + ? mapGlobalViewsSettings(globalSettings.getViews()) + : new GlobalViewsSettings(); + } catch (Exception e) { + throw new RuntimeException("Failed to retrieve Global Views Settings", e); + } + }); + } + + private static GlobalViewsSettings mapGlobalViewsSettings(@Nonnull final com.linkedin.settings.global.GlobalViewsSettings settings) { + final GlobalViewsSettings result = new GlobalViewsSettings(); + if (settings.hasDefaultView()) { + result.setDefaultView(settings.getDefaultView().toString()); + } + return result; + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/settings/view/UpdateGlobalViewsSettingsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/settings/view/UpdateGlobalViewsSettingsResolver.java new file mode 100644 index 00000000000000..c90ec04b3a2dfc --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/settings/view/UpdateGlobalViewsSettingsResolver.java @@ -0,0 +1,76 @@ +package com.linkedin.datahub.graphql.resolvers.settings.view; + +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.SetMode; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.UpdateGlobalViewsSettingsInput; +import com.linkedin.metadata.service.SettingsService; +import com.linkedin.settings.global.GlobalSettingsInfo; +import com.linkedin.settings.global.GlobalViewsSettings; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; + + +/** + * Resolver responsible for updating the Global Views settings. + * + * This capability requires the 'MANAGE_GLOBAL_VIEWS' Platform Privilege. + */ +public class UpdateGlobalViewsSettingsResolver implements DataFetcher> { + + private final SettingsService _settingsService; + + public UpdateGlobalViewsSettingsResolver(@Nonnull final SettingsService settingsService) { + _settingsService = Objects.requireNonNull(settingsService, "settingsService must not be null"); + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + final UpdateGlobalViewsSettingsInput input = bindArgument(environment.getArgument("input"), UpdateGlobalViewsSettingsInput.class); + + return CompletableFuture.supplyAsync(() -> { + if (AuthorizationUtils.canManageGlobalViews(context)) { + try { + // First, fetch the existing global settings. This does a R-M-F. + final GlobalSettingsInfo maybeGlobalSettings = _settingsService.getGlobalSettings(context.getAuthentication()); + + final GlobalSettingsInfo newGlobalSettings = maybeGlobalSettings != null + ? maybeGlobalSettings + : new GlobalSettingsInfo(); + + final GlobalViewsSettings newGlobalViewsSettings = newGlobalSettings.hasViews() + ? newGlobalSettings.getViews() + : new GlobalViewsSettings(); + + // Next, patch the global views settings. + updateViewsSettings(newGlobalViewsSettings, input); + newGlobalSettings.setViews(newGlobalViewsSettings); + + // Finally, write back to GMS. + _settingsService.updateGlobalSettings(newGlobalSettings, context.getAuthentication()); + return true; + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to update global view settings! %s", input), e); + } + } + throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); + }); + } + + private static void updateViewsSettings( + @Nonnull final com.linkedin.settings.global.GlobalViewsSettings settings, + @Nonnull final UpdateGlobalViewsSettingsInput input) { + settings.setDefaultView(input.getDefaultView() != null + ? UrnUtils.getUrn(input.getDefaultView()) + : null, + SetMode.REMOVE_IF_NULL); + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/CreateViewResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/CreateViewResolver.java new file mode 100644 index 00000000000000..11dbcc390af3d4 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/CreateViewResolver.java @@ -0,0 +1,78 @@ +package com.linkedin.datahub.graphql.resolvers.view; + +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.CreateViewInput; +import com.linkedin.datahub.graphql.generated.DataHubView; +import com.linkedin.datahub.graphql.generated.DataHubViewDefinition; +import com.linkedin.datahub.graphql.generated.DataHubViewFilter; +import com.linkedin.datahub.graphql.generated.FacetFilter; +import com.linkedin.metadata.service.ViewService; +import com.linkedin.view.DataHubViewType; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; + + +/** + * Resolver responsible for updating a particular DataHub View + */ +@Slf4j +public class CreateViewResolver implements DataFetcher> { + + private final ViewService _viewService; + + public CreateViewResolver(@Nonnull final ViewService viewService) { + _viewService = Objects.requireNonNull(viewService); + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + final CreateViewInput input = bindArgument(environment.getArgument("input"), CreateViewInput.class); + + return CompletableFuture.supplyAsync(() -> { + if (ViewUtils.canCreateView( + DataHubViewType.valueOf(input.getViewType().toString()), + context)) { + try { + final Urn urn = _viewService.createView( + DataHubViewType.valueOf(input.getViewType().toString()), + input.getName(), + input.getDescription(), + ViewUtils.mapDefinition(input.getDefinition()), + context.getAuthentication(), + System.currentTimeMillis()); + return createView(urn, input); + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to create View with input: %s", input), e); + } + } + throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); + }); + } + + private DataHubView createView(@Nonnull final Urn urn, @Nonnull final CreateViewInput input) { + return new DataHubView( + urn.toString(), + com.linkedin.datahub.graphql.generated.EntityType.DATAHUB_VIEW, + input.getViewType(), + input.getName(), + input.getDescription(), + new DataHubViewDefinition( + input.getDefinition().getEntityTypes(), + new DataHubViewFilter( + input.getDefinition().getFilter().getOperator(), + input.getDefinition().getFilter().getFilters().stream().map(filterInput -> + new FacetFilter(filterInput.getField(), filterInput.getCondition(), filterInput.getValues(), + filterInput.getNegated())) + .collect(Collectors.toList())))); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/DeleteViewResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/DeleteViewResolver.java new file mode 100644 index 00000000000000..2b8c3b8640aa88 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/DeleteViewResolver.java @@ -0,0 +1,48 @@ +package com.linkedin.datahub.graphql.resolvers.view; + +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.metadata.service.ViewService; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; + + +/** + * Resolver responsible for hard deleting a particular DataHub View + */ +@Slf4j +public class DeleteViewResolver implements DataFetcher> { + + private final ViewService _viewService; + + public DeleteViewResolver(@Nonnull final ViewService viewService) { + _viewService = Objects.requireNonNull(viewService, "viewService must not be null"); + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + final String urnStr = environment.getArgument("urn"); + final Urn urn = Urn.createFromString(urnStr); + return CompletableFuture.supplyAsync(() -> { + try { + if (ViewUtils.canUpdateView(_viewService, urn, context)) { + _viewService.deleteView(urn, context.getAuthentication()); + log.info(String.format("Successfully deleted View %s with urn", urn)); + return true; + } + throw new AuthorizationException( + "Unauthorized to perform this action. Please contact your DataHub administrator."); + } catch (AuthorizationException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to perform delete against View with urn %s", urn), e); + } + }); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolver.java new file mode 100644 index 00000000000000..bfe7b178c655a3 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolver.java @@ -0,0 +1,112 @@ +package com.linkedin.datahub.graphql.resolvers.view; + +import com.google.common.collect.ImmutableList; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.AndFilterInput; +import com.linkedin.datahub.graphql.generated.DataHubView; +import com.linkedin.datahub.graphql.generated.DataHubViewType; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.FacetFilterInput; +import com.linkedin.datahub.graphql.generated.FilterOperator; +import com.linkedin.datahub.graphql.generated.ListGlobalViewsInput; +import com.linkedin.datahub.graphql.generated.ListViewsResult; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.query.filter.SortCriterion; +import com.linkedin.metadata.query.filter.SortOrder; +import com.linkedin.metadata.search.SearchEntity; +import com.linkedin.metadata.search.SearchResult; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; + + +/** + * Resolver used for listing global DataHub Views. + */ +@Slf4j +public class ListGlobalViewsResolver implements DataFetcher> { + + private static final String CREATED_AT_FIELD = "createdAt"; + private static final String VIEW_TYPE_FIELD = "type"; + private static final SortCriterion DEFAULT_SORT_CRITERION = new SortCriterion() + .setField(CREATED_AT_FIELD) + .setOrder(SortOrder.DESCENDING); + private static final Integer DEFAULT_START = 0; + private static final Integer DEFAULT_COUNT = 20; + private static final String DEFAULT_QUERY = ""; + + private final EntityClient _entityClient; + + public ListGlobalViewsResolver(@Nonnull final EntityClient entityClient) { + _entityClient = Objects.requireNonNull(entityClient, "entityClient must not be null"); + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { + + final QueryContext context = environment.getContext(); + final ListGlobalViewsInput input = bindArgument(environment.getArgument("input"), ListGlobalViewsInput.class); + + return CompletableFuture.supplyAsync(() -> { + final Integer start = input.getStart() == null ? DEFAULT_START : input.getStart(); + final Integer count = input.getCount() == null ? DEFAULT_COUNT : input.getCount(); + final String query = input.getQuery() == null ? DEFAULT_QUERY : input.getQuery(); + + try { + + final SearchResult gmsResult = _entityClient.search( + Constants.DATAHUB_VIEW_ENTITY_NAME, + query, + buildFilters(), + DEFAULT_SORT_CRITERION, + start, + count, + context.getAuthentication()); + + final ListViewsResult result = new ListViewsResult(); + result.setStart(gmsResult.getFrom()); + result.setCount(gmsResult.getPageSize()); + result.setTotal(gmsResult.getNumEntities()); + result.setViews(mapUnresolvedViews(gmsResult.getEntities().stream() + .map(SearchEntity::getEntity) + .collect(Collectors.toList()))); + return result; + } catch (Exception e) { + throw new RuntimeException("Failed to list global Views", e); + } + }); + } + + // This method maps urns returned from the list endpoint into Partial View objects which will be resolved be a separate Batch resolver. + private List mapUnresolvedViews(final List entityUrns) { + final List results = new ArrayList<>(); + for (final Urn urn : entityUrns) { + final DataHubView unresolvedView = new DataHubView(); + unresolvedView.setUrn(urn.toString()); + unresolvedView.setType(EntityType.DATAHUB_VIEW); + results.add(unresolvedView); + } + return results; + } + + private Filter buildFilters() { + final AndFilterInput globalCriteria = new AndFilterInput(); + List andConditions = new ArrayList<>(); + andConditions.add( + new FacetFilterInput(VIEW_TYPE_FIELD, null, ImmutableList.of(DataHubViewType.GLOBAL.name()), false, FilterOperator.EQUAL)); + globalCriteria.setAnd(andConditions); + return buildFilter(Collections.emptyList(), ImmutableList.of(globalCriteria)); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolver.java new file mode 100644 index 00000000000000..2b7bde0b7ce85c --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolver.java @@ -0,0 +1,125 @@ +package com.linkedin.datahub.graphql.resolvers.view; + +import com.google.common.collect.ImmutableList; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.AndFilterInput; +import com.linkedin.datahub.graphql.generated.DataHubView; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.FacetFilterInput; +import com.linkedin.datahub.graphql.generated.FilterOperator; +import com.linkedin.datahub.graphql.generated.ListMyViewsInput; +import com.linkedin.datahub.graphql.generated.ListViewsResult; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.query.filter.SortCriterion; +import com.linkedin.metadata.query.filter.SortOrder; +import com.linkedin.metadata.search.SearchEntity; +import com.linkedin.metadata.search.SearchResult; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; + + +/** + * Resolver used for listing the current user's DataHub Views. + */ +@Slf4j +public class ListMyViewsResolver implements DataFetcher> { + + private static final String CREATED_AT_FIELD = "createdAt"; + private static final String VIEW_TYPE_FIELD = "type"; + private static final String CREATOR_URN_FIELD = "createdBy"; + private static final SortCriterion DEFAULT_SORT_CRITERION = new SortCriterion() + .setField(CREATED_AT_FIELD) + .setOrder(SortOrder.DESCENDING); + private static final Integer DEFAULT_START = 0; + private static final Integer DEFAULT_COUNT = 20; + private static final String DEFAULT_QUERY = ""; + + private final EntityClient _entityClient; + + public ListMyViewsResolver(@Nonnull final EntityClient entityClient) { + _entityClient = Objects.requireNonNull(entityClient, "entityClient must not be null"); + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { + + final QueryContext context = environment.getContext(); + final ListMyViewsInput input = bindArgument(environment.getArgument("input"), ListMyViewsInput.class); + + return CompletableFuture.supplyAsync(() -> { + final Integer start = input.getStart() == null ? DEFAULT_START : input.getStart(); + final Integer count = input.getCount() == null ? DEFAULT_COUNT : input.getCount(); + final String query = input.getQuery() == null ? DEFAULT_QUERY : input.getQuery(); + final String viewType = input.getViewType() == null ? null : input.getViewType().toString(); + + try { + + final SearchResult gmsResult = _entityClient.search( + Constants.DATAHUB_VIEW_ENTITY_NAME, + query, + buildFilters(viewType, context.getActorUrn()), + DEFAULT_SORT_CRITERION, + start, + count, + context.getAuthentication()); + + final ListViewsResult result = new ListViewsResult(); + result.setStart(gmsResult.getFrom()); + result.setCount(gmsResult.getPageSize()); + result.setTotal(gmsResult.getNumEntities()); + result.setViews(mapUnresolvedViews(gmsResult.getEntities().stream() + .map(SearchEntity::getEntity) + .collect(Collectors.toList()))); + return result; + } catch (Exception e) { + throw new RuntimeException("Failed to list Views", e); + } + }); + } + + // This method maps urns returned from the list endpoint into Partial View objects which will be resolved be a separate Batch resolver. + private List mapUnresolvedViews(final List entityUrns) { + final List results = new ArrayList<>(); + for (final Urn urn : entityUrns) { + final DataHubView unresolvedView = new DataHubView(); + unresolvedView.setUrn(urn.toString()); + unresolvedView.setType(EntityType.DATAHUB_VIEW); + results.add(unresolvedView); + } + return results; + } + + private Filter buildFilters(@Nullable final String viewType, final String creatorUrn) { + // And GLOBAL views for the authenticated actor. + final AndFilterInput filterCriteria = new AndFilterInput(); + final List andConditions = new ArrayList<>(); + andConditions.add( + new FacetFilterInput(CREATOR_URN_FIELD, + null, + ImmutableList.of(creatorUrn), + false, + FilterOperator.EQUAL)); + if (viewType != null) { + andConditions.add( + new FacetFilterInput(VIEW_TYPE_FIELD, null, ImmutableList.of(viewType), false, FilterOperator.EQUAL)); + } + filterCriteria.setAnd(andConditions); + + // Currently, there is no way to fetch the views belonging to another user. + return buildFilter(Collections.emptyList(), ImmutableList.of(filterCriteria)); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/UpdateViewResolver.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/UpdateViewResolver.java new file mode 100644 index 00000000000000..61e22da3c94447 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/UpdateViewResolver.java @@ -0,0 +1,71 @@ +package com.linkedin.datahub.graphql.resolvers.view; + +import com.datahub.authentication.Authentication; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.exception.AuthorizationException; +import com.linkedin.datahub.graphql.generated.DataHubView; +import com.linkedin.datahub.graphql.generated.UpdateViewInput; +import com.linkedin.datahub.graphql.types.view.DataHubViewMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.metadata.service.ViewService; +import graphql.schema.DataFetcher; +import graphql.schema.DataFetchingEnvironment; +import java.util.Objects; +import java.util.concurrent.CompletableFuture; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*; + +/** + * Resolver responsible for updating a particular DataHub View + */ +@Slf4j +public class UpdateViewResolver implements DataFetcher> { + + private final ViewService _viewService; + + public UpdateViewResolver(@Nonnull final ViewService viewService) { + _viewService = Objects.requireNonNull(viewService, "viewService must not be null"); + } + + @Override + public CompletableFuture get(final DataFetchingEnvironment environment) throws Exception { + final QueryContext context = environment.getContext(); + final String urnStr = environment.getArgument("urn"); + final UpdateViewInput input = bindArgument(environment.getArgument("input"), UpdateViewInput.class); + + final Urn urn = Urn.createFromString(urnStr); + return CompletableFuture.supplyAsync(() -> { + try { + if (ViewUtils.canUpdateView(_viewService, urn, context)) { + _viewService.updateView( + urn, + input.getName(), + input.getDescription(), + ViewUtils.mapDefinition(input.getDefinition()), + context.getAuthentication(), + System.currentTimeMillis()); + log.info(String.format("Successfully updated View %s with urn", urn)); + return getView(urn, context.getAuthentication()); + } + throw new AuthorizationException("Unauthorized to perform this action. Please contact your DataHub administrator."); + } catch (AuthorizationException e) { + throw e; + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to perform update against View with urn %s", urn), e); + } + }); + } + + private DataHubView getView(@Nonnull final Urn urn, @Nonnull final Authentication authentication) { + final EntityResponse maybeResponse = _viewService.getViewEntityResponse(urn, authentication); + // If there is no response, there is a problem. + if (maybeResponse == null) { + throw new RuntimeException( + String.format("Failed to perform update to View with urn %s. Failed to find view in GMS.", urn)); + } + return DataHubViewMapper.map(maybeResponse); + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtils.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtils.java new file mode 100644 index 00000000000000..dda0c3bebc2ebe --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtils.java @@ -0,0 +1,146 @@ +package com.linkedin.datahub.graphql.resolvers.view; + +import com.google.common.collect.ImmutableList; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.SetMode; +import com.linkedin.data.template.StringArray; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.authorization.AuthorizationUtils; +import com.linkedin.datahub.graphql.generated.DataHubViewDefinitionInput; +import com.linkedin.datahub.graphql.generated.DataHubViewFilterInput; +import com.linkedin.datahub.graphql.generated.FacetFilterInput; +import com.linkedin.datahub.graphql.generated.LogicalOperator; +import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; +import com.linkedin.datahub.graphql.resolvers.ResolverUtils; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.CriterionArray; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.service.ViewService; +import com.linkedin.view.DataHubViewDefinition; +import com.linkedin.view.DataHubViewInfo; +import com.linkedin.view.DataHubViewType; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; + + +public class ViewUtils { + + /** + * Returns true if the authenticated actor is allowed to create a view with the given parameters. + * + * The user can create a View if it's a personal View specific to them, or + * if it's a Global view and they have the correct Platform privileges. + * + * @param type the type of the new View + * @param context the current GraphQL {@link QueryContext} + * @return true if the authenticator actor is allowed to change or delete the view, false otherwise. + */ + public static boolean canCreateView( + @Nonnull DataHubViewType type, + @Nonnull QueryContext context) { + Objects.requireNonNull(type, "type must not be null"); + Objects.requireNonNull(context, "context must not be null"); + return DataHubViewType.PERSONAL.equals(type) + || (DataHubViewType.GLOBAL.equals(type) && AuthorizationUtils.canManageGlobalViews(context)); + } + + + /** + * Returns true if the authenticated actor is allowed to update or delete + * the View with the specified urn. + * + * @param viewService an instance of {@link ViewService} + * @param viewUrn the urn of the View + * @param context the current GraphQL {@link QueryContext} + * @return true if the authenticator actor is allowed to change or delete the view, false otherwise. + */ + public static boolean canUpdateView(@Nonnull ViewService viewService, @Nonnull Urn viewUrn, @Nonnull QueryContext context) { + Objects.requireNonNull(viewService, "viewService must not be null"); + Objects.requireNonNull(viewUrn, "viewUrn must not be null"); + Objects.requireNonNull(context, "context must not be null"); + + // Retrieve the view, determine it's type, and then go from there. + final DataHubViewInfo viewInfo = viewService.getViewInfo(viewUrn, context.getAuthentication()); + + if (viewInfo == null) { + throw new IllegalArgumentException(String.format("Failed to modify View. View with urn %s does not exist.", viewUrn)); + } + + // If the View is Global, then the user must have ability to manage global views OR must be its owner + if (DataHubViewType.GLOBAL.equals(viewInfo.getType()) && AuthorizationUtils.canManageGlobalViews(context)) { + return true; + } + + // If the View is Personal, then the current actor must be the owner. + return isViewOwner(viewInfo.getCreated().getActor(), UrnUtils.getUrn(context.getAuthentication().getActor().toUrnStr())); + } + + /** + * Map a GraphQL {@link DataHubViewDefinition} to the GMS equivalent. + * + * @param input the GraphQL model + * @return the GMS model + */ + @Nonnull + public static DataHubViewDefinition mapDefinition(@Nonnull final DataHubViewDefinitionInput input) { + Objects.requireNonNull(input, "input must not be null"); + + final DataHubViewDefinition result = new DataHubViewDefinition(); + if (input.getFilter() != null) { + result.setFilter(mapFilter(input.getFilter()), SetMode.IGNORE_NULL); + } + result.setEntityTypes(new StringArray(input.getEntityTypes().stream().map(EntityTypeMapper::getName).collect( + Collectors.toList()))); + return result; + } + + /** + * Converts an instance of {@link DataHubViewFilterInput} into the corresponding {@link Filter} object, + * which is then persisted to the backend in an aspect. + * + * We intentionally convert from a more rigid model to something more flexible to hedge for the case + * in which the views feature evolves to require more advanced filter capabilities. + * + * The risk we run is that people ingest Views through the Rest.li ingestion APIs (back door), which cannot be + * rendered in full by the UI. We account for this on the read path by logging a warning and returning an empty + * View in such cases. + */ + private static Filter mapFilter(@Nonnull DataHubViewFilterInput input) { + if (LogicalOperator.AND.equals(input.getOperator())) { + // AND + return buildAndFilter(input.getFilters()); + } else { + // OR + return buildOrFilter(input.getFilters()); + } + } + + private static Filter buildAndFilter(@Nonnull List input) { + final Filter result = new Filter(); + result.setOr(new ConjunctiveCriterionArray(ImmutableList.of( + new ConjunctiveCriterion().setAnd( + new CriterionArray(input.stream().map(ResolverUtils::criterionFromFilter).collect(Collectors.toList())))) + )); + return result; + } + + private static Filter buildOrFilter(@Nonnull List input) { + final Filter result = new Filter(); + result.setOr(new ConjunctiveCriterionArray(input.stream().map(filter -> + new ConjunctiveCriterion().setAnd(new CriterionArray(ImmutableList.of(ResolverUtils.criterionFromFilter(filter)))) + ) + .collect(Collectors.toList()))); + return result; + } + + private static boolean isViewOwner(Urn creatorUrn, Urn actorUrn) { + return creatorUrn.equals(actorUrn); + } + + private ViewUtils() { } + +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java index cc7b43e77dfb61..e519fc06c5471f 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/common/mappers/UrnToEntityMapper.java @@ -10,6 +10,7 @@ import com.linkedin.datahub.graphql.generated.DataFlow; import com.linkedin.datahub.graphql.generated.DataHubPolicy; import com.linkedin.datahub.graphql.generated.DataHubRole; +import com.linkedin.datahub.graphql.generated.DataHubView; import com.linkedin.datahub.graphql.generated.DataJob; import com.linkedin.datahub.graphql.generated.DataPlatform; import com.linkedin.datahub.graphql.generated.DataPlatformInstance; @@ -169,6 +170,11 @@ public Entity apply(Urn input) { ((SchemaFieldEntity) partialEntity).setUrn(input.toString()); ((SchemaFieldEntity) partialEntity).setType(EntityType.SCHEMA_FIELD); } + if (input.getEntityType().equals(DATAHUB_VIEW_ENTITY_NAME)) { + partialEntity = new DataHubView(); + ((DataHubView) partialEntity).setUrn(input.toString()); + ((DataHubView) partialEntity).setType(EntityType.DATAHUB_VIEW); + } return partialEntity; } } diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/mappers/CorpUserMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/mappers/CorpUserMapper.java index edb7d33b235780..02ae9a616e3e01 100644 --- a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/mappers/CorpUserMapper.java +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/corpuser/mappers/CorpUserMapper.java @@ -7,6 +7,8 @@ import com.linkedin.datahub.graphql.featureflags.FeatureFlags; import com.linkedin.datahub.graphql.generated.CorpUser; import com.linkedin.datahub.graphql.generated.CorpUserAppearanceSettings; +import com.linkedin.datahub.graphql.generated.CorpUserViewsSettings; +import com.linkedin.datahub.graphql.generated.DataHubView; import com.linkedin.datahub.graphql.generated.EntityType; import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; import com.linkedin.datahub.graphql.types.tag.mappers.GlobalTagsMapper; @@ -70,10 +72,25 @@ private void mapCorpUserSettings(@Nonnull CorpUser corpUser, EnvelopedAspect env if (envelopedAspect != null) { corpUserSettings = new CorpUserSettings(envelopedAspect.getValue().data()); } - com.linkedin.datahub.graphql.generated.CorpUserSettings result = new com.linkedin.datahub.graphql.generated.CorpUserSettings(); + // Map Appearance Settings -- Appearance settings always exist. + result.setAppearance(mapCorpUserAppearanceSettings(corpUserSettings, featureFlags)); + + // Map Views Settings. + if (corpUserSettings.hasViews()) { + result.setViews(mapCorpUserViewsSettings(corpUserSettings.getViews())); + } + + corpUser.setSettings(result); + } + + @Nonnull + private CorpUserAppearanceSettings mapCorpUserAppearanceSettings( + @Nonnull final CorpUserSettings corpUserSettings, + @Nullable final FeatureFlags featureFlags + ) { CorpUserAppearanceSettings appearanceResult = new CorpUserAppearanceSettings(); if (featureFlags != null) { appearanceResult.setShowSimplifiedHomepage(featureFlags.isShowSimplifiedHomepageByDefault()); @@ -84,10 +101,21 @@ private void mapCorpUserSettings(@Nonnull CorpUser corpUser, EnvelopedAspect env if (corpUserSettings.hasAppearance()) { appearanceResult.setShowSimplifiedHomepage(corpUserSettings.getAppearance().isShowSimplifiedHomepage()); } + return appearanceResult; + } - result.setAppearance(appearanceResult); + @Nonnull + private CorpUserViewsSettings mapCorpUserViewsSettings(@Nonnull final com.linkedin.identity.CorpUserViewsSettings viewsSettings) { + CorpUserViewsSettings viewsResult = new CorpUserViewsSettings(); - corpUser.setSettings(result); + if (viewsSettings.hasDefaultView()) { + final DataHubView unresolvedView = new DataHubView(); + unresolvedView.setUrn(viewsSettings.getDefaultView().toString()); + unresolvedView.setType(EntityType.DATAHUB_VIEW); + viewsResult.setDefaultView(unresolvedView); + } + + return viewsResult; } private void mapCorpUserKey(@Nonnull CorpUser corpUser, @Nonnull DataMap dataMap) { diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/view/DataHubViewMapper.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/view/DataHubViewMapper.java new file mode 100644 index 00000000000000..f6c348937c7a55 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/view/DataHubViewMapper.java @@ -0,0 +1,146 @@ +package com.linkedin.datahub.graphql.types.view; + +import com.linkedin.data.DataMap; +import com.linkedin.datahub.graphql.generated.DataHubView; +import com.linkedin.datahub.graphql.generated.DataHubViewDefinition; +import com.linkedin.datahub.graphql.generated.DataHubViewFilter; +import com.linkedin.datahub.graphql.generated.DataHubViewType; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.FacetFilter; +import com.linkedin.datahub.graphql.generated.FilterOperator; +import com.linkedin.datahub.graphql.generated.LogicalOperator; +import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; +import com.linkedin.datahub.graphql.types.common.mappers.util.MappingHelper; +import com.linkedin.datahub.graphql.types.mappers.ModelMapper; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.view.DataHubViewInfo; +import java.util.Collections; +import java.util.List; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.metadata.Constants.*; + +@Slf4j +public class DataHubViewMapper implements ModelMapper { + + private static final String KEYWORD_FILTER_SUFFIX = ".keyword"; + public static final DataHubViewMapper INSTANCE = new DataHubViewMapper(); + + public static DataHubView map(@Nonnull final EntityResponse entityResponse) { + return INSTANCE.apply(entityResponse); + } + + @Override + public DataHubView apply(@Nonnull final EntityResponse entityResponse) { + final DataHubView result = new DataHubView(); + result.setUrn(entityResponse.getUrn().toString()); + result.setType(EntityType.DATAHUB_VIEW); + EnvelopedAspectMap aspectMap = entityResponse.getAspects(); + MappingHelper mappingHelper = new MappingHelper<>(aspectMap, result); + mappingHelper.mapToResult(DATAHUB_VIEW_INFO_ASPECT_NAME, this::mapDataHubViewInfo); + return mappingHelper.getResult(); + } + + private void mapDataHubViewInfo(@Nonnull final DataHubView view, @Nonnull final DataMap dataMap) { + DataHubViewInfo viewInfo = new DataHubViewInfo(dataMap); + view.setDescription(viewInfo.getDescription()); + view.setViewType(DataHubViewType.valueOf(viewInfo.getType().toString())); + view.setName(viewInfo.getName()); + view.setDescription(viewInfo.getDescription()); + view.setDefinition(mapViewDefinition(viewInfo.getDefinition())); + } + + @Nonnull + private DataHubViewDefinition mapViewDefinition(@Nonnull final com.linkedin.view.DataHubViewDefinition definition) { + final DataHubViewDefinition result = new DataHubViewDefinition(); + result.setFilter(mapFilter(definition.getFilter())); + result.setEntityTypes(definition.getEntityTypes().stream().map(EntityTypeMapper::getType).collect( + Collectors.toList())); + return result; + } + + @Nullable + private DataHubViewFilter mapFilter(@Nonnull final com.linkedin.metadata.query.filter.Filter filter) { + // This assumes that people DO NOT emit Views on their own, since we expect that the Filter structure is within + // a finite set of possibilities. + // + // If we find a View that was ingested manually and malformed, then we log that and return a default. + final DataHubViewFilter result = new DataHubViewFilter(); + if (filter.hasOr() && filter.getOr().size() == 1) { + // Then we are looking at an AND with multiple sub conditions. + result.setOperator(LogicalOperator.AND); + result.setFilters(mapAndFilters(filter.getOr().get(0).getAnd())); + } else { + result.setOperator(LogicalOperator.OR); + // Then we are looking at an OR with a group of sub conditions. + result.setFilters(mapOrFilters(filter.getOr())); + } + return result; + } + + /** + * This simply converts a List of leaf criterion into the FacetFiler equivalent. + */ + @Nonnull + private List mapAndFilters(@Nullable final List ands) { + // If the array is missing, return empty array. + if (ands == null) { + log.warn("Found a View without any AND filter criteria. Returning empty filter list."); + return Collections.emptyList(); + } + return ands.stream().map(this::mapCriterion).collect(Collectors.toList()); + } + + /** + * This converts a list of Conjunctive Criterion into a flattened list + * of FacetFilters. This method makes the assumption that WE (our GraphQL API) + * has minted the View and that each or criterion contains at maximum one nested condition. + */ + @Nonnull + private List mapOrFilters(@Nullable final List ors) { + if (ors == null) { + log.warn("Found a View without any OR filter criteria. Returning empty filter list."); + return Collections.emptyList(); + } + if (ors.stream().anyMatch(or -> or.hasAnd() && or.getAnd().size() > 1)) { + log.warn(String.format( + "Detected a View with a malformed filter clause. OR view has children conjunctions with more than one Criterion. Returning empty filters. %s", ors)); + return Collections.emptyList(); + } + // It is assumed that in this case, the view is a flat list of ORs. Thus, we filter + // for all nested AND conditions containing only 1 entry. + return ors.stream() + .filter(or -> or.hasAnd() && or.getAnd().size() == 1) + .map(or -> or.getAnd().get(0)) + .map(this::mapCriterion) + .collect(Collectors.toList()); + } + + @Nonnull + private FacetFilter mapCriterion(@Nonnull final Criterion andFilter) { + final FacetFilter result = new FacetFilter(); + result.setField(stripKeyword(andFilter.getField())); + result.setValues(andFilter.getValues()); + if (andFilter.hasCondition()) { + result.setCondition(FilterOperator.valueOf(andFilter.getCondition().toString())); + } + if (andFilter.hasNegated()) { + result.setNegated(andFilter.isNegated()); + } + return result; + } + + private String stripKeyword(final String fieldName) { + // When the Filter is persisted, it may include a .keyword suffix. + if (fieldName.endsWith(KEYWORD_FILTER_SUFFIX)) { + return fieldName.substring(0, fieldName.length() - KEYWORD_FILTER_SUFFIX.length()); + } + return fieldName; + } +} diff --git a/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/view/DataHubViewType.java b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/view/DataHubViewType.java new file mode 100644 index 00000000000000..21a80e3f900d41 --- /dev/null +++ b/datahub-graphql-core/src/main/java/com/linkedin/datahub/graphql/types/view/DataHubViewType.java @@ -0,0 +1,76 @@ +package com.linkedin.datahub.graphql.types.view; + +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.DataHubView; +import com.linkedin.datahub.graphql.generated.Entity; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import graphql.execution.DataFetcherResult; +import java.net.URISyntaxException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.function.Function; +import java.util.stream.Collectors; +import javax.annotation.Nonnull; +import lombok.RequiredArgsConstructor; + +import static com.linkedin.metadata.Constants.*; + + +@RequiredArgsConstructor +public class DataHubViewType implements com.linkedin.datahub.graphql.types.EntityType { + public static final Set ASPECTS_TO_FETCH = ImmutableSet.of(DATAHUB_VIEW_INFO_ASPECT_NAME); + private final EntityClient _entityClient; + + @Override + public EntityType type() { + return EntityType.DATAHUB_VIEW; + } + + @Override + public Function getKeyProvider() { + return Entity::getUrn; + } + + @Override + public Class objectClass() { + return DataHubView.class; + } + + @Override + public List> batchLoad(@Nonnull List urns, @Nonnull QueryContext context) + throws Exception { + final List viewUrns = urns.stream().map(this::getUrn).collect(Collectors.toList()); + + try { + final Map entities = + _entityClient.batchGetV2(DATAHUB_VIEW_ENTITY_NAME, new HashSet<>(viewUrns), ASPECTS_TO_FETCH, + context.getAuthentication()); + + final List gmsResults = new ArrayList<>(); + for (Urn urn : viewUrns) { + gmsResults.add(entities.getOrDefault(urn, null)); + } + return gmsResults.stream() + .map(gmsResult -> gmsResult == null ? null + : DataFetcherResult.newResult().data(DataHubViewMapper.map(gmsResult)).build()) + .collect(Collectors.toList()); + } catch (Exception e) { + throw new RuntimeException("Failed to batch load Views", e); + } + } + + private Urn getUrn(final String urnStr) { + try { + return Urn.createFromString(urnStr); + } catch (URISyntaxException e) { + throw new RuntimeException(String.format("Failed to convert urn string %s into Urn", urnStr)); + } + } +} diff --git a/datahub-graphql-core/src/main/resources/app.graphql b/datahub-graphql-core/src/main/resources/app.graphql index 6db846217839c2..333c932c0d1ada 100644 --- a/datahub-graphql-core/src/main/resources/app.graphql +++ b/datahub-graphql-core/src/main/resources/app.graphql @@ -11,6 +11,20 @@ extend type Query { Used by DataHub UI """ appConfig: AppConfig + + """ + Fetch the Global Settings related to the Views feature. + Requires the 'Manage Global Views' Platform Privilege. + """ + globalViewsSettings: GlobalViewsSettings +} + +extend type Mutation { + """ + Update the global settings related to the Views feature. + Requires the 'Manage Global Views' Platform Privilege. + """ + updateGlobalViewsSettings(input: UpdateGlobalViewsSettingsInput!): Boolean! } """ @@ -101,6 +115,11 @@ type PlatformPrivileges { Whether the user should be able to create and delete all Tags """ manageTags: Boolean! + + """ + Whether the user should be able to create, update, and delete global views. + """ + manageGlobalViews: Boolean! } """ @@ -157,6 +176,11 @@ type AppConfig { Configurations related to DataHub tests """ testsConfig: TestsConfig! + + """ + Configurations related to DataHub Views + """ + viewsConfig: ViewsConfig! } """ @@ -310,3 +334,35 @@ type TestsConfig { """ enabled: Boolean! } + +""" +Configurations related to DataHub Views feature +""" +type ViewsConfig { + """ + Whether Views feature is enabled + """ + enabled: Boolean! +} + +""" +Input required to update Global View Settings. +""" +input UpdateGlobalViewsSettingsInput { + """ + The URN of the View that serves as the Global, or organization-wide, default. + If this field is not provided, the existing Global Default will be cleared. + """ + defaultView: String +} + +""" +Global (platform-level) settings related to the Views feature +""" +type GlobalViewsSettings { + """ + The global default View. If a user does not have a personal default, then + this will be the default view. + """ + defaultView: String +} diff --git a/datahub-graphql-core/src/main/resources/entity.graphql b/datahub-graphql-core/src/main/resources/entity.graphql index 4104fa02198632..bd19d143bcd4d4 100644 --- a/datahub-graphql-core/src/main/resources/entity.graphql +++ b/datahub-graphql-core/src/main/resources/entity.graphql @@ -188,6 +188,16 @@ type Query { List all Posts """ listPosts(input: ListPostsInput!): ListPostsResult + + """ + List DataHub Views owned by the current user + """ + listMyViews(input: ListMyViewsInput!): ListViewsResult + + """ + List Global DataHub Views + """ + listGlobalViews(input: ListGlobalViewsInput!): ListViewsResult } """ @@ -504,6 +514,11 @@ type Mutation { """ batchUpdateSoftDeleted(input: BatchUpdateSoftDeletedInput!): Boolean + """ + Update the View-related settings for a user. + """ + updateCorpUserViewsSettings(input: UpdateCorpUserViewsSettingsInput!): Boolean + """ Update a user setting """ @@ -528,6 +543,29 @@ type Mutation { Create a post """ createPost(input: CreatePostInput!): Boolean + + """ + Create a new DataHub View (Saved Filter) + """ + createView( + "Input required to create a new DataHub View" + input: CreateViewInput!): DataHubView + + """ + Delete a DataHub View (Saved Filter) + """ + updateView( + "The urn of the View to update" + urn: String!, + "Input required to updat an existing DataHub View" + input: UpdateViewInput!): DataHubView + + """ + Delete a DataHub View (Saved Filter) + """ + deleteView( + "The urn of the View to delete" + urn: String!): Boolean } """ @@ -713,6 +751,11 @@ enum EntityType { A Schema Field """ SCHEMA_FIELD + + """ + A DataHub View + """ + DATAHUB_VIEW } """ @@ -2991,6 +3034,11 @@ type CorpUserSettings { Settings that control look and feel of the DataHub UI for the user """ appearance: CorpUserAppearanceSettings + + """ + Settings related to the DataHub Views feature + """ + views: CorpUserViewsSettings } """ @@ -3004,6 +3052,16 @@ type CorpUserAppearanceSettings { showSimplifiedHomepage: Boolean } +""" +Settings related to the Views feature of DataHub. +""" +type CorpUserViewsSettings { + """ + The default view for the User. + """ + defaultView: DataHubView +} + """ Deprecated, use CorpUserProperties instead Additional read only info about a user @@ -9300,6 +9358,9 @@ type InputField { schemaField: SchemaField } +""" +An individual setting type for a Corp User. +""" enum UserSetting { """ Show simplified homepage @@ -9307,7 +9368,6 @@ enum UserSetting { SHOW_SIMPLIFIED_HOMEPAGE } - """ Input for updating a user setting """ @@ -9662,3 +9722,259 @@ type Media { """ location: String! } + +""" +The type of a DataHub View +""" +enum DataHubViewType { + """ + A personal view - e.g. saved filters + """ + PERSONAL + + """ + A global view, e.g. role view + """ + GLOBAL +} + +""" +An DataHub View - Filters that are applied across the application automatically. +""" +type DataHubView implements Entity { + """ + The primary key of the View + """ + urn: String! + + """ + The standard Entity Type + """ + type: EntityType! + + """ + The type of the View + """ + viewType: DataHubViewType! + + """ + The name of the View + """ + name: String! + + """ + The description of the View + """ + description: String + + """ + The definition of the View + """ + definition: DataHubViewDefinition! + + """ + Granular API for querying edges extending from the View + """ + relationships(input: RelationshipsInput!): EntityRelationshipsResult +} + +""" +An DataHub View Definition +""" +type DataHubViewDefinition { + """ + A set of filters to apply. If left empty, then ALL entity types are in scope. + """ + entityTypes: [EntityType!]! + + """ + A set of filters to apply. If left empty, then no filters will be applied. + """ + filter: DataHubViewFilter! +} + +""" +A Logical Operator, AND or OR. +""" +enum LogicalOperator { + """ + An AND operator. + """ + AND + + """ + An OR operator. + """ + OR +} + +""" +A DataHub View Filter. Note that +""" +type DataHubViewFilter { + """ + The operator used to combine the filters. + """ + operator: LogicalOperator! + + """ + A set of filters combined using the operator. If left empty, then no filters will be applied. + """ + filters: [FacetFilter!]! +} + +""" +Input provided when listing DataHub Views +""" +input ListMyViewsInput { + """ + The starting offset of the result set returned + """ + start: Int + + """ + The maximum number of Views to be returned in the result set + """ + count: Int + + """ + Optional search query + """ + query: String + + """ + Optional - List the type of View to filter for. + """ + viewType: DataHubViewType +} + +""" +Input provided when listing DataHub Global Views +""" +input ListGlobalViewsInput { + """ + The starting offset of the result set returned + """ + start: Int + + """ + The maximum number of Views to be returned in the result set + """ + count: Int + + """ + Optional search query + """ + query: String +} + +""" +The result obtained when listing DataHub Views +""" +type ListViewsResult { + """ + The starting offset of the result set returned + """ + start: Int! + + """ + The number of Views in the returned result set + """ + count: Int! + + """ + The total number of Views in the result set + """ + total: Int! + + """ + The Views themselves + """ + views: [DataHubView!]! +} + +""" +Input provided when creating a DataHub View +""" +input CreateViewInput { + """ + The type of View + """ + viewType: DataHubViewType! + + """ + The name of the View + """ + name: String! + + """ + An optional description of the View + """ + description: String + + """ + The view definition itself + """ + definition: DataHubViewDefinitionInput! +} + +""" +Input required for creating a DataHub View Definition +""" +input DataHubViewDefinitionInput { + """ + A set of entity types that the view applies for. If left empty, then ALL entities will be in scope. + """ + entityTypes: [EntityType!]! + + """ + A set of filters to apply. + """ + filter: DataHubViewFilterInput! +} + +""" +Input required for creating a DataHub View Definition +""" +input DataHubViewFilterInput { + """ + The operator used to combine the filters. + """ + operator: LogicalOperator! + + """ + A set of filters combined via an operator. If left empty, then no filters will be applied. + """ + filters: [FacetFilterInput!]! +} + +""" +Input provided when updating a DataHub View +""" +input UpdateViewInput { + """ + The name of the View + """ + name: String + + """ + An optional description of the View + """ + description: String + + """ + The view definition itself + """ + definition: DataHubViewDefinitionInput +} + +""" +Input required to update a users settings. +""" +input UpdateCorpUserViewsSettingsInput { + """ + The URN of the View that serves as this user's personal default. + If not provided, any existing default view will be removed. + """ + defaultView: String +} \ No newline at end of file diff --git a/datahub-graphql-core/src/main/resources/recommendation.graphql b/datahub-graphql-core/src/main/resources/recommendation.graphql index 4e4bd14052aff9..439b22142b0cb8 100644 --- a/datahub-graphql-core/src/main/resources/recommendation.graphql +++ b/datahub-graphql-core/src/main/resources/recommendation.graphql @@ -227,32 +227,7 @@ type SearchParams { """ Filters """ - filters: [Filter!] -} - -""" -Facet filters to apply to search results -""" -type Filter { - """ - Name of field to filter by - """ - field: String! - - """ - Values, one of which the intended field should match. - """ - values: [String!]! - - """ - If the filter should or should not be matched - """ - negated: Boolean - - """ - Condition for the values. How to If unset, assumed to be equality - """ - condition: FilterOperator + filters: [FacetFilter!] } """ diff --git a/datahub-graphql-core/src/main/resources/search.graphql b/datahub-graphql-core/src/main/resources/search.graphql index 8faed5655492a9..b8fa47a460584b 100644 --- a/datahub-graphql-core/src/main/resources/search.graphql +++ b/datahub-graphql-core/src/main/resources/search.graphql @@ -73,7 +73,7 @@ input SearchInput { """ A list of disjunctive criterion for the filter. (or operation to combine filters) """ - orFilters: [OrFilter!] + orFilters: [AndFilterInput!] } """ @@ -109,7 +109,12 @@ input SearchAcrossEntitiesInput { """ A list of disjunctive criterion for the filter. (or operation to combine filters) """ - orFilters: [OrFilter!] + orFilters: [AndFilterInput!] + + """ + Optional - A View to apply when generating results + """ + viewUrn: String } """ @@ -155,13 +160,13 @@ input SearchAcrossLineageInput { """ A list of disjunctive criterion for the filter. (or operation to combine filters) """ - orFilters: [OrFilter!] + orFilters: [AndFilterInput!] } """ A list of disjunctive criterion for the filter. (or operation to combine filters) """ -input OrFilter { +input AndFilterInput { """ A list of and criteria the filter applies to the query """ @@ -556,7 +561,7 @@ input BrowseInput { """ A list of disjunctive criterion for the filter. (or operation to combine filters) """ - orFilters: [OrFilter!] + orFilters: [AndFilterInput!] } """ @@ -658,3 +663,28 @@ input FilterInput { """ and: [FacetFilterInput!]! } + +""" +A single filter value +""" +type FacetFilter { + """ + Name of field to filter by + """ + field: String! + + """ + Condition for the values. + """ + condition: FilterOperator + + """ + Values, one of which the intended field should match. + """ + values: [String!]! + + """ + If the filter should or should not be matched + """ + negated: Boolean +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/TestUtils.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/TestUtils.java index e9b12937ddf817..cb11100a53946f 100644 --- a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/TestUtils.java +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/TestUtils.java @@ -1,9 +1,12 @@ package com.linkedin.datahub.graphql; +import com.datahub.authentication.Actor; +import com.datahub.authentication.ActorType; import com.datahub.authentication.Authentication; import com.datahub.authorization.AuthorizationResult; import com.datahub.plugins.auth.authorization.Authorizer; import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.UrnUtils; import com.linkedin.metadata.entity.EntityService; import com.linkedin.mxe.MetadataChangeProposal; import org.mockito.Mockito; @@ -12,8 +15,12 @@ public class TestUtils { public static QueryContext getMockAllowContext() { + return getMockAllowContext("urn:li:corpuser:test"); + } + + public static QueryContext getMockAllowContext(String actorUrn) { QueryContext mockContext = Mockito.mock(QueryContext.class); - Mockito.when(mockContext.getActorUrn()).thenReturn("urn:li:corpuser:test"); + Mockito.when(mockContext.getActorUrn()).thenReturn(actorUrn); Authorizer mockAuthorizer = Mockito.mock(Authorizer.class); AuthorizationResult result = Mockito.mock(AuthorizationResult.class); @@ -21,13 +28,19 @@ public static QueryContext getMockAllowContext() { Mockito.when(mockAuthorizer.authorize(Mockito.any())).thenReturn(result); Mockito.when(mockContext.getAuthorizer()).thenReturn(mockAuthorizer); - Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); + Mockito.when(mockContext.getAuthentication()).thenReturn( + new Authentication(new Actor(ActorType.USER, UrnUtils.getUrn(actorUrn).getId()), "creds") + ); return mockContext; } public static QueryContext getMockDenyContext() { + return getMockDenyContext("urn:li:corpuser:test"); + } + + public static QueryContext getMockDenyContext(String actorUrn) { QueryContext mockContext = Mockito.mock(QueryContext.class); - Mockito.when(mockContext.getActorUrn()).thenReturn("urn:li:corpuser:test"); + Mockito.when(mockContext.getActorUrn()).thenReturn(actorUrn); Authorizer mockAuthorizer = Mockito.mock(Authorizer.class); AuthorizationResult result = Mockito.mock(AuthorizationResult.class); @@ -35,7 +48,9 @@ public static QueryContext getMockDenyContext() { Mockito.when(mockAuthorizer.authorize(Mockito.any())).thenReturn(result); Mockito.when(mockContext.getAuthorizer()).thenReturn(mockAuthorizer); - Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); + Mockito.when(mockContext.getAuthentication()).thenReturn( + new Authentication(new Actor(ActorType.USER, UrnUtils.getUrn(actorUrn).getId()), "creds") + ); return mockContext; } diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java new file mode 100644 index 00000000000000..57e3b646235c1c --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchAcrossEntitiesResolverTest.java @@ -0,0 +1,527 @@ +package com.linkedin.datahub.graphql.resolvers.search; + +import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableList; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.StringArray; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.AndFilterInput; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.FacetFilterInput; +import com.linkedin.datahub.graphql.generated.FilterOperator; +import com.linkedin.datahub.graphql.generated.SearchAcrossEntitiesInput; +import com.linkedin.datahub.graphql.resolvers.EntityTypeMapper; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.search.SearchEntityArray; +import com.linkedin.metadata.search.SearchResult; +import com.linkedin.metadata.search.SearchResultMetadata; +import com.linkedin.metadata.service.ViewService; +import com.linkedin.r2.RemoteInvocationException; +import com.linkedin.view.DataHubViewDefinition; +import com.linkedin.view.DataHubViewInfo; +import com.linkedin.view.DataHubViewType; +import graphql.schema.DataFetchingEnvironment; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.CompletionException; +import java.util.stream.Collectors; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static com.linkedin.datahub.graphql.resolvers.search.SearchUtils.*; + + +public class SearchAcrossEntitiesResolverTest { + + private static final Urn TEST_VIEW_URN = UrnUtils.getUrn("urn:li:dataHubView:test"); + private static final Urn TEST_USER_URN = UrnUtils.getUrn("urn:li:corpuser:test"); + + + @Test + public static void testApplyViewNullBaseFilter() throws Exception { + + Filter viewFilter = new Filter() + .setOr(new ConjunctiveCriterionArray( + new ConjunctiveCriterion().setAnd( + new CriterionArray(ImmutableList.of( + new Criterion() + .setField("field") + .setValue("test") + .setValues(new StringArray(ImmutableList.of("test"))) + )) + ))); + + DataHubViewInfo info = new DataHubViewInfo(); + info.setName("test"); + info.setType(DataHubViewType.GLOBAL); + info.setCreated(new AuditStamp().setTime(0L).setActor(TEST_USER_URN)); + info.setLastModified(new AuditStamp().setTime(0L).setActor(TEST_USER_URN)); + info.setDefinition(new DataHubViewDefinition() + .setEntityTypes(new StringArray(ImmutableList.of(Constants.DATASET_ENTITY_NAME, Constants.DASHBOARD_ENTITY_NAME))) + .setFilter(viewFilter) + ); + + ViewService mockService = initMockViewService( + TEST_VIEW_URN, + info + ); + + EntityClient mockClient = initMockEntityClient( + ImmutableList.of(Constants.DATASET_ENTITY_NAME), + "", + viewFilter, + 0, + 10, + new SearchResult() + .setEntities(new SearchEntityArray()) + .setNumEntities(0) + .setFrom(0) + .setPageSize(0) + .setMetadata(new SearchResultMetadata()) + ); + + final SearchAcrossEntitiesResolver resolver = new SearchAcrossEntitiesResolver(mockClient, mockService); + + final SearchAcrossEntitiesInput testInput = new SearchAcrossEntitiesInput( + ImmutableList.of(EntityType.DATASET), + "", + 0, + 10, + null, + null, + TEST_VIEW_URN.toString() + ); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(testInput); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + resolver.get(mockEnv).get(); + + verifyMockEntityClient( + mockClient, + ImmutableList.of(Constants.DATASET_ENTITY_NAME), // Verify that merged entity types were used. + "", + viewFilter, // Verify that view filter was used. + 0, + 10 + ); + + verifyMockViewService( + mockService, + TEST_VIEW_URN + ); + } + + @Test + public static void testApplyViewBaseFilter() throws Exception { + + Filter viewFilter = new Filter() + .setOr(new ConjunctiveCriterionArray( + new ConjunctiveCriterion().setAnd( + new CriterionArray(ImmutableList.of( + new Criterion() + .setField("field") + .setValue("test") + .setValues(new StringArray(ImmutableList.of("test"))) + )) + ))); + + DataHubViewInfo info = new DataHubViewInfo(); + info.setName("test"); + info.setType(DataHubViewType.GLOBAL); + info.setCreated(new AuditStamp().setTime(0L).setActor(TEST_USER_URN)); + info.setLastModified(new AuditStamp().setTime(0L).setActor(TEST_USER_URN)); + info.setDefinition(new DataHubViewDefinition() + .setEntityTypes(new StringArray(ImmutableList.of(Constants.DATASET_ENTITY_NAME, Constants.DASHBOARD_ENTITY_NAME))) + .setFilter(viewFilter) + ); + + ViewService mockService = initMockViewService( + TEST_VIEW_URN, + info + ); + + Filter baseFilter = new Filter() + .setOr(new ConjunctiveCriterionArray( + new ConjunctiveCriterion().setAnd( + new CriterionArray(ImmutableList.of( + new Criterion() + .setField("baseField.keyword") + .setValue("baseTest") + .setCondition(Condition.EQUAL) + .setNegated(false) + .setValues(new StringArray(ImmutableList.of("baseTest"))) + )) + ))); + + EntityClient mockClient = initMockEntityClient( + ImmutableList.of(Constants.DATASET_ENTITY_NAME), + "", + SearchUtils.combineFilters(baseFilter, viewFilter), + 0, + 10, + new SearchResult() + .setEntities(new SearchEntityArray()) + .setNumEntities(0) + .setFrom(0) + .setPageSize(0) + .setMetadata(new SearchResultMetadata()) + ); + + final SearchAcrossEntitiesResolver resolver = new SearchAcrossEntitiesResolver(mockClient, mockService); + + final SearchAcrossEntitiesInput testInput = new SearchAcrossEntitiesInput( + ImmutableList.of(EntityType.DATASET), + "", + 0, + 10, + null, + ImmutableList.of( + new AndFilterInput(ImmutableList.of( + new FacetFilterInput("baseField", "baseTest", ImmutableList.of("baseTest"), false, FilterOperator.EQUAL) + )) + ), + TEST_VIEW_URN.toString() + ); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(testInput); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + resolver.get(mockEnv).get(); + + verifyMockEntityClient( + mockClient, + ImmutableList.of(Constants.DATASET_ENTITY_NAME), // Verify that merged entity types were used. + "", + SearchUtils.combineFilters(baseFilter, viewFilter), // Verify that merged filters were used. + 0, + 10 + ); + + verifyMockViewService( + mockService, + TEST_VIEW_URN + ); + } + + @Test + public static void testApplyViewNullBaseEntityTypes() throws Exception { + Filter viewFilter = new Filter() + .setOr(new ConjunctiveCriterionArray( + new ConjunctiveCriterion().setAnd( + new CriterionArray(ImmutableList.of( + new Criterion() + .setField("field") + .setValue("test") + .setValues(new StringArray(ImmutableList.of("test"))) + )) + ))); + + DataHubViewInfo info = new DataHubViewInfo(); + info.setName("test"); + info.setType(DataHubViewType.GLOBAL); + info.setCreated(new AuditStamp().setTime(0L).setActor(TEST_USER_URN)); + info.setLastModified(new AuditStamp().setTime(0L).setActor(TEST_USER_URN)); + info.setDefinition(new DataHubViewDefinition() + .setEntityTypes(new StringArray(ImmutableList.of(Constants.DATASET_ENTITY_NAME, Constants.DASHBOARD_ENTITY_NAME))) + .setFilter(viewFilter) + ); + + ViewService mockService = initMockViewService( + TEST_VIEW_URN, + info + ); + + EntityClient mockClient = initMockEntityClient( + ImmutableList.of(Constants.DATASET_ENTITY_NAME, Constants.DASHBOARD_ENTITY_NAME), + "", + viewFilter, + 0, + 10, + new SearchResult() + .setEntities(new SearchEntityArray()) + .setNumEntities(0) + .setFrom(0) + .setPageSize(0) + .setMetadata(new SearchResultMetadata()) + ); + + final SearchAcrossEntitiesResolver resolver = new SearchAcrossEntitiesResolver(mockClient, mockService); + + final SearchAcrossEntitiesInput testInput = new SearchAcrossEntitiesInput( + null, + "", + 0, + 10, + null, + null, + TEST_VIEW_URN.toString() + ); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(testInput); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + resolver.get(mockEnv).get(); + + verifyMockEntityClient( + mockClient, + ImmutableList.of(Constants.DATASET_ENTITY_NAME, Constants.DASHBOARD_ENTITY_NAME), // Verify that view entity types were honored. + "", + viewFilter, // Verify that merged filters were used. + 0, + 10 + ); + + verifyMockViewService( + mockService, + TEST_VIEW_URN + ); + } + + @Test + public static void testApplyViewEmptyBaseEntityTypes() throws Exception { + Filter viewFilter = new Filter() + .setOr(new ConjunctiveCriterionArray( + new ConjunctiveCriterion().setAnd( + new CriterionArray(ImmutableList.of( + new Criterion() + .setField("field") + .setValue("test") + .setValues(new StringArray(ImmutableList.of("test"))) + )) + ))); + + DataHubViewInfo info = new DataHubViewInfo(); + info.setName("test"); + info.setType(DataHubViewType.GLOBAL); + info.setCreated(new AuditStamp().setTime(0L).setActor(TEST_USER_URN)); + info.setLastModified(new AuditStamp().setTime(0L).setActor(TEST_USER_URN)); + info.setDefinition(new DataHubViewDefinition() + .setEntityTypes(new StringArray(ImmutableList.of(Constants.DATASET_ENTITY_NAME, Constants.DASHBOARD_ENTITY_NAME))) + .setFilter(viewFilter) + ); + + ViewService mockService = initMockViewService( + TEST_VIEW_URN, + info + ); + + EntityClient mockClient = initMockEntityClient( + ImmutableList.of(Constants.DATASET_ENTITY_NAME, Constants.DASHBOARD_ENTITY_NAME), + "", + viewFilter, + 0, + 10, + new SearchResult() + .setEntities(new SearchEntityArray()) + .setNumEntities(0) + .setFrom(0) + .setPageSize(0) + .setMetadata(new SearchResultMetadata()) + ); + + final SearchAcrossEntitiesResolver resolver = new SearchAcrossEntitiesResolver(mockClient, mockService); + + final SearchAcrossEntitiesInput testInput = new SearchAcrossEntitiesInput( + Collections.emptyList(), // Empty Entity Types + "", + 0, + 10, + null, + null, + TEST_VIEW_URN.toString() + ); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(testInput); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + resolver.get(mockEnv).get(); + + verifyMockEntityClient( + mockClient, + ImmutableList.of(Constants.DATASET_ENTITY_NAME, Constants.DASHBOARD_ENTITY_NAME), // Verify that view entity types were honored. + "", + viewFilter, // Verify that merged filters were used. + 0, + 10 + ); + + verifyMockViewService( + mockService, + TEST_VIEW_URN + ); + } + + @Test + public static void testApplyViewViewDoesNotExist() throws Exception { + // When a view does not exist, the endpoint should WARN and not apply the view. + + ViewService mockService = initMockViewService( + TEST_VIEW_URN, + null + ); + + List searchEntityTypes = SEARCHABLE_ENTITY_TYPES.stream().map(EntityTypeMapper::getName).collect(Collectors.toList()); + + EntityClient mockClient = initMockEntityClient( + searchEntityTypes, + "", + null, + 0, + 10, + new SearchResult() + .setEntities(new SearchEntityArray()) + .setNumEntities(0) + .setFrom(0) + .setPageSize(0) + .setMetadata(new SearchResultMetadata()) + ); + + final SearchAcrossEntitiesResolver resolver = new SearchAcrossEntitiesResolver(mockClient, mockService); + final SearchAcrossEntitiesInput testInput = new SearchAcrossEntitiesInput( + Collections.emptyList(), // Empty Entity Types + "", + 0, + 10, + null, + null, + TEST_VIEW_URN.toString() + ); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(testInput); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + resolver.get(mockEnv).get(); + + verifyMockEntityClient( + mockClient, + searchEntityTypes, + "", + null, + 0, + 10 + ); + } + + @Test + public static void testApplyViewErrorFetchingView() throws Exception { + // When a view cannot be successfully resolved, the endpoint show THROW. + + ViewService mockService = initMockViewService( + TEST_VIEW_URN, + null + ); + + EntityClient mockClient = Mockito.mock(EntityClient.class); + Mockito.when(mockClient.searchAcrossEntities( + Mockito.anyList(), + Mockito.anyString(), + Mockito.any(), + Mockito.anyInt(), + Mockito.anyInt(), + Mockito.any(Authentication.class) + )).thenThrow(new RemoteInvocationException()); + + final SearchAcrossEntitiesResolver resolver = new SearchAcrossEntitiesResolver(mockClient, mockService); + final SearchAcrossEntitiesInput testInput = new SearchAcrossEntitiesInput( + Collections.emptyList(), // Empty Entity Types + "", + 0, + 10, + null, + null, + TEST_VIEW_URN.toString() + ); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(testInput); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + Assert.assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } + + private static ViewService initMockViewService( + Urn viewUrn, + DataHubViewInfo viewInfo + ) { + ViewService service = Mockito.mock(ViewService.class); + Mockito.when(service.getViewInfo( + Mockito.eq(viewUrn), + Mockito.any(Authentication.class) + )).thenReturn( + viewInfo + ); + return service; + } + + private static EntityClient initMockEntityClient( + List entityTypes, + String query, + Filter filter, + int start, + int limit, + SearchResult result + ) throws Exception { + EntityClient client = Mockito.mock(EntityClient.class); + Mockito.when(client.searchAcrossEntities( + Mockito.eq(entityTypes), + Mockito.eq(query), + Mockito.eq(filter), + Mockito.eq(start), + Mockito.eq(limit), + Mockito.any(Authentication.class) + )).thenReturn( + result + ); + return client; + } + + private static void verifyMockEntityClient( + EntityClient mockClient, + List entityTypes, + String query, + Filter filter, + int start, + int limit + ) throws Exception { + Mockito.verify(mockClient, Mockito.times(1)) + .searchAcrossEntities( + Mockito.eq(entityTypes), + Mockito.eq(query), + Mockito.eq(filter), + Mockito.eq(start), + Mockito.eq(limit), + Mockito.any(Authentication.class) + ); + } + + private static void verifyMockViewService( + ViewService mockService, + Urn viewUrn + ) { + Mockito.verify(mockService, Mockito.times(1)) + .getViewInfo( + Mockito.eq(viewUrn), + Mockito.any(Authentication.class) + ); + } + + private SearchAcrossEntitiesResolverTest() { } + +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtilsTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtilsTest.java new file mode 100644 index 00000000000000..b35f7a77b209c9 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/search/SearchUtilsTest.java @@ -0,0 +1,352 @@ +package com.linkedin.datahub.graphql.resolvers.search; + +import com.google.common.collect.ImmutableList; +import com.linkedin.data.template.StringArray; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; +import com.linkedin.metadata.query.filter.Filter; +import java.util.List; +import org.testng.Assert; +import org.testng.annotations.Test; + +public class SearchUtilsTest { + + @Test + public static void testApplyViewToFilterNullBaseFilter() { + + Filter viewFilter = new Filter() + .setOr(new ConjunctiveCriterionArray( + new ConjunctiveCriterion().setAnd( + new CriterionArray(ImmutableList.of( + new Criterion() + .setField("field") + .setValue("test") + .setValues(new StringArray(ImmutableList.of("test"))) + )) + ))); + + Filter result = SearchUtils.combineFilters(null, viewFilter); + Assert.assertEquals(viewFilter, result); + } + + @Test + public static void testApplyViewToFilterComplexBaseFilter() { + Filter baseFilter = new Filter() + .setOr(new ConjunctiveCriterionArray( + ImmutableList.of( + new ConjunctiveCriterion().setAnd( + new CriterionArray(ImmutableList.of( + new Criterion() + .setField("field1") + .setValue("test1") + .setValues(new StringArray(ImmutableList.of("test1"))), + new Criterion() + .setField("field2") + .setValue("test2") + .setValues(new StringArray(ImmutableList.of("test2"))) + )) + ), + new ConjunctiveCriterion().setAnd( + new CriterionArray(ImmutableList.of( + new Criterion() + .setField("field3") + .setValue("test3") + .setValues(new StringArray(ImmutableList.of("test3"))), + new Criterion() + .setField("field4") + .setValue("test4") + .setValues(new StringArray(ImmutableList.of("test4"))) + )) + ) + ))); + + Filter viewFilter = new Filter() + .setOr(new ConjunctiveCriterionArray( + new ConjunctiveCriterion().setAnd( + new CriterionArray(ImmutableList.of( + new Criterion() + .setField("field") + .setValue("test") + .setValues(new StringArray(ImmutableList.of("test"))) + )) + ))); + + Filter result = SearchUtils.combineFilters(baseFilter, viewFilter); + + Filter expectedResult = new Filter() + .setOr(new ConjunctiveCriterionArray( + ImmutableList.of( + new ConjunctiveCriterion().setAnd( + new CriterionArray(ImmutableList.of( + new Criterion() + .setField("field1") + .setValue("test1") + .setValues(new StringArray(ImmutableList.of("test1"))), + new Criterion() + .setField("field2") + .setValue("test2") + .setValues(new StringArray(ImmutableList.of("test2"))), + new Criterion() + .setField("field") + .setValue("test") + .setValues(new StringArray(ImmutableList.of("test"))) + )) + ), + new ConjunctiveCriterion().setAnd( + new CriterionArray(ImmutableList.of( + new Criterion() + .setField("field3") + .setValue("test3") + .setValues(new StringArray(ImmutableList.of("test3"))), + new Criterion() + .setField("field4") + .setValue("test4") + .setValues(new StringArray(ImmutableList.of("test4"))), + new Criterion() + .setField("field") + .setValue("test") + .setValues(new StringArray(ImmutableList.of("test"))) + )) + ) + ))); + + Assert.assertEquals(expectedResult, result); + } + + @Test + public static void testApplyViewToFilterComplexViewFilter() { + Filter baseFilter = new Filter() + .setOr(new ConjunctiveCriterionArray( + ImmutableList.of( + new ConjunctiveCriterion().setAnd( + new CriterionArray(ImmutableList.of( + new Criterion() + .setField("field1") + .setValue("test1") + .setValues(new StringArray(ImmutableList.of("test1"))), + new Criterion() + .setField("field2") + .setValue("test2") + .setValues(new StringArray(ImmutableList.of("test2"))) + )) + ), + new ConjunctiveCriterion().setAnd( + new CriterionArray(ImmutableList.of( + new Criterion() + .setField("field3") + .setValue("test3") + .setValues(new StringArray(ImmutableList.of("test3"))), + new Criterion() + .setField("field4") + .setValue("test4") + .setValues(new StringArray(ImmutableList.of("test4"))) + )) + ) + ))); + + Filter viewFilter = new Filter() + .setOr(new ConjunctiveCriterionArray( + ImmutableList.of( + new ConjunctiveCriterion().setAnd( + new CriterionArray(ImmutableList.of( + new Criterion() + .setField("viewField1") + .setValue("viewTest1") + .setValues(new StringArray(ImmutableList.of("viewTest1"))), + new Criterion() + .setField("viewField2") + .setValue("viewTest2") + .setValues(new StringArray(ImmutableList.of("viewTest2"))) + )) + ), + new ConjunctiveCriterion().setAnd( + new CriterionArray(ImmutableList.of( + new Criterion() + .setField("viewField3") + .setValue("viewTest3") + .setValues(new StringArray(ImmutableList.of("viewTest3"))), + new Criterion() + .setField("viewField4") + .setValue("viewTest4") + .setValues(new StringArray(ImmutableList.of("viewTest4"))) + )) + ) + ))); + + Filter result = SearchUtils.combineFilters(baseFilter, viewFilter); + + Filter expectedResult = new Filter() + .setOr(new ConjunctiveCriterionArray( + ImmutableList.of( + new ConjunctiveCriterion().setAnd( + new CriterionArray(ImmutableList.of( + new Criterion() + .setField("field1") + .setValue("test1") + .setValues(new StringArray(ImmutableList.of("test1"))), + new Criterion() + .setField("field2") + .setValue("test2") + .setValues(new StringArray(ImmutableList.of("test2"))), + new Criterion() + .setField("viewField1") + .setValue("viewTest1") + .setValues(new StringArray(ImmutableList.of("viewTest1"))), + new Criterion() + .setField("viewField2") + .setValue("viewTest2") + .setValues(new StringArray(ImmutableList.of("viewTest2"))) + )) + ), + new ConjunctiveCriterion().setAnd( + new CriterionArray(ImmutableList.of( + new Criterion() + .setField("field1") + .setValue("test1") + .setValues(new StringArray(ImmutableList.of("test1"))), + new Criterion() + .setField("field2") + .setValue("test2") + .setValues(new StringArray(ImmutableList.of("test2"))), + new Criterion() + .setField("viewField3") + .setValue("viewTest3") + .setValues(new StringArray(ImmutableList.of("viewTest3"))), + new Criterion() + .setField("viewField4") + .setValue("viewTest4") + .setValues(new StringArray(ImmutableList.of("viewTest4"))) + )) + ), + new ConjunctiveCriterion().setAnd( + new CriterionArray(ImmutableList.of( + new Criterion() + .setField("field3") + .setValue("test3") + .setValues(new StringArray(ImmutableList.of("test3"))), + new Criterion() + .setField("field4") + .setValue("test4") + .setValues(new StringArray(ImmutableList.of("test4"))), + new Criterion() + .setField("viewField1") + .setValue("viewTest1") + .setValues(new StringArray(ImmutableList.of("viewTest1"))), + new Criterion() + .setField("viewField2") + .setValue("viewTest2") + .setValues(new StringArray(ImmutableList.of("viewTest2"))) + )) + ), + new ConjunctiveCriterion().setAnd( + new CriterionArray(ImmutableList.of( + new Criterion() + .setField("field3") + .setValue("test3") + .setValues(new StringArray(ImmutableList.of("test3"))), + new Criterion() + .setField("field4") + .setValue("test4") + .setValues(new StringArray(ImmutableList.of("test4"))), + new Criterion() + .setField("viewField3") + .setValue("viewTest3") + .setValues(new StringArray(ImmutableList.of("viewTest3"))), + new Criterion() + .setField("viewField4") + .setValue("viewTest4") + .setValues(new StringArray(ImmutableList.of("viewTest4"))) + )) + ) + ))); + + Assert.assertEquals(expectedResult, result); + } + + @Test + public static void testApplyViewToFilterV1Filter() { + Filter baseFilter = new Filter() + .setCriteria( + new CriterionArray(ImmutableList.of( + new Criterion() + .setField("field1") + .setValue("test1") + .setValues(new StringArray(ImmutableList.of("test1"))), + new Criterion() + .setField("field2") + .setValue("test2") + .setValues(new StringArray(ImmutableList.of("test2"))) + )) + ); + + Filter viewFilter = new Filter() + .setCriteria( + new CriterionArray(ImmutableList.of( + new Criterion() + .setField("viewField1") + .setValue("viewTest1") + .setValues(new StringArray(ImmutableList.of("viewTest1"))), + new Criterion() + .setField("viewField2") + .setValue("viewTest2") + .setValues(new StringArray(ImmutableList.of("viewTest2"))) + )) + ); + + Filter result = SearchUtils.combineFilters(baseFilter, viewFilter); + + Filter expectedResult = new Filter() + .setOr(new ConjunctiveCriterionArray( + ImmutableList.of( + new ConjunctiveCriterion().setAnd( + new CriterionArray(ImmutableList.of( + new Criterion() + .setField("field1") + .setValue("test1") + .setValues(new StringArray(ImmutableList.of("test1"))), + new Criterion() + .setField("field2") + .setValue("test2") + .setValues(new StringArray(ImmutableList.of("test2"))), + new Criterion() + .setField("viewField1") + .setValue("viewTest1") + .setValues(new StringArray(ImmutableList.of("viewTest1"))), + new Criterion() + .setField("viewField2") + .setValue("viewTest2") + .setValues(new StringArray(ImmutableList.of("viewTest2"))) + )) + ) + ))); + + Assert.assertEquals(expectedResult, result); + } + + @Test + public static void testApplyViewToEntityTypes() { + + List baseEntityTypes = ImmutableList.of( + Constants.CHART_ENTITY_NAME, + Constants.DATASET_ENTITY_NAME + ); + + List viewEntityTypes = ImmutableList.of( + Constants.DATASET_ENTITY_NAME, + Constants.DASHBOARD_ENTITY_NAME + ); + + final List result = SearchUtils.intersectEntityTypes(baseEntityTypes, viewEntityTypes); + + final List expectedResult = ImmutableList.of( + Constants.DATASET_ENTITY_NAME + ); + Assert.assertEquals(expectedResult, result); + } + + private SearchUtilsTest() { } + +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/settings/user/UpdateCorpUserViewsSettingsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/settings/user/UpdateCorpUserViewsSettingsResolverTest.java new file mode 100644 index 00000000000000..905e913fba909c --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/settings/user/UpdateCorpUserViewsSettingsResolverTest.java @@ -0,0 +1,164 @@ +package com.linkedin.datahub.graphql.resolvers.settings.user; + +import com.datahub.authentication.Authentication; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.SetMode; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.UpdateCorpUserViewsSettingsInput; +import com.linkedin.identity.CorpUserAppearanceSettings; +import com.linkedin.identity.CorpUserSettings; +import com.linkedin.identity.CorpUserViewsSettings; +import com.linkedin.metadata.service.SettingsService; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.testng.Assert.*; + + +public class UpdateCorpUserViewsSettingsResolverTest { + + private static final Urn TEST_URN = UrnUtils.getUrn("urn:li:dataHubView:test-id"); + private static final Urn TEST_USER_URN = UrnUtils.getUrn("urn:li:corpuser:test"); + private static final UpdateCorpUserViewsSettingsInput TEST_VIEWS_INPUT = new UpdateCorpUserViewsSettingsInput( + TEST_URN.toString() + ); + private static final UpdateCorpUserViewsSettingsInput TEST_VIEWS_INPUT_NULL = new UpdateCorpUserViewsSettingsInput( + null + ); + + @Test + public void testGetSuccessViewSettingsNoExistingSettings() throws Exception { + SettingsService mockService = initSettingsService( + TEST_USER_URN, + new CorpUserSettings() + .setAppearance(new CorpUserAppearanceSettings().setShowSimplifiedHomepage(true)) + ); + UpdateCorpUserViewsSettingsResolver resolver = new UpdateCorpUserViewsSettingsResolver(mockService); + + QueryContext mockContext = getMockAllowContext(TEST_USER_URN.toString()); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_VIEWS_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertTrue(resolver.get(mockEnv).get()); + + Mockito.verify(mockService, Mockito.times(1)).updateCorpUserSettings( + Mockito.eq(TEST_USER_URN), + Mockito.eq(new CorpUserSettings() + .setAppearance(new CorpUserAppearanceSettings().setShowSimplifiedHomepage(true)) + .setViews(new CorpUserViewsSettings().setDefaultView(TEST_URN))), + Mockito.any(Authentication.class)); + } + + @Test + public void testGetSuccessViewSettingsExistingSettings() throws Exception { + SettingsService mockService = initSettingsService( + TEST_USER_URN, + new CorpUserSettings() + .setAppearance(new CorpUserAppearanceSettings().setShowSimplifiedHomepage(true)) + .setViews(new CorpUserViewsSettings().setDefaultView(UrnUtils.getUrn( + "urn:li:dataHubView:otherView" + ))) + ); + UpdateCorpUserViewsSettingsResolver resolver = new UpdateCorpUserViewsSettingsResolver(mockService); + + QueryContext mockContext = getMockAllowContext(TEST_USER_URN.toString()); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_VIEWS_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertTrue(resolver.get(mockEnv).get()); + + Mockito.verify(mockService, Mockito.times(1)).updateCorpUserSettings( + Mockito.eq(TEST_USER_URN), + Mockito.eq(new CorpUserSettings() + .setAppearance(new CorpUserAppearanceSettings().setShowSimplifiedHomepage(true)) + .setViews(new CorpUserViewsSettings().setDefaultView(TEST_URN))), + Mockito.any(Authentication.class)); + } + + + @Test + public void testGetSuccessViewSettingsRemoveDefaultView() throws Exception { + SettingsService mockService = initSettingsService( + TEST_USER_URN, + new CorpUserSettings() + .setAppearance(new CorpUserAppearanceSettings().setShowSimplifiedHomepage(true)) + .setViews(new CorpUserViewsSettings().setDefaultView(UrnUtils.getUrn( + "urn:li:dataHubView:otherView" + ))) + ); + UpdateCorpUserViewsSettingsResolver resolver = new UpdateCorpUserViewsSettingsResolver(mockService); + + QueryContext mockContext = getMockAllowContext(TEST_USER_URN.toString()); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_VIEWS_INPUT_NULL); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertTrue(resolver.get(mockEnv).get()); + + Mockito.verify(mockService, Mockito.times(1)).updateCorpUserSettings( + Mockito.eq(TEST_USER_URN), + Mockito.eq(new CorpUserSettings() + .setAppearance(new CorpUserAppearanceSettings().setShowSimplifiedHomepage(true)) + .setViews(new CorpUserViewsSettings().setDefaultView(null, SetMode.IGNORE_NULL))), + Mockito.any(Authentication.class)); + } + + @Test + public void testGetCorpUserSettingsException() throws Exception { + SettingsService mockService = Mockito.mock(SettingsService.class); + Mockito.doThrow(RuntimeException.class).when(mockService).getCorpUserSettings( + Mockito.eq(TEST_USER_URN), + Mockito.any(Authentication.class)); + + UpdateCorpUserViewsSettingsResolver resolver = new UpdateCorpUserViewsSettingsResolver(mockService); + + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(TEST_USER_URN.toString()); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_VIEWS_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } + + + @Test + public void testUpdateCorpUserSettingsException() throws Exception { + SettingsService mockService = initSettingsService( + TEST_USER_URN, + null + ); + Mockito.doThrow(RuntimeException.class).when(mockService).updateCorpUserSettings( + Mockito.eq(TEST_USER_URN), + Mockito.any(CorpUserSettings.class), + Mockito.any(Authentication.class)); + + UpdateCorpUserViewsSettingsResolver resolver = new UpdateCorpUserViewsSettingsResolver(mockService); + + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_VIEWS_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } + + private static SettingsService initSettingsService( + Urn user, + CorpUserSettings existingSettings + ) { + SettingsService mockService = Mockito.mock(SettingsService.class); + + Mockito.when(mockService.getCorpUserSettings( + Mockito.eq(user), + Mockito.any(Authentication.class))) + .thenReturn(existingSettings); + + return mockService; + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/settings/view/GlobalViewsSettingsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/settings/view/GlobalViewsSettingsResolverTest.java new file mode 100644 index 00000000000000..4e2283735b8c97 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/settings/view/GlobalViewsSettingsResolverTest.java @@ -0,0 +1,121 @@ +package com.linkedin.datahub.graphql.resolvers.settings.view; + +import com.datahub.authentication.Authentication; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.SetMode; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.metadata.service.SettingsService; +import com.linkedin.settings.global.GlobalSettingsInfo; +import com.linkedin.settings.global.GlobalViewsSettings; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.testng.Assert.*; + + +public class GlobalViewsSettingsResolverTest { + + private static final Urn TEST_URN = UrnUtils.getUrn("urn:li:dataHubView:test-id"); + private static final Urn TEST_USER_URN = UrnUtils.getUrn("urn:li:corpuser:test"); + + @Test + public void testGetSuccessNullSettings() throws Exception { + SettingsService mockService = initSettingsService( + null + ); + GlobalViewsSettingsResolver resolver = new GlobalViewsSettingsResolver(mockService); + + QueryContext mockContext = getMockAllowContext(TEST_USER_URN.toString()); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + com.linkedin.datahub.graphql.generated.GlobalViewsSettings result = resolver.get(mockEnv).get(); + + Assert.assertNotNull(result); // Empty settings + Assert.assertNull(result.getDefaultView()); + } + + @Test + public void testGetSuccessEmptySettings() throws Exception { + SettingsService mockService = initSettingsService( + new GlobalViewsSettings() + ); + GlobalViewsSettingsResolver resolver = new GlobalViewsSettingsResolver(mockService); + + QueryContext mockContext = getMockAllowContext(TEST_USER_URN.toString()); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + com.linkedin.datahub.graphql.generated.GlobalViewsSettings result = resolver.get(mockEnv).get(); + + Assert.assertNull( + result.getDefaultView() + ); + } + + @Test + public void testGetSuccessExistingSettings() throws Exception { + SettingsService mockService = initSettingsService( + new GlobalViewsSettings().setDefaultView(TEST_URN) + ); + GlobalViewsSettingsResolver resolver = new GlobalViewsSettingsResolver(mockService); + + QueryContext mockContext = getMockAllowContext(TEST_USER_URN.toString()); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + com.linkedin.datahub.graphql.generated.GlobalViewsSettings result = resolver.get(mockEnv).get(); + + Assert.assertEquals( + result.getDefaultView(), + TEST_URN.toString() + ); + } + + @Test + public void testGetException() throws Exception { + SettingsService mockService = Mockito.mock(SettingsService.class); + Mockito.doThrow(RuntimeException.class).when(mockService).getGlobalSettings( + Mockito.any(Authentication.class)); + + GlobalViewsSettingsResolver resolver = new GlobalViewsSettingsResolver(mockService); + + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(TEST_USER_URN.toString()); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } + + @Test + public void testGetUnauthorized() throws Exception { + SettingsService mockService = initSettingsService( + new GlobalViewsSettings() + ); + UpdateGlobalViewsSettingsResolver resolver = new UpdateGlobalViewsSettingsResolver(mockService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockDenyContext(TEST_USER_URN.toString()); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } + + private static SettingsService initSettingsService( + GlobalViewsSettings existingViewSettings + ) { + SettingsService mockService = Mockito.mock(SettingsService.class); + + Mockito.when(mockService.getGlobalSettings( + Mockito.any(Authentication.class))) + .thenReturn(new GlobalSettingsInfo().setViews(existingViewSettings, SetMode.IGNORE_NULL)); + + return mockService; + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/settings/view/UpdateGlobalViewsSettingsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/settings/view/UpdateGlobalViewsSettingsResolverTest.java new file mode 100644 index 00000000000000..9ea3c223559cd2 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/settings/view/UpdateGlobalViewsSettingsResolverTest.java @@ -0,0 +1,168 @@ +package com.linkedin.datahub.graphql.resolvers.settings.view; + +import com.datahub.authentication.Authentication; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.SetMode; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.UpdateGlobalViewsSettingsInput; +import com.linkedin.metadata.service.SettingsService; +import com.linkedin.settings.global.GlobalSettingsInfo; +import com.linkedin.settings.global.GlobalViewsSettings; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.testng.Assert.*; + + +public class UpdateGlobalViewsSettingsResolverTest { + + private static final Urn TEST_URN = UrnUtils.getUrn("urn:li:dataHubView:test-id"); + private static final UpdateGlobalViewsSettingsInput TEST_INPUT = new UpdateGlobalViewsSettingsInput( + TEST_URN.toString() + ); + + @Test + public void testGetSuccessNoExistingSettings() throws Exception { + SettingsService mockService = initSettingsService( + null + ); + UpdateGlobalViewsSettingsResolver resolver = new UpdateGlobalViewsSettingsResolver(mockService); + + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertTrue(resolver.get(mockEnv).get()); + + Mockito.verify(mockService, Mockito.times(1)).updateGlobalSettings( + Mockito.eq(new GlobalSettingsInfo().setViews(new GlobalViewsSettings().setDefaultView(TEST_URN))), + Mockito.any(Authentication.class)); + } + + @Test + public void testGetSuccessNoDefaultView() throws Exception { + SettingsService mockService = initSettingsService( + new GlobalViewsSettings() + ); + UpdateGlobalViewsSettingsResolver resolver = new UpdateGlobalViewsSettingsResolver(mockService); + + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertTrue(resolver.get(mockEnv).get()); + + Mockito.verify(mockService, Mockito.times(1)).updateGlobalSettings( + Mockito.eq(new GlobalSettingsInfo().setViews(new GlobalViewsSettings().setDefaultView(TEST_URN))), + Mockito.any(Authentication.class)); + } + + @Test + public void testGetSuccessExistingDefaultView() throws Exception { + SettingsService mockService = initSettingsService( + new GlobalViewsSettings().setDefaultView(UrnUtils.getUrn( + "urn:li:dataHubView:otherView" + )) + ); + UpdateGlobalViewsSettingsResolver resolver = new UpdateGlobalViewsSettingsResolver(mockService); + + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertTrue(resolver.get(mockEnv).get()); + + Mockito.verify(mockService, Mockito.times(1)).updateGlobalSettings( + Mockito.eq(new GlobalSettingsInfo().setViews(new GlobalViewsSettings().setDefaultView(TEST_URN))), + Mockito.any(Authentication.class)); + } + + @Test + public void testGetGlobalViewsSettingsException() throws Exception { + SettingsService mockService = Mockito.mock(SettingsService.class); + Mockito.doThrow(RuntimeException.class).when(mockService).getGlobalSettings( + Mockito.any(Authentication.class)); + + UpdateGlobalViewsSettingsResolver resolver = new UpdateGlobalViewsSettingsResolver(mockService); + + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } + + + @Test + public void testUpdateGlobalViewsSettingsException() throws Exception { + SettingsService mockService = initSettingsService( + new GlobalViewsSettings() + ); + Mockito.doThrow(RuntimeException.class).when(mockService).updateGlobalSettings( + Mockito.any(GlobalSettingsInfo.class), + Mockito.any(Authentication.class)); + + UpdateGlobalViewsSettingsResolver resolver = new UpdateGlobalViewsSettingsResolver(mockService); + + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } + + @Test + public void testGetGlobalViewsSettingsNoSettingsException() throws Exception { + SettingsService mockService = initSettingsService( + null // Should never be null. + ); + Mockito.doThrow(RuntimeException.class).when(mockService).getGlobalSettings( + Mockito.any(Authentication.class)); + + UpdateGlobalViewsSettingsResolver resolver = new UpdateGlobalViewsSettingsResolver(mockService); + + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } + + @Test + public void testGetUnauthorized() throws Exception { + SettingsService mockService = initSettingsService( + new GlobalViewsSettings() + ); + UpdateGlobalViewsSettingsResolver resolver = new UpdateGlobalViewsSettingsResolver(mockService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockDenyContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } + + private static SettingsService initSettingsService( + GlobalViewsSettings existingViewSettings + ) { + SettingsService mockService = Mockito.mock(SettingsService.class); + + Mockito.when(mockService.getGlobalSettings( + Mockito.any(Authentication.class))) + .thenReturn(new GlobalSettingsInfo().setViews(existingViewSettings, SetMode.IGNORE_NULL)); + + return mockService; + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/CreateViewResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/CreateViewResolverTest.java new file mode 100644 index 00000000000000..0957acf0cbbb30 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/CreateViewResolverTest.java @@ -0,0 +1,161 @@ +package com.linkedin.datahub.graphql.resolvers.view; + +import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableList; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.StringArray; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.CreateViewInput; +import com.linkedin.datahub.graphql.generated.DataHubView; +import com.linkedin.datahub.graphql.generated.DataHubViewDefinitionInput; +import com.linkedin.datahub.graphql.generated.DataHubViewFilterInput; +import com.linkedin.datahub.graphql.generated.DataHubViewType; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.FacetFilterInput; +import com.linkedin.datahub.graphql.generated.FilterOperator; +import com.linkedin.datahub.graphql.generated.LogicalOperator; +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.service.ViewService; +import com.linkedin.view.DataHubViewDefinition; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.testng.Assert.*; + + +public class CreateViewResolverTest { + + private static final CreateViewInput TEST_INPUT = new CreateViewInput( + DataHubViewType.PERSONAL, + "test-name", + "test-description", + new DataHubViewDefinitionInput( + ImmutableList.of(EntityType.DATASET, EntityType.DASHBOARD), + new DataHubViewFilterInput( + LogicalOperator.AND, + ImmutableList.of( + new FacetFilterInput("test1", null, ImmutableList.of("value1", "value2"), false, FilterOperator.EQUAL), + new FacetFilterInput("test2", null, ImmutableList.of("value1", "value2"), true, FilterOperator.IN) + ) + ) + ) + ); + + private static final Urn TEST_VIEW_URN = UrnUtils.getUrn("urn:li:dataHubView:test"); + + @Test + public void testGetSuccess() throws Exception { + // Create resolver + ViewService mockService = initMockService(); + CreateViewResolver resolver = new CreateViewResolver(mockService); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + DataHubView view = resolver.get(mockEnv).get(); + assertEquals(view.getName(), TEST_INPUT.getName()); + assertEquals(view.getDescription(), TEST_INPUT.getDescription()); + assertEquals(view.getViewType(), TEST_INPUT.getViewType()); + assertEquals(view.getType(), EntityType.DATAHUB_VIEW); + assertEquals(view.getDefinition().getEntityTypes(), TEST_INPUT.getDefinition().getEntityTypes()); + assertEquals(view.getDefinition().getFilter().getOperator(), TEST_INPUT.getDefinition().getFilter().getOperator()); + assertEquals(view.getDefinition().getFilter().getFilters().size(), TEST_INPUT.getDefinition().getFilter().getFilters().size()); + + Mockito.verify(mockService, Mockito.times(1)).createView( + Mockito.eq(com.linkedin.view.DataHubViewType.PERSONAL), + Mockito.eq(TEST_INPUT.getName()), + Mockito.eq(TEST_INPUT.getDescription()), + Mockito.eq( + new DataHubViewDefinition() + .setEntityTypes(new StringArray(ImmutableList.of(Constants.DATASET_ENTITY_NAME, Constants.DASHBOARD_ENTITY_NAME))) + .setFilter(new Filter() + .setOr(new ConjunctiveCriterionArray(ImmutableList.of( + new ConjunctiveCriterion() + .setAnd(new CriterionArray(ImmutableList.of( + new Criterion() + .setCondition(Condition.EQUAL) + .setField("test1.keyword") + .setValue("value1") // Unfortunate --- For backwards compat. + .setValues(new StringArray(ImmutableList.of("value1", "value2"))) + .setNegated(false), + new Criterion() + .setCondition(Condition.IN) + .setField("test2.keyword") + .setValue("value1") // Unfortunate --- For backwards compat. + .setValues(new StringArray(ImmutableList.of("value1", "value2"))) + .setNegated(true) + ))) + )) + ) + )), Mockito.any(Authentication.class), Mockito.anyLong()); + } + + @Test + public void testGetUnauthorized() throws Exception { + // Create resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + ViewService mockService = Mockito.mock(ViewService.class); + CreateViewResolver resolver = new CreateViewResolver(mockService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockDenyContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockClient, Mockito.times(0)).ingestProposal( + Mockito.any(), + Mockito.any(Authentication.class)); + } + + @Test + public void testGetViewServiceException() throws Exception { + // Create resolver + ViewService mockService = Mockito.mock(ViewService.class); + Mockito.doThrow(RuntimeException.class).when(mockService).createView( + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(Authentication.class), + Mockito.anyLong()); + + CreateViewResolver resolver = new CreateViewResolver(mockService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } + + private ViewService initMockService() { + ViewService service = Mockito.mock(ViewService.class); + Mockito.when(service.createView( + Mockito.eq(com.linkedin.view.DataHubViewType.PERSONAL), + Mockito.eq(TEST_INPUT.getName()), + Mockito.eq(TEST_INPUT.getDescription()), + Mockito.any(), + Mockito.any(Authentication.class), + Mockito.anyLong() + )).thenReturn(TEST_VIEW_URN); + return service; + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/DeleteViewResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/DeleteViewResolverTest.java new file mode 100644 index 00000000000000..afb4c16767f47c --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/DeleteViewResolverTest.java @@ -0,0 +1,164 @@ +package com.linkedin.datahub.graphql.resolvers.view; + +import com.datahub.authentication.Authentication; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.StringArray; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.service.ViewService; +import com.linkedin.view.DataHubViewDefinition; +import com.linkedin.view.DataHubViewInfo; +import com.linkedin.view.DataHubViewType; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import java.util.concurrent.ExecutionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.testng.Assert.*; + + +public class DeleteViewResolverTest { + + private static final Urn TEST_URN = UrnUtils.getUrn("urn:li:dataHubView:test-id"); + private static final Urn TEST_AUTHORIZED_USER = UrnUtils.getUrn("urn:li:corpuser:auth"); + private static final Urn TEST_UNAUTHORIZED_USER = UrnUtils.getUrn("urn:li:corpuser:no-auth"); + + @Test + public void testGetSuccessGlobalViewIsCreator() throws Exception { + ViewService mockService = initViewService(DataHubViewType.GLOBAL); + DeleteViewResolver resolver = new DeleteViewResolver(mockService); + + // Execute resolver - user is allowed since he owns the thing. + QueryContext mockContext = getMockDenyContext(TEST_AUTHORIZED_USER.toString()); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))).thenReturn(TEST_URN.toString()); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertTrue(resolver.get(mockEnv).get()); + + Mockito.verify(mockService, Mockito.times(1)).deleteView( + Mockito.eq(TEST_URN), + Mockito.any(Authentication.class) + ); + } + + @Test + public void testGetSuccessGlobalViewCanManager() throws Exception { + ViewService mockService = initViewService(DataHubViewType.GLOBAL); + DeleteViewResolver resolver = new DeleteViewResolver(mockService); + + // Execute resolver - user is allowed since they have the Manage Global View priv + // even though they are not the owner. + QueryContext mockContext = getMockAllowContext(TEST_UNAUTHORIZED_USER.toString()); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))).thenReturn(TEST_URN.toString()); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertTrue(resolver.get(mockEnv).get()); + + Mockito.verify(mockService, Mockito.times(1)).deleteView( + Mockito.eq(TEST_URN), + Mockito.any(Authentication.class) + ); + } + + @Test + public void testGetFailureGlobalViewIsNotCreatorOrManager() throws Exception { + ViewService mockService = initViewService(DataHubViewType.GLOBAL); + DeleteViewResolver resolver = new DeleteViewResolver(mockService); + + // Execute resolver - user is allowed since he owns the thing. + QueryContext mockContext = getMockDenyContext(TEST_UNAUTHORIZED_USER.toString()); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))).thenReturn(TEST_URN.toString()); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(ExecutionException.class, () -> resolver.get(mockEnv).get()); + + Mockito.verify(mockService, Mockito.times(0)).deleteView( + Mockito.eq(TEST_URN), + Mockito.any(Authentication.class) + ); + } + + + @Test + public void testGetSuccessPersonalViewIsCreator() throws Exception { + ViewService mockService = initViewService(DataHubViewType.PERSONAL); + DeleteViewResolver resolver = new DeleteViewResolver(mockService); + + // Execute resolver - user is allowed since he owns the thing. + QueryContext mockContext = getMockDenyContext(TEST_AUTHORIZED_USER.toString()); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))).thenReturn(TEST_URN.toString()); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertTrue(resolver.get(mockEnv).get()); + + Mockito.verify(mockService, Mockito.times(1)).deleteView( + Mockito.eq(TEST_URN), + Mockito.any(Authentication.class) + ); + } + + @Test + public void testGetFailurePersonalViewIsNotCreator() throws Exception { + ViewService mockService = initViewService(DataHubViewType.PERSONAL); + DeleteViewResolver resolver = new DeleteViewResolver(mockService); + + // Execute resolver - user is allowed since he owns the thing. + QueryContext mockContext = getMockDenyContext(TEST_UNAUTHORIZED_USER.toString()); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))).thenReturn(TEST_URN.toString()); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(ExecutionException.class, () -> resolver.get(mockEnv).get()); + + Mockito.verify(mockService, Mockito.times(0)).deleteView( + Mockito.eq(TEST_URN), + Mockito.any(Authentication.class) + ); + } + + @Test + public void testGetViewServiceException() throws Exception { + // Create resolver + ViewService mockService = Mockito.mock(ViewService.class); + Mockito.doThrow(RuntimeException.class).when(mockService).deleteView( + Mockito.any(), + Mockito.any(Authentication.class)); + + DeleteViewResolver resolver = new DeleteViewResolver(mockService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))).thenReturn(TEST_URN.toString()); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } + + private static ViewService initViewService(DataHubViewType viewType) { + ViewService mockService = Mockito.mock(ViewService.class); + + DataHubViewInfo testInfo = new DataHubViewInfo() + .setType(viewType) + .setName("test-name") + .setDescription("test-description") + .setCreated(new AuditStamp().setActor(TEST_AUTHORIZED_USER).setTime(0L)) + .setLastModified(new AuditStamp().setActor(TEST_AUTHORIZED_USER).setTime(0L)) + .setDefinition(new DataHubViewDefinition().setEntityTypes(new StringArray()).setFilter(new Filter())); + + Mockito.when(mockService.getViewInfo( + Mockito.eq(TEST_URN), + Mockito.any(Authentication.class))) + .thenReturn(testInfo); + + return mockService; + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolverTest.java new file mode 100644 index 00000000000000..cda9275ba9d071 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ListGlobalViewsResolverTest.java @@ -0,0 +1,138 @@ +package com.linkedin.datahub.graphql.resolvers.view; + +import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.StringArray; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.DataHubViewType; +import com.linkedin.datahub.graphql.generated.ListGlobalViewsInput; +import com.linkedin.datahub.graphql.generated.ListViewsResult; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.search.SearchEntity; +import com.linkedin.metadata.search.SearchEntityArray; +import com.linkedin.metadata.search.SearchResult; +import com.linkedin.r2.RemoteInvocationException; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.testng.Assert.*; + + +public class ListGlobalViewsResolverTest { + + private static final Urn TEST_URN = Urn.createFromTuple("dataHubView", "test-id"); + private static final Urn TEST_USER = UrnUtils.getUrn("urn:li:corpuser:test"); + + private static final ListGlobalViewsInput TEST_INPUT = new ListGlobalViewsInput( + 0, 20, "" + ); + + @Test + public void testGetSuccessInput() throws Exception { + EntityClient mockClient = Mockito.mock(EntityClient.class); + + Mockito.when(mockClient.search( + Mockito.eq(Constants.DATAHUB_VIEW_ENTITY_NAME), + Mockito.eq(""), + Mockito.eq( + new Filter() + .setOr(new ConjunctiveCriterionArray(ImmutableList.of( + new ConjunctiveCriterion() + .setAnd(new CriterionArray(ImmutableList.of( + new Criterion() + .setField("type.keyword") + .setValue(DataHubViewType.GLOBAL.toString()) + .setValues(new StringArray( + ImmutableList.of(DataHubViewType.GLOBAL.toString()))) + .setCondition(Condition.EQUAL) + .setNegated(false) + ))) + ))) + ), + Mockito.any(), + Mockito.eq(0), + Mockito.eq(20), + Mockito.any(Authentication.class) + )).thenReturn( + new SearchResult() + .setFrom(0) + .setPageSize(1) + .setNumEntities(1) + .setEntities(new SearchEntityArray(ImmutableSet.of(new SearchEntity().setEntity(TEST_URN)))) + ); + + ListGlobalViewsResolver resolver = new ListGlobalViewsResolver(mockClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(TEST_USER.toString()); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + ListViewsResult result = resolver.get(mockEnv).get(); + + // Data Assertions + assertEquals(result.getStart(), 0); + assertEquals(result.getCount(), 1); + assertEquals(result.getTotal(), 1); + assertEquals(result.getViews().size(), 1); + assertEquals(result.getViews().get(0).getUrn(), TEST_URN.toString()); + } + + @Test + public void testGetUnauthorized() throws Exception { + // Create resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + ListMyViewsResolver resolver = new ListMyViewsResolver(mockClient); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockDenyContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockClient, Mockito.times(0)).search( + Mockito.any(), + Mockito.eq(""), + Mockito.anyMap(), + Mockito.anyInt(), + Mockito.anyInt(), + Mockito.any(Authentication.class)); + } + + @Test + public void testGetEntityClientException() throws Exception { + // Create resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + Mockito.doThrow(RemoteInvocationException.class).when(mockClient).search( + Mockito.any(), + Mockito.eq(""), + Mockito.anyMap(), + Mockito.anyInt(), + Mockito.anyInt(), + Mockito.any(Authentication.class)); + ListMyViewsResolver resolver = new ListMyViewsResolver(mockClient); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolverTest.java new file mode 100644 index 00000000000000..8f69ed2675a897 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ListMyViewsResolverTest.java @@ -0,0 +1,195 @@ +package com.linkedin.datahub.graphql.resolvers.view; + +import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.StringArray; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.DataHubViewType; +import com.linkedin.datahub.graphql.generated.ListMyViewsInput; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.search.SearchEntity; +import com.linkedin.metadata.search.SearchEntityArray; +import com.linkedin.metadata.search.SearchResult; +import com.linkedin.r2.RemoteInvocationException; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.testng.Assert.*; + + +public class ListMyViewsResolverTest { + + private static final Urn TEST_URN = Urn.createFromTuple("dataHubView", "test-id"); + private static final Urn TEST_USER = UrnUtils.getUrn("urn:li:corpuser:test"); + + private static final ListMyViewsInput TEST_INPUT_1 = new ListMyViewsInput( + 0, 20, "", DataHubViewType.GLOBAL + ); + + private static final ListMyViewsInput TEST_INPUT_2 = new ListMyViewsInput( + 0, 20, "", null + ); + + @Test + public void testGetSuccessInput1() throws Exception { + EntityClient mockClient = Mockito.mock(EntityClient.class); + + Mockito.when(mockClient.search( + Mockito.eq(Constants.DATAHUB_VIEW_ENTITY_NAME), + Mockito.eq(""), + Mockito.eq( + new Filter() + .setOr(new ConjunctiveCriterionArray(ImmutableList.of( + new ConjunctiveCriterion() + .setAnd(new CriterionArray(ImmutableList.of( + new Criterion() + .setField("createdBy.keyword") + .setValue(TEST_USER.toString()) + .setValues(new StringArray(ImmutableList.of(TEST_USER.toString()))) + .setCondition(Condition.EQUAL) + .setNegated(false), + new Criterion() + .setField("type.keyword") + .setValue(DataHubViewType.GLOBAL.toString()) + .setValues(new StringArray( + ImmutableList.of(DataHubViewType.GLOBAL.toString()))) + .setCondition(Condition.EQUAL) + .setNegated(false) + ))) + ))) + ), + Mockito.any(), + Mockito.eq(0), + Mockito.eq(20), + Mockito.any(Authentication.class) + )).thenReturn( + new SearchResult() + .setFrom(0) + .setPageSize(1) + .setNumEntities(1) + .setEntities(new SearchEntityArray(ImmutableSet.of(new SearchEntity().setEntity(TEST_URN)))) + ); + + ListMyViewsResolver resolver = new ListMyViewsResolver(mockClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(TEST_USER.toString()); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT_1); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + // Data Assertions + assertEquals(resolver.get(mockEnv).get().getStart(), 0); + assertEquals(resolver.get(mockEnv).get().getCount(), 1); + assertEquals(resolver.get(mockEnv).get().getTotal(), 1); + assertEquals(resolver.get(mockEnv).get().getViews().size(), 1); + assertEquals(resolver.get(mockEnv).get().getViews().get(0).getUrn(), TEST_URN.toString()); + } + + @Test + public void testGetSuccessInput2() throws Exception { + // Create resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + + Mockito.when(mockClient.search( + Mockito.eq(Constants.DATAHUB_VIEW_ENTITY_NAME), + Mockito.eq(""), + Mockito.eq( + new Filter() + .setOr(new ConjunctiveCriterionArray(ImmutableList.of( + new ConjunctiveCriterion() + .setAnd(new CriterionArray(ImmutableList.of( + new Criterion() + .setField("createdBy.keyword") + .setValue(TEST_USER.toString()) + .setValues(new StringArray(ImmutableList.of(TEST_USER.toString()))) + .setCondition(Condition.EQUAL) + .setNegated(false) + ))) + ))) + ), + Mockito.any(), + Mockito.eq(0), + Mockito.eq(20), + Mockito.any(Authentication.class) + )).thenReturn( + new SearchResult() + .setFrom(0) + .setPageSize(1) + .setNumEntities(1) + .setEntities(new SearchEntityArray(ImmutableSet.of(new SearchEntity().setEntity(TEST_URN)))) + ); + + ListMyViewsResolver resolver = new ListMyViewsResolver(mockClient); + + // Execute resolver + QueryContext mockContext = getMockAllowContext(TEST_USER.toString()); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT_2); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + // Data Assertions + assertEquals(resolver.get(mockEnv).get().getStart(), 0); + assertEquals(resolver.get(mockEnv).get().getCount(), 1); + assertEquals(resolver.get(mockEnv).get().getTotal(), 1); + assertEquals(resolver.get(mockEnv).get().getViews().size(), 1); + assertEquals(resolver.get(mockEnv).get().getViews().get(0).getUrn(), TEST_URN.toString()); + } + + @Test + public void testGetUnauthorized() throws Exception { + // Create resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + ListMyViewsResolver resolver = new ListMyViewsResolver(mockClient); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockDenyContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT_1); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockClient, Mockito.times(0)).search( + Mockito.any(), + Mockito.eq(""), + Mockito.anyMap(), + Mockito.anyInt(), + Mockito.anyInt(), + Mockito.any(Authentication.class)); + } + + @Test + public void testGetEntityClientException() throws Exception { + // Create resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + Mockito.doThrow(RemoteInvocationException.class).when(mockClient).search( + Mockito.any(), + Mockito.eq(""), + Mockito.anyMap(), + Mockito.anyInt(), + Mockito.anyInt(), + Mockito.any(Authentication.class)); + ListMyViewsResolver resolver = new ListMyViewsResolver(mockClient); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT_1); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/UpdateViewResolverTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/UpdateViewResolverTest.java new file mode 100644 index 00000000000000..b4895982ae7801 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/UpdateViewResolverTest.java @@ -0,0 +1,238 @@ +package com.linkedin.datahub.graphql.resolvers.view; + +import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.StringArray; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.DataHubView; +import com.linkedin.datahub.graphql.generated.DataHubViewDefinitionInput; +import com.linkedin.datahub.graphql.generated.DataHubViewFilterInput; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.FacetFilterInput; +import com.linkedin.datahub.graphql.generated.FilterOperator; +import com.linkedin.datahub.graphql.generated.LogicalOperator; +import com.linkedin.datahub.graphql.generated.UpdateViewInput; +import com.linkedin.entity.Aspect; +import com.linkedin.entity.AspectType; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.service.ViewService; +import com.linkedin.view.DataHubViewDefinition; +import com.linkedin.view.DataHubViewInfo; +import com.linkedin.view.DataHubViewType; +import graphql.schema.DataFetchingEnvironment; +import java.util.concurrent.CompletionException; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.testng.Assert.*; + + +public class UpdateViewResolverTest { + + private static final Urn TEST_URN = UrnUtils.getUrn("urn:li:dataHubView:test-id"); + private static final Urn TEST_AUTHORIZED_USER = UrnUtils.getUrn("urn:li:corpuser:auth"); + private static final Urn TEST_UNAUTHORIZED_USER = UrnUtils.getUrn("urn:li:corpuser:no-auth"); + + private static final UpdateViewInput TEST_INPUT = new UpdateViewInput( + "test-name", + "test-description", + new DataHubViewDefinitionInput( + ImmutableList.of(EntityType.DATASET, EntityType.DASHBOARD), + new DataHubViewFilterInput( + LogicalOperator.AND, + ImmutableList.of( + new FacetFilterInput("test1", null, ImmutableList.of("value1", "value2"), false, FilterOperator.EQUAL), + new FacetFilterInput("test2", null, ImmutableList.of("value1", "value2"), true, FilterOperator.IN) + ) + ) + ) + ); + + @Test + public void testGetSuccessGlobalViewIsCreator() throws Exception { + ViewService mockService = initViewService(DataHubViewType.GLOBAL); + UpdateViewResolver resolver = new UpdateViewResolver(mockService); + + // Execute resolver - user is allowed since he owns the thing. + QueryContext mockContext = getMockDenyContext(TEST_AUTHORIZED_USER.toString()); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))).thenReturn(TEST_URN.toString()); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + DataHubView view = resolver.get(mockEnv).get(); + assertEquals(view.getName(), TEST_INPUT.getName()); + assertEquals(view.getDescription(), TEST_INPUT.getDescription()); + assertEquals(view.getViewType(), com.linkedin.datahub.graphql.generated.DataHubViewType.GLOBAL); + assertEquals(view.getType(), EntityType.DATAHUB_VIEW); + + Mockito.verify(mockService, Mockito.times(1)).updateView( + Mockito.eq(TEST_URN), + Mockito.eq(TEST_INPUT.getName()), + Mockito.eq(TEST_INPUT.getDescription()), + Mockito.eq( + new DataHubViewDefinition() + .setEntityTypes(new StringArray(ImmutableList.of(Constants.DATASET_ENTITY_NAME, Constants.DASHBOARD_ENTITY_NAME))) + .setFilter(new Filter() + .setOr(new ConjunctiveCriterionArray(ImmutableList.of( + new ConjunctiveCriterion() + .setAnd(new CriterionArray(ImmutableList.of( + new Criterion() + .setCondition(Condition.EQUAL) + .setField("test1.keyword") + .setValue("value1") // Unfortunate --- For backwards compat. + .setValues(new StringArray(ImmutableList.of("value1", "value2"))) + .setNegated(false), + new Criterion() + .setCondition(Condition.IN) + .setField("test2.keyword") + .setValue("value1") // Unfortunate --- For backwards compat. + .setValues(new StringArray(ImmutableList.of("value1", "value2"))) + .setNegated(true) + ))) + )) + ) + )), Mockito.any(Authentication.class), Mockito.anyLong()); + } + + @Test + public void testGetSuccessGlobalViewManageGlobalViews() throws Exception { + ViewService mockService = initViewService(DataHubViewType.GLOBAL); + UpdateViewResolver resolver = new UpdateViewResolver(mockService); + + // Execute resolver - user is allowed since he owns the thing. + QueryContext mockContext = getMockAllowContext(TEST_UNAUTHORIZED_USER.toString()); + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))).thenReturn(TEST_URN.toString()); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + DataHubView view = resolver.get(mockEnv).get(); + assertEquals(view.getName(), TEST_INPUT.getName()); + assertEquals(view.getDescription(), TEST_INPUT.getDescription()); + assertEquals(view.getViewType(), com.linkedin.datahub.graphql.generated.DataHubViewType.GLOBAL); + assertEquals(view.getType(), EntityType.DATAHUB_VIEW); + + Mockito.verify(mockService, Mockito.times(1)).updateView( + Mockito.eq(TEST_URN), + Mockito.eq(TEST_INPUT.getName()), + Mockito.eq(TEST_INPUT.getDescription()), + Mockito.eq( + new DataHubViewDefinition() + .setEntityTypes(new StringArray(ImmutableList.of(Constants.DATASET_ENTITY_NAME, Constants.DASHBOARD_ENTITY_NAME))) + .setFilter(new Filter() + .setOr(new ConjunctiveCriterionArray(ImmutableList.of( + new ConjunctiveCriterion() + .setAnd(new CriterionArray(ImmutableList.of( + new Criterion() + .setCondition(Condition.EQUAL) + .setField("test1.keyword") + .setValue("value1") // Unfortunate --- For backwards compat. + .setValues(new StringArray(ImmutableList.of("value1", "value2"))) + .setNegated(false), + new Criterion() + .setCondition(Condition.IN) + .setField("test2.keyword") + .setValue("value1") // Unfortunate --- For backwards compat. + .setValues(new StringArray(ImmutableList.of("value1", "value2"))) + .setNegated(true) + ))) + )) + ) + )), Mockito.any(Authentication.class), Mockito.anyLong()); + } + + @Test + public void testGetViewServiceException() throws Exception { + // Update resolver + ViewService mockService = Mockito.mock(ViewService.class); + Mockito.doThrow(RuntimeException.class).when(mockService).updateView( + Mockito.any(Urn.class), + Mockito.any(), + Mockito.any(), + Mockito.any(), + Mockito.any(Authentication.class), + Mockito.anyLong()); + + UpdateViewResolver resolver = new UpdateViewResolver(mockService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockAllowContext(); + Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))).thenReturn(TEST_URN.toString()); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + } + + @Test + public void testGetUnauthorized() throws Exception { + // Update resolver + EntityClient mockClient = Mockito.mock(EntityClient.class); + ViewService mockService = initViewService(DataHubViewType.GLOBAL); + UpdateViewResolver resolver = new UpdateViewResolver(mockService); + + // Execute resolver + DataFetchingEnvironment mockEnv = Mockito.mock(DataFetchingEnvironment.class); + QueryContext mockContext = getMockDenyContext(TEST_UNAUTHORIZED_USER.toString()); + Mockito.when(mockEnv.getArgument(Mockito.eq("urn"))).thenReturn(TEST_URN.toString()); + Mockito.when(mockEnv.getArgument(Mockito.eq("input"))).thenReturn(TEST_INPUT); + Mockito.when(mockEnv.getContext()).thenReturn(mockContext); + + assertThrows(CompletionException.class, () -> resolver.get(mockEnv).join()); + Mockito.verify(mockClient, Mockito.times(0)).ingestProposal( + Mockito.any(), + Mockito.any(Authentication.class)); + } + + private static ViewService initViewService(DataHubViewType viewType) { + ViewService mockService = Mockito.mock(ViewService.class); + + DataHubViewInfo testInfo = new DataHubViewInfo() + .setType(viewType) + .setName(TEST_INPUT.getName()) + .setDescription(TEST_INPUT.getDescription()) + .setCreated(new AuditStamp().setActor(TEST_AUTHORIZED_USER).setTime(0L)) + .setLastModified(new AuditStamp().setActor(TEST_AUTHORIZED_USER).setTime(0L)) + .setDefinition(new DataHubViewDefinition().setEntityTypes(new StringArray()).setFilter(new Filter())); + + EntityResponse testEntityResponse = new EntityResponse() + .setUrn(TEST_URN) + .setEntityName(Constants.DATAHUB_VIEW_ENTITY_NAME) + .setAspects(new EnvelopedAspectMap(ImmutableMap.of( + Constants.DATAHUB_VIEW_INFO_ASPECT_NAME, + new EnvelopedAspect() + .setName(Constants.DATAHUB_VIEW_INFO_ASPECT_NAME) + .setType(AspectType.VERSIONED) + .setValue(new Aspect(testInfo.data())) + ))); + + Mockito.when(mockService.getViewInfo( + Mockito.eq(TEST_URN), + Mockito.any(Authentication.class))) + .thenReturn(testInfo); + + Mockito.when(mockService.getViewEntityResponse( + Mockito.eq(TEST_URN), + Mockito.any(Authentication.class))) + .thenReturn(testEntityResponse); + + return mockService; + } +} \ No newline at end of file diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtilsTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtilsTest.java new file mode 100644 index 00000000000000..9578ff201ca194 --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/resolvers/view/ViewUtilsTest.java @@ -0,0 +1,188 @@ +package com.linkedin.datahub.graphql.resolvers.view; + +import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableList; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.StringArray; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.DataHubViewDefinitionInput; +import com.linkedin.datahub.graphql.generated.DataHubViewFilterInput; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.datahub.graphql.generated.FacetFilterInput; +import com.linkedin.datahub.graphql.generated.FilterOperator; +import com.linkedin.datahub.graphql.generated.LogicalOperator; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.service.ViewService; +import com.linkedin.view.DataHubViewDefinition; +import com.linkedin.view.DataHubViewInfo; +import com.linkedin.view.DataHubViewType; +import graphql.Assert; +import org.testng.annotations.Test; +import org.mockito.Mockito; + +import static com.linkedin.datahub.graphql.TestUtils.*; +import static org.testng.Assert.*; + + +public class ViewUtilsTest { + + private static final Urn TEST_AUTHORIZED_USER = UrnUtils.getUrn("urn:li:corpuser:auth"); + private static final Urn TEST_UNAUTHORIZED_USER = UrnUtils.getUrn("urn:li:corpuser:no-auth"); + + private static final Urn TEST_VIEW_URN = UrnUtils.getUrn("urn:li:dataHubView:test"); + + + @Test + public static void testCanCreatePersonalViewAllowed() { + boolean res = ViewUtils.canCreateView(DataHubViewType.PERSONAL, Mockito.mock(QueryContext.class)); + Assert.assertTrue(res); + } + + @Test + public static void testCanCreateGlobalViewAllowed() { + QueryContext context = getMockAllowContext(TEST_AUTHORIZED_USER.toString()); + boolean res = ViewUtils.canCreateView(DataHubViewType.GLOBAL, context); + Assert.assertTrue(res); + } + + @Test + public static void testCanCreateGlobalViewDenied() { + QueryContext context = getMockDenyContext(TEST_AUTHORIZED_USER.toString()); + boolean res = ViewUtils.canCreateView(DataHubViewType.GLOBAL, context); + Assert.assertFalse(res); + } + + @Test + public void testCanUpdateViewSuccessGlobalViewIsCreator() { + ViewService mockService = initViewService(DataHubViewType.GLOBAL); + QueryContext mockContext = getMockDenyContext(TEST_AUTHORIZED_USER.toString()); + + assertTrue(ViewUtils.canUpdateView(mockService, TEST_VIEW_URN, mockContext)); + + Mockito.verify(mockService, Mockito.times(1)).getViewInfo( + Mockito.eq(TEST_VIEW_URN), + Mockito.any(Authentication.class) + ); + } + + @Test + public void testCanUpdateViewSuccessGlobalViewCanManageGlobalViews() { + ViewService mockService = initViewService(DataHubViewType.GLOBAL); + QueryContext mockContext = getMockDenyContext(TEST_AUTHORIZED_USER.toString()); + + assertTrue(ViewUtils.canUpdateView(mockService, TEST_VIEW_URN, mockContext)); + + Mockito.verify(mockService, Mockito.times(1)).getViewInfo( + Mockito.eq(TEST_VIEW_URN), + Mockito.any(Authentication.class) + ); + } + + @Test + public void testGetFailureGlobalViewIsNotCreatorOrManager() { + ViewService mockService = initViewService(DataHubViewType.GLOBAL); + QueryContext mockContext = getMockDenyContext(TEST_UNAUTHORIZED_USER.toString()); + + assertFalse(ViewUtils.canUpdateView(mockService, TEST_VIEW_URN, mockContext)); + + Mockito.verify(mockService, Mockito.times(1)).getViewInfo( + Mockito.eq(TEST_VIEW_URN), + Mockito.any(Authentication.class) + ); + } + + @Test + public void testGetSuccessPersonalViewIsCreator() { + ViewService mockService = initViewService(DataHubViewType.PERSONAL); + QueryContext mockContext = getMockDenyContext(TEST_AUTHORIZED_USER.toString()); + + assertTrue(ViewUtils.canUpdateView(mockService, TEST_VIEW_URN, mockContext)); + + Mockito.verify(mockService, Mockito.times(1)).getViewInfo( + Mockito.eq(TEST_VIEW_URN), + Mockito.any(Authentication.class) + ); + } + + @Test + public void testGetFailurePersonalViewIsNotCreator() { + ViewService mockService = initViewService(DataHubViewType.PERSONAL); + QueryContext mockContext = getMockDenyContext(TEST_UNAUTHORIZED_USER.toString()); + + assertFalse(ViewUtils.canUpdateView(mockService, TEST_VIEW_URN, mockContext)); + + Mockito.verify(mockService, Mockito.times(1)).getViewInfo( + Mockito.eq(TEST_VIEW_URN), + Mockito.any(Authentication.class) + ); + } + + @Test + public void testMapDefinition() throws Exception { + + DataHubViewDefinitionInput input = new DataHubViewDefinitionInput( + ImmutableList.of(EntityType.DATASET, EntityType.DASHBOARD), + new DataHubViewFilterInput( + LogicalOperator.AND, + ImmutableList.of( + new FacetFilterInput("test1", null, ImmutableList.of("value1", "value2"), false, FilterOperator.IN), + new FacetFilterInput("test2", null, ImmutableList.of("value3", "value4"), true, FilterOperator.CONTAIN) + ) + ) + ); + + DataHubViewDefinition expectedResult = new DataHubViewDefinition() + .setEntityTypes(new StringArray(ImmutableList.of(Constants.DATASET_ENTITY_NAME, Constants.DASHBOARD_ENTITY_NAME))) + .setFilter(new Filter() + .setOr(new ConjunctiveCriterionArray( + ImmutableList.of(new ConjunctiveCriterion() + .setAnd( + new CriterionArray(ImmutableList.of( + new Criterion() + .setNegated(false) + .setValues(new StringArray(ImmutableList.of("value1", "value2"))) + .setValue("value1") // Disgraceful + .setField("test1.keyword") // Consider whether we should NOT go through the keyword mapping. + .setCondition(Condition.IN), + new Criterion() + .setNegated(true) + .setValues(new StringArray(ImmutableList.of("value3", "value4"))) + .setValue("value3") // Disgraceful + .setField("test2.keyword") // Consider whether we should NOT go through the keyword mapping. + .setCondition(Condition.CONTAIN) + )) + ) + ) + )) + ); + + assertEquals(ViewUtils.mapDefinition(input), expectedResult); + } + + private static ViewService initViewService(DataHubViewType viewType) { + ViewService mockService = Mockito.mock(ViewService.class); + + DataHubViewInfo testInfo = new DataHubViewInfo() + .setType(viewType) + .setName("test-name") + .setDescription("test-description") + .setCreated(new AuditStamp().setActor(TEST_AUTHORIZED_USER).setTime(0L)) + .setLastModified(new AuditStamp().setActor(TEST_AUTHORIZED_USER).setTime(0L)) + .setDefinition(new DataHubViewDefinition().setEntityTypes(new StringArray()).setFilter(new Filter())); + + Mockito.when(mockService.getViewInfo( + Mockito.eq(TEST_VIEW_URN), + Mockito.any(Authentication.class))) + .thenReturn(testInfo); + + return mockService; + } +} diff --git a/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/view/DataHubViewTypeTest.java b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/view/DataHubViewTypeTest.java new file mode 100644 index 00000000000000..7f3c8f99f6593a --- /dev/null +++ b/datahub-graphql-core/src/test/java/com/linkedin/datahub/graphql/types/view/DataHubViewTypeTest.java @@ -0,0 +1,243 @@ +package com.linkedin.datahub.graphql.types.view; + +import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.StringArray; +import com.linkedin.datahub.graphql.generated.FilterOperator; +import com.linkedin.datahub.graphql.generated.LogicalOperator; +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.view.DataHubViewDefinition; +import com.linkedin.view.DataHubViewInfo; +import com.linkedin.view.DataHubViewType; +import com.linkedin.common.urn.Urn; +import com.linkedin.datahub.graphql.QueryContext; +import com.linkedin.datahub.graphql.generated.DataHubView; +import com.linkedin.datahub.graphql.generated.EntityType; +import com.linkedin.entity.Aspect; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.r2.RemoteInvocationException; +import graphql.execution.DataFetcherResult; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import org.mockito.Mockito; + +import org.testng.annotations.Test; + +import static org.testng.Assert.*; + +public class DataHubViewTypeTest { + + private static final String TEST_VIEW_URN = "urn:li:dataHubView:test"; + private static final Urn TEST_USER_URN = UrnUtils.getUrn("urn:li:corpuser:test"); + /** + * A Valid View is one which is minted by the createView or updateView GraphQL resolvers. + * + * View Definitions currently support a limited Filter structure, which includes a single Logical filter set. + * Either a set of OR criteria with 1 value in each nested "and", or a single OR criteria with a set of nested ANDs. + * + * This enables us to easily support merging more complex View predicates in the future without a data migration, + * should the need arise. + */ + private static final DataHubViewInfo TEST_VALID_VIEW_INFO = new DataHubViewInfo() + .setType(DataHubViewType.PERSONAL) + .setName("test") + .setDescription("test description") + .setCreated(new AuditStamp().setTime(0L).setActor(TEST_USER_URN)) + .setLastModified(new AuditStamp().setTime(0L).setActor(TEST_USER_URN)) + .setDefinition(new DataHubViewDefinition() + .setFilter(new Filter() + .setOr(new ConjunctiveCriterionArray(ImmutableList.of( + new ConjunctiveCriterion().setAnd(new CriterionArray(ImmutableList.of( + new Criterion() + .setValues(new StringArray(ImmutableList.of("value1", "value2"))) + .setField("test") + .setCondition(Condition.EQUAL) + ))) + ))) + ) + .setEntityTypes(new StringArray(ImmutableList.of(Constants.DATASET_ENTITY_NAME, Constants.DASHBOARD_ENTITY_NAME)))); + + /** + * An Invalid View is on which has been ingested manually, which should not occur under normal operation of DataHub. + * + * This would be a complex view with multiple OR and nested AND predicates. + */ + private static final DataHubViewInfo TEST_INVALID_VIEW_INFO = new DataHubViewInfo() + .setType(DataHubViewType.PERSONAL) + .setName("test") + .setDescription("test description") + .setCreated(new AuditStamp().setTime(0L).setActor(TEST_USER_URN)) + .setLastModified(new AuditStamp().setTime(0L).setActor(TEST_USER_URN)) + .setDefinition(new DataHubViewDefinition() + .setFilter(new Filter() + .setOr(new ConjunctiveCriterionArray(ImmutableList.of( + new ConjunctiveCriterion().setAnd(new CriterionArray(ImmutableList.of( + new Criterion() + .setValues(new StringArray(ImmutableList.of("value1", "value2"))) + .setField("test") + .setCondition(Condition.EQUAL), + new Criterion() + .setValues(new StringArray(ImmutableList.of("value1", "value2"))) + .setField("test2") + .setCondition(Condition.EQUAL) + ))), + new ConjunctiveCriterion().setAnd(new CriterionArray(ImmutableList.of( + new Criterion() + .setValues(new StringArray(ImmutableList.of("value1", "value2"))) + .setField("test2") + .setCondition(Condition.EQUAL), + new Criterion() + .setValues(new StringArray(ImmutableList.of("value1", "value2"))) + .setField("test2") + .setCondition(Condition.EQUAL) + ))) + ))) + ) + .setEntityTypes(new StringArray(ImmutableList.of(Constants.DATASET_ENTITY_NAME, Constants.DASHBOARD_ENTITY_NAME)))); + + private static final String TEST_VIEW_URN_2 = "urn:li:dataHubView:test2"; + + @Test + public void testBatchLoadValidView() throws Exception { + + EntityClient client = Mockito.mock(EntityClient.class); + + Urn viewUrn1 = Urn.createFromString(TEST_VIEW_URN); + Urn viewUrn2 = Urn.createFromString(TEST_VIEW_URN_2); + + Map view1Aspects = new HashMap<>(); + view1Aspects.put( + Constants.DATAHUB_VIEW_INFO_ASPECT_NAME, + new EnvelopedAspect().setValue(new Aspect(TEST_VALID_VIEW_INFO.data())) + ); + Mockito.when(client.batchGetV2( + Mockito.eq(Constants.DATAHUB_VIEW_ENTITY_NAME), + Mockito.eq(new HashSet<>(ImmutableSet.of(viewUrn1, viewUrn2))), + Mockito.eq(com.linkedin.datahub.graphql.types.view.DataHubViewType.ASPECTS_TO_FETCH), + Mockito.any(Authentication.class))) + .thenReturn(ImmutableMap.of( + viewUrn1, + new EntityResponse() + .setEntityName(Constants.DATAHUB_VIEW_ENTITY_NAME) + .setUrn(viewUrn1) + .setAspects(new EnvelopedAspectMap(view1Aspects)))); + + com.linkedin.datahub.graphql.types.view.DataHubViewType type = new com.linkedin.datahub.graphql.types.view.DataHubViewType(client); + + QueryContext mockContext = Mockito.mock(QueryContext.class); + Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); + List> result = type.batchLoad(ImmutableList.of(TEST_VIEW_URN, TEST_VIEW_URN_2), mockContext); + + // Verify response + Mockito.verify(client, Mockito.times(1)).batchGetV2( + Mockito.eq(Constants.DATAHUB_VIEW_ENTITY_NAME), + Mockito.eq(ImmutableSet.of(viewUrn1, viewUrn2)), + Mockito.eq(com.linkedin.datahub.graphql.types.view.DataHubViewType.ASPECTS_TO_FETCH), + Mockito.any(Authentication.class) + ); + + assertEquals(result.size(), 2); + + DataHubView view = result.get(0).getData(); + assertEquals(view.getUrn(), TEST_VIEW_URN); + assertEquals(view.getType(), EntityType.DATAHUB_VIEW); + assertEquals(view.getViewType().toString(), DataHubViewType.PERSONAL.toString()); + assertEquals(view.getName(), TEST_VALID_VIEW_INFO.getName()); + assertEquals(view.getDescription(), TEST_VALID_VIEW_INFO.getDescription()); + assertEquals(view.getDefinition().getEntityTypes().size(), 2); + assertEquals(view.getDefinition().getEntityTypes().get(0), EntityType.DATASET); + assertEquals(view.getDefinition().getEntityTypes().get(1), EntityType.DASHBOARD); + assertEquals(view.getDefinition().getFilter().getOperator(), LogicalOperator.AND); + assertEquals(view.getDefinition().getFilter().getFilters().size(), 1); + assertEquals(view.getDefinition().getFilter().getFilters().get(0).getCondition(), FilterOperator.EQUAL); + assertEquals(view.getDefinition().getFilter().getFilters().get(0).getField(), "test"); + assertEquals(view.getDefinition().getFilter().getFilters().get(0).getValues(), ImmutableList.of("value1", "value2")); + + // Assert second element is null. + assertNull(result.get(1)); + } + + @Test + public void testBatchLoadInvalidView() throws Exception { + // If an Invalid View Definition is found in MySQL, we will return an Empty no-op View. (and log a warning). + EntityClient client = Mockito.mock(EntityClient.class); + Urn invalidViewUrn = Urn.createFromString(TEST_VIEW_URN); + + Map view1Aspects = new HashMap<>(); + view1Aspects.put( + Constants.DATAHUB_VIEW_INFO_ASPECT_NAME, + new EnvelopedAspect().setValue(new Aspect(TEST_INVALID_VIEW_INFO.data())) + ); + Mockito.when(client.batchGetV2( + Mockito.eq(Constants.DATAHUB_VIEW_ENTITY_NAME), + Mockito.eq(new HashSet<>(ImmutableSet.of(invalidViewUrn))), + Mockito.eq(com.linkedin.datahub.graphql.types.view.DataHubViewType.ASPECTS_TO_FETCH), + Mockito.any(Authentication.class))) + .thenReturn(ImmutableMap.of( + invalidViewUrn, + new EntityResponse() + .setEntityName(Constants.DATAHUB_VIEW_ENTITY_NAME) + .setUrn(invalidViewUrn) + .setAspects(new EnvelopedAspectMap(view1Aspects)))); + + com.linkedin.datahub.graphql.types.view.DataHubViewType type = new com.linkedin.datahub.graphql.types.view.DataHubViewType(client); + + QueryContext mockContext = Mockito.mock(QueryContext.class); + Mockito.when(mockContext.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); + List> result = type.batchLoad(ImmutableList.of(TEST_VIEW_URN), mockContext); + + // Verify response + Mockito.verify(client, Mockito.times(1)).batchGetV2( + Mockito.eq(Constants.DATAHUB_VIEW_ENTITY_NAME), + Mockito.eq(ImmutableSet.of(invalidViewUrn)), + Mockito.eq(com.linkedin.datahub.graphql.types.view.DataHubViewType.ASPECTS_TO_FETCH), + Mockito.any(Authentication.class) + ); + + assertEquals(result.size(), 1); + + DataHubView view = result.get(0).getData(); + assertEquals(view.getUrn(), TEST_VIEW_URN); + assertEquals(view.getType(), EntityType.DATAHUB_VIEW); + assertEquals(view.getViewType().toString(), DataHubViewType.PERSONAL.toString()); + assertEquals(view.getName(), TEST_INVALID_VIEW_INFO.getName()); + assertEquals(view.getDescription(), TEST_INVALID_VIEW_INFO.getDescription()); + assertEquals(view.getDefinition().getEntityTypes().size(), 2); + assertEquals(view.getDefinition().getEntityTypes().get(0), EntityType.DATASET); + assertEquals(view.getDefinition().getEntityTypes().get(1), EntityType.DASHBOARD); + assertEquals(view.getDefinition().getFilter().getOperator(), LogicalOperator.OR); + assertEquals(view.getDefinition().getFilter().getFilters().size(), 0); + } + + @Test + public void testBatchLoadClientException() throws Exception { + EntityClient mockClient = Mockito.mock(EntityClient.class); + Mockito.doThrow(RemoteInvocationException.class).when(mockClient).batchGetV2( + Mockito.anyString(), + Mockito.anySet(), + Mockito.anySet(), + Mockito.any(Authentication.class)); + com.linkedin.datahub.graphql.types.view.DataHubViewType type = new com.linkedin.datahub.graphql.types.view.DataHubViewType(mockClient); + + // Execute Batch load + QueryContext context = Mockito.mock(QueryContext.class); + Mockito.when(context.getAuthentication()).thenReturn(Mockito.mock(Authentication.class)); + assertThrows(RuntimeException.class, () -> type.batchLoad(ImmutableList.of(TEST_VIEW_URN, TEST_VIEW_URN_2), + context)); + } +} \ No newline at end of file diff --git a/datahub-web-react/src/AppConfigProvider.tsx b/datahub-web-react/src/AppConfigProvider.tsx index 9a03b38584b79d..2984116cf20281 100644 --- a/datahub-web-react/src/AppConfigProvider.tsx +++ b/datahub-web-react/src/AppConfigProvider.tsx @@ -19,7 +19,7 @@ function changeFavicon(src) { } const AppConfigProvider = ({ children }: { children: React.ReactNode }) => { - const { data: appConfigData, refetch } = useAppConfigQuery(); + const { data: appConfigData, refetch } = useAppConfigQuery({ fetchPolicy: 'no-cache' }); const refreshAppConfig = () => { refetch(); diff --git a/datahub-web-react/src/Mocks.tsx b/datahub-web-react/src/Mocks.tsx index 69795e759bc3bc..9615723c981343 100644 --- a/datahub-web-react/src/Mocks.tsx +++ b/datahub-web-react/src/Mocks.tsx @@ -72,7 +72,7 @@ const user1 = { }, ], }, - settings: { appearance: { showSimplifiedHomepage: false } }, + settings: { appearance: { showSimplifiedHomepage: false }, views: { defaultView: null } }, }; const user2 = { @@ -116,7 +116,7 @@ const user2 = { }, ], }, - settings: { appearance: { showSimplifiedHomepage: false } }, + settings: { appearance: { showSimplifiedHomepage: false }, views: { defaultView: null } }, }; const dataPlatform = { @@ -3213,6 +3213,7 @@ export const mocks = [ manageSecrets: true, manageIngestion: true, generatePersonalAccessTokens: true, + manageGlobalViews: true, }, }, }, @@ -3437,4 +3438,5 @@ export const platformPrivileges: PlatformPrivileges = { manageTags: true, createTags: true, createDomains: true, + manageGlobalViews: true, }; diff --git a/datahub-web-react/src/app/ProtectedRoutes.tsx b/datahub-web-react/src/app/ProtectedRoutes.tsx index d1370ab882f68b..f5830d4bc06987 100644 --- a/datahub-web-react/src/app/ProtectedRoutes.tsx +++ b/datahub-web-react/src/app/ProtectedRoutes.tsx @@ -5,6 +5,7 @@ import { HomePage } from './home/HomePage'; import AppConfigProvider from '../AppConfigProvider'; import { SearchRoutes } from './SearchRoutes'; import { EducationStepsProvider } from '../providers/EducationStepsProvider'; +import UserContextProvider from './context/UserContextProvider'; /** * Container for all views behind an authentication wall. @@ -12,16 +13,18 @@ import { EducationStepsProvider } from '../providers/EducationStepsProvider'; export const ProtectedRoutes = (): JSX.Element => { return ( - - - - - } /> - } /> - + + + + + + } /> + } /> + + - - + + ); }; diff --git a/datahub-web-react/src/app/analytics/event.ts b/datahub-web-react/src/app/analytics/event.ts index 3ae1bd6191218e..47a390ac122698 100644 --- a/datahub-web-react/src/app/analytics/event.ts +++ b/datahub-web-react/src/app/analytics/event.ts @@ -1,4 +1,4 @@ -import { EntityType, RecommendationRenderType, ScenarioType } from '../../types.generated'; +import { DataHubViewType, EntityType, RecommendationRenderType, ScenarioType } from '../../types.generated'; /** * Valid event types. @@ -49,6 +49,10 @@ export enum EventType { DeleteIngestionSourceEvent, ExecuteIngestionSourceEvent, SsoEvent, + CreateViewEvent, + UpdateViewEvent, + SetGlobalDefaultViewEvent, + SetUserDefaultViewEvent, } /** @@ -382,6 +386,40 @@ export interface SsoEvent extends BaseEvent { type: EventType.SsoEvent; } +/** + * Emitted when a new View is created. + */ +export interface CreateViewEvent extends BaseEvent { + type: EventType.CreateViewEvent; + viewType: DataHubViewType; +} + +/** + * Emitted when an existing View is updated. + */ +export interface UpdateViewEvent extends BaseEvent { + type: EventType.UpdateViewEvent; + viewType: DataHubViewType; + urn: string; +} + +/** + * Emitted when a user sets or clears their personal default view. + */ +export interface SetUserDefaultViewEvent extends BaseEvent { + type: EventType.SetUserDefaultViewEvent; + urn: string | null; + viewType: DataHubViewType | null; +} + +/** + * Emitted when a user sets or clears the global default view. + */ +export interface SetGlobalDefaultViewEvent extends BaseEvent { + type: EventType.SetGlobalDefaultViewEvent; + urn: string | null; +} + /** * Event consisting of a union of specific event types. */ @@ -430,4 +468,8 @@ export type Event = | DeleteIngestionSourceEvent | ExecuteIngestionSourceEvent | ShowStandardHomepageEvent - | SsoEvent; + | SsoEvent + | CreateViewEvent + | UpdateViewEvent + | SetUserDefaultViewEvent + | SetGlobalDefaultViewEvent; diff --git a/datahub-web-react/src/app/context/UserContextProvider.tsx b/datahub-web-react/src/app/context/UserContextProvider.tsx new file mode 100644 index 00000000000000..3bcff15cc27485 --- /dev/null +++ b/datahub-web-react/src/app/context/UserContextProvider.tsx @@ -0,0 +1,145 @@ +import React, { useCallback, useEffect, useState } from 'react'; +import { useGetMeLazyQuery } from '../../graphql/me.generated'; +import { useGetGlobalViewsSettingsLazyQuery } from '../../graphql/app.generated'; +import { CorpUser, PlatformPrivileges } from '../../types.generated'; +import { UserContext, LocalState, DEFAULT_STATE, State } from './userContext'; + +// TODO: Migrate all usage of useAuthenticatedUser to using this provider. + +/** + * Key used when writing user state to local browser state. + */ +const LOCAL_STATE_KEY = 'userState'; + +/** + * Loads a persisted object from the local browser storage. + */ +const loadLocalState = () => { + return JSON.parse(localStorage.getItem(LOCAL_STATE_KEY) || '{}'); +}; + +/** + * Saves an object to local browser storage. + */ +const saveLocalState = (newState: LocalState) => { + return localStorage.setItem(LOCAL_STATE_KEY, JSON.stringify(newState)); +}; + +/** + * A provider of context related to the currently authenticated user. + */ +const UserContextProvider = ({ children }: { children: React.ReactNode }) => { + /** + * Stores transient session state, and browser-persistent local state. + */ + const [state, setState] = useState(DEFAULT_STATE); + const [localState, setLocalState] = useState(loadLocalState()); + + /** + * Retrieve the current user details once on component mount. + */ + const [getMe, { data: meData, refetch }] = useGetMeLazyQuery({ fetchPolicy: 'cache-first' }); + useEffect(() => getMe(), [getMe]); + + /** + * Retrieve the Global View settings once on component mount. + */ + const [getGlobalViewSettings, { data: settingsData }] = useGetGlobalViewsSettingsLazyQuery({ + fetchPolicy: 'cache-first', + }); + useEffect(() => getGlobalViewSettings(), [getGlobalViewSettings]); + + const updateLocalState = (newState: LocalState) => { + saveLocalState(newState); + setLocalState(newState); + }; + + const setDefaultSelectedView = useCallback( + (newViewUrn) => { + updateLocalState({ + ...localState, + selectedViewUrn: newViewUrn, + }); + }, + [localState], + ); + + // Update the global default views in local state + useEffect(() => { + if (!state.views.loadedGlobalDefaultViewUrn && settingsData?.globalViewsSettings) { + setState({ + ...state, + views: { + ...state.views, + globalDefaultViewUrn: settingsData?.globalViewsSettings?.defaultView, + loadedGlobalDefaultViewUrn: true, + }, + }); + } + }, [settingsData, state]); + + // Update the personal default views in local state + useEffect(() => { + if (!state.views.loadedPersonalDefaultViewUrn && meData?.me?.corpUser?.settings) { + setState({ + ...state, + views: { + ...state.views, + personalDefaultViewUrn: meData?.me?.corpUser?.settings?.views?.defaultView?.urn, + loadedPersonalDefaultViewUrn: true, + }, + }); + } + }, [meData, state]); + + /** + * Initialize the default selected view for the logged in user. + * + * This is computed as either the user's personal default view (if one is set) + * else the global default view (if one is set) else undefined as normal. + * + * This logic should only run once at initial page load because if a user + * unselects the current active view, it should NOT be reset to the default they've selected. + */ + useEffect(() => { + const shouldSetDefaultView = + !state.views.hasSetDefaultView && + state.views.loadedPersonalDefaultViewUrn && + state.views.loadedGlobalDefaultViewUrn; + if (shouldSetDefaultView) { + if (localState.selectedViewUrn === undefined) { + if (state.views.personalDefaultViewUrn) { + setDefaultSelectedView(state.views.personalDefaultViewUrn); + } else if (state.views.globalDefaultViewUrn) { + setDefaultSelectedView(state.views.globalDefaultViewUrn); + } + } + setState({ + ...state, + views: { + ...state.views, + hasSetDefaultView: true, + }, + }); + } + }, [state, localState.selectedViewUrn, setDefaultSelectedView]); + + return ( + + {children} + + ); +}; + +export default UserContextProvider; diff --git a/datahub-web-react/src/app/context/useUserContext.tsx b/datahub-web-react/src/app/context/useUserContext.tsx new file mode 100644 index 00000000000000..28f2f399ec9fee --- /dev/null +++ b/datahub-web-react/src/app/context/useUserContext.tsx @@ -0,0 +1,9 @@ +import { useContext } from 'react'; +import { UserContext } from './userContext'; + +/** + * Fetch an instance of User Context + */ +export function useUserContext() { + return useContext(UserContext); +} diff --git a/datahub-web-react/src/app/context/userContext.tsx b/datahub-web-react/src/app/context/userContext.tsx new file mode 100644 index 00000000000000..ebd62d3a1fea5f --- /dev/null +++ b/datahub-web-react/src/app/context/userContext.tsx @@ -0,0 +1,62 @@ +import React from 'react'; +import { CorpUser, PlatformPrivileges } from '../../types.generated'; + +/** + * Local State is persisted to local storage. + */ +export type LocalState = { + selectedViewUrn?: string | null; +}; + +/** + * State is transient, it is refreshed on browser refesh. + */ +export type State = { + views: { + globalDefaultViewUrn?: string | null; + personalDefaultViewUrn?: string | null; + loadedGlobalDefaultViewUrn: boolean; + loadedPersonalDefaultViewUrn: boolean; + hasSetDefaultView: boolean; + }; +}; + +/** + * Context about the currently-authenticated user. + */ +export type UserContextType = { + urn?: string | null; + user?: CorpUser | null; + platformPrivileges?: PlatformPrivileges | null; + localState: LocalState; + state: State; + updateLocalState: (newState: LocalState) => void; + updateState: (newState: State) => void; + refetchUser: () => any; +}; + +export const DEFAULT_LOCAL_STATE: LocalState = { + selectedViewUrn: undefined, +}; + +export const DEFAULT_STATE: State = { + views: { + globalDefaultViewUrn: undefined, + personalDefaultViewUrn: undefined, + loadedGlobalDefaultViewUrn: false, + loadedPersonalDefaultViewUrn: false, + hasSetDefaultView: false, + }, +}; + +export const DEFAULT_CONTEXT = { + urn: undefined, + user: undefined, + state: DEFAULT_STATE, + localState: DEFAULT_LOCAL_STATE, + updateLocalState: (_: LocalState) => null, + updateState: (_: State) => null, + refetchUser: () => null, +}; + +export const UserContext = React.createContext(DEFAULT_CONTEXT); diff --git a/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx b/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx index d814ce0743fbf8..738ffcb566afcc 100644 --- a/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx +++ b/datahub-web-react/src/app/entity/dataset/DatasetEntity.tsx @@ -245,6 +245,7 @@ export class DatasetEntity implements Entity { glossaryTerms={data.glossaryTerms} domain={data.domain?.domain} container={data.container} + externalUrl={data.properties?.externalUrl} /> ); }; diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/DownloadAsCsvModal.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/DownloadAsCsvModal.tsx index 1ea6b6bd6b4c12..3f727d4138490a 100644 --- a/datahub-web-react/src/app/entity/shared/components/styled/search/DownloadAsCsvModal.tsx +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/DownloadAsCsvModal.tsx @@ -2,7 +2,7 @@ import React, { useState } from 'react'; import { Button, Input, Modal } from 'antd'; import { useLocation } from 'react-router'; -import { EntityType, OrFilter, SearchAcrossEntitiesInput } from '../../../../../../types.generated'; +import { EntityType, AndFilterInput, SearchAcrossEntitiesInput } from '../../../../../../types.generated'; import { SearchResultsInterface } from './types'; import { getSearchCsvDownloadHeader, transformResultsToCsvRow } from './downloadAsCsvUtil'; import { downloadRowsAsCsv } from '../../../../../search/utils/csvUtils'; @@ -15,7 +15,7 @@ type Props = { input: SearchAcrossEntitiesInput; }) => Promise; entityFilters: EntityType[]; - filters: OrFilter[]; + filters: AndFilterInput[]; query: string; setIsDownloadingCsv: (isDownloadingCsv: boolean) => any; showDownloadAsCsvModal: boolean; diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchHeader.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchHeader.tsx index c253351e2f81ba..882e43473db8c3 100644 --- a/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchHeader.tsx +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/EmbeddedListSearchHeader.tsx @@ -5,7 +5,7 @@ import styled from 'styled-components'; import TabToolbar from '../TabToolbar'; import { SearchBar } from '../../../../../search/SearchBar'; import { useEntityRegistry } from '../../../../../useEntityRegistry'; -import { EntityType, OrFilter, SearchAcrossEntitiesInput } from '../../../../../../types.generated'; +import { EntityType, AndFilterInput, SearchAcrossEntitiesInput } from '../../../../../../types.generated'; import { SearchResultsInterface } from './types'; import SearchExtendedMenu from './SearchExtendedMenu'; import { SearchSelectBar } from './SearchSelectBar'; @@ -36,7 +36,7 @@ type Props = { input: SearchAcrossEntitiesInput; }) => Promise; entityFilters: EntityType[]; - filters: OrFilter[]; + filters: AndFilterInput[]; query: string; isSelectMode: boolean; isSelectAll: boolean; diff --git a/datahub-web-react/src/app/entity/shared/components/styled/search/SearchExtendedMenu.tsx b/datahub-web-react/src/app/entity/shared/components/styled/search/SearchExtendedMenu.tsx index 61d2d3ed6527f5..6cb74aececf9ca 100644 --- a/datahub-web-react/src/app/entity/shared/components/styled/search/SearchExtendedMenu.tsx +++ b/datahub-web-react/src/app/entity/shared/components/styled/search/SearchExtendedMenu.tsx @@ -2,7 +2,7 @@ import React, { useState } from 'react'; import { Button, Dropdown, Menu } from 'antd'; import { FormOutlined, MoreOutlined } from '@ant-design/icons'; import styled from 'styled-components'; -import { EntityType, OrFilter, SearchAcrossEntitiesInput } from '../../../../../../types.generated'; +import { EntityType, AndFilterInput, SearchAcrossEntitiesInput } from '../../../../../../types.generated'; import { SearchResultsInterface } from './types'; import DownloadAsCsvButton from './DownloadAsCsvButton'; import DownloadAsCsvModal from './DownloadAsCsvModal'; @@ -27,7 +27,7 @@ type Props = { input: SearchAcrossEntitiesInput; }) => Promise; entityFilters: EntityType[]; - filters: OrFilter[]; + filters: AndFilterInput[]; query: string; setShowSelectMode?: (showSelectMode: boolean) => any; }; diff --git a/datahub-web-react/src/app/entity/view/ManageViews.tsx b/datahub-web-react/src/app/entity/view/ManageViews.tsx new file mode 100644 index 00000000000000..b31d3869ab1581 --- /dev/null +++ b/datahub-web-react/src/app/entity/view/ManageViews.tsx @@ -0,0 +1,43 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Typography } from 'antd'; +import { ViewsList } from './ViewsList'; + +const PageContainer = styled.div` + padding-top: 20px; + width: 100%; +`; + +const PageHeaderContainer = styled.div` + && { + padding-left: 24px; + } +`; + +const PageTitle = styled(Typography.Title)` + && { + margin-bottom: 12px; + } +`; + +const ListContainer = styled.div``; + +/** + * Component used for displaying the 'Manage Views' experience. + */ +export const ManageViews = () => { + return ( + + + Manage Views + + Create, edit, and remove your Views. Views allow you to save and share sets of filters for reuse + when browsing DataHub. + + + + + + + ); +}; diff --git a/datahub-web-react/src/app/entity/view/ViewTypeLabel.tsx b/datahub-web-react/src/app/entity/view/ViewTypeLabel.tsx new file mode 100644 index 00000000000000..698195ef25990f --- /dev/null +++ b/datahub-web-react/src/app/entity/view/ViewTypeLabel.tsx @@ -0,0 +1,54 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Typography } from 'antd'; +import { GlobalOutlined, LockOutlined } from '@ant-design/icons'; +import { DataHubViewType } from '../../../types.generated'; + +const StyledLockOutlined = styled(LockOutlined)<{ color }>` + color: ${(props) => props.color}; + margin-right: 4px; +`; + +const StyledGlobalOutlined = styled(GlobalOutlined)<{ color }>` + color: ${(props) => props.color}; + margin-right: 4px; +`; + +const StyledText = styled(Typography.Text)<{ color }>` + && { + color: ${(props) => props.color}; + } +`; + +type Props = { + type: DataHubViewType; + color: string; +}; + +/** + * Label used to describe View Types + * + * @param param0 the color of the text and iconography + */ +export const ViewTypeLabel = ({ type, color }: Props) => { + const copy = + type === DataHubViewType.Personal ? ( + <> + Private - only visible to you. + + ) : ( + <> + Public - visible to everyone. + + ); + const Icon = type === DataHubViewType.Global ? StyledGlobalOutlined : StyledLockOutlined; + + return ( + <> + + + {copy} + + + ); +}; diff --git a/datahub-web-react/src/app/entity/view/ViewsList.tsx b/datahub-web-react/src/app/entity/view/ViewsList.tsx new file mode 100644 index 00000000000000..2487783fff38b3 --- /dev/null +++ b/datahub-web-react/src/app/entity/view/ViewsList.tsx @@ -0,0 +1,144 @@ +import React, { useEffect, useState } from 'react'; +import styled from 'styled-components'; +import { useLocation } from 'react-router'; +import { Button, message, Pagination } from 'antd'; +import { PlusOutlined } from '@ant-design/icons'; +import * as QueryString from 'query-string'; +import { useListMyViewsQuery } from '../../../graphql/view.generated'; +import { SearchBar } from '../../search/SearchBar'; +import TabToolbar from '../shared/components/styled/TabToolbar'; +import { Message } from '../../shared/Message'; +import { useEntityRegistry } from '../../useEntityRegistry'; +import { scrollToTop } from '../../shared/searchUtils'; +import { ViewsTable } from './ViewsTable'; +import { DEFAULT_LIST_VIEWS_PAGE_SIZE, searchViews } from './utils'; +import { ViewBuilder } from './builder/ViewBuilder'; +import { ViewBuilderMode } from './builder/types'; + +const PaginationContainer = styled.div` + display: flex; + justify-content: center; +`; + +const StyledPagination = styled(Pagination)` + margin: 40px; +`; + +const searchBarStyle = { + maxWidth: 220, + padding: 0, +}; + +const searchBarInputStyle = { + height: 32, + fontSize: 12, +}; + +/** + * This component renders a paginated, searchable list of Views. + */ +export const ViewsList = () => { + /** + * Context + */ + const location = useLocation(); + const entityRegistry = useEntityRegistry(); + + /** + * Query Params + */ + const params = QueryString.parse(location.search, { arrayFormat: 'comma' }); + const paramsQuery = (params?.query as string) || undefined; + + /** + * State + */ + const [page, setPage] = useState(1); + const [selectedViewUrn, setSelectedViewUrn] = useState(undefined); + const [showViewBuilder, setShowViewBuilder] = useState(false); + const [query, setQuery] = useState(undefined); + useEffect(() => setQuery(paramsQuery), [paramsQuery]); + + /** + * Queries + */ + const pageSize = DEFAULT_LIST_VIEWS_PAGE_SIZE; + const start = (page - 1) * pageSize; + const { loading, error, data } = useListMyViewsQuery({ + variables: { + start, + count: pageSize, + }, + fetchPolicy: 'cache-first', + }); + + const onClickCreateView = () => { + setShowViewBuilder(true); + }; + + const onClickEditView = (urn: string) => { + setShowViewBuilder(true); + setSelectedViewUrn(urn); + }; + + const onCloseModal = () => { + setShowViewBuilder(false); + setSelectedViewUrn(undefined); + }; + + const onChangePage = (newPage: number) => { + scrollToTop(); + setPage(newPage); + }; + + /** + * Render variables. + */ + const totalViews = data?.listMyViews?.total || 0; + const views = searchViews(data?.listMyViews?.views || [], query); + const selectedView = (selectedViewUrn && views.find((view) => view.urn === selectedViewUrn)) || undefined; + + return ( + <> + {!data && loading && } + {error && message.error({ content: `Failed to load Views! An unexpected error occurred.`, duration: 3 })} + + + null} + onQueryChange={(q) => setQuery(q.length > 0 ? q : undefined)} + entityRegistry={entityRegistry} + /> + + + {totalViews >= pageSize && ( + + + + )} + {showViewBuilder && ( + + )} + + ); +}; diff --git a/datahub-web-react/src/app/entity/view/ViewsTable.tsx b/datahub-web-react/src/app/entity/view/ViewsTable.tsx new file mode 100644 index 00000000000000..97d1fe18fe0695 --- /dev/null +++ b/datahub-web-react/src/app/entity/view/ViewsTable.tsx @@ -0,0 +1,61 @@ +import React from 'react'; +import { Empty } from 'antd'; +import { StyledTable } from '../shared/components/styled/StyledTable'; +import { ActionsColumn, DescriptionColumn, NameColumn, ViewTypeColumn } from './select/ViewsTableColumns'; +import { DataHubView } from '../../../types.generated'; + +type ViewsTableProps = { + views: DataHubView[]; + onEditView: (urn) => void; +}; + +/** + * This component renders a table of Views. + */ +export const ViewsTable = ({ views, onEditView }: ViewsTableProps) => { + const tableColumns = [ + { + title: 'Name', + dataIndex: 'name', + key: 'name', + render: (name, record) => , + }, + { + title: 'Description', + dataIndex: 'description', + key: 'description', + render: (description) => , + }, + { + title: 'Type', + dataIndex: 'viewType', + key: 'viewType', + render: (viewType) => , + }, + { + title: '', + dataIndex: '', + key: 'x', + render: (record) => , + }, + ]; + + /** + * The data for the Views List. + */ + const tableData = views.map((view) => ({ + ...view, + })); + + return ( + , + }} + pagination={false} + /> + ); +}; diff --git a/datahub-web-react/src/app/entity/view/builder/ViewBuilder.tsx b/datahub-web-react/src/app/entity/view/builder/ViewBuilder.tsx new file mode 100644 index 00000000000000..ac1107d735e8be --- /dev/null +++ b/datahub-web-react/src/app/entity/view/builder/ViewBuilder.tsx @@ -0,0 +1,123 @@ +import React from 'react'; +import { message } from 'antd'; +import { useApolloClient } from '@apollo/client'; +import { useCreateViewMutation, useUpdateViewMutation } from '../../../../graphql/view.generated'; +import { ViewBuilderState } from '../types'; +import { ViewBuilderModal } from './ViewBuilderModal'; +import { updateViewSelectCache, updateListMyViewsCache } from '../cacheUtils'; +import { convertStateToUpdateInput, DEFAULT_LIST_VIEWS_PAGE_SIZE } from '../utils'; +import { useUserContext } from '../../../context/useUserContext'; +import { ViewBuilderMode } from './types'; +import analytics, { Event, EventType } from '../../../analytics'; +import { DataHubView } from '../../../../types.generated'; + +type Props = { + mode: ViewBuilderMode; + urn?: string | null; // When editing an existing view, this is provided. + initialState?: ViewBuilderState; + onSubmit?: (state: ViewBuilderState) => void; + onCancel?: () => void; +}; + +/** + * This component handles creating and editing DataHub Views. + */ +export const ViewBuilder = ({ mode, urn, initialState, onSubmit, onCancel }: Props) => { + const userContext = useUserContext(); + + const client = useApolloClient(); + const [updateViewMutation] = useUpdateViewMutation(); + const [createViewMutation] = useCreateViewMutation(); + + const emitTrackingEvent = (viewUrn: string, state: ViewBuilderState, isCreate: boolean) => { + analytics.event({ + type: isCreate ? EventType.CreateViewEvent : EventType.UpdateViewEvent, + urn: viewUrn, + viewType: state.viewType, + } as Event); + }; + + /** + * Updates the selected View to a new urn. When a View is created or edited, + * we automatically update the currently applied View to be that view. + */ + const updatedSelectedView = (viewUrn: string) => { + // Hack: Force a re-render of the pages dependent on the value of the currently selected View + // (e.g. Search results) + userContext.updateLocalState({ + ...userContext.localState, + selectedViewUrn: undefined, + }); + userContext.updateLocalState({ + ...userContext.localState, + selectedViewUrn: viewUrn, + }); + }; + + const addViewToCaches = (viewUrn: string, view: DataHubView) => { + updateListMyViewsCache(viewUrn, view, client, 1, DEFAULT_LIST_VIEWS_PAGE_SIZE, undefined, undefined); + updateViewSelectCache(viewUrn, view, client); + }; + + const updateViewInCaches = (viewUrn: string, view: DataHubView) => { + updateListMyViewsCache(viewUrn, view, client, 1, DEFAULT_LIST_VIEWS_PAGE_SIZE, undefined, undefined); + updateViewSelectCache(viewUrn, view, client); + }; + + /** + * Updates the caches that power the Manage My Views and Views Select + * experiences when a new View has been create or an existing View has been updated. + */ + const updateViewCaches = (viewUrn: string, view: DataHubView, isCreate: boolean) => { + if (isCreate) { + addViewToCaches(viewUrn, view); + } else { + updateViewInCaches(viewUrn, view); + } + }; + + /** + * Create a new View, or update an existing one. + * + * @param state the state, which defines the fields of the new View. + */ + const upsertView = (state: ViewBuilderState) => { + const viewInput = convertStateToUpdateInput(state); + const isCreate = urn === undefined; + const mutation = isCreate ? createViewMutation : updateViewMutation; + const variables = urn + ? { urn, input: { ...viewInput, viewType: undefined } } + : { + input: viewInput, + }; + ( + mutation({ + variables: variables as any, + }) as any + ) + .then((res) => { + const updatedView = isCreate ? res.data?.createView : res.data?.updateView; + emitTrackingEvent(updatedView.urn, updatedView, isCreate); + updateViewCaches(updatedView.urn, updatedView, isCreate); + updatedSelectedView(updatedView.urn); + onSubmit?.(state); + }) + .catch((_) => { + message.destroy(); + message.error({ + content: `Failed to save View! An unexpected error occurred.`, + duration: 3, + }); + }); + }; + + return ( + + ); +}; diff --git a/datahub-web-react/src/app/entity/view/builder/ViewBuilderForm.tsx b/datahub-web-react/src/app/entity/view/builder/ViewBuilderForm.tsx new file mode 100644 index 00000000000000..7192ad68434b0c --- /dev/null +++ b/datahub-web-react/src/app/entity/view/builder/ViewBuilderForm.tsx @@ -0,0 +1,115 @@ +import React, { useEffect } from 'react'; +import styled from 'styled-components'; +import { Form, Input, Select, Typography } from 'antd'; +import { ViewBuilderState } from '../types'; +import { DataHubViewType } from '../../../../types.generated'; +import { ViewTypeLabel } from '../ViewTypeLabel'; +import { ViewDefinitionBuilder } from './ViewDefinitionBuilder'; +import { ANTD_GRAY } from '../../shared/constants'; +import { useUserContext } from '../../../context/useUserContext'; +import { ViewBuilderMode } from './types'; + +const StyledFormItem = styled(Form.Item)` + margin-bottom: 8px; +`; + +type Props = { + urn?: string; + mode: ViewBuilderMode; + state: ViewBuilderState; + updateState: (newState: ViewBuilderState) => void; +}; + +export const ViewBuilderForm = ({ urn, mode, state, updateState }: Props) => { + const userContext = useUserContext(); + const [form] = Form.useForm(); + + useEffect(() => { + form.setFieldsValue(state); + }, [state, form]); + + const setName = (name: string) => { + updateState({ + ...state, + name, + }); + }; + + const setDescription = (description: string) => { + updateState({ + ...state, + description, + }); + }; + + const setViewType = (viewType: DataHubViewType) => { + updateState({ + ...state, + viewType, + }); + }; + + const canManageGlobalViews = userContext?.platformPrivileges?.manageGlobalViews || false; + const isEditing = urn !== undefined; + + return ( + <> +
+ Name}> + Give your new View a name. + + setName(event.target.value)} + disabled={mode === ViewBuilderMode.PREVIEW} + /> + + + Description}> + Write a description for your View. + + setDescription(event.target.value)} + disabled={mode === ViewBuilderMode.PREVIEW} + /> + + + Type}> + Select the type of your new View. + + + + + Filters} style={{ marginBottom: 8 }}> + + Select the filters that are applied when this View is selected. Assets that match these filters + will be shown when the View is applied. + + +
+ + + ); +}; diff --git a/datahub-web-react/src/app/entity/view/builder/ViewBuilderModal.tsx b/datahub-web-react/src/app/entity/view/builder/ViewBuilderModal.tsx new file mode 100644 index 00000000000000..4a5e563fa81bde --- /dev/null +++ b/datahub-web-react/src/app/entity/view/builder/ViewBuilderModal.tsx @@ -0,0 +1,102 @@ +import React, { useEffect, useState } from 'react'; +import styled from 'styled-components'; +import { Button, Modal, Typography } from 'antd'; +import { DEFAULT_BUILDER_STATE, ViewBuilderState } from '../types'; +import { ViewBuilderForm } from './ViewBuilderForm'; +import ClickOutside from '../../../shared/ClickOutside'; +import { ViewBuilderMode } from './types'; + +const modalWidth = 700; +const modalStyle = { top: 40 }; +const modalBodyStyle = { paddingRight: 60, paddingLeft: 60, paddingBottom: 40 }; + +const TitleContainer = styled.div` + display: flex; + justify-content: space-between; +`; + +const SaveButtonContainer = styled.div` + width: 100%; + display: flex; + justify-content: right; +`; + +const CancelButton = styled(Button)` + margin-right: 12px; +`; + +type Props = { + mode: ViewBuilderMode; + urn?: string; + initialState?: ViewBuilderState; + onSubmit: (input: ViewBuilderState) => void; + onCancel?: () => void; +}; + +const getTitleText = (mode, urn) => { + if (mode === ViewBuilderMode.PREVIEW) { + return 'Preview View'; + } + return urn !== undefined ? 'Edit View' : 'Create new View'; +}; + +export const ViewBuilderModal = ({ mode, urn, initialState, onSubmit, onCancel }: Props) => { + const [viewBuilderState, setViewBuilderState] = useState(initialState || DEFAULT_BUILDER_STATE); + + useEffect(() => { + setViewBuilderState(initialState || DEFAULT_BUILDER_STATE); + }, [initialState]); + + const confirmClose = () => { + Modal.confirm({ + title: 'Exit View Editor', + content: `Are you sure you want to exit View editor? All changes will be lost`, + onOk() { + onCancel?.(); + }, + onCancel() {}, + okText: 'Yes', + maskClosable: true, + closable: true, + }); + }; + + const canSave = viewBuilderState.name && viewBuilderState.viewType && viewBuilderState?.definition?.filter; + const titleText = getTitleText(mode, urn); + + return ( + + + {titleText} + + } + style={modalStyle} + bodyStyle={modalBodyStyle} + visible + width={modalWidth} + onCancel={onCancel} + > + + + + Cancel + + {mode === ViewBuilderMode.EDITOR && ( + + )} + + + + ); +}; diff --git a/datahub-web-react/src/app/entity/view/builder/ViewDefinitionBuilder.tsx b/datahub-web-react/src/app/entity/view/builder/ViewDefinitionBuilder.tsx new file mode 100644 index 00000000000000..5f632d113cb98f --- /dev/null +++ b/datahub-web-react/src/app/entity/view/builder/ViewDefinitionBuilder.tsx @@ -0,0 +1,154 @@ +import React, { useEffect, useMemo, useState } from 'react'; +import styled from 'styled-components'; +import { useGetEntitiesLazyQuery } from '../../../../graphql/entity.generated'; +import { Entity, FacetFilter, FacetFilterInput, LogicalOperator } from '../../../../types.generated'; +import { AdvancedSearchFilters, LayoutDirection } from '../../../search/AdvancedSearchFilters'; +import { ENTITY_FILTER_NAME } from '../../../search/utils/constants'; +import { ANTD_GRAY } from '../../shared/constants'; +import { ViewBuilderState } from '../types'; +import { ViewBuilderMode } from './types'; +import { + buildEntityCache, + extractEntityTypesFilterValues, + fromUnionType, + isResolutionRequired, + toUnionType, +} from './utils'; + +const Container = styled.div` + border-radius: 4px; + padding: 12px; + box-shadow: ${(props) => props.theme.styles['box-shadow']}; + border: 1px solid ${ANTD_GRAY[4]}; + margin-bottom: 20px; +`; + +/** + * Filter fields representing entity URN criteria. + */ +const FIELDS_FOR_ENTITY_RESOLUTION = [ + 'tags', + 'domains', + 'glossaryTerms', + 'owners', + 'container', + 'platform', + 'fieldGlossaryTerms', + 'fieldTags', +]; + +type Props = { + mode: ViewBuilderMode; + state: ViewBuilderState; + updateState: (newState: ViewBuilderState) => void; +}; + +export const ViewDefinitionBuilder = ({ mode, state, updateState }: Props) => { + // Stores an URN to the resolved entity. + const [entityCache, setEntityCache] = useState>(new Map()); + + // Find the filters requiring entity resolution. + const filtersToResolve = useMemo( + () => + state.definition?.filter?.filters?.filter((filter) => + FIELDS_FOR_ENTITY_RESOLUTION.includes(filter.field), + ) || [], + [state], + ); + + // Create an array of all urns requiring resolution + const urnsToResolve: string[] = useMemo(() => { + return filtersToResolve.flatMap((filter) => { + return filter.values; + }); + }, [filtersToResolve]); + + /** + * Bootstrap by resolving all URNs that are not in the cache yet. + */ + const [getEntities, { data: resolvedEntitiesData }] = useGetEntitiesLazyQuery(); + + useEffect(() => { + if (isResolutionRequired(urnsToResolve, entityCache)) { + getEntities({ variables: { urns: urnsToResolve } }); + } + }, [urnsToResolve, entityCache, getEntities]); + + /** + * If some entities need to be resolved, simply build the cache from them. + * + * This should only happen once at component bootstrap. Typically + * all URNs will be missing from the cache. + */ + useEffect(() => { + if (resolvedEntitiesData && resolvedEntitiesData.entities?.length) { + const entities: Entity[] = (resolvedEntitiesData?.entities as Entity[]) || []; + setEntityCache(buildEntityCache(entities)); + } + }, [resolvedEntitiesData]); + + // Resolve "no-op" entity aggregations --> TODO: Migrate Advanced Search away from using aggregations. + const facets = filtersToResolve.map((filter) => ({ + field: filter.field, + aggregations: filter.values + .filter((value) => entityCache.has(value)) + .map((value) => ({ + value, + count: 0, + entity: entityCache.get(value), + })), + })); + + const operatorType = state.definition?.filter?.operator || LogicalOperator.Or; + const selectedFilters = state.definition?.filter?.filters || []; + const entityTypeFilter = state?.definition?.entityTypes?.length && { + field: ENTITY_FILTER_NAME, + values: state?.definition?.entityTypes, + }; + const finalFilters = (entityTypeFilter && [entityTypeFilter, ...selectedFilters]) || selectedFilters; + + const updateOperatorType = (newOperatorType) => { + const newDefinition = { + ...state.definition, + filter: { + operator: newOperatorType, + filters: state.definition?.filter?.filters || [], + }, + }; + updateState({ + ...state, + definition: newDefinition, + }); + }; + + const updateFilters = (newFilters: Array) => { + const entityTypes = extractEntityTypesFilterValues(newFilters); + const filteredFilters = newFilters.filter((filter) => filter.field !== ENTITY_FILTER_NAME); + const newDefinition = { + entityTypes, + filter: { + operator: operatorType, + filters: (filteredFilters.length > 0 ? filteredFilters : []) as FacetFilter[], + }, + }; + updateState({ + ...state, + definition: newDefinition, + }); + }; + + return ( + + updateOperatorType(fromUnionType(unionType))} + unionType={toUnionType(operatorType)} + loading={false} + direction={LayoutDirection.Horizontal} + disabled={mode === ViewBuilderMode.PREVIEW} + /> + + ); +}; diff --git a/datahub-web-react/src/app/entity/view/builder/types.ts b/datahub-web-react/src/app/entity/view/builder/types.ts new file mode 100644 index 00000000000000..1d3cf3875a8609 --- /dev/null +++ b/datahub-web-react/src/app/entity/view/builder/types.ts @@ -0,0 +1,10 @@ +export enum ViewBuilderMode { + /** + * See a View definition in Preview Mode. + */ + PREVIEW, + /** + * Create or Edit a View. + */ + EDITOR, +} diff --git a/datahub-web-react/src/app/entity/view/builder/utils.ts b/datahub-web-react/src/app/entity/view/builder/utils.ts new file mode 100644 index 00000000000000..3b67face7f33e1 --- /dev/null +++ b/datahub-web-react/src/app/entity/view/builder/utils.ts @@ -0,0 +1,92 @@ +import { + DataHubViewType, + Entity, + EntityType, + FacetFilter, + FacetFilterInput, + LogicalOperator, +} from '../../../../types.generated'; +import { ENTITY_FILTER_NAME, UnionType } from '../../../search/utils/constants'; + +/** + * Extract the special "Entity Type" filter values from a list + * of filters. + */ +export const extractEntityTypesFilterValues = (filters: Array) => { + // Currently we only support 1 entity type filter. + return filters + .filter((filter) => filter.field === ENTITY_FILTER_NAME) + .flatMap((filter) => filter.values as EntityType[]); +}; + +/** + * Build an object representation of a View Definition, which consists of a list of entity types + + * a set of filters joined in either conjunction or disjunction. + * + * @param filters a list of Facet Filter Inputs representing the view filters. This can include the entity type filter. + * @param operatorType a logical operator to be used when joining the filters into the View definition. + */ +export const buildViewDefinition = (filters: Array, operatorType: LogicalOperator) => { + const entityTypes = extractEntityTypesFilterValues(filters); + const filteredFilters = filters.filter((filter) => filter.field !== ENTITY_FILTER_NAME); + return { + entityTypes, + filter: { + operator: operatorType, + filters: (filteredFilters.length > 0 ? filteredFilters : []) as FacetFilter[], + }, + }; +}; + +/** + * Build an object representation of a View Definition, which consists of a list of entity types + + * a set of filters joined in either conjunction or disjunction. + * + * @param filters a list of Facet Filter Inputs representing the view filters. This can include the entity type filter. + * @param operatorType a logical operator to be used when joining the filters into the View definition. + */ +export const buildInitialViewState = (filters: Array, operatorType: LogicalOperator) => { + return { + viewType: DataHubViewType.Personal, + name: '', + description: null, + definition: buildViewDefinition(filters, operatorType), + }; +}; + +/** + * Convert a LogicalOperator to the equivalent UnionType. + */ +export const toUnionType = (operator: LogicalOperator) => { + if (operator === LogicalOperator.And) { + return UnionType.AND; + } + return UnionType.OR; +}; + +/** + * Convert a UnionType to the equivalent LogicalOperator. + */ +export const fromUnionType = (unionType: UnionType) => { + if (unionType === 0) { + return LogicalOperator.And; + } + return LogicalOperator.Or; +}; + +/** + * Returns a map of entity urn to entity from a list of entities. + */ +export const buildEntityCache = (entities: Entity[]) => { + const cache = new Map(); + entities.forEach((entity) => cache.set(entity.urn, entity)); + return cache; +}; + +/** + * Returns 'true' if any urns are not present in an entity cache, 'false' otherwise. + */ +export const isResolutionRequired = (urns: string[], cache: Map) => { + const uncachedUrns = urns.filter((urn) => !cache.has(urn)); + return uncachedUrns.length > 0; +}; diff --git a/datahub-web-react/src/app/entity/view/cacheUtils.ts b/datahub-web-react/src/app/entity/view/cacheUtils.ts new file mode 100644 index 00000000000000..c8f6d03ef98681 --- /dev/null +++ b/datahub-web-react/src/app/entity/view/cacheUtils.ts @@ -0,0 +1,250 @@ +import { + ListGlobalViewsDocument, + ListGlobalViewsQuery, + ListMyViewsDocument, + ListMyViewsQuery, +} from '../../../graphql/view.generated'; +import { DataHubViewType, DataHubView } from '../../../types.generated'; +import { DEFAULT_LIST_VIEWS_PAGE_SIZE } from './utils'; + +/** + * This file contains utility classes for manipulating the Apollo Cache + * when Views are added, updated, or removed. + */ + +const addViewToList = (existingViews, newView) => { + return [newView, ...existingViews]; +}; + +const addOrUpdateViewInList = (existingViews, newView) => { + const newViews = [...existingViews]; + let didUpdate = false; + const updatedViews = newViews.map((view) => { + if (view.urn === newView.urn) { + didUpdate = true; + return newView; + } + return view; + }); + return didUpdate ? updatedViews : addViewToList(existingViews, newView); +}; + +export const updateListMyViewsCache = ( + urn: string, + newView: DataHubView, + client, + page, + pageSize, + viewType?: DataHubViewType, + query?: string, +) => { + // Read the data from our cache for this query. + const currData: ListMyViewsQuery | null = client.readQuery({ + query: ListMyViewsDocument, + variables: { + start: (page - 1) * pageSize, + count: pageSize, + viewType, + query, + }, + }); + + if (currData === null) { + // If there's no cached data, the first load has not occurred. Let it occur naturally. + return; + } + + const existingViews = currData?.listMyViews?.views || []; + const newViews = addOrUpdateViewInList(existingViews, newView); + + const currCount = currData?.listMyViews?.count || 0; + const currTotal = currData?.listMyViews?.total || 0; + const didAdd = newViews.length > existingViews.length; + + // Write our data back to the cache. + client.writeQuery({ + query: ListMyViewsDocument, + variables: { + viewType, + start: (page - 1) * pageSize, + count: pageSize, + query, + }, + data: { + listMyViews: { + __typename: 'ListViewsResult', + start: (page - 1) * pageSize, + count: currCount + (didAdd ? 1 : 0), + total: currTotal + (didAdd ? 1 : 0), + views: newViews, + }, + }, + }); +}; + +const updateListGlobalViewsCache = (urn: string, newView: DataHubView, client, page, pageSize, query?: string) => { + // Read the data from our cache for this query. + const currData: ListGlobalViewsQuery | null = client.readQuery({ + query: ListGlobalViewsDocument, + variables: { + start: (page - 1) * pageSize, + count: pageSize, + query, + }, + }); + + if (currData === null) { + // If there's no cached data, the first load has not occurred. Let it occur naturally. + return; + } + + const existingViews = currData?.listGlobalViews?.views || []; + const newViews = addOrUpdateViewInList(existingViews, newView); + const didAdd = newViews.length > existingViews.length; + + const currCount = currData?.listGlobalViews?.count || 0; + const currTotal = currData?.listGlobalViews?.total || 0; + + // Write our data back to the cache. + client.writeQuery({ + query: ListGlobalViewsDocument, + variables: { + start: (page - 1) * pageSize, + count: pageSize, + query, + }, + data: { + listGlobalViews: { + __typename: 'ListViewsResult', + start: (page - 1) * pageSize, + count: currCount + (didAdd ? 1 : 0), + total: currTotal + (didAdd ? 1 : 0), + views: newViews, + }, + }, + }); +}; + +export const removeFromListMyViewsCache = ( + urn: string, + client, + page: number, + pageSize: number, + viewType?: DataHubViewType, + query?: string, +) => { + const currData: ListMyViewsQuery | null = client.readQuery({ + query: ListMyViewsDocument, + variables: { + start: (page - 1) * pageSize, + count: pageSize, + viewType, + query, + }, + }); + + if (currData === null) { + // If there's no cached data, the first load has not occurred. Let it occur naturally. + return; + } + + const existingViews = currData?.listMyViews?.views || []; + const newViews = [...existingViews.filter((view) => view.urn !== urn)]; + const didRemove = existingViews.length !== newViews.length; + + const currCount = currData?.listMyViews?.count || 0; + const currTotal = currData?.listMyViews?.total || 0; + + // Write our data back to the cache. + client.writeQuery({ + query: ListMyViewsDocument, + variables: { + viewType, + start: (page - 1) * pageSize, + count: pageSize, + query, + }, + data: { + listMyViews: { + __typename: 'ListViewsResult', + start: (page - 1) * pageSize, + count: didRemove ? currCount - 1 : currCount, + total: didRemove ? currTotal - 1 : currCount, + views: newViews, + }, + }, + }); +}; + +const removeFromListGlobalViewsCache = (urn: string, client, page: number, pageSize: number, query?: string) => { + const currData: ListGlobalViewsQuery | null = client.readQuery({ + query: ListGlobalViewsDocument, + variables: { + start: (page - 1) * pageSize, + count: pageSize, + query, + }, + }); + + if (currData === null) { + // If there's no cached data, the first load has not occurred. Let it occur naturally. + return; + } + + const existingViews = currData?.listGlobalViews?.views || []; + const newViews = [...existingViews.filter((view) => view.urn !== urn)]; + const didRemove = existingViews.length !== newViews.length; + + const currCount = currData?.listGlobalViews?.count || 0; + const currTotal = currData?.listGlobalViews?.total || 0; + + // Write our data back to the cache. + client.writeQuery({ + query: ListGlobalViewsDocument, + variables: { + start: (page - 1) * pageSize, + count: pageSize, + query, + }, + data: { + listGlobalViews: { + __typename: 'ListViewsResult', + start: (page - 1) * pageSize, + count: didRemove ? currCount - 1 : currCount, + total: didRemove ? currTotal - 1 : currCount, + views: newViews, + }, + }, + }); +}; + +/** + * Updates an entry in the list views caches, which are used + * to power the View Select component. + * + * @param urn the urn of the updated view + * @param view the updated view + * @param client the Apollo client + */ +export const updateViewSelectCache = (urn: string, view: DataHubView, client) => { + if (view.viewType === DataHubViewType.Personal) { + // Add or Update in Personal Views Cache, remove from the opposite. + updateListMyViewsCache(urn, view, client, 1, DEFAULT_LIST_VIEWS_PAGE_SIZE, DataHubViewType.Personal, undefined); + removeFromListGlobalViewsCache(urn, client, 1, DEFAULT_LIST_VIEWS_PAGE_SIZE, undefined); + } else { + // Add or Update in Global Views Cache, remove from the opposite. + updateListGlobalViewsCache(urn, view, client, 1, DEFAULT_LIST_VIEWS_PAGE_SIZE, undefined); + removeFromListMyViewsCache(urn, client, 1, DEFAULT_LIST_VIEWS_PAGE_SIZE, DataHubViewType.Personal, undefined); + } +}; + +/** + * Removes an URN from the queries used to power the View Select dropdown. + * + * @param urn the urn of the updated view + * @param client the Apollo client + */ +export const removeFromViewSelectCaches = (urn: string, client) => { + removeFromListMyViewsCache(urn, client, 1, DEFAULT_LIST_VIEWS_PAGE_SIZE, DataHubViewType.Personal, undefined); + removeFromListGlobalViewsCache(urn, client, 1, DEFAULT_LIST_VIEWS_PAGE_SIZE, undefined); +}; diff --git a/datahub-web-react/src/app/entity/view/menu/ViewDropdownMenu.tsx b/datahub-web-react/src/app/entity/view/menu/ViewDropdownMenu.tsx new file mode 100644 index 00000000000000..1c05f82ed08046 --- /dev/null +++ b/datahub-web-react/src/app/entity/view/menu/ViewDropdownMenu.tsx @@ -0,0 +1,267 @@ +import React, { useState } from 'react'; +import styled from 'styled-components'; +import { useApolloClient } from '@apollo/client'; +import { MoreOutlined } from '@ant-design/icons'; +import { Dropdown, Menu, message, Modal } from 'antd'; +import { DataHubView, DataHubViewType } from '../../../../types.generated'; +import { useUserContext } from '../../../context/useUserContext'; +import { useUpdateCorpUserViewsSettingsMutation } from '../../../../graphql/user.generated'; +import { useUpdateGlobalViewsSettingsMutation } from '../../../../graphql/app.generated'; +import { useDeleteViewMutation } from '../../../../graphql/view.generated'; +import { removeFromListMyViewsCache, removeFromViewSelectCaches } from '../cacheUtils'; +import { DEFAULT_LIST_VIEWS_PAGE_SIZE } from '../utils'; +import { ViewBuilderMode } from '../builder/types'; +import { ViewBuilder } from '../builder/ViewBuilder'; +import { EditViewItem } from './item/EditViewItem'; +import { PreviewViewItem } from './item/PreviewViewItem'; +import { RemoveUserDefaultItem } from './item/RemoveUserDefaultItem'; +import { SetUserDefaultItem } from './item/SetUserDefaultItem'; +import { RemoveGlobalDefaultItem } from './item/RemoveGlobalDefaultItem'; +import { SetGlobalDefaultItem } from './item/SetGlobalDefaultItem'; +import { DeleteViewItem } from './item/DeleteViewItem'; +import analytics, { EventType } from '../../../analytics'; + +const MenuButton = styled(MoreOutlined)` + width: 20px; + &&& { + padding-left: 0px; + padding-right: 0px; + font-size: 18px; + } + :hover { + cursor: pointer; + } +`; + +const DEFAULT_VIEW_BUILDER_STATE = { + mode: ViewBuilderMode.EDITOR, + visible: false, +}; + +type Props = { + view: DataHubView; + visible?: boolean; + isOwnedByUser?: boolean; + trigger?: 'hover' | 'click'; + // Custom Action Handlers - useful if you do NOT want the Menu to handle Modal rendering. + onClickEdit?: () => void; + onClickPreview?: () => void; + onClickDelete?: () => void; +}; + +export const ViewDropdownMenu = ({ + view, + visible, + isOwnedByUser, + trigger = 'hover', + onClickEdit, + onClickPreview, + onClickDelete, +}: Props) => { + const userContext = useUserContext(); + const client = useApolloClient(); + + const [updateUserViewSettingMutation] = useUpdateCorpUserViewsSettingsMutation(); + const [updateGlobalViewSettingMutation] = useUpdateGlobalViewsSettingsMutation(); + const [deleteViewMutation] = useDeleteViewMutation(); + + const [viewBuilderState, setViewBuilderState] = useState(DEFAULT_VIEW_BUILDER_STATE); + + /** + * Updates the User's Personal Default View via mutation. + * + * Then updates the User Context state to contain the new default. + */ + const setUserDefault = (viewUrn: string | null) => { + updateUserViewSettingMutation({ + variables: { + input: { + defaultView: viewUrn, + }, + }, + }) + .then(({ errors }) => { + if (!errors) { + userContext.updateState({ + ...userContext.state, + views: { + ...userContext.state.views, + personalDefaultViewUrn: viewUrn, + }, + }); + analytics.event({ + type: EventType.SetUserDefaultViewEvent, + urn: viewUrn, + viewType: (viewUrn && view.viewType) || null, + }); + } + }) + .catch((_) => { + message.destroy(); + message.error({ + content: `Failed to make this your default view. An unexpected error occurred.`, + duration: 3, + }); + }); + }; + + /** + * Updates the Global Default View via mutation. + * + * Then updates the User Context state to contain the new default. + */ + const setGlobalDefault = (viewUrn: string | null) => { + updateGlobalViewSettingMutation({ + variables: { + input: { + defaultView: viewUrn, + }, + }, + }) + .then(({ errors }) => { + if (!errors) { + userContext.updateState({ + ...userContext.state, + views: { + ...userContext.state.views, + globalDefaultViewUrn: viewUrn, + }, + }); + analytics.event({ + type: EventType.SetGlobalDefaultViewEvent, + urn: viewUrn, + }); + } + }) + .catch((_) => { + message.destroy(); + message.error({ + content: `Failed to make this your organization's default view. An unexpected error occurred.`, + duration: 3, + }); + }); + }; + + const onEditView = () => { + if (onClickEdit) { + onClickEdit?.(); + } else { + setViewBuilderState({ + mode: ViewBuilderMode.EDITOR, + visible: true, + }); + } + }; + + const onPreviewView = () => { + if (onClickPreview) { + onClickPreview?.(); + } else { + setViewBuilderState({ + mode: ViewBuilderMode.PREVIEW, + visible: true, + }); + } + }; + + const onViewBuilderClose = () => { + setViewBuilderState(DEFAULT_VIEW_BUILDER_STATE); + }; + + const deleteView = (viewUrn: string) => { + deleteViewMutation({ + variables: { urn: viewUrn }, + }) + .then(({ errors }) => { + if (!errors) { + removeFromViewSelectCaches(viewUrn, client); + removeFromListMyViewsCache(viewUrn, client, 1, DEFAULT_LIST_VIEWS_PAGE_SIZE, undefined, undefined); + /** + * Clear the selected view urn from local state, + * if the deleted view was that urn. + */ + if (viewUrn === userContext.localState?.selectedViewUrn) { + userContext.updateLocalState({ + ...userContext.localState, + selectedViewUrn: undefined, + }); + } + message.success({ content: 'Removed View!', duration: 2 }); + } + }) + .catch(() => { + message.destroy(); + message.error({ + content: `Failed to delete View. An unexpected error occurred.`, + duration: 3, + }); + }); + }; + + const confirmDeleteView = () => { + if (onClickDelete) { + onClickDelete?.(); + } else { + Modal.confirm({ + title: `Confirm Remove ${view.name}`, + content: `Are you sure you want to remove this View?`, + onOk() { + deleteView(view.urn); + }, + onCancel() {}, + okText: 'Yes', + maskClosable: true, + closable: true, + }); + } + }; + + const canManageGlobalViews = userContext.platformPrivileges?.manageGlobalViews; + const canManageView = isOwnedByUser || canManageGlobalViews; + + const maybePersonalDefaultViewUrn = userContext.state?.views?.personalDefaultViewUrn; + const maybeGlobalDefaultViewUrn = userContext.state?.views?.globalDefaultViewUrn; + + const isGlobalView = view.viewType === DataHubViewType.Global; + const isUserDefault = view.urn === maybePersonalDefaultViewUrn; + const isGlobalDefault = view.urn === maybeGlobalDefaultViewUrn; + + const showRemoveGlobalDefaultView = canManageGlobalViews && isGlobalView && isGlobalDefault; + const showSetGlobalDefaultView = canManageGlobalViews && isGlobalView && !isGlobalDefault; + + return ( + <> + + {(canManageView && ) || ( + + )} + {(isUserDefault && setUserDefault(null)} />) || ( + setUserDefault(view.urn)} /> + )} + {showRemoveGlobalDefaultView && ( + setGlobalDefault(null)} /> + )} + {showSetGlobalDefaultView && ( + setGlobalDefault(view.urn)} /> + )} + {canManageView && } + + } + trigger={[trigger]} + > + + + {viewBuilderState.visible && ( + + )} + + ); +}; diff --git a/datahub-web-react/src/app/entity/view/menu/item/DeleteViewItem.tsx b/datahub-web-react/src/app/entity/view/menu/item/DeleteViewItem.tsx new file mode 100644 index 00000000000000..b8513c23ebe4fb --- /dev/null +++ b/datahub-web-react/src/app/entity/view/menu/item/DeleteViewItem.tsx @@ -0,0 +1,20 @@ +import React from 'react'; +import { DeleteOutlined } from '@ant-design/icons'; +import { Menu } from 'antd'; +import { IconItemTitle } from './IconItemTitle'; + +type Props = { + key: string; + onClick: () => void; +}; + +/** + * Delete a View Item + */ +export const DeleteViewItem = ({ key, onClick }: Props) => { + return ( + + } /> + + ); +}; diff --git a/datahub-web-react/src/app/entity/view/menu/item/EditViewItem.tsx b/datahub-web-react/src/app/entity/view/menu/item/EditViewItem.tsx new file mode 100644 index 00000000000000..650bc239957ab6 --- /dev/null +++ b/datahub-web-react/src/app/entity/view/menu/item/EditViewItem.tsx @@ -0,0 +1,20 @@ +import React from 'react'; +import { FormOutlined } from '@ant-design/icons'; +import { Menu } from 'antd'; +import { IconItemTitle } from './IconItemTitle'; + +type Props = { + key: string; + onClick: () => void; +}; + +/** + * Edit View Menu Item + */ +export const EditViewItem = ({ key, onClick }: Props) => { + return ( + + } /> + + ); +}; diff --git a/datahub-web-react/src/app/entity/view/menu/item/IconItemTitle.tsx b/datahub-web-react/src/app/entity/view/menu/item/IconItemTitle.tsx new file mode 100644 index 00000000000000..2d366c47c59c24 --- /dev/null +++ b/datahub-web-react/src/app/entity/view/menu/item/IconItemTitle.tsx @@ -0,0 +1,38 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Tooltip } from 'antd'; +import { ANTD_GRAY } from '../../../shared/constants'; + +const TitleContainer = styled.span` + display: flex; + align-items: center; + justify-content: left; + padding-left: 0px; +`; + +const IconContainer = styled.span` + && { + color: ${ANTD_GRAY[8]}; + margin-right: 12px; + } +`; + +type Props = { + tip?: React.ReactNode; + title: string; + icon: React.ReactNode; +}; + +/** + * Base Item Title for the menu + */ +export const IconItemTitle = ({ tip, title, icon }: Props) => { + return ( + + + {icon} + {title} + + + ); +}; diff --git a/datahub-web-react/src/app/entity/view/menu/item/PreviewViewItem.tsx b/datahub-web-react/src/app/entity/view/menu/item/PreviewViewItem.tsx new file mode 100644 index 00000000000000..24b122feeb2354 --- /dev/null +++ b/datahub-web-react/src/app/entity/view/menu/item/PreviewViewItem.tsx @@ -0,0 +1,20 @@ +import React from 'react'; +import { Menu } from 'antd'; +import { EyeOutlined } from '@ant-design/icons'; +import { IconItemTitle } from './IconItemTitle'; + +type Props = { + key: string; + onClick: () => void; +}; + +/** + * Preview View Menu Item + */ +export const PreviewViewItem = ({ key, onClick }: Props) => { + return ( + + } /> + + ); +}; diff --git a/datahub-web-react/src/app/entity/view/menu/item/RemoveGlobalDefaultItem.tsx b/datahub-web-react/src/app/entity/view/menu/item/RemoveGlobalDefaultItem.tsx new file mode 100644 index 00000000000000..10977fcc4470e9 --- /dev/null +++ b/datahub-web-react/src/app/entity/view/menu/item/RemoveGlobalDefaultItem.tsx @@ -0,0 +1,24 @@ +import React from 'react'; +import { StopOutlined } from '@ant-design/icons'; +import { Menu } from 'antd'; +import { IconItemTitle } from './IconItemTitle'; + +type Props = { + key: string; + onClick: () => void; +}; + +/** + * Remove the Global View + */ +export const RemoveGlobalDefaultItem = ({ key, onClick }: Props) => { + return ( + + } + /> + + ); +}; diff --git a/datahub-web-react/src/app/entity/view/menu/item/RemoveUserDefaultItem.tsx b/datahub-web-react/src/app/entity/view/menu/item/RemoveUserDefaultItem.tsx new file mode 100644 index 00000000000000..d07d54f6b929b3 --- /dev/null +++ b/datahub-web-react/src/app/entity/view/menu/item/RemoveUserDefaultItem.tsx @@ -0,0 +1,24 @@ +import React from 'react'; +import { StopOutlined } from '@ant-design/icons'; +import { Menu } from 'antd'; +import { IconItemTitle } from './IconItemTitle'; + +type Props = { + key: string; + onClick: () => void; +}; + +/** + * Remove the User's default view item + */ +export const RemoveUserDefaultItem = ({ key, onClick }: Props) => { + return ( + + } + /> + + ); +}; diff --git a/datahub-web-react/src/app/entity/view/menu/item/SetGlobalDefaultItem.tsx b/datahub-web-react/src/app/entity/view/menu/item/SetGlobalDefaultItem.tsx new file mode 100644 index 00000000000000..7461565c22ec5a --- /dev/null +++ b/datahub-web-react/src/app/entity/view/menu/item/SetGlobalDefaultItem.tsx @@ -0,0 +1,24 @@ +import React from 'react'; +import { Menu } from 'antd'; +import { GlobalDefaultViewIcon } from '../../shared/GlobalDefaultViewIcon'; +import { IconItemTitle } from './IconItemTitle'; + +type Props = { + key: string; + onClick: () => void; +}; + +/** + * Set the Global Default Item + */ +export const SetGlobalDefaultItem = ({ key, onClick }: Props) => { + return ( + + } + /> + + ); +}; diff --git a/datahub-web-react/src/app/entity/view/menu/item/SetUserDefaultItem.tsx b/datahub-web-react/src/app/entity/view/menu/item/SetUserDefaultItem.tsx new file mode 100644 index 00000000000000..3149f06294c009 --- /dev/null +++ b/datahub-web-react/src/app/entity/view/menu/item/SetUserDefaultItem.tsx @@ -0,0 +1,24 @@ +import React from 'react'; +import { Menu } from 'antd'; +import { UserDefaultViewIcon } from '../../shared/UserDefaultViewIcon'; +import { IconItemTitle } from './IconItemTitle'; + +type Props = { + key: string; + onClick: () => void; +}; + +/** + * Set the User's default view item + */ +export const SetUserDefaultItem = ({ key, onClick }: Props) => { + return ( + + } + /> + + ); +}; diff --git a/datahub-web-react/src/app/entity/view/select/ViewOption.tsx b/datahub-web-react/src/app/entity/view/select/ViewOption.tsx new file mode 100644 index 00000000000000..b4c7e7db2b05db --- /dev/null +++ b/datahub-web-react/src/app/entity/view/select/ViewOption.tsx @@ -0,0 +1,60 @@ +import React from 'react'; +import styled from 'styled-components'; +import { DataHubView } from '../../../../types.generated'; +import { ViewOptionName } from './ViewOptionName'; +import { ViewDropdownMenu } from '../menu/ViewDropdownMenu'; +import { UserDefaultViewIcon } from '../shared/UserDefaultViewIcon'; +import { GlobalDefaultViewIcon } from '../shared/GlobalDefaultViewIcon'; + +const ICON_WIDTH = 30; + +const Container = styled.div` + display: flex; + align-items: center; + justify-content: stretch; + width: 100%; +`; + +const IconPlaceholder = styled.div` + width: ${ICON_WIDTH}px; + display: flex; + align-items: center; + justify-content: center; +`; + +type Props = { + view: DataHubView; + showOptions: boolean; + isGlobalDefault: boolean; + isUserDefault: boolean; + isOwnedByUser?: boolean; + onClickEdit: () => void; + onClickPreview: () => void; +}; + +export const ViewOption = ({ + view, + showOptions, + isGlobalDefault, + isUserDefault, + isOwnedByUser, + onClickEdit, + onClickPreview, +}: Props) => { + return ( + + + {isUserDefault && } + {isGlobalDefault && } + + + + + ); +}; diff --git a/datahub-web-react/src/app/entity/view/select/ViewOptionName.tsx b/datahub-web-react/src/app/entity/view/select/ViewOptionName.tsx new file mode 100644 index 00000000000000..ff89b0005912f3 --- /dev/null +++ b/datahub-web-react/src/app/entity/view/select/ViewOptionName.tsx @@ -0,0 +1,24 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Tooltip } from 'antd'; +import { ViewOptionTooltipTitle } from './ViewOptionTooltipTitle'; + +const ViewName = styled.span` + width: 200px; + white-space: nowrap; + overflow: hidden; + text-overflow: ellipsis; +`; + +type Props = { + name: string; + description?: string | null; +}; + +export const ViewOptionName = ({ name, description }: Props) => { + return ( + }> + {name} + + ); +}; diff --git a/datahub-web-react/src/app/entity/view/select/ViewOptionTooltipTitle.tsx b/datahub-web-react/src/app/entity/view/select/ViewOptionTooltipTitle.tsx new file mode 100644 index 00000000000000..4949dc9d7d7ce2 --- /dev/null +++ b/datahub-web-react/src/app/entity/view/select/ViewOptionTooltipTitle.tsx @@ -0,0 +1,22 @@ +import React from 'react'; +import styled from 'styled-components'; + +const NameContainer = styled.div` + margin-bottom: 8px; +`; + +type Props = { + name: string; + description?: string | null; +}; + +export const ViewOptionTooltipTitle = ({ name, description }: Props) => { + return ( + <> + + {name} + +
{description}
+ + ); +}; diff --git a/datahub-web-react/src/app/entity/view/select/ViewSelect.tsx b/datahub-web-react/src/app/entity/view/select/ViewSelect.tsx new file mode 100644 index 00000000000000..d5d779aabe2f1f --- /dev/null +++ b/datahub-web-react/src/app/entity/view/select/ViewSelect.tsx @@ -0,0 +1,215 @@ +import React, { useEffect, useRef, useState } from 'react'; +import { useHistory } from 'react-router'; +import { Select } from 'antd'; +import { useListMyViewsQuery, useListGlobalViewsQuery } from '../../../../graphql/view.generated'; +import { useUserContext } from '../../../context/useUserContext'; +import { DataHubView, DataHubViewType } from '../../../../types.generated'; +import { ViewBuilder } from '../builder/ViewBuilder'; +import { DEFAULT_LIST_VIEWS_PAGE_SIZE } from '../utils'; +import { PageRoutes } from '../../../../conf/Global'; +import { ViewSelectToolTip } from './ViewSelectToolTip'; +import { ViewBuilderMode } from '../builder/types'; +import { ViewSelectDropdown } from './ViewSelectDropdown'; +import { renderViewOptionGroup } from './renderViewOptionGroup'; + +const selectStyle = { + width: 240, +}; + +const dropdownStyle = { + position: 'fixed', +} as any; + +type ViewBuilderDisplayState = { + mode: ViewBuilderMode; + visible: boolean; + view?: DataHubView; +}; + +const DEFAULT_VIEW_BUILDER_DISPLAY_STATE = { + mode: ViewBuilderMode.EDITOR, + visible: false, + view: undefined, +}; + +/** + * The View Select component allows you to select a View to apply to query on the current page. For example, + * search, recommendations, and browse. + * + * The current state of the View select includes an urn that must be forwarded with search, browse, and recommendations + * requests. As we navigate around the app, the state of the selected View should not change. + * + * In the event that a user refreshes their browser, the state of the view should be saved as well. + */ +export const ViewSelect = () => { + const history = useHistory(); + const userContext = useUserContext(); + const [viewBuilderDisplayState, setViewBuilderDisplayState] = useState( + DEFAULT_VIEW_BUILDER_DISPLAY_STATE, + ); + const [selectedUrn, setSelectedUrn] = useState( + userContext.localState?.selectedViewUrn || undefined, + ); + const [hoverViewUrn, setHoverViewUrn] = useState(undefined); + + useEffect(() => { + setSelectedUrn(userContext.localState?.selectedViewUrn || undefined); + }, [userContext.localState?.selectedViewUrn, setSelectedUrn]); + + const selectRef = useRef(null); + + /** + * Queries - Notice, each of these queries is cached. Here we fetch both the user's private views, + * along with all public views. + */ + + const { data: privateViewsData } = useListMyViewsQuery({ + variables: { + start: 0, + count: DEFAULT_LIST_VIEWS_PAGE_SIZE, + viewType: DataHubViewType.Personal, + }, + fetchPolicy: 'cache-first', + }); + + // Fetch Public Views + const { data: publicViewsData } = useListGlobalViewsQuery({ + variables: { + start: 0, + count: DEFAULT_LIST_VIEWS_PAGE_SIZE, + }, + fetchPolicy: 'cache-first', + }); + + /** + * Event Handlers + */ + + const onSelectView = (newUrn) => { + userContext.updateLocalState({ + ...userContext.localState, + selectedViewUrn: newUrn, + }); + }; + + const onClickCreateView = () => { + setViewBuilderDisplayState({ + visible: true, + mode: ViewBuilderMode.EDITOR, + view: undefined, + }); + }; + + const onClickEditView = (view) => { + setViewBuilderDisplayState({ + visible: true, + mode: ViewBuilderMode.EDITOR, + view, + }); + }; + + const onCloseViewBuilder = () => { + setViewBuilderDisplayState(DEFAULT_VIEW_BUILDER_DISPLAY_STATE); + }; + + const onClickPreviewView = (view) => { + setViewBuilderDisplayState({ + visible: true, + mode: ViewBuilderMode.PREVIEW, + view, + }); + }; + + const onClear = () => { + setSelectedUrn(undefined); + userContext.updateLocalState({ + ...userContext.localState, + selectedViewUrn: undefined, + }); + }; + + const onClickManageViews = () => { + history.push(PageRoutes.SETTINGS_VIEWS); + }; + + /** + * Render variables + */ + const privateViews = privateViewsData?.listMyViews?.views || []; + const publicViews = publicViewsData?.listGlobalViews?.views || []; + const privateViewCount = privateViews?.length || 0; + const publicViewCount = publicViews?.length || 0; + const hasViews = privateViewCount > 0 || publicViewCount > 0 || false; + + /** + * Notice - we assume that we will find the selected View urn in the list + * of Views retrieved for the user (private or public). If this is not the case, + * the view will be set to undefined. + * + * This may become a problem if a list of public views exceeds the default pagination size of 1,000. + */ + const foundSelectedUrn = + (privateViews.filter((view) => view.urn === selectedUrn)?.length || 0) > 0 || + (publicViews.filter((view) => view.urn === selectedUrn)?.length || 0) > 0 || + false; + + return ( + <> + + + + {viewBuilderDisplayState.visible && ( + + )} + + ); +}; diff --git a/datahub-web-react/src/app/entity/view/select/ViewSelectDropdown.tsx b/datahub-web-react/src/app/entity/view/select/ViewSelectDropdown.tsx new file mode 100644 index 00000000000000..60a229d8d97a1d --- /dev/null +++ b/datahub-web-react/src/app/entity/view/select/ViewSelectDropdown.tsx @@ -0,0 +1,21 @@ +import React from 'react'; +import { ViewSelectFooter } from './ViewSelectFooter'; +import { ViewSelectHeader } from './ViewSelectHeader'; + +type Props = { + menu: React.ReactNode; + hasViews: boolean; + onClickCreateView: () => void; + onClickManageViews: () => void; + onClickClear: () => void; +}; + +export const ViewSelectDropdown = ({ menu, hasViews, onClickCreateView, onClickManageViews, onClickClear }: Props) => { + return ( + <> + + {hasViews && menu} + + + ); +}; diff --git a/datahub-web-react/src/app/entity/view/select/ViewSelectFooter.tsx b/datahub-web-react/src/app/entity/view/select/ViewSelectFooter.tsx new file mode 100644 index 00000000000000..c30511d80d52f4 --- /dev/null +++ b/datahub-web-react/src/app/entity/view/select/ViewSelectFooter.tsx @@ -0,0 +1,46 @@ +import React, { useRef } from 'react'; +import styled from 'styled-components'; +import { Button } from 'antd'; +import { RightOutlined } from '@ant-design/icons'; +import { ANTD_GRAY } from '../../shared/constants'; + +const ButtonContainer = styled.div` + display: flex; + color: ${ANTD_GRAY[6]}; + width: 100%; + justify-content: left; +`; + +const StyledRightOutlined = styled(RightOutlined)` + && { + font-size: 8px; + color: ${ANTD_GRAY[7]}; + } +`; + +const ManageViewsButton = styled(Button)` + font-weight: normal; + color: ${ANTD_GRAY[7]}; +`; + +type Props = { + onClickManageViews: () => void; +}; + +export const ViewSelectFooter = ({ onClickManageViews }: Props) => { + const manageViewsButtonRef = useRef(null); + + const onHandleClickManageViews = () => { + (manageViewsButtonRef?.current as any)?.blur(); + onClickManageViews(); + }; + + return ( + + + Manage Views + + + + ); +}; diff --git a/datahub-web-react/src/app/entity/view/select/ViewSelectHeader.tsx b/datahub-web-react/src/app/entity/view/select/ViewSelectHeader.tsx new file mode 100644 index 00000000000000..75cca161c4ae4d --- /dev/null +++ b/datahub-web-react/src/app/entity/view/select/ViewSelectHeader.tsx @@ -0,0 +1,46 @@ +import React, { useRef } from 'react'; +import styled from 'styled-components'; +import { Button, Typography } from 'antd'; +import { PlusOutlined } from '@ant-design/icons'; + +const ButtonContainer = styled.div` + display: flex; + justify-content: space-between; +`; + +const NoMarginButton = styled(Button)` + && { + margin: 0px; + } +`; + +type Props = { + onClickCreateView: () => void; + onClickClear: () => void; +}; + +export const ViewSelectHeader = ({ onClickCreateView, onClickClear }: Props) => { + const clearButtonRef = useRef(null); + + const onHandleClickClear = () => { + (clearButtonRef?.current as any)?.blur(); + onClickClear(); + }; + + return ( + + + + Create View + + + Clear + + + ); +}; diff --git a/datahub-web-react/src/app/entity/view/select/ViewSelectToolTip.tsx b/datahub-web-react/src/app/entity/view/select/ViewSelectToolTip.tsx new file mode 100644 index 00000000000000..5e04250cc21b87 --- /dev/null +++ b/datahub-web-react/src/app/entity/view/select/ViewSelectToolTip.tsx @@ -0,0 +1,29 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Tooltip } from 'antd'; + +const HeaderText = styled.div` + margin-bottom: 8px; +`; + +type Props = { + children: React.ReactNode; + visible?: boolean; +}; + +export const ViewSelectToolTip = ({ children, visible = true }: Props) => { + return ( + + Select a View to apply to search results. +
Views help narrow down search results to those that matter most to you.
+ + } + > + {children} + + ); +}; diff --git a/datahub-web-react/src/app/entity/view/select/ViewsTableColumns.tsx b/datahub-web-react/src/app/entity/view/select/ViewsTableColumns.tsx new file mode 100644 index 00000000000000..48eb3b9add87fb --- /dev/null +++ b/datahub-web-react/src/app/entity/view/select/ViewsTableColumns.tsx @@ -0,0 +1,92 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Button, Typography } from 'antd'; +import { DataHubViewType } from '../../../../types.generated'; +import { ANTD_GRAY } from '../../shared/constants'; +import { ViewTypeLabel } from '../ViewTypeLabel'; +import { ViewDropdownMenu } from '../menu/ViewDropdownMenu'; +import { UserDefaultViewIcon } from '../shared/UserDefaultViewIcon'; +import { GlobalDefaultViewIcon } from '../shared/GlobalDefaultViewIcon'; +import { useUserContext } from '../../../context/useUserContext'; + +const StyledDescription = styled.div` + max-width: 300px; +`; + +const ActionButtonsContainer = styled.div` + display: flex; + justify-content: center; + align-items: center; + padding-right: 8px; +`; + +const NameContainer = styled.span` + display: flex; + align-items: center; + justify-content: left; +`; + +const IconPlaceholder = styled.span` + display: flex; + align-items: center; + justify-content: center; +`; + +type NameColumnProps = { + name: string; + record: any; + onEditView: (urn) => void; +}; + +export function NameColumn({ name, record, onEditView }: NameColumnProps) { + const userContext = useUserContext(); + const maybePersonalDefaultViewUrn = userContext.state?.views?.personalDefaultViewUrn; + const maybeGlobalDefaultViewUrn = userContext.state?.views?.globalDefaultViewUrn; + + const isUserDefault = record.urn === maybePersonalDefaultViewUrn; + const isGlobalDefault = record.urn === maybeGlobalDefaultViewUrn; + + return ( + + + {isUserDefault && } + {isGlobalDefault && } + + + + ); +} + +type DescriptionColumnProps = { + description: string; +}; + +export function DescriptionColumn({ description }: DescriptionColumnProps) { + return ( + + {description || No description} + + ); +} + +type ViewTypeColumnProps = { + viewType: DataHubViewType; +}; + +export function ViewTypeColumn({ viewType }: ViewTypeColumnProps) { + return ; +} + +type ActionColumnProps = { + record: any; +}; + +export function ActionsColumn({ record }: ActionColumnProps) { + return ( + + + + ); +} diff --git a/datahub-web-react/src/app/entity/view/select/renderViewOptionGroup.tsx b/datahub-web-react/src/app/entity/view/select/renderViewOptionGroup.tsx new file mode 100644 index 00000000000000..3cc260bda1cf5e --- /dev/null +++ b/datahub-web-react/src/app/entity/view/select/renderViewOptionGroup.tsx @@ -0,0 +1,57 @@ +import React from 'react'; +import { Select } from 'antd'; +import { DataHubView } from '../../../../types.generated'; +import { ViewOption } from './ViewOption'; +import { UserContextType } from '../../../context/userContext'; + +const selectOptionStyle = { paddingLeft: 0 }; + +type Args = { + views: Array; + label: string; + userContext: UserContextType; + hoverViewUrn?: string; + isOwnedByUser?: boolean; + setHoverViewUrn: (viewUrn: string) => void; + onClickEditView: (view: DataHubView) => void; + onClickPreviewView: (view: DataHubView) => void; +}; + +export const renderViewOptionGroup = ({ + views, + label, + userContext, + hoverViewUrn, + isOwnedByUser, + setHoverViewUrn, + onClickEditView, + onClickPreviewView, +}: Args) => { + const maybePersonalDefaultViewUrn = userContext.state?.views?.personalDefaultViewUrn; + const maybeGlobalDefaultViewUrn = userContext.state?.views?.globalDefaultViewUrn; + + return ( + + {views.map((view) => ( + setHoverViewUrn(view.urn)} + key={view.urn} + label={view.name} + value={view.urn} + style={selectOptionStyle} + data-testid="view-select-item" + > + onClickEditView(view)} + onClickPreview={() => onClickPreviewView(view)} + /> + + ))} + + ); +}; diff --git a/datahub-web-react/src/app/entity/view/shared/DefaultViewIcon.tsx b/datahub-web-react/src/app/entity/view/shared/DefaultViewIcon.tsx new file mode 100644 index 00000000000000..99e7234b02ae86 --- /dev/null +++ b/datahub-web-react/src/app/entity/view/shared/DefaultViewIcon.tsx @@ -0,0 +1,36 @@ +import React from 'react'; +import styled from 'styled-components'; +import { Tooltip } from 'antd'; + +const CircleContainer = styled.span` + width: 10px; + height: 10px; + display: flex; + align-items: center; + justify-content: center; +`; + +const Circle = styled.span<{ color: string }>` + width: 6px; + height: 6px; + border-radius: 50%; + padding: 0px; + margin: 0px; + background-color: ${(props) => props.color}; + opacity: 1; +`; + +type Props = { + title?: React.ReactNode; + color: string; +}; + +export const DefaultViewIcon = ({ title, color }: Props) => { + return ( + + + + + + ); +}; diff --git a/datahub-web-react/src/app/entity/view/shared/GlobalDefaultViewIcon.tsx b/datahub-web-react/src/app/entity/view/shared/GlobalDefaultViewIcon.tsx new file mode 100644 index 00000000000000..1b2720d75b868a --- /dev/null +++ b/datahub-web-react/src/app/entity/view/shared/GlobalDefaultViewIcon.tsx @@ -0,0 +1,11 @@ +import React from 'react'; +import { ANTD_GRAY } from '../../shared/constants'; +import { DefaultViewIcon } from './DefaultViewIcon'; + +type Props = { + title?: React.ReactNode; +}; + +export const GlobalDefaultViewIcon = ({ title }: Props) => { + return ; +}; diff --git a/datahub-web-react/src/app/entity/view/shared/UserDefaultViewIcon.tsx b/datahub-web-react/src/app/entity/view/shared/UserDefaultViewIcon.tsx new file mode 100644 index 00000000000000..3d78e1c49c314f --- /dev/null +++ b/datahub-web-react/src/app/entity/view/shared/UserDefaultViewIcon.tsx @@ -0,0 +1,11 @@ +import React from 'react'; +import { REDESIGN_COLORS } from '../../shared/constants'; +import { DefaultViewIcon } from './DefaultViewIcon'; + +type Props = { + title?: React.ReactNode; +}; + +export const UserDefaultViewIcon = ({ title }: Props) => { + return ; +}; diff --git a/datahub-web-react/src/app/entity/view/types.ts b/datahub-web-react/src/app/entity/view/types.ts new file mode 100644 index 00000000000000..7cb89869d3c0a9 --- /dev/null +++ b/datahub-web-react/src/app/entity/view/types.ts @@ -0,0 +1,52 @@ +import { DataHubViewFilter, DataHubViewType, EntityType, LogicalOperator } from '../../../types.generated'; + +/** + * Default builder state when creating a new View. + */ +export const DEFAULT_BUILDER_STATE = { + viewType: DataHubViewType.Personal, + name: '', + description: null, + definition: { + entityTypes: [], + filter: { + operator: LogicalOperator.And, + filters: [], + }, + }, +} as ViewBuilderState; + +/** + * The object represents the state of the Test Builder form. + */ +export interface ViewBuilderState { + /** + * The type of the View + */ + viewType?: DataHubViewType; + + /** + * The name of the View. + */ + name?: string; + + /** + * An optional description for the View. + */ + description?: string | null; + + /** + * The definition of the View + */ + definition?: { + /** + * The Entity Types in scope for the View. + */ + entityTypes?: EntityType[] | null; + + /** + * The Filter for the View. + */ + filter?: DataHubViewFilter; + }; +} diff --git a/datahub-web-react/src/app/entity/view/utils.ts b/datahub-web-react/src/app/entity/view/utils.ts new file mode 100644 index 00000000000000..68b64714c7ce99 --- /dev/null +++ b/datahub-web-react/src/app/entity/view/utils.ts @@ -0,0 +1,83 @@ +import { DataHubView, DataHubViewType, EntityType, LogicalOperator } from '../../../types.generated'; +import { ViewBuilderState } from './types'; + +/** + * The max single-page results in both the View Select and Manage Views list. + * + * The explicit assumption is that the number of public + personal views for a user + * will not exceed this number. + * + * In the future, we will need to consider pagination, or bumping this + * limit if we find that this maximum is reached. + */ +export const DEFAULT_LIST_VIEWS_PAGE_SIZE = 1000; + +/** + * Converts an instance of the View builder state + * into the input required to create or update a View in + * GraphQL. + * + * @param state the builder state + */ +export const convertStateToUpdateInput = (state: ViewBuilderState) => { + return { + viewType: state.viewType, + name: state.name as string, + description: state.description as string, + definition: { + entityTypes: state?.definition?.entityTypes, + filter: { + operator: state?.definition?.filter?.operator, + filters: state?.definition?.filter?.filters.map((filter) => ({ + field: filter.field, + condition: filter.condition, + values: filter.values, + negated: filter.negated, + })), + }, + }, + }; +}; + +/** + * Convert ViewBuilderState and an URN into a DataHubView object. + * + * @param urn urn of the View + * @param state state of the View + */ +export const convertStateToView = (urn: string, state: ViewBuilderState): DataHubView => { + return { + urn, + type: EntityType.DatahubView, + viewType: state.viewType as DataHubViewType, + name: state.name as string, + description: state.description, + definition: { + entityTypes: state.definition?.entityTypes || [], + filter: { + operator: state?.definition?.filter?.operator as LogicalOperator, + filters: state?.definition?.filter?.filters?.map((filter) => ({ + field: filter.field, + condition: filter.condition, + values: filter.values, + negated: filter.negated || false, + })) as any, + }, + }, + }; +}; + +/** + * Search through a list of Views by a text string by comparing + * against View name and descriptions. + * + * @param views: A list of DataHub View objects. + * @param q: An optional search query. + */ +export const searchViews = (views: Array, q?: string) => { + if (q && q.length > 0) { + const qLower = q.toLowerCase(); + return views.filter((view) => view.name.toLowerCase().includes(qLower) || view.description?.includes(qLower)); + } + return views; +}; diff --git a/datahub-web-react/src/app/search/AdvancedFilterSelectValueModal.tsx b/datahub-web-react/src/app/search/AdvancedFilterSelectValueModal.tsx index 3fa8e5c35b8244..c562fc6e8349a9 100644 --- a/datahub-web-react/src/app/search/AdvancedFilterSelectValueModal.tsx +++ b/datahub-web-react/src/app/search/AdvancedFilterSelectValueModal.tsx @@ -7,6 +7,23 @@ import { SelectPlatformModal } from '../entity/shared/containers/profile/sidebar import EditTagTermsModal from '../shared/tags/AddTagsTermsModal'; import { ChooseEntityTypeModal } from './ChooseEntityTypeModal'; import { EditTextModal } from './EditTextModal'; +import { + CONTAINER_FILTER_NAME, + DESCRIPTION_FILTER_NAME, + DOMAINS_FILTER_NAME, + ENTITY_FILTER_NAME, + FIELD_DESCRIPTIONS_FILTER_NAME, + FIELD_GLOSSARY_TERMS_FILTER_NAME, + FIELD_PATHS_FILTER_NAME, + FIELD_TAGS_FILTER_NAME, + GLOSSARY_TERMS_FILTER_NAME, + ORIGIN_FILTER_NAME, + OWNERS_FILTER_NAME, + PLATFORM_FILTER_NAME, + REMOVED_FILTER_NAME, + TAGS_FILTER_NAME, + TYPE_NAMES_FILTER_NAME, +} from './utils/constants'; type Props = { facet?: FacetMetadata | null; @@ -23,7 +40,7 @@ export const AdvancedFilterSelectValueModal = ({ initialValues, facet, }: Props) => { - if (filterField === 'owners') { + if (filterField === OWNERS_FILTER_NAME) { return ( ); } - if (filterField === 'domains') { + if (filterField === DOMAINS_FILTER_NAME) { return ( ({ @@ -95,7 +112,7 @@ export const AdvancedFilterSelectValueModal = ({ ); } - if (filterField === 'fieldPaths') { + if (filterField === FIELD_PATHS_FILTER_NAME) { return ( { @@ -151,21 +168,21 @@ export const AdvancedFilterSelectValueModal = ({ ); } - if (filterField === 'entity') { + if (filterField === ENTITY_FILTER_NAME) { return ( { - onSelect([newValue]); + onOk={(newValues) => { + onSelect(newValues); onCloseModal(); }} /> ); } - if (filterField === 'tags' || filterField === 'fieldTags') { + if (filterField === TAGS_FILTER_NAME || filterField === FIELD_TAGS_FILTER_NAME) { return ( ; onFilterFieldSelect: (value) => void; @@ -30,24 +32,22 @@ export const AdvancedSearchAddFilterSelect = ({ selectedFilters, onFilterFieldSe ), }} labelInValue - style={{ padding: 6, fontWeight: 500 }} + style={selectStyle} onChange={onFilterFieldSelect} dropdownMatchSelectWidth={false} filterOption={(_, option) => option?.value === 'null'} > - {Object.keys(FIELD_TO_LABEL) - .sort((a, b) => FIELD_TO_LABEL[a].localeCompare(FIELD_TO_LABEL[b])) - .filter((key) => key !== 'degree') - .map((key) => ( - - ))} + {ORDERED_FIELDS.filter((key) => key !== DEGREE_FILTER_NAME).map((key) => ( + + ))} ); }; diff --git a/datahub-web-react/src/app/search/AdvancedSearchFilter.tsx b/datahub-web-react/src/app/search/AdvancedSearchFilter.tsx index ebf0e585a4fea8..3ed1cf8c9a6dbe 100644 --- a/datahub-web-react/src/app/search/AdvancedSearchFilter.tsx +++ b/datahub-web-react/src/app/search/AdvancedSearchFilter.tsx @@ -15,13 +15,14 @@ type Props = { onClose: () => void; onUpdate: (newValue: FacetFilterInput) => void; loading: boolean; + disabled?: boolean; }; const FilterContainer = styled.div` box-shadow: 0px 0px 4px 0px #00000010; border-radius: 10px; border: 1px solid ${ANTD_GRAY[4]}; - padding: 5px; + padding: 4px; margin: 4px; :hover { cursor: pointer; @@ -36,6 +37,10 @@ const FieldFilterSection = styled.span` justify-content: space-between; `; +const FieldFilterSelect = styled.span` + padding-right: 8px; +`; + const CloseSpan = styled.span` :hover { color: black; @@ -47,7 +52,7 @@ const FilterFieldLabel = styled.span` margin-right: 2px; `; -export const AdvancedSearchFilter = ({ facet, filter, onClose, onUpdate, loading }: Props) => { +export const AdvancedSearchFilter = ({ facet, filter, onClose, onUpdate, loading, disabled = false }: Props) => { const [isEditing, setIsEditing] = useState(false); return ( <> @@ -57,26 +62,28 @@ export const AdvancedSearchFilter = ({ facet, filter, onClose, onUpdate, loading }} > - + {FIELD_TO_LABEL[filter.field]} - - { - e.preventDefault(); - e.stopPropagation(); - onClose(); - }} - tabIndex={0} - onKeyPress={onClose} - > - - + + {!disabled && ( + { + e.preventDefault(); + e.stopPropagation(); + onClose(); + }} + tabIndex={0} + onKeyPress={onClose} + > + + + )} {!loading && } - {isEditing && ( + {!disabled && isEditing && ( setIsEditing(false)} diff --git a/datahub-web-react/src/app/search/AdvancedSearchFilterConditionSelect.tsx b/datahub-web-react/src/app/search/AdvancedSearchFilterConditionSelect.tsx index 073c3d9617572b..49faa8a7af04ed 100644 --- a/datahub-web-react/src/app/search/AdvancedSearchFilterConditionSelect.tsx +++ b/datahub-web-react/src/app/search/AdvancedSearchFilterConditionSelect.tsx @@ -1,10 +1,19 @@ import { Select } from 'antd'; import React from 'react'; import styled from 'styled-components/macro'; - import { FacetFilterInput } from '../../types.generated'; import { ANTD_GRAY } from '../entity/shared/constants'; -import { FIELDS_THAT_USE_CONTAINS_OPERATOR } from './utils/constants'; +import { + DESCRIPTION_FILTER_NAME, + DOMAINS_FILTER_NAME, + ENTITY_FILTER_NAME, + FIELDS_THAT_USE_CONTAINS_OPERATOR, + FIELD_DESCRIPTIONS_FILTER_NAME, + FIELD_PATHS_FILTER_NAME, + ORIGIN_FILTER_NAME, + REMOVED_FILTER_NAME, + TYPE_NAMES_FILTER_NAME, +} from './utils/constants'; type Props = { filter: FacetFilterInput; @@ -16,14 +25,13 @@ const { Option } = Select; // We track which fields are collection fields for the purpose of printing the conditions // in a more gramatically correct way. On the backend they are handled the same. const filtersOnNonCollectionFields = [ - 'description', - 'fieldDescriptions', - 'fieldPaths', - 'removed', - 'typeNames', - 'entity', - 'domains', - 'origin', + DESCRIPTION_FILTER_NAME, + FIELD_DESCRIPTIONS_FILTER_NAME, + FIELD_PATHS_FILTER_NAME, + REMOVED_FILTER_NAME, + TYPE_NAMES_FILTER_NAME, + DOMAINS_FILTER_NAME, + ORIGIN_FILTER_NAME, ]; function getLabelsForField(field: string) { @@ -42,17 +50,19 @@ function getLabelsForField(field: string) { // collection field return { - default: 'is either of', + default: 'is any of', negated: 'is not', }; } const StyledSelect = styled(Select)` border-radius: 5px; - background: ${ANTD_GRAY[4]}; + color: ${ANTD_GRAY[9]}; + background: ${ANTD_GRAY[3]}; :hover { background: ${ANTD_GRAY[4.5]}; } + width: auto; `; export const AdvancedSearchFilterConditionSelect = ({ filter, onUpdate }: Props) => { @@ -81,7 +91,7 @@ export const AdvancedSearchFilterConditionSelect = ({ filter, onUpdate }: Props) } }} size="small" - disabled={filter.field === 'entity'} + disabled={filter.field === ENTITY_FILTER_NAME} dropdownMatchSelectWidth={false} > {Object.keys(labelsForField).map((labelKey) => ( diff --git a/datahub-web-react/src/app/search/AdvancedSearchFilterOverallUnionTypeSelect.tsx b/datahub-web-react/src/app/search/AdvancedSearchFilterOverallUnionTypeSelect.tsx index d74a4c2bc3229b..a37605182f3814 100644 --- a/datahub-web-react/src/app/search/AdvancedSearchFilterOverallUnionTypeSelect.tsx +++ b/datahub-web-react/src/app/search/AdvancedSearchFilterOverallUnionTypeSelect.tsx @@ -8,6 +8,7 @@ import { UnionType } from './utils/constants'; type Props = { unionType: UnionType; onUpdate: (newValue: UnionType) => void; + disabled?: boolean; }; const { Option } = Select; @@ -20,12 +21,13 @@ const StyledSelect = styled(Select)` } `; -export const AdvancedSearchFilterOverallUnionTypeSelect = ({ unionType, onUpdate }: Props) => { +export const AdvancedSearchFilterOverallUnionTypeSelect = ({ unionType, onUpdate, disabled = false }: Props) => { return ( <> { diff --git a/datahub-web-react/src/app/search/AdvancedSearchFilterValuesSection.tsx b/datahub-web-react/src/app/search/AdvancedSearchFilterValuesSection.tsx index eaa656338331d4..f24c6f18c2793e 100644 --- a/datahub-web-react/src/app/search/AdvancedSearchFilterValuesSection.tsx +++ b/datahub-web-react/src/app/search/AdvancedSearchFilterValuesSection.tsx @@ -2,7 +2,6 @@ import React from 'react'; import styled from 'styled-components'; import { FacetFilterInput, FacetMetadata } from '../../types.generated'; -import { ANTD_GRAY } from '../entity/shared/constants'; import { SearchFilterLabel } from './SearchFilterLabel'; type Props = { @@ -14,7 +13,7 @@ const ValueFilterSection = styled.div` :hover { cursor: pointer; } - border-top: 1px solid ${ANTD_GRAY[3]}; + margin: 4px; `; const StyledSearchFilterLabel = styled.div` @@ -26,11 +25,14 @@ export const AdvancedSearchFilterValuesSection = ({ facet, filter }: Props) => { {filter?.values?.map((value) => { const matchedAggregation = facet?.aggregations?.find((aggregation) => aggregation.value === value); - if (!matchedAggregation) return {value}; - return ( - - + + ); })} diff --git a/datahub-web-react/src/app/search/AdvancedSearchFilters.tsx b/datahub-web-react/src/app/search/AdvancedSearchFilters.tsx index ab242b49fed69f..0c6c7e32a5a19d 100644 --- a/datahub-web-react/src/app/search/AdvancedSearchFilters.tsx +++ b/datahub-web-react/src/app/search/AdvancedSearchFilters.tsx @@ -1,7 +1,6 @@ import * as React from 'react'; import { useState } from 'react'; import styled from 'styled-components'; - import { FacetFilterInput, FacetMetadata, FilterOperator } from '../../types.generated'; import { ANTD_GRAY } from '../entity/shared/constants'; import { AdvancedSearchFilter } from './AdvancedSearchFilter'; @@ -10,23 +9,6 @@ import { AdvancedFilterSelectValueModal } from './AdvancedFilterSelectValueModal import { FIELDS_THAT_USE_CONTAINS_OPERATOR, UnionType } from './utils/constants'; import { AdvancedSearchAddFilterSelect } from './AdvancedSearchAddFilterSelect'; -export const SearchFilterWrapper = styled.div` - flex: 1; - padding: 6px 12px 10px 12px; - overflow: auto; - - &::-webkit-scrollbar { - height: 12px; - width: 1px; - background: #f2f2f2; - } - &::-webkit-scrollbar-thumb { - background: #cccccc; - -webkit-border-radius: 1ex; - -webkit-box-shadow: 0px 1px 2px rgba(0, 0, 0, 0.75); - } -`; - const AnyAllSection = styled.div` padding: 6px; color: ${ANTD_GRAY[8]}; @@ -39,6 +21,16 @@ const EmptyStateSection = styled.div` margin-top: 10px; `; +const AdvancedSearchFiltersGroup = styled.div` + display: flex; + flex-wrap: wrap; +`; + +export enum LayoutDirection { + Horizontal = 'horizontal', + Vertical = 'vertical', +} + interface Props { selectedFilters: Array; facets: Array; @@ -46,6 +38,8 @@ interface Props { onChangeUnionType: (unionType: UnionType) => void; unionType?: UnionType; loading: boolean; + direction?: LayoutDirection; + disabled?: boolean; } export const AdvancedSearchFilters = ({ @@ -55,6 +49,8 @@ export const AdvancedSearchFilters = ({ onFilterSelect, onChangeUnionType, loading, + direction = LayoutDirection.Vertical, + disabled = false, }: Props) => { const [filterField, setFilterField] = useState(null); @@ -76,49 +72,58 @@ export const AdvancedSearchFilters = ({ }; return ( - - + <> + {!disabled && ( + + )} + + {selectedFilters.map((filter) => ( + facet.field === filter.field) || facets[0]} + loading={loading} + filter={filter} + onClose={() => { + onFilterSelect(selectedFilters.filter((f) => f !== filter)); + }} + onUpdate={(newValue) => { + onFilterSelect( + selectedFilters.map((f) => { + if (f === filter) { + return newValue; + } + return f; + }), + ); + }} + disabled={disabled} + /> + ))} + + {filterField && ( + facet.field === filterField) || null} + onCloseModal={() => setFilterField(null)} + filterField={filterField} + onSelect={onSelectValueFromModal} + /> + )} {selectedFilters?.length >= 2 && ( Show results that match{' '} onChangeUnionType(newValue)} + disabled={disabled} /> )} - {selectedFilters.map((filter) => ( - facet.field === filter.field) || facets[0]} - loading={loading} - filter={filter} - onClose={() => { - onFilterSelect(selectedFilters.filter((f) => f !== filter)); - }} - onUpdate={(newValue) => { - onFilterSelect( - selectedFilters.map((f) => { - if (f === filter) { - return newValue; - } - return f; - }), - ); - }} - /> - ))} - {filterField && ( - facet.field === filterField) || null} - onCloseModal={() => setFilterField(null)} - filterField={filterField} - onSelect={onSelectValueFromModal} - /> + {selectedFilters?.length === 0 && direction === LayoutDirection.Vertical && ( + No filters applied. )} - {selectedFilters?.length === 0 && No filters applied, add one above.} - + ); }; diff --git a/datahub-web-react/src/app/search/ChooseEntityTypeModal.tsx b/datahub-web-react/src/app/search/ChooseEntityTypeModal.tsx index fc0f2cdf22f089..645f6723668546 100644 --- a/datahub-web-react/src/app/search/ChooseEntityTypeModal.tsx +++ b/datahub-web-react/src/app/search/ChooseEntityTypeModal.tsx @@ -4,18 +4,26 @@ import { useEntityRegistry } from '../useEntityRegistry'; type Props = { onCloseModal: () => void; - onOk?: (result: string) => void; + onOk?: (results: string[]) => void; title?: string; - defaultValue?: string; + defaultValues?: string[]; }; const { Option } = Select; -export const ChooseEntityTypeModal = ({ defaultValue, onCloseModal, onOk, title }: Props) => { +export const ChooseEntityTypeModal = ({ defaultValues, onCloseModal, onOk, title }: Props) => { const entityRegistry = useEntityRegistry(); const entityTypes = entityRegistry.getSearchEntityTypes(); - const [stagedValue, setStagedValue] = useState(defaultValue || entityTypes[0]); + const [stagedValues, setStagedValues] = useState(defaultValues || []); + + const addEntityType = (newType) => { + setStagedValues([...stagedValues, newType]); + }; + + const removeEntityType = (type) => { + setStagedValues(stagedValues.filter((stagedValue) => stagedValue !== type)); + }; return ( Cancel - } > diff --git a/datahub-web-react/src/app/search/SaveAsViewButton.tsx b/datahub-web-react/src/app/search/SaveAsViewButton.tsx new file mode 100644 index 00000000000000..de5abac5980791 --- /dev/null +++ b/datahub-web-react/src/app/search/SaveAsViewButton.tsx @@ -0,0 +1,51 @@ +import { FilterOutlined } from '@ant-design/icons'; +import { Button, Tooltip } from 'antd'; +import React from 'react'; +import styled from 'styled-components'; + +const StyledButton = styled(Button)` + && { + margin: 0px; + margin-left: 6px; + padding: 0px; + } +`; + +const StyledFilterOutlined = styled(FilterOutlined)` + && { + font-size: 12px; + } +`; + +const SaveAsViewText = styled.span` + &&& { + margin-left: 4px; + } +`; + +const ToolTipHeader = styled.div` + margin-bottom: 12px; +`; + +type Props = { + onClick: () => void; +}; + +export const SaveAsViewButton = ({ onClick }: Props) => { + return ( + + Save these filters as a new View. +
Views allow you to easily save or share search filters.
+ + } + > + + + Save as View + + + ); +}; diff --git a/datahub-web-react/src/app/search/SearchFilterLabel.tsx b/datahub-web-react/src/app/search/SearchFilterLabel.tsx index bcd5249c360ecc..81bdab81626d62 100644 --- a/datahub-web-react/src/app/search/SearchFilterLabel.tsx +++ b/datahub-web-react/src/app/search/SearchFilterLabel.tsx @@ -3,7 +3,6 @@ import { BookOutlined } from '@ant-design/icons'; import { Tag, Tooltip } from 'antd'; import styled from 'styled-components'; import { - AggregationMetadata, Domain, Container, DataPlatform, @@ -13,6 +12,7 @@ import { CorpUser, CorpGroup, DataPlatformInstance, + Entity, } from '../../types.generated'; import { StyledTag } from '../entity/shared/components/styled/StyledTag'; import { capitalizeFirstLetter } from '../shared/textUtil'; @@ -24,8 +24,10 @@ import { IconStyleType } from '../entity/Entity'; import { formatNumber } from '../shared/formatNumber'; type Props = { - aggregation: AggregationMetadata; field: string; + value: string; + count?: number; + entity?: Entity | null; // TODO: If the entity is not provided, we should hydrate it. hideCount?: boolean; }; @@ -40,26 +42,22 @@ const PreviewImage = styled.img` const MAX_COUNT_VAL = 10000; // SearchFilterLabel renders custom labels for entity, tag, term & data platform filters. All other filters use the default behavior. -export const SearchFilterLabel = ({ aggregation, field, hideCount }: Props) => { +export const SearchFilterLabel = ({ field, value, entity, count, hideCount }: Props) => { const entityRegistry = useEntityRegistry(); - const countText = hideCount - ? '' - : ` (${aggregation.count === MAX_COUNT_VAL ? '10k+' : formatNumber(aggregation.count)})`; - - if (!aggregation) return <>; + const countText = hideCount ? '' : ` (${count === MAX_COUNT_VAL ? '10k+' : formatNumber(count)})`; if (field === ENTITY_FILTER_NAME) { - const entityType = aggregation.value.toUpperCase() as EntityType; + const entityType = value.toUpperCase() as EntityType; return ( - {entityType ? entityRegistry.getCollectionName(entityType) : aggregation.value} + {entityType ? entityRegistry.getCollectionName(entityType) : value} {countText} ); } - if (aggregation.entity?.type === EntityType.Tag) { - const tag = aggregation.entity as TagType; + if (entity?.type === EntityType.Tag) { + const tag = entity as TagType; const displayName = entityRegistry.getDisplayName(EntityType.Tag, tag); const truncatedDisplayName = displayName.length > 25 ? `${displayName.slice(0, 25)}...` : displayName; return ( @@ -72,8 +70,8 @@ export const SearchFilterLabel = ({ aggregation, field, hideCount }: Props) => { ); } - if (aggregation.entity?.type === EntityType.CorpUser) { - const user = aggregation.entity as CorpUser; + if (entity?.type === EntityType.CorpUser) { + const user = entity as CorpUser; const displayName = entityRegistry.getDisplayName(EntityType.CorpUser, user); const truncatedDisplayName = displayName.length > 25 ? `${displayName.slice(0, 25)}...` : displayName; return ( @@ -93,8 +91,8 @@ export const SearchFilterLabel = ({ aggregation, field, hideCount }: Props) => { ); } - if (aggregation.entity?.type === EntityType.CorpGroup) { - const group = aggregation.entity as CorpGroup; + if (entity?.type === EntityType.CorpGroup) { + const group = entity as CorpGroup; const displayName = entityRegistry.getDisplayName(EntityType.CorpGroup, group); const truncatedDisplayName = displayName.length > 25 ? `${displayName.slice(0, 25)}...` : displayName; return ( @@ -108,8 +106,8 @@ export const SearchFilterLabel = ({ aggregation, field, hideCount }: Props) => { ); } - if (aggregation.entity?.type === EntityType.GlossaryTerm) { - const term = aggregation.entity as GlossaryTerm; + if (entity?.type === EntityType.GlossaryTerm) { + const term = entity as GlossaryTerm; const displayName = entityRegistry.getDisplayName(EntityType.GlossaryTerm, term); const truncatedDisplayName = displayName.length > 25 ? `${displayName.slice(0, 25)}...` : displayName; return ( @@ -123,8 +121,8 @@ export const SearchFilterLabel = ({ aggregation, field, hideCount }: Props) => { ); } - if (aggregation.entity?.type === EntityType.DataPlatform) { - const platform = aggregation.entity as DataPlatform; + if (entity?.type === EntityType.DataPlatform) { + const platform = entity as DataPlatform; const displayName = platform.properties?.displayName || platform.info?.displayName || platform.name; const truncatedDisplayName = displayName.length > 25 ? `${displayName.slice(0, 25)}...` : displayName; return ( @@ -140,8 +138,8 @@ export const SearchFilterLabel = ({ aggregation, field, hideCount }: Props) => { ); } - if (aggregation.entity?.type === EntityType.DataPlatformInstance) { - const platform = aggregation.entity as DataPlatformInstance; + if (entity?.type === EntityType.DataPlatformInstance) { + const platform = entity as DataPlatformInstance; const displayName = platform.instanceId; const truncatedDisplayName = displayName.length > 25 ? `${displayName.slice(0, 25)}...` : displayName; return ( @@ -152,8 +150,8 @@ export const SearchFilterLabel = ({ aggregation, field, hideCount }: Props) => { ); } - if (aggregation.entity?.type === EntityType.Container) { - const container = aggregation.entity as Container; + if (entity?.type === EntityType.Container) { + const container = entity as Container; const displayName = entityRegistry.getDisplayName(EntityType.Container, container); const truncatedDisplayName = displayName.length > 25 ? `${displayName.slice(0, 25)}...` : displayName; return ( @@ -169,8 +167,8 @@ export const SearchFilterLabel = ({ aggregation, field, hideCount }: Props) => { ); } - if (aggregation.entity?.type === EntityType.Domain) { - const domain = aggregation.entity as Domain; + if (entity?.type === EntityType.Domain) { + const domain = entity as Domain; const displayName = entityRegistry.getDisplayName(EntityType.Domain, domain); const truncatedDomainName = displayName.length > 25 ? `${displayName.slice(0, 25)}...` : displayName; return ( @@ -183,7 +181,7 @@ export const SearchFilterLabel = ({ aggregation, field, hideCount }: Props) => { // Warning: Special casing for Sub-Types if (field === 'typeNames') { - const displayName = capitalizeFirstLetter(aggregation.value) || ''; + const displayName = capitalizeFirstLetter(value) || ''; const truncatedDomainName = displayName.length > 25 ? `${displayName.slice(0, 25)}...` : displayName; return ( @@ -196,11 +194,11 @@ export const SearchFilterLabel = ({ aggregation, field, hideCount }: Props) => { } if (field === 'degree') { - return <>{aggregation.value}; + return <>{value}; } return ( <> - {aggregation.value} + {value} {countText} ); diff --git a/datahub-web-react/src/app/search/SearchFiltersSection.tsx b/datahub-web-react/src/app/search/SearchFiltersSection.tsx index b315e55d7ebba3..4d04477fe889ef 100644 --- a/datahub-web-react/src/app/search/SearchFiltersSection.tsx +++ b/datahub-web-react/src/app/search/SearchFiltersSection.tsx @@ -7,6 +7,11 @@ import { hasAdvancedFilters } from './utils/hasAdvancedFilters'; import { AdvancedSearchFilters } from './AdvancedSearchFilters'; import { SimpleSearchFilters } from './SimpleSearchFilters'; import { SEARCH_RESULTS_ADVANCED_SEARCH_ID } from '../onboarding/config/SearchOnboardingConfig'; +import { ViewBuilder } from '../entity/view/builder/ViewBuilder'; +import { buildInitialViewState, fromUnionType } from '../entity/view/builder/utils'; +import { SaveAsViewButton } from './SaveAsViewButton'; +import { useUserContext } from '../context/useUserContext'; +import { ViewBuilderMode } from '../entity/view/builder/types'; type Props = { filters?: Array | null; @@ -25,7 +30,7 @@ const FiltersContainer = styled.div` overflow-wrap: break-word; border-right: 1px solid; border-color: ${(props) => props.theme.styles['border-color-base']}; - max-height: 100%; + height: 100%; `; const FiltersHeader = styled.div` @@ -46,9 +51,27 @@ const FiltersHeader = styled.div` display: flex; `; -const SearchFilterContainer = styled.div` - flex: 1; +const SearchFiltersWrapper = styled.div` + max-height: 100%; + padding-top: 10px; overflow: auto; + + &::-webkit-scrollbar { + height: 12px; + width: 1px; + background: #f2f2f2; + } + &::-webkit-scrollbar-thumb { + background: #cccccc; + -webkit-border-radius: 1ex; + -webkit-box-shadow: 0px 1px 2px rgba(0, 0, 0, 0.75); + } +`; + +const AdvancedSearchFiltersWrapper = styled.div` + margin-top: 6px; + margin-left: 12px; + margin-right: 12px; `; // This component renders the entire filters section that allows toggling @@ -61,9 +84,20 @@ export const SearchFiltersSection = ({ onChangeFilters, onChangeUnionType, }: Props) => { + const userContext = useUserContext(); const onlyShowAdvancedFilters = hasAdvancedFilters(selectedFilters, unionType); - + const [showViewBuilder, setShowViewBuilder] = useState(false); const [seeAdvancedFilters, setSeeAdvancedFilters] = useState(onlyShowAdvancedFilters); + + const onSaveAsView = () => { + setShowViewBuilder(true); + }; + + // Only show "Save As View" if there are selected Filters, and there is no + // current View applied (creating a new Filter on top of an existing View is not currently supported). + const selectedViewUrn = userContext?.localState?.selectedViewUrn; + const showSaveAsView = selectedFilters?.length > 0 && selectedViewUrn === undefined; + return ( @@ -79,25 +113,36 @@ export const SearchFiltersSection = ({ - {seeAdvancedFilters ? ( - onChangeFilters(newFilters)} - onChangeUnionType={onChangeUnionType} - facets={filters || []} - loading={loading} - /> - ) : ( - + + {seeAdvancedFilters ? ( + + onChangeFilters(newFilters)} + onChangeUnionType={onChangeUnionType} + facets={filters || []} + loading={loading} + /> + {showSaveAsView && } + {showViewBuilder && ( + setShowViewBuilder(false)} + onCancel={() => setShowViewBuilder(false)} + /> + )} + + ) : ( onChangeFilters(newFilters)} /> - - )} + )} + ); }; diff --git a/datahub-web-react/src/app/search/SearchHeader.tsx b/datahub-web-react/src/app/search/SearchHeader.tsx index 1372c29778ce69..b06b42792bced3 100644 --- a/datahub-web-react/src/app/search/SearchHeader.tsx +++ b/datahub-web-react/src/app/search/SearchHeader.tsx @@ -11,6 +11,7 @@ import { ANTD_GRAY } from '../entity/shared/constants'; import { HeaderLinks } from '../shared/admin/HeaderLinks'; import { useAppConfig } from '../useAppConfig'; import { DEFAULT_APP_CONFIG } from '../../appConfigContext'; +import { ViewSelect } from '../entity/view/select/ViewSelect'; const { Header } = Layout; @@ -47,6 +48,10 @@ const NavGroup = styled.div` min-width: 200px; `; +const ViewSelectContainer = styled.span` + margin-right: 14px; +`; + type Props = { initialQuery: string; placeholderText: string; @@ -78,6 +83,7 @@ export const SearchHeader = ({ const [isSearchBarFocused, setIsSearchBarFocused] = useState(false); const themeConfig = useTheme(); const appConfig = useAppConfig(); + const viewsEnabled = appConfig.config?.viewsConfig?.enabled; return (
@@ -104,6 +110,11 @@ export const SearchHeader = ({ /> + {viewsEnabled && ( + + + + )} diff --git a/datahub-web-react/src/app/search/SearchPage.tsx b/datahub-web-react/src/app/search/SearchPage.tsx index ed9377af97f9d6..3e73d53972e467 100644 --- a/datahub-web-react/src/app/search/SearchPage.tsx +++ b/datahub-web-react/src/app/search/SearchPage.tsx @@ -19,6 +19,7 @@ import { SEARCH_RESULTS_ADVANCED_SEARCH_ID, SEARCH_RESULTS_FILTERS_ID, } from '../onboarding/config/SearchOnboardingConfig'; +import { useUserContext } from '../context/useUserContext'; type SearchPageParams = { type?: string; @@ -30,6 +31,7 @@ type SearchPageParams = { export const SearchPage = () => { const history = useHistory(); const location = useLocation(); + const userContext = useUserContext(); const entityRegistry = useEntityRegistry(); const params = QueryString.parse(location.search, { arrayFormat: 'comma' }); @@ -37,6 +39,7 @@ export const SearchPage = () => { const activeType = entityRegistry.getTypeOrDefaultFromPathName(useParams().type || '', undefined); const page: number = params.page && Number(params.page as string) > 0 ? Number(params.page as string) : 1; const unionType: UnionType = Number(params.unionType as any as UnionType) || UnionType.AND; + const viewUrn = userContext.localState?.selectedViewUrn; const filters: Array = useFilters(params); const filtersWithoutEntities: Array = filters.filter( @@ -44,7 +47,7 @@ export const SearchPage = () => { ); const entityFilters: Array = filters .filter((filter) => filter.field === ENTITY_FILTER_NAME) - .flatMap((filter) => filter.values?.map((value) => value?.toUpperCase() as EntityType) || []); + .flatMap((filter) => (filter.values || []).map((value) => value?.toUpperCase() as EntityType)); const [numResultsPerPage, setNumResultsPerPage] = useState(SearchCfg.RESULTS_PER_PAGE); const [isSelectMode, setIsSelectMode] = useState(false); @@ -64,6 +67,7 @@ export const SearchPage = () => { count: numResultsPerPage, filters: [], orFilters: generateOrFilters(unionType, filtersWithoutEntities), + viewUrn, }, }, }); @@ -86,6 +90,7 @@ export const SearchPage = () => { count: SearchCfg.RESULTS_PER_PAGE, filters: [], orFilters: generateOrFilters(unionType, filtersWithoutEntities), + viewUrn, }, }, }); diff --git a/datahub-web-react/src/app/search/SimpleSearchFilter.tsx b/datahub-web-react/src/app/search/SimpleSearchFilter.tsx index f0c8f17e288604..9d7c32c4065384 100644 --- a/datahub-web-react/src/app/search/SimpleSearchFilter.tsx +++ b/datahub-web-react/src/app/search/SimpleSearchFilter.tsx @@ -101,7 +101,12 @@ export const SimpleSearchFilter = ({ facet, selectedFilters, onFilterSelect, def onFilterSelect(e.target.checked, facet.field, aggregation.value) } > - +
diff --git a/datahub-web-react/src/app/search/SimpleSearchFilters.tsx b/datahub-web-react/src/app/search/SimpleSearchFilters.tsx index 654341be7715c5..ef31ec4210d19d 100644 --- a/datahub-web-react/src/app/search/SimpleSearchFilters.tsx +++ b/datahub-web-react/src/app/search/SimpleSearchFilters.tsx @@ -1,28 +1,10 @@ import * as React from 'react'; -import styled from 'styled-components'; import { useEffect, useState } from 'react'; import { FacetFilterInput, FacetMetadata } from '../../types.generated'; import { SimpleSearchFilter } from './SimpleSearchFilter'; const TOP_FILTERS = ['degree', 'entity', 'tags', 'glossaryTerms', 'domains', 'owners']; -export const SearchFilterWrapper = styled.div` - padding-top: 10px; - max-height: 100%; - overflow: auto; - - &::-webkit-scrollbar { - height: 12px; - width: 1px; - background: #f2f2f2; - } - &::-webkit-scrollbar-thumb { - background: #cccccc; - -webkit-border-radius: 1ex; - -webkit-box-shadow: 0px 1px 2px rgba(0, 0, 0, 0.75); - } -`; - interface Props { facets: Array; selectedFilters: Array; @@ -67,7 +49,7 @@ export const SimpleSearchFilters = ({ facets, selectedFilters, onFilterSelect, l }); return ( - + <> {sortedFacets.map((facet) => ( ))} - + ); }; diff --git a/datahub-web-react/src/app/search/__tests__/constants.test.tsx b/datahub-web-react/src/app/search/__tests__/constants.test.tsx new file mode 100644 index 00000000000000..ad5f38e94483eb --- /dev/null +++ b/datahub-web-react/src/app/search/__tests__/constants.test.tsx @@ -0,0 +1,7 @@ +import { FIELD_TO_LABEL, ORDERED_FIELDS } from '../utils/constants'; + +describe('constants', () => { + it('ensure that all ordered fields have a corresponding label', () => { + expect(ORDERED_FIELDS.filter((field) => !Object.keys(FIELD_TO_LABEL).includes(field)).length).toEqual(0); + }); +}); diff --git a/datahub-web-react/src/app/search/utils/constants.ts b/datahub-web-react/src/app/search/utils/constants.ts index fdac81c659e3aa..b8aed7ec47b420 100644 --- a/datahub-web-react/src/app/search/utils/constants.ts +++ b/datahub-web-react/src/app/search/utils/constants.ts @@ -3,26 +3,56 @@ export const SEARCH_FOR_ENTITY_PREFIX = 'SEARCH__'; export const EXACT_SEARCH_PREFIX = 'EXACT__'; export const ENTITY_FILTER_NAME = 'entity'; -export const TAG_FILTER_NAME = 'tags'; -export const GLOSSARY_FILTER_NAME = 'glossaryTerms'; +export const TAGS_FILTER_NAME = 'tags'; +export const GLOSSARY_TERMS_FILTER_NAME = 'glossaryTerms'; export const CONTAINER_FILTER_NAME = 'container'; export const DOMAINS_FILTER_NAME = 'domains'; export const OWNERS_FILTER_NAME = 'owners'; -export const TYPE_FILTER_NAME = 'typeNames'; +export const TYPE_NAMES_FILTER_NAME = 'typeNames'; export const PLATFORM_FILTER_NAME = 'platform'; +export const FIELD_TAGS_FILTER_NAME = 'fieldTags'; +export const EDITED_FIELD_TAGS_FILTER_NAME = 'editedFieldTags'; +export const FIELD_GLOSSARY_TERMS_FILTER_NAME = 'fieldGlossaryTerms'; +export const EDITED_FIELD_GLOSSARY_TERMS_FILTER_NAME = 'editedFieldGlossaryTerms'; +export const FIELD_PATHS_FILTER_NAME = 'fieldPaths'; +export const FIELD_DESCRIPTIONS_FILTER_NAME = 'fieldDescriptions'; +export const EDITED_FIELD_DESCRIPTIONS_FILTER_NAME = 'editedFieldDescriptions'; +export const DESCRIPTION_FILTER_NAME = 'description'; +export const REMOVED_FILTER_NAME = 'removed'; +export const ORIGIN_FILTER_NAME = 'origin'; +export const DEGREE_FILTER_NAME = 'degree'; export const FILTERS_TO_TRUNCATE = [ - TAG_FILTER_NAME, - GLOSSARY_FILTER_NAME, + TAGS_FILTER_NAME, + GLOSSARY_TERMS_FILTER_NAME, CONTAINER_FILTER_NAME, DOMAINS_FILTER_NAME, OWNERS_FILTER_NAME, ENTITY_FILTER_NAME, - TYPE_FILTER_NAME, + TYPE_NAMES_FILTER_NAME, PLATFORM_FILTER_NAME, ]; export const TRUNCATED_FILTER_LENGTH = 5; +export const ORDERED_FIELDS = [ + ENTITY_FILTER_NAME, + PLATFORM_FILTER_NAME, + OWNERS_FILTER_NAME, + TAGS_FILTER_NAME, + GLOSSARY_TERMS_FILTER_NAME, + DOMAINS_FILTER_NAME, + FIELD_TAGS_FILTER_NAME, + FIELD_GLOSSARY_TERMS_FILTER_NAME, + FIELD_PATHS_FILTER_NAME, + FIELD_DESCRIPTIONS_FILTER_NAME, + DESCRIPTION_FILTER_NAME, + CONTAINER_FILTER_NAME, + REMOVED_FILTER_NAME, + TYPE_NAMES_FILTER_NAME, + ORIGIN_FILTER_NAME, + DEGREE_FILTER_NAME, +]; + export const FIELD_TO_LABEL = { owners: 'Owner', tags: 'Tag', @@ -37,26 +67,29 @@ export const FIELD_TO_LABEL = { removed: 'Soft Deleted', entity: 'Entity Type', container: 'Container', - typeNames: 'Subtype', + typeNames: 'Sub Type', origin: 'Environment', degree: 'Degree', }; -export const FIELDS_THAT_USE_CONTAINS_OPERATOR = ['description', 'fieldDescriptions']; +export const FIELDS_THAT_USE_CONTAINS_OPERATOR = [ + DESCRIPTION_FILTER_NAME, + FIELD_DESCRIPTIONS_FILTER_NAME, + FIELD_PATHS_FILTER_NAME, +]; export const ADVANCED_SEARCH_ONLY_FILTERS = [ - 'fieldGlossaryTerms', - 'editedFieldGlossaryTerms', - 'fieldTags', - 'editedFieldTags', - 'fieldPaths', - 'description', - 'fieldDescriptions', - 'removed', + FIELD_GLOSSARY_TERMS_FILTER_NAME, + EDITED_FIELD_GLOSSARY_TERMS_FILTER_NAME, + FIELD_TAGS_FILTER_NAME, + EDITED_FIELD_TAGS_FILTER_NAME, + FIELD_PATHS_FILTER_NAME, + DESCRIPTION_FILTER_NAME, + FIELD_DESCRIPTIONS_FILTER_NAME, + EDITED_FIELD_DESCRIPTIONS_FILTER_NAME, + REMOVED_FILTER_NAME, ]; -export const DEGREE_FILTER = 'degree'; - export enum UnionType { AND, OR, diff --git a/datahub-web-react/src/app/search/utils/filterUtils.ts b/datahub-web-react/src/app/search/utils/filterUtils.ts index ce0b3de88c94f3..85dfcde6114a24 100644 --- a/datahub-web-react/src/app/search/utils/filterUtils.ts +++ b/datahub-web-react/src/app/search/utils/filterUtils.ts @@ -1,4 +1,4 @@ -import { FacetFilterInput, OrFilter } from '../../../types.generated'; +import { FacetFilterInput, AndFilterInput } from '../../../types.generated'; import { FilterSet } from '../../entity/shared/components/styled/search/types'; import { UnionType } from './constants'; @@ -19,7 +19,7 @@ import { UnionType } from './constants'; * @param conjunction1 a conjunctive set of filters * @param conjunction2 a conjunctive set of filters */ -const mergeConjunctions = (conjunction1: FacetFilterInput[], conjunction2: FacetFilterInput[]): OrFilter[] => { +const mergeConjunctions = (conjunction1: FacetFilterInput[], conjunction2: FacetFilterInput[]): AndFilterInput[] => { return [ { and: [...conjunction1, ...conjunction2], @@ -55,8 +55,8 @@ const mergeConjunctions = (conjunction1: FacetFilterInput[], conjunction2: Facet * @param disjunction1 a disjunctive set of filters * @param disjunction2 a disjunctive set of filters */ -const mergeDisjunctions = (disjunction1: FacetFilterInput[], disjunction2: FacetFilterInput[]): OrFilter[] => { - const finalOrFilters: OrFilter[] = []; +const mergeDisjunctions = (disjunction1: FacetFilterInput[], disjunction2: FacetFilterInput[]): AndFilterInput[] => { + const finalOrFilters: AndFilterInput[] = []; disjunction1.forEach((d1) => { disjunction2.forEach((d2) => { @@ -90,8 +90,11 @@ const mergeDisjunctions = (disjunction1: FacetFilterInput[], disjunction2: Facet * @param conjunction a conjunctive set of filters * @param disjunction a disjunctive set of filters */ -const mergeConjunctionDisjunction = (conjunction: FacetFilterInput[], disjunction: FacetFilterInput[]): OrFilter[] => { - const finalOrFilters: OrFilter[] = []; +const mergeConjunctionDisjunction = ( + conjunction: FacetFilterInput[], + disjunction: FacetFilterInput[], +): AndFilterInput[] => { + const finalOrFilters: AndFilterInput[] = []; disjunction.forEach((filter) => { const andFilters = [filter, ...conjunction]; @@ -112,7 +115,7 @@ const mergeConjunctionDisjunction = (conjunction: FacetFilterInput[], disjunctio * @param orFilters the fixed set of filters in disjunction * @param baseFilters a base set of filters in either conjunction or disjunction */ -const mergeOrFilters = (orFilters: FacetFilterInput[], baseFilters: FilterSet): OrFilter[] => { +const mergeOrFilters = (orFilters: FacetFilterInput[], baseFilters: FilterSet): AndFilterInput[] => { // If the user-provided union type is AND, we need to treat differenty // than if user-provided union type is OR. if (baseFilters.unionType === UnionType.AND) { @@ -132,7 +135,7 @@ const mergeOrFilters = (orFilters: FacetFilterInput[], baseFilters: FilterSet): * @param andFilters the fixed set of filters in conjunction * @param baseFilters a base set of filters in either conjunction or disjunction */ -const mergeAndFilters = (andFilters: FacetFilterInput[], baseFilters: FilterSet): OrFilter[] => { +const mergeAndFilters = (andFilters: FacetFilterInput[], baseFilters: FilterSet): AndFilterInput[] => { // If the user-provided union type is AND, we need to treat differenty // than if user-provided union type is OR. if (baseFilters.unionType === UnionType.AND) { @@ -148,7 +151,7 @@ const mergeAndFilters = (andFilters: FacetFilterInput[], baseFilters: FilterSet) * @param filterSet1 the fixed set of filters to be merged. * @param filterSet2 the set of base filters to merge into. */ -export const mergeFilterSets = (filterSet1: FilterSet, filterSet2: FilterSet): OrFilter[] => { +export const mergeFilterSets = (filterSet1: FilterSet, filterSet2: FilterSet): AndFilterInput[] => { if (filterSet1 && filterSet2) { if (filterSet1.unionType === UnionType.AND) { // Inject fixed AND filters. diff --git a/datahub-web-react/src/app/search/utils/filtersToQueryStringParams.ts b/datahub-web-react/src/app/search/utils/filtersToQueryStringParams.ts index eec8afdf8a1262..fc34dad1467a36 100644 --- a/datahub-web-react/src/app/search/utils/filtersToQueryStringParams.ts +++ b/datahub-web-react/src/app/search/utils/filtersToQueryStringParams.ts @@ -1,6 +1,6 @@ import { FacetFilterInput, FilterOperator } from '../../../types.generated'; import { encodeComma } from '../../entity/shared/utils'; -import { DEGREE_FILTER, FILTER_URL_PREFIX } from './constants'; +import { DEGREE_FILTER_NAME, FILTER_URL_PREFIX } from './constants'; export const URL_PARAM_SEPARATOR = '___'; @@ -9,10 +9,10 @@ export const URL_PARAM_SEPARATOR = '___'; // we need to special case `degree` filter since it is a OR grouping vs the others which are ANDS by default function reduceFiltersToCombineDegreeFilters(acc: FacetFilterInput[], filter: FacetFilterInput) { // if we see a `degree` filter and we already have one, combine it with the other degree filter - if (filter.field === DEGREE_FILTER && acc.filter((f) => f.field === DEGREE_FILTER).length > 0) { + if (filter.field === DEGREE_FILTER_NAME && acc.filter((f) => f.field === DEGREE_FILTER_NAME).length > 0) { // instead of appending this new degree filter, combine it with the previous one and continue return acc.map((f) => - f.field === DEGREE_FILTER ? { ...f, values: [...(f.values || []), ...(filter.values || [])] } : f, + f.field === DEGREE_FILTER_NAME ? { ...f, values: [...(f.values || []), ...(filter.values || [])] } : f, ) as FacetFilterInput[]; } return [...acc, filter] as FacetFilterInput[]; diff --git a/datahub-web-react/src/app/search/utils/generateOrFilters.ts b/datahub-web-react/src/app/search/utils/generateOrFilters.ts index a798a6ada4b2a8..6f034eec14a929 100644 --- a/datahub-web-react/src/app/search/utils/generateOrFilters.ts +++ b/datahub-web-react/src/app/search/utils/generateOrFilters.ts @@ -1,7 +1,7 @@ -import { FacetFilterInput, OrFilter } from '../../../types.generated'; +import { FacetFilterInput, AndFilterInput } from '../../../types.generated'; import { UnionType } from './constants'; -export function generateOrFilters(unionType: UnionType, filters: FacetFilterInput[]): OrFilter[] { +export function generateOrFilters(unionType: UnionType, filters: FacetFilterInput[]): AndFilterInput[] { if ((filters?.length || 0) === 0) { return []; } diff --git a/datahub-web-react/src/app/search/utils/navigateToSearchUrl.ts b/datahub-web-react/src/app/search/utils/navigateToSearchUrl.ts index 73f797900419cf..28b1e9b917c78d 100644 --- a/datahub-web-react/src/app/search/utils/navigateToSearchUrl.ts +++ b/datahub-web-react/src/app/search/utils/navigateToSearchUrl.ts @@ -1,6 +1,5 @@ import * as QueryString from 'query-string'; import { RouteComponentProps } from 'react-router-dom'; - import filtersToQueryStringParams from './filtersToQueryStringParams'; import { EntityType, FacetFilterInput } from '../../../types.generated'; import { PageRoutes } from '../../../conf/Global'; diff --git a/datahub-web-react/src/app/settings/SettingsPage.tsx b/datahub-web-react/src/app/settings/SettingsPage.tsx index 59e4868e565c00..bde73e98dd42a2 100644 --- a/datahub-web-react/src/app/settings/SettingsPage.tsx +++ b/datahub-web-react/src/app/settings/SettingsPage.tsx @@ -1,6 +1,12 @@ import React from 'react'; import { Menu, Typography, Divider } from 'antd'; -import { BankOutlined, SafetyCertificateOutlined, UsergroupAddOutlined, ToolOutlined } from '@ant-design/icons'; +import { + BankOutlined, + SafetyCertificateOutlined, + UsergroupAddOutlined, + ToolOutlined, + FilterOutlined, +} from '@ant-design/icons'; import { Redirect, Route, useHistory, useLocation, useRouteMatch, Switch } from 'react-router'; import styled from 'styled-components'; import { ANTD_GRAY } from '../entity/shared/constants'; @@ -10,6 +16,7 @@ import { useAppConfig } from '../useAppConfig'; import { useGetAuthenticatedUser } from '../useGetAuthenticatedUser'; import { AccessTokens } from './AccessTokens'; import { Preferences } from './Preferences'; +import { ManageViews } from '../entity/view/ManageViews'; const PageContainer = styled.div` display: flex; @@ -51,6 +58,7 @@ const PATHS = [ { path: 'identities', content: }, { path: 'permissions', content: }, { path: 'preferences', content: }, + { path: 'views', content: }, ]; /** @@ -74,9 +82,11 @@ export const SettingsPage = () => { const isPoliciesEnabled = config?.policiesConfig.enabled; const isIdentityManagementEnabled = config?.identityManagementConfig.enabled; + const isViewsEnabled = config?.viewsConfig.enabled; const showPolicies = (isPoliciesEnabled && me && me.platformPrivileges.managePolicies) || false; const showUsersGroups = (isIdentityManagementEnabled && me && me.platformPrivileges.manageIdentities) || false; + const showViews = isViewsEnabled || false; return ( @@ -117,6 +127,13 @@ export const SettingsPage = () => { )} )} + {showViews && ( + + + My Views + + + )} diff --git a/datahub-web-react/src/app/shared/ManageAccount.tsx b/datahub-web-react/src/app/shared/ManageAccount.tsx index b4f18300e897e9..cc278bee3ce5b9 100644 --- a/datahub-web-react/src/app/shared/ManageAccount.tsx +++ b/datahub-web-react/src/app/shared/ManageAccount.tsx @@ -11,6 +11,7 @@ import CustomAvatar from './avatar/CustomAvatar'; import analytics, { EventType } from '../analytics'; import { ANTD_GRAY } from '../entity/shared/constants'; import { useAppConfig } from '../useAppConfig'; +import { useUserContext } from '../context/useUserContext'; const MenuItem = styled(Menu.Item)` display: flex; @@ -54,10 +55,12 @@ export const ManageAccount = ({ urn: _urn, pictureLink: _pictureLink, name }: Pr const entityRegistry = useEntityRegistry(); const themeConfig = useTheme(); const { config } = useAppConfig(); + const userContext = useUserContext(); const handleLogout = () => { analytics.event({ type: EventType.LogOutEvent }); isLoggedInVar(false); Cookies.remove(GlobalCfg.CLIENT_AUTH_COOKIE); + userContext.updateLocalState({ selectedViewUrn: undefined }); }; const version = config?.appVersion; const menu = ( diff --git a/datahub-web-react/src/appConfigContext.tsx b/datahub-web-react/src/appConfigContext.tsx index 7fd391d75a1ad1..7376fdbe2afe78 100644 --- a/datahub-web-react/src/appConfigContext.tsx +++ b/datahub-web-react/src/appConfigContext.tsx @@ -31,6 +31,9 @@ export const DEFAULT_APP_CONFIG = { testsConfig: { enabled: false, }, + viewsConfig: { + enabled: false, + }, }; export const AppConfigContext = React.createContext<{ diff --git a/datahub-web-react/src/conf/Global.ts b/datahub-web-react/src/conf/Global.ts index ba25907a287ca7..9281aaa4622f06 100644 --- a/datahub-web-react/src/conf/Global.ts +++ b/datahub-web-react/src/conf/Global.ts @@ -25,6 +25,7 @@ export enum PageRoutes { SETTINGS = '/settings', DOMAINS = '/domains', GLOSSARY = '/glossary', + SETTINGS_VIEWS = '/settings/views', } /** diff --git a/datahub-web-react/src/graphql/app.graphql b/datahub-web-react/src/graphql/app.graphql index 0c6bf0d34ee983..b09b27f0ed54db 100644 --- a/datahub-web-react/src/graphql/app.graphql +++ b/datahub-web-react/src/graphql/app.graphql @@ -44,6 +44,9 @@ query appConfig { testsConfig { enabled } + viewsConfig { + enabled + } } } @@ -55,3 +58,13 @@ query getEntityCounts($input: EntityCountInput) { } } } + +query getGlobalViewsSettings { + globalViewsSettings { + defaultView + } +} + +mutation updateGlobalViewsSettings($input: UpdateGlobalViewsSettingsInput!) { + updateGlobalViewsSettings(input: $input) +} diff --git a/datahub-web-react/src/graphql/entity.graphql b/datahub-web-react/src/graphql/entity.graphql new file mode 100644 index 00000000000000..6b75db064ca7a2 --- /dev/null +++ b/datahub-web-react/src/graphql/entity.graphql @@ -0,0 +1,7 @@ +query getEntities($urns: [String!]!) { + entities(urns: $urns) { + urn + type + ...searchResultFields + } +} diff --git a/datahub-web-react/src/graphql/me.graphql b/datahub-web-react/src/graphql/me.graphql index e771b012784bcc..cdbeb737a4937e 100644 --- a/datahub-web-react/src/graphql/me.graphql +++ b/datahub-web-react/src/graphql/me.graphql @@ -23,6 +23,11 @@ query getMe { appearance { showSimplifiedHomepage } + views { + defaultView { + urn + } + } } } platformPrivileges { @@ -39,6 +44,7 @@ query getMe { manageTags createDomains createTags + manageGlobalViews } } } diff --git a/datahub-web-react/src/graphql/user.graphql b/datahub-web-react/src/graphql/user.graphql index 8e3d33e2bc34e9..4757b8a7e28dd5 100644 --- a/datahub-web-react/src/graphql/user.graphql +++ b/datahub-web-react/src/graphql/user.graphql @@ -185,4 +185,8 @@ mutation createNativeUserResetToken($input: CreateNativeUserResetTokenInput!) { createNativeUserResetToken(input: $input) { resetToken } -} \ No newline at end of file +} + +mutation updateCorpUserViewsSettings($input: UpdateCorpUserViewsSettingsInput!) { + updateCorpUserViewsSettings(input: $input) +} diff --git a/datahub-web-react/src/graphql/view.graphql b/datahub-web-react/src/graphql/view.graphql new file mode 100644 index 00000000000000..9d38a09b8280bc --- /dev/null +++ b/datahub-web-react/src/graphql/view.graphql @@ -0,0 +1,56 @@ +fragment view on DataHubView { + urn + type + viewType + name + description + definition { + entityTypes + filter { + operator + filters { + field + condition + values + negated + } + } + } +} + +fragment listViewResults on ListViewsResult { + start + count + total + views { + ...view + } +} + +query listMyViews($viewType: DataHubViewType, $start: Int!, $count: Int!, $query: String) { + listMyViews(input: { viewType: $viewType, start: $start, count: $count, query: $query }) { + ...listViewResults + } +} + +query listGlobalViews($start: Int!, $count: Int!, $query: String) { + listGlobalViews(input: { start: $start, count: $count, query: $query }) { + ...listViewResults + } +} + +mutation createView($input: CreateViewInput!) { + createView(input: $input) { + ...view + } +} + +mutation updateView($urn: String!, $input: UpdateViewInput!) { + updateView(urn: $urn, input: $input) { + ...view + } +} + +mutation deleteView($urn: String!) { + deleteView(urn: $urn) +} diff --git a/docs/authorization/access-policies-guide.md b/docs/authorization/access-policies-guide.md index df9ba8954e0880..a659132d900364 100644 --- a/docs/authorization/access-policies-guide.md +++ b/docs/authorization/access-policies-guide.md @@ -108,6 +108,7 @@ In the second step, we can simply select the Privileges that this Platform Polic | Manage Glossaries | Allow the actor to create, edit, move, and delete Glossary Terms and Term Groups | | Create Tags | Allow the actor to create new Tags | | Manage Tags | Allow the actor to create and remove any Tags | +| Manage Public Views | Allow the actor to create, edit, and remove all Public Views. | #### Step 3: Choose Policy Actors diff --git a/li-utils/src/main/java/com/linkedin/metadata/Constants.java b/li-utils/src/main/java/com/linkedin/metadata/Constants.java index 751a4553754f78..3f450390258770 100644 --- a/li-utils/src/main/java/com/linkedin/metadata/Constants.java +++ b/li-utils/src/main/java/com/linkedin/metadata/Constants.java @@ -1,5 +1,8 @@ package com.linkedin.metadata; +import com.linkedin.common.urn.Urn; + + /** * Static class containing commonly-used constants across DataHub services. */ @@ -50,7 +53,7 @@ public class Constants { public static final String POST_ENTITY_NAME = "post"; public static final String SCHEMA_FIELD_ENTITY_NAME = "schemaField"; public static final String DATAHUB_STEP_STATE_ENTITY_NAME = "dataHubStepState"; - + public static final String DATAHUB_VIEW_ENTITY_NAME = "dataHubView"; /** * Aspects @@ -175,6 +178,7 @@ public class Constants { // Policy public static final String DATAHUB_POLICY_INFO_ASPECT_NAME = "dataHubPolicyInfo"; + // Role public static final String DATAHUB_ROLE_INFO_ASPECT_NAME = "dataHubRoleInfo"; @@ -236,13 +240,20 @@ public class Constants { public static final String DATA_HUB_UPGRADE_REQUEST_ASPECT_NAME = "dataHubUpgradeRequest"; public static final String DATA_HUB_UPGRADE_RESULT_ASPECT_NAME = "dataHubUpgradeResult"; - // Invite Token public static final String INVITE_TOKEN_ASPECT_NAME = "inviteToken"; public static final int INVITE_TOKEN_LENGTH = 32; public static final int SALT_TOKEN_LENGTH = 16; public static final int PASSWORD_RESET_TOKEN_LENGTH = 32; + // Views + public static final String DATAHUB_VIEW_KEY_ASPECT_NAME = "dataHubViewKey"; + public static final String DATAHUB_VIEW_INFO_ASPECT_NAME = "dataHubViewInfo"; + + // Settings + public static final String GLOBAL_SETTINGS_ENTITY_NAME = "globalSettings"; + public static final String GLOBAL_SETTINGS_INFO_ASPECT_NAME = "globalSettingsInfo"; + public static final Urn GLOBAL_SETTINGS_URN = Urn.createFromTuple(GLOBAL_SETTINGS_ENTITY_NAME, 0); // Relationships public static final String IS_MEMBER_OF_GROUP_RELATIONSHIP_NAME = "IsMemberOfGroup"; diff --git a/metadata-io/src/main/java/com/linkedin/metadata/config/ViewsConfiguration.java b/metadata-io/src/main/java/com/linkedin/metadata/config/ViewsConfiguration.java new file mode 100644 index 00000000000000..89c7376dfd110d --- /dev/null +++ b/metadata-io/src/main/java/com/linkedin/metadata/config/ViewsConfiguration.java @@ -0,0 +1,14 @@ +package com.linkedin.metadata.config; + +import lombok.Data; + +/** + * POJO representing the "views" configuration block in application.yml.on.yml + */ +@Data +public class ViewsConfiguration { + /** + * Whether Views are enabled + */ + public boolean enabled; +} \ No newline at end of file diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/SettingsService.java b/metadata-io/src/main/java/com/linkedin/metadata/service/SettingsService.java new file mode 100644 index 00000000000000..58645166a21ef4 --- /dev/null +++ b/metadata-io/src/main/java/com/linkedin/metadata/service/SettingsService.java @@ -0,0 +1,145 @@ +package com.linkedin.metadata.service; + +import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.urn.Urn; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.identity.CorpUserSettings; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.AspectUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.settings.global.GlobalSettingsInfo; +import java.util.Objects; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.extern.slf4j.Slf4j; + +import static com.linkedin.metadata.Constants.*; + + +/** + * This class is used to permit easy CRUD operations on both Global and Personal + * DataHub settings. + * + * Note that no Authorization is performed within the service. The expectation + * is that the caller has already verified the permissions of the active Actor. + */ +@Slf4j +public class SettingsService extends BaseService { + + public SettingsService(@Nonnull final EntityClient entityClient, @Nonnull final Authentication systemAuthentication) { + super(entityClient, systemAuthentication); + } + + /** + * Returns the settings for a particular user, or null if they do not exist yet. + * + * @param user the urn of the user to fetch settings for + * @param authentication the current authentication + * + * @return an instance of {@link CorpUserSettings} for the specified user, or null if none exists. + */ + @Nullable + public CorpUserSettings getCorpUserSettings( + @Nonnull final Urn user, + @Nonnull final Authentication authentication) { + Objects.requireNonNull(user, "user must not be null"); + Objects.requireNonNull(authentication, "authentication must not be null"); + try { + EntityResponse response = this.entityClient.getV2( + CORP_USER_ENTITY_NAME, + user, + ImmutableSet.of(CORP_USER_SETTINGS_ASPECT_NAME), + authentication + ); + if (response != null && response.getAspects().containsKey(Constants.CORP_USER_SETTINGS_ASPECT_NAME)) { + return new CorpUserSettings(response.getAspects().get(Constants.CORP_USER_SETTINGS_ASPECT_NAME).getValue().data()); + } + // No aspect found + return null; + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to retrieve Corp User settings for user with urn %s", user), e); + } + } + + /** + * Updates the settings for a given user. + * + * Note that this method does not do authorization validation. + * It is assumed that users of this class have already authorized the operation. + * + * @param user the urn of the user + * @param authentication the current authentication + */ + public void updateCorpUserSettings( + @Nonnull final Urn user, + @Nonnull final CorpUserSettings newSettings, + @Nonnull final Authentication authentication) { + Objects.requireNonNull(user, "user must not be null"); + Objects.requireNonNull(newSettings, "newSettings must not be null"); + Objects.requireNonNull(authentication, "authentication must not be null"); + try { + MetadataChangeProposal proposal = AspectUtils.buildMetadataChangeProposal( + user, + CORP_USER_SETTINGS_ASPECT_NAME, + newSettings); + this.entityClient.ingestProposal(proposal, authentication, false); + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to update Corp User settings for user with urn %s", user), e); + } + } + + /** + * Returns the Global Settings. They are expected to exist. + * + * @param authentication the current authentication + * @return an instance of {@link GlobalSettingsInfo}, or null if none exists. + */ + public GlobalSettingsInfo getGlobalSettings(@Nonnull final Authentication authentication) { + Objects.requireNonNull(authentication, "authentication must not be null"); + try { + EntityResponse response = this.entityClient.getV2( + GLOBAL_SETTINGS_ENTITY_NAME, + GLOBAL_SETTINGS_URN, + ImmutableSet.of(GLOBAL_SETTINGS_INFO_ASPECT_NAME), + authentication + ); + if (response != null && response.getAspects().containsKey(Constants.GLOBAL_SETTINGS_INFO_ASPECT_NAME)) { + return new GlobalSettingsInfo(response.getAspects().get(Constants.GLOBAL_SETTINGS_INFO_ASPECT_NAME).getValue().data()); + } + // No aspect found + log.warn("Failed to retrieve Global Settings. No settings exist, but they should. Returning null"); + return null; + } catch (Exception e) { + throw new RuntimeException("Failed to retrieve Global Settings!", e); + } + } + + /** + * Updates the Global settings. + * + * This performs a read-modify-write of the underlying GlobalSettingsInfo aspect. + * + * Note that this method does not do authorization validation. + * It is assumed that users of this class have already authorized the operation. + * + * @param newSettings the new value for the global settings. + * @param authentication the current authentication + */ + public void updateGlobalSettings( + @Nonnull final GlobalSettingsInfo newSettings, + @Nonnull final Authentication authentication) { + Objects.requireNonNull(newSettings, "newSettings must not be null"); + Objects.requireNonNull(authentication, "authentication must not be null"); + try { + MetadataChangeProposal proposal = AspectUtils.buildMetadataChangeProposal( + GLOBAL_SETTINGS_URN, + GLOBAL_SETTINGS_INFO_ASPECT_NAME, + newSettings); + this.entityClient.ingestProposal(proposal, authentication, false); + } catch (Exception e) { + throw new RuntimeException("Failed to update Global settings", e); + } + } +} \ No newline at end of file diff --git a/metadata-io/src/main/java/com/linkedin/metadata/service/ViewService.java b/metadata-io/src/main/java/com/linkedin/metadata/service/ViewService.java new file mode 100644 index 00000000000000..026eb3cd61def3 --- /dev/null +++ b/metadata-io/src/main/java/com/linkedin/metadata/service/ViewService.java @@ -0,0 +1,225 @@ +package com.linkedin.metadata.service; + +import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.SetMode; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.entity.AspectUtils; +import com.linkedin.metadata.key.DataHubViewKey; +import com.linkedin.metadata.utils.EntityKeyUtils; +import com.linkedin.view.DataHubViewDefinition; +import com.linkedin.view.DataHubViewInfo; +import com.linkedin.view.DataHubViewType; +import java.util.Objects; +import java.util.UUID; +import javax.annotation.Nonnull; +import javax.annotation.Nullable; +import lombok.extern.slf4j.Slf4j; + + +/** + * This class is used to permit easy CRUD operations on a DataHub View. + * Currently it supports creating, updating, and removing a View. + * + * Note that no Authorization is performed within the service. The expectation + * is that the caller has already verified the permissions of the active Actor. + * + * TODO: Ideally we have some basic caching of the view information inside of this class. + */ +@Slf4j +public class ViewService extends BaseService { + + public ViewService(@Nonnull EntityClient entityClient, @Nonnull Authentication systemAuthentication) { + super(entityClient, systemAuthentication); + } + + /** + * Creates a new DataHub View. + * + * Note that this method does not do authorization validation. + * It is assumed that users of this class have already authorized the operation. + * + * @param type the type of the View + * @param name the name of the View + * @param description the description of the View + * @param definition the view definition, a.k.a. the View definition + * @param authentication the current authentication + * + * @return the urn of the newly created View + */ + public Urn createView( + @Nonnull DataHubViewType type, + @Nonnull String name, + @Nullable String description, + @Nonnull DataHubViewDefinition definition, + @Nonnull Authentication authentication, + long currentTimeMs) { + Objects.requireNonNull(type, "type must not be null"); + Objects.requireNonNull(name, "name must not be null"); + Objects.requireNonNull(definition, "definition must not be null"); + Objects.requireNonNull(authentication, "authentication must not be null"); + + // 1. Generate a unique id for the new View. + final DataHubViewKey key = new DataHubViewKey(); + key.setId(UUID.randomUUID().toString()); + + // 2. Create a new instance of DataHubViewInfo + final DataHubViewInfo newView = new DataHubViewInfo(); + newView.setType(type); + newView.setName(name); + newView.setDescription(description, SetMode.IGNORE_NULL); + newView.setDefinition(definition); + final AuditStamp auditStamp = new AuditStamp() + .setActor(UrnUtils.getUrn(authentication.getActor().toUrnStr())) + .setTime(currentTimeMs); + newView.setCreated(auditStamp); + newView.setLastModified(auditStamp); + + + // 3. Write the new view to GMS, return the new URN. + try { + return UrnUtils.getUrn(this.entityClient.ingestProposal(AspectUtils.buildMetadataChangeProposal( + EntityKeyUtils.convertEntityKeyToUrn(key, Constants.DATAHUB_VIEW_ENTITY_NAME), Constants.DATAHUB_VIEW_INFO_ASPECT_NAME, newView), authentication, + false)); + } catch (Exception e) { + throw new RuntimeException("Failed to create View", e); + } + } + + /** + * Updates an existing DataHub View with a specific urn. The overwrites only the fields + * which are not null (provided). + * + * Note that this method does not do authorization validation. + * It is assumed that users of this class have already authorized the operation. + * + * The View with the provided urn must exist, else an {@link IllegalArgumentException} will be + * thrown. + * + * This method will perform a read-modify-write. This can cause concurrent writes + * to conflict, and overwrite one another. The expected frequency of writes + * for views is very low, however. TODO: Convert this into a safer patch. + * + * @param viewUrn the urn of the View + * @param name the name of the View + * @param description the description of the View + * @param definition the view definition itself + * @param authentication the current authentication + * @param currentTimeMs the current time in milliseconds, used for populating the lastUpdatedAt field. + */ + public void updateView( + @Nonnull Urn viewUrn, + @Nullable String name, + @Nullable String description, + @Nullable DataHubViewDefinition definition, + @Nonnull Authentication authentication, + long currentTimeMs) { + Objects.requireNonNull(viewUrn, "viewUrn must not be null"); + Objects.requireNonNull(authentication, "authentication must not be null"); + + // 1. Check whether the View exists + DataHubViewInfo existingInfo = getViewInfo(viewUrn, authentication); + + if (existingInfo == null) { + throw new IllegalArgumentException(String.format("Failed to update View. View with urn %s does not exist.", viewUrn)); + } + + // 2. Apply changes to existing View + if (name != null) { + existingInfo.setName(name); + } + if (description != null) { + existingInfo.setDescription(description); + } + if (definition != null) { + existingInfo.setDefinition(definition); + } + + existingInfo.setLastModified(new AuditStamp() + .setTime(currentTimeMs) + .setActor(UrnUtils.getUrn(authentication.getActor().toUrnStr()))); + + // 3. Write changes to GMS + try { + this.entityClient.ingestProposal( + AspectUtils.buildMetadataChangeProposal(viewUrn, Constants.DATAHUB_VIEW_INFO_ASPECT_NAME, existingInfo), + authentication, false); + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to update View with urn %s", viewUrn), e); + } + } + + /** + * Deletes an existing DataHub View with a specific urn. + * + * Note that this method does not do authorization validation. + * It is assumed that users of this class have already authorized the operation + * + * If the View does not exist, no exception will be thrown. + * + * @param viewUrn the urn of the View + * @param authentication the current authentication + */ + public void deleteView( + @Nonnull Urn viewUrn, + @Nonnull Authentication authentication) { + try { + this.entityClient.deleteEntity( + Objects.requireNonNull(viewUrn, "viewUrn must not be null"), + Objects.requireNonNull(authentication, "authentication must not be null")); + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to delete View with urn %s", viewUrn), e); + } + } + + /** + * Returns an instance of {@link DataHubViewInfo} for the specified View urn, + * or null if one cannot be found. + * + * @param viewUrn the urn of the View + * @param authentication the authentication to use + * + * @return an instance of {@link DataHubViewInfo} for the View, null if it does not exist. + */ + @Nullable + public DataHubViewInfo getViewInfo(@Nonnull final Urn viewUrn, @Nonnull final Authentication authentication) { + Objects.requireNonNull(viewUrn, "viewUrn must not be null"); + Objects.requireNonNull(authentication, "authentication must not be null"); + final EntityResponse response = getViewEntityResponse(viewUrn, authentication); + if (response != null && response.getAspects().containsKey(Constants.DATAHUB_VIEW_INFO_ASPECT_NAME)) { + return new DataHubViewInfo(response.getAspects().get(Constants.DATAHUB_VIEW_INFO_ASPECT_NAME).getValue().data()); + } + // No aspect found + return null; + } + + /** + * Returns an instance of {@link EntityResponse} for the specified View urn, + * or null if one cannot be found. + * + * @param viewUrn the urn of the View + * @param authentication the authentication to use + * + * @return an instance of {@link EntityResponse} for the View, null if it does not exist. + */ + @Nullable + public EntityResponse getViewEntityResponse(@Nonnull final Urn viewUrn, @Nonnull final Authentication authentication) { + Objects.requireNonNull(viewUrn, "viewUrn must not be null"); + Objects.requireNonNull(authentication, "authentication must not be null"); + try { + return this.entityClient.getV2( + Constants.DATAHUB_VIEW_ENTITY_NAME, + viewUrn, + ImmutableSet.of(Constants.DATAHUB_VIEW_INFO_ASPECT_NAME), + authentication + ); + } catch (Exception e) { + throw new RuntimeException(String.format("Failed to retrieve View with urn %s", viewUrn), e); + } + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/service/SettingsServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/service/SettingsServiceTest.java new file mode 100644 index 00000000000000..43ebc53385ad4f --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/service/SettingsServiceTest.java @@ -0,0 +1,317 @@ +package com.linkedin.metadata.service; + +import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.entity.Aspect; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.identity.CorpUserAppearanceSettings; +import com.linkedin.identity.CorpUserSettings; +import com.linkedin.identity.CorpUserViewsSettings; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.r2.RemoteInvocationException; +import com.linkedin.settings.global.GlobalSettingsInfo; +import com.linkedin.settings.global.GlobalViewsSettings; +import javax.annotation.Nullable; +import org.junit.Assert; +import org.mockito.Mockito; +import org.testng.annotations.Test; + +import static com.linkedin.metadata.Constants.*; + + +public class SettingsServiceTest { + + private static final Urn TEST_VIEW_URN = UrnUtils.getUrn("urn:li:dataHubView:test"); + private static final Urn TEST_USER_URN = UrnUtils.getUrn("urn:li:corpuser:test"); + + @Test + private static void testGetCorpUserSettingsNullSettings() throws Exception { + final SettingsService service = new SettingsService( + getCorpUserSettingsEntityClientMock(null), + Mockito.mock(Authentication.class) + ); + final CorpUserSettings res = service.getCorpUserSettings(TEST_USER_URN, Mockito.mock(Authentication.class)); + Assert.assertNull(res); + } + + @Test + private static void testGetCorpUserSettingsValidSettings() throws Exception { + final CorpUserSettings existingSettings = new CorpUserSettings() + .setViews(new CorpUserViewsSettings().setDefaultView(TEST_VIEW_URN)) + .setAppearance(new CorpUserAppearanceSettings().setShowSimplifiedHomepage(true)); + + final SettingsService service = new SettingsService( + getCorpUserSettingsEntityClientMock(existingSettings), + Mockito.mock(Authentication.class) + ); + + final CorpUserSettings res = service.getCorpUserSettings(TEST_USER_URN, Mockito.mock(Authentication.class)); + Assert.assertEquals(existingSettings, res); + } + + @Test + private static void testGetCorpUserSettingsSettingsException() throws Exception { + final EntityClient mockClient = Mockito.mock(EntityClient.class); + + Mockito.when(mockClient.getV2( + Mockito.eq(Constants.CORP_USER_ENTITY_NAME), + Mockito.eq(TEST_USER_URN), + Mockito.eq(ImmutableSet.of(Constants.CORP_USER_SETTINGS_ASPECT_NAME)), + Mockito.any(Authentication.class) + )).thenThrow(new RemoteInvocationException()); + + final SettingsService service = new SettingsService( + mockClient, + Mockito.mock(Authentication.class) + ); + + Assert.assertThrows(RuntimeException.class, () -> service.getCorpUserSettings(TEST_USER_URN, Mockito.mock(Authentication.class))); + } + + @Test + private static void testUpdateCorpUserSettingsValidSettings() throws Exception { + + final CorpUserSettings newSettings = new CorpUserSettings() + .setViews(new CorpUserViewsSettings().setDefaultView(TEST_VIEW_URN)) + .setAppearance(new CorpUserAppearanceSettings().setShowSimplifiedHomepage(true)); + + final MetadataChangeProposal expectedProposal = buildUpdateCorpUserSettingsChangeProposal( + TEST_USER_URN, + newSettings + ); + + final EntityClient mockClient = Mockito.mock(EntityClient.class); + Mockito.when(mockClient.ingestProposal( + Mockito.eq(expectedProposal), + Mockito.any(Authentication.class), + Mockito.eq(false) + )).thenReturn(TEST_USER_URN.toString()); + + final SettingsService service = new SettingsService( + mockClient, + Mockito.mock(Authentication.class) + ); + + service.updateCorpUserSettings( + TEST_USER_URN, + newSettings, + Mockito.mock(Authentication.class)); + + Mockito.verify(mockClient, Mockito.times(1)) + .ingestProposal( + Mockito.eq(expectedProposal), + Mockito.any(Authentication.class), + Mockito.eq(false) + ); + } + + @Test + private static void testUpdateCorpUserSettingsSettingsException() throws Exception { + + final CorpUserSettings newSettings = new CorpUserSettings() + .setViews(new CorpUserViewsSettings().setDefaultView(TEST_VIEW_URN)) + .setAppearance(new CorpUserAppearanceSettings().setShowSimplifiedHomepage(true)); + + final MetadataChangeProposal expectedProposal = buildUpdateCorpUserSettingsChangeProposal( + TEST_USER_URN, + newSettings + ); + + final EntityClient mockClient = Mockito.mock(EntityClient.class); + Mockito.when(mockClient.ingestProposal( + Mockito.eq(expectedProposal), + Mockito.any(Authentication.class), + Mockito.eq(false) + )).thenThrow(new RemoteInvocationException()); + + final SettingsService service = new SettingsService( + mockClient, + Mockito.mock(Authentication.class) + ); + + Assert.assertThrows(RuntimeException.class, () -> service.updateCorpUserSettings( + TEST_USER_URN, + newSettings, + Mockito.mock(Authentication.class))); + } + + @Test + private static void testGetGlobalSettingsNullSettings() throws Exception { + final SettingsService service = new SettingsService( + getGlobalSettingsEntityClientMock(null), + Mockito.mock(Authentication.class) + ); + final GlobalSettingsInfo res = service.getGlobalSettings(Mockito.mock(Authentication.class)); + Assert.assertNull(res); + } + + @Test + private static void testGetGlobalSettingsValidSettings() throws Exception { + final GlobalSettingsInfo existingSettings = new GlobalSettingsInfo() + .setViews(new GlobalViewsSettings().setDefaultView(TEST_VIEW_URN)); + + final SettingsService service = new SettingsService( + getGlobalSettingsEntityClientMock(existingSettings), + Mockito.mock(Authentication.class) + ); + + final GlobalSettingsInfo res = service.getGlobalSettings(Mockito.mock(Authentication.class)); + Assert.assertEquals(existingSettings, res); + } + + @Test + private static void testGetGlobalSettingsSettingsException() throws Exception { + final EntityClient mockClient = Mockito.mock(EntityClient.class); + + Mockito.when(mockClient.getV2( + Mockito.eq(GLOBAL_SETTINGS_ENTITY_NAME), + Mockito.eq(GLOBAL_SETTINGS_URN), + Mockito.eq(ImmutableSet.of(GLOBAL_SETTINGS_INFO_ASPECT_NAME)), + Mockito.any(Authentication.class) + )).thenThrow(new RemoteInvocationException()); + + final SettingsService service = new SettingsService( + mockClient, + Mockito.mock(Authentication.class) + ); + + Assert.assertThrows(RuntimeException.class, () -> service.getGlobalSettings(Mockito.mock(Authentication.class))); + } + + @Test + private static void testUpdateGlobalSettingsValidSettings() throws Exception { + + final GlobalSettingsInfo newSettings = new GlobalSettingsInfo() + .setViews(new GlobalViewsSettings().setDefaultView(TEST_VIEW_URN)); + + final MetadataChangeProposal expectedProposal = buildUpdateGlobalSettingsChangeProposal(newSettings); + + final EntityClient mockClient = Mockito.mock(EntityClient.class); + Mockito.when(mockClient.ingestProposal( + Mockito.eq(expectedProposal), + Mockito.any(Authentication.class), + Mockito.eq(false) + )).thenReturn(GLOBAL_SETTINGS_URN.toString()); + + final SettingsService service = new SettingsService( + mockClient, + Mockito.mock(Authentication.class) + ); + + service.updateGlobalSettings( + newSettings, + Mockito.mock(Authentication.class)); + + Mockito.verify(mockClient, Mockito.times(1)) + .ingestProposal( + Mockito.eq(expectedProposal), + Mockito.any(Authentication.class), + Mockito.eq(false) + ); + } + + @Test + private static void testUpdateGlobalSettingsSettingsException() throws Exception { + + final GlobalSettingsInfo newSettings = new GlobalSettingsInfo() + .setViews(new GlobalViewsSettings().setDefaultView(TEST_VIEW_URN)); + + final MetadataChangeProposal expectedProposal = buildUpdateGlobalSettingsChangeProposal( + newSettings + ); + + final EntityClient mockClient = Mockito.mock(EntityClient.class); + Mockito.when(mockClient.ingestProposal( + Mockito.eq(expectedProposal), + Mockito.any(Authentication.class), + Mockito.eq(false) + )).thenThrow(new RemoteInvocationException()); + + final SettingsService service = new SettingsService( + mockClient, + Mockito.mock(Authentication.class) + ); + + Assert.assertThrows(RuntimeException.class, () -> service.updateGlobalSettings( + newSettings, + Mockito.mock(Authentication.class))); + } + + private static EntityClient getCorpUserSettingsEntityClientMock(@Nullable final CorpUserSettings settings) + throws Exception { + EntityClient mockClient = Mockito.mock(EntityClient.class); + + EnvelopedAspectMap aspectMap = settings != null ? new EnvelopedAspectMap(ImmutableMap.of( + Constants.CORP_USER_SETTINGS_ASPECT_NAME, + new EnvelopedAspect().setValue(new Aspect(settings.data())) + )) : new EnvelopedAspectMap(); + + Mockito.when(mockClient.getV2( + Mockito.eq(Constants.CORP_USER_ENTITY_NAME), + Mockito.eq(TEST_USER_URN), + Mockito.eq(ImmutableSet.of(Constants.CORP_USER_SETTINGS_ASPECT_NAME)), + Mockito.any(Authentication.class) + )).thenReturn( + new EntityResponse() + .setEntityName(Constants.CORP_USER_ENTITY_NAME) + .setUrn(TEST_USER_URN) + .setAspects(aspectMap) + ); + return mockClient; + } + + private static EntityClient getGlobalSettingsEntityClientMock(@Nullable final GlobalSettingsInfo settings) + throws Exception { + EntityClient mockClient = Mockito.mock(EntityClient.class); + + EnvelopedAspectMap aspectMap = settings != null ? new EnvelopedAspectMap(ImmutableMap.of( + GLOBAL_SETTINGS_INFO_ASPECT_NAME, + new EnvelopedAspect().setValue(new Aspect(settings.data())) + )) : new EnvelopedAspectMap(); + + Mockito.when(mockClient.getV2( + Mockito.eq(GLOBAL_SETTINGS_ENTITY_NAME), + Mockito.eq(GLOBAL_SETTINGS_URN), + Mockito.eq(ImmutableSet.of(GLOBAL_SETTINGS_INFO_ASPECT_NAME)), + Mockito.any(Authentication.class) + )).thenReturn( + new EntityResponse() + .setEntityName(Constants.GLOBAL_SETTINGS_INFO_ASPECT_NAME) + .setUrn(GLOBAL_SETTINGS_URN) + .setAspects(aspectMap) + ); + return mockClient; + } + + private static MetadataChangeProposal buildUpdateCorpUserSettingsChangeProposal( + final Urn urn, + final CorpUserSettings newSettings) { + final MetadataChangeProposal mcp = new MetadataChangeProposal(); + mcp.setEntityUrn(urn); + mcp.setEntityType(CORP_USER_ENTITY_NAME); + mcp.setAspectName(CORP_USER_SETTINGS_ASPECT_NAME); + mcp.setChangeType(ChangeType.UPSERT); + mcp.setAspect(GenericRecordUtils.serializeAspect(newSettings)); + return mcp; + } + + private static MetadataChangeProposal buildUpdateGlobalSettingsChangeProposal( + final GlobalSettingsInfo newSettings) { + final MetadataChangeProposal mcp = new MetadataChangeProposal(); + mcp.setEntityUrn(GLOBAL_SETTINGS_URN); + mcp.setEntityType(GLOBAL_SETTINGS_ENTITY_NAME); + mcp.setAspectName(GLOBAL_SETTINGS_INFO_ASPECT_NAME); + mcp.setChangeType(ChangeType.UPSERT); + mcp.setAspect(GenericRecordUtils.serializeAspect(newSettings)); + return mcp; + } +} diff --git a/metadata-io/src/test/java/com/linkedin/metadata/service/ViewServiceTest.java b/metadata-io/src/test/java/com/linkedin/metadata/service/ViewServiceTest.java new file mode 100644 index 00000000000000..79449c1ff62c1f --- /dev/null +++ b/metadata-io/src/test/java/com/linkedin/metadata/service/ViewServiceTest.java @@ -0,0 +1,617 @@ +package com.linkedin.metadata.service; + +import com.datahub.authentication.Actor; +import com.datahub.authentication.ActorType; +import com.datahub.authentication.Authentication; +import com.google.common.collect.ImmutableSet; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.data.template.StringArray; +import com.linkedin.entity.Aspect; +import com.linkedin.entity.EntityResponse; +import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.entity.EnvelopedAspectMap; +import com.linkedin.entity.client.EntityClient; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.query.filter.Condition; +import com.linkedin.metadata.query.filter.ConjunctiveCriterion; +import com.linkedin.metadata.query.filter.ConjunctiveCriterionArray; +import com.linkedin.metadata.query.filter.Criterion; +import com.linkedin.metadata.query.filter.CriterionArray; +import com.linkedin.metadata.query.filter.Filter; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.r2.RemoteInvocationException; +import com.linkedin.view.DataHubViewDefinition; +import com.linkedin.view.DataHubViewInfo; +import com.linkedin.view.DataHubViewType; +import java.util.Collections; +import org.mockito.Mockito; +import org.testcontainers.shaded.com.google.common.collect.ImmutableList; +import org.testcontainers.shaded.com.google.common.collect.ImmutableMap; +import org.testng.Assert; +import org.testng.annotations.Test; + +import static com.linkedin.metadata.Constants.*; + + +public class ViewServiceTest { + + private static final Urn TEST_VIEW_URN = UrnUtils.getUrn("urn:li:dataHubView:test"); + private static final Urn TEST_USER_URN = UrnUtils.getUrn("urn:li:corpuser:test"); + + @Test + private void testCreateViewSuccess() throws Exception { + + final EntityClient mockClient = createViewMockEntityClient(); + final ViewService service = new ViewService( + mockClient, + Mockito.mock(Authentication.class)); + + // Case 1: With description + Urn urn = service.createView(DataHubViewType.PERSONAL, + "test view", + "my description", + new DataHubViewDefinition() + .setEntityTypes(new StringArray(ImmutableList.of(DATASET_ENTITY_NAME, DASHBOARD_ENTITY_NAME))) + .setFilter(new Filter() + .setOr(new ConjunctiveCriterionArray(ImmutableList.of(new ConjunctiveCriterion() + .setAnd(new CriterionArray(ImmutableList.of(new Criterion() + .setField("field") + .setCondition(Condition.EQUAL) + .setValue("value") + ))))))), + mockAuthentication(), + 0L + ); + + Assert.assertEquals(urn, TEST_VIEW_URN); + Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( + Mockito.any(MetadataChangeProposal.class), + Mockito.any(Authentication.class), + Mockito.eq(false) + ); + + // Case 2: Without description + urn = service.createView(DataHubViewType.PERSONAL, + "test view", + null, + new DataHubViewDefinition() + .setEntityTypes(new StringArray(ImmutableList.of(DATASET_ENTITY_NAME, DASHBOARD_ENTITY_NAME))) + .setFilter(new Filter() + .setOr(new ConjunctiveCriterionArray(ImmutableList.of(new ConjunctiveCriterion() + .setAnd(new CriterionArray(ImmutableList.of(new Criterion() + .setField("field") + .setCondition(Condition.EQUAL) + .setValue("value") + ))))))), + mockAuthentication(), + 0L + ); + + Assert.assertEquals(urn, TEST_VIEW_URN); + Mockito.verify(mockClient, Mockito.times(2)).ingestProposal( + Mockito.any(MetadataChangeProposal.class), + Mockito.any(Authentication.class), + Mockito.eq(false) + ); + } + + @Test + private void testCreateViewErrorMissingInputs() throws Exception { + final EntityClient mockClient = createViewMockEntityClient(); + final ViewService service = new ViewService( + mockClient, + Mockito.mock(Authentication.class)); + + // Case 1: missing View Type + Assert.assertThrows( + RuntimeException.class, + () -> service.createView(null, + "test view", + "my description", + new DataHubViewDefinition() + .setEntityTypes(new StringArray(ImmutableList.of(DATASET_ENTITY_NAME, DASHBOARD_ENTITY_NAME))) + .setFilter(new Filter() + .setOr(new ConjunctiveCriterionArray(ImmutableList.of(new ConjunctiveCriterion() + .setAnd(new CriterionArray(ImmutableList.of(new Criterion() + .setField("field") + .setCondition(Condition.EQUAL) + .setValue("value") + ))))))), + mockAuthentication(), + 0L + ) + ); + + + // Case 2: missing View name + Assert.assertThrows( + RuntimeException.class, + () -> service.createView(DataHubViewType.PERSONAL, + null, + "my description", + new DataHubViewDefinition() + .setEntityTypes(new StringArray(ImmutableList.of(DATASET_ENTITY_NAME, DASHBOARD_ENTITY_NAME))) + .setFilter(new Filter() + .setOr(new ConjunctiveCriterionArray(ImmutableList.of(new ConjunctiveCriterion() + .setAnd(new CriterionArray(ImmutableList.of(new Criterion() + .setField("field") + .setCondition(Condition.EQUAL) + .setValue("value") + ))))))), + mockAuthentication(), + 0L + ) + ); + + // Case 3: missing View definition + Assert.assertThrows( + RuntimeException.class, + () -> service.createView(DataHubViewType.PERSONAL, + "My name", + "my description", + null, + mockAuthentication(), + 0L + ) + ); + } + + @Test + private void testCreateViewError() throws Exception { + final EntityClient mockClient = Mockito.mock(EntityClient.class); + + Mockito.doThrow(new RemoteInvocationException()).when(mockClient).ingestProposal( + Mockito.any(MetadataChangeProposal.class), + Mockito.any(Authentication.class), + Mockito.eq(false)); + + final ViewService service = new ViewService( + mockClient, + Mockito.mock(Authentication.class)); + + // Throws wrapped exception + Assert.assertThrows(RuntimeException.class, () -> service.createView( + DataHubViewType.PERSONAL, + "new name", + "my description", + new DataHubViewDefinition() + .setEntityTypes(new StringArray(ImmutableList.of(DATASET_ENTITY_NAME, DASHBOARD_ENTITY_NAME))) + .setFilter(new Filter() + .setOr(new ConjunctiveCriterionArray(ImmutableList.of(new ConjunctiveCriterion() + .setAnd(new CriterionArray(ImmutableList.of(new Criterion() + .setField("field") + .setCondition(Condition.EQUAL) + .setValue("value") + ))))))), + mockAuthentication(), + 1L + )); + } + + @Test + private void testUpdateViewSuccess() throws Exception { + final DataHubViewType type = DataHubViewType.PERSONAL; + final String oldName = "old name"; + final String oldDescription = "old description"; + final DataHubViewDefinition oldDefinition = new DataHubViewDefinition() + .setEntityTypes(new StringArray()) + .setFilter(new Filter().setOr(new ConjunctiveCriterionArray(Collections.emptyList()))); + + final EntityClient mockClient = Mockito.mock(EntityClient.class); + + resetUpdateViewMockEntityClient( + mockClient, + TEST_VIEW_URN, + type, + oldName, + oldDescription, + oldDefinition, + TEST_USER_URN, + 0L, + 0L + ); + + final ViewService service = new ViewService( + mockClient, + Mockito.mock(Authentication.class)); + final String newName = "new name"; + final String newDescription = "new description"; + final DataHubViewDefinition newDefinition = new DataHubViewDefinition() + .setEntityTypes(new StringArray(ImmutableList.of(DATASET_ENTITY_NAME, DASHBOARD_ENTITY_NAME))) + .setFilter(new Filter() + .setOr(new ConjunctiveCriterionArray(ImmutableList.of(new ConjunctiveCriterion() + .setAnd(new CriterionArray(ImmutableList.of(new Criterion() + .setField("field") + .setCondition(Condition.EQUAL) + .setValue("value") + ))))))); + + // Case 1: Update name only + service.updateView( + TEST_VIEW_URN, + newName, + null, + null, + mockAuthentication(), + 1L + ); + + Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( + Mockito.eq(buildUpdateViewProposal(TEST_VIEW_URN, type, newName, oldDescription, oldDefinition, 0L, 1L)), + Mockito.any(Authentication.class), + Mockito.eq(false) + ); + + resetUpdateViewMockEntityClient( + mockClient, + TEST_VIEW_URN, + type, + oldName, + oldDescription, + oldDefinition, + TEST_USER_URN, + 0L, + 0L + ); + + // Case 2: Update description only + service.updateView( + TEST_VIEW_URN, + null, + newDescription, + null, + mockAuthentication(), + 1L + ); + + Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( + Mockito.eq(buildUpdateViewProposal(TEST_VIEW_URN, type, oldName, newDescription, oldDefinition, 0L, 1L)), + Mockito.any(Authentication.class), + Mockito.eq(false) + ); + + resetUpdateViewMockEntityClient( + mockClient, + TEST_VIEW_URN, + type, + oldName, + oldDescription, + oldDefinition, + TEST_USER_URN, + 0L, + 0L + ); + + // Case 3: Update definition only + service.updateView(TEST_VIEW_URN, + null, + null, + newDefinition, + mockAuthentication(), + 1L + ); + + Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( + Mockito.eq(buildUpdateViewProposal(TEST_VIEW_URN, type, oldName, oldDescription, newDefinition, 0L, 1L)), + Mockito.any(Authentication.class), + Mockito.eq(false) + ); + + resetUpdateViewMockEntityClient( + mockClient, + TEST_VIEW_URN, + type, + oldName, + oldDescription, + oldDefinition, + TEST_USER_URN, + 0L, + 0L + ); + + // Case 4: Update all fields at once + service.updateView( + TEST_VIEW_URN, + newName, + newDescription, + newDefinition, + mockAuthentication(), + 1L + ); + + Mockito.verify(mockClient, Mockito.times(1)).ingestProposal( + Mockito.eq(buildUpdateViewProposal(TEST_VIEW_URN, type, newName, newDescription, newDefinition, 0L, 1L)), + Mockito.any(Authentication.class), + Mockito.eq(false) + ); + } + + @Test + private void testUpdateViewMissingView() throws Exception { + final EntityClient mockClient = Mockito.mock(EntityClient.class); + + Mockito.when(mockClient.getV2( + Mockito.eq(DATAHUB_VIEW_ENTITY_NAME), + Mockito.eq(TEST_VIEW_URN), + Mockito.eq(ImmutableSet.of(DATAHUB_VIEW_INFO_ASPECT_NAME)), + Mockito.any(Authentication.class))) + .thenReturn(null); + + final ViewService service = new ViewService( + mockClient, + Mockito.mock(Authentication.class)); + + final String newName = "new name"; + + // Throws wrapped exception + Assert.assertThrows(RuntimeException.class, () -> service.updateView( + TEST_VIEW_URN, + newName, + null, + null, + mockAuthentication(), + 1L + )); + } + + @Test + private void testUpdateViewError() throws Exception { + final EntityClient mockClient = Mockito.mock(EntityClient.class); + + Mockito.doThrow(new RemoteInvocationException()).when(mockClient).getV2( + Mockito.eq(DATAHUB_VIEW_ENTITY_NAME), + Mockito.eq(TEST_VIEW_URN), + Mockito.eq(ImmutableSet.of(DATAHUB_VIEW_INFO_ASPECT_NAME)), + Mockito.any(Authentication.class)); + + final ViewService service = new ViewService( + mockClient, + Mockito.mock(Authentication.class)); + + // Throws wrapped exception + Assert.assertThrows(RuntimeException.class, () -> service.updateView( + TEST_VIEW_URN, + "new name", + null, + null, + mockAuthentication(), + 1L + )); + } + + @Test + private void testDeleteViewSuccess() throws Exception { + final EntityClient mockClient = Mockito.mock(EntityClient.class); + + final ViewService service = new ViewService( + mockClient, + Mockito.mock(Authentication.class)); + + service.deleteView(TEST_VIEW_URN, mockAuthentication()); + + Mockito.verify(mockClient, Mockito.times(1)).deleteEntity( + Mockito.eq(TEST_VIEW_URN), + Mockito.any(Authentication.class) + ); + } + + @Test + private void testDeleteViewError() throws Exception { + final EntityClient mockClient = Mockito.mock(EntityClient.class); + + final ViewService service = new ViewService( + mockClient, + Mockito.mock(Authentication.class)); + + Mockito.doThrow(new RemoteInvocationException()).when(mockClient).deleteEntity( + Mockito.eq(TEST_VIEW_URN), + Mockito.any(Authentication.class) + ); + + // Throws wrapped exception + Assert.assertThrows(RuntimeException.class, () -> service.deleteView(TEST_VIEW_URN, mockAuthentication())); + } + + @Test + private void testGetViewInfoSuccess() throws Exception { + final EntityClient mockClient = Mockito.mock(EntityClient.class); + + final DataHubViewType type = DataHubViewType.PERSONAL; + final String name = "name"; + final String description = "description"; + final DataHubViewDefinition definition = new DataHubViewDefinition() + .setEntityTypes(new StringArray(ImmutableList.of(DATASET_ENTITY_NAME, DASHBOARD_ENTITY_NAME))) + .setFilter(new Filter() + .setOr(new ConjunctiveCriterionArray(ImmutableList.of(new ConjunctiveCriterion() + .setAnd(new CriterionArray(ImmutableList.of(new Criterion() + .setField("field") + .setCondition(Condition.EQUAL) + .setValue("value") + ))))))); + + resetGetViewInfoMockEntityClient( + mockClient, + TEST_VIEW_URN, + type, + name, + description, + definition, + TEST_USER_URN, + 0L, + 1L + ); + + final ViewService service = new ViewService( + mockClient, + Mockito.mock(Authentication.class)); + + final DataHubViewInfo info = service.getViewInfo(TEST_VIEW_URN, mockAuthentication()); + + // Assert that the info is correct. + Assert.assertEquals(info.getType(), type); + Assert.assertEquals((long) info.getCreated().getTime(), 0L); + Assert.assertEquals((long) info.getLastModified().getTime(), 1L); + Assert.assertEquals(info.getName(), name); + Assert.assertEquals(info.getDescription(), description); + Assert.assertEquals(info.getCreated().getActor(), TEST_USER_URN); + Assert.assertEquals(info.getDefinition(), definition); + } + + @Test + private void testGetViewInfoNoViewExists() throws Exception { + final EntityClient mockClient = Mockito.mock(EntityClient.class); + + Mockito.when(mockClient.getV2( + Mockito.eq(DATAHUB_VIEW_ENTITY_NAME), + Mockito.eq(TEST_VIEW_URN), + Mockito.eq(ImmutableSet.of(DATAHUB_VIEW_INFO_ASPECT_NAME)), + Mockito.any(Authentication.class))) + .thenReturn(null); + + final ViewService service = new ViewService( + mockClient, + Mockito.mock(Authentication.class)); + + Assert.assertNull(service.getViewInfo(TEST_VIEW_URN, mockAuthentication())); + + } + + @Test + private void testGetViewInfoError() throws Exception { + final EntityClient mockClient = Mockito.mock(EntityClient.class); + + Mockito.doThrow(new RemoteInvocationException()).when(mockClient).getV2( + Mockito.eq(DATAHUB_VIEW_ENTITY_NAME), + Mockito.eq(TEST_VIEW_URN), + Mockito.eq(ImmutableSet.of(DATAHUB_VIEW_INFO_ASPECT_NAME)), + Mockito.any(Authentication.class)); + + final ViewService service = new ViewService( + mockClient, + Mockito.mock(Authentication.class)); + + // Throws wrapped exception + Assert.assertThrows(RuntimeException.class, () -> service.getViewInfo(TEST_VIEW_URN, mockAuthentication())); + } + + private static MetadataChangeProposal buildUpdateViewProposal( + final Urn urn, + final DataHubViewType newType, + final String newName, + final String newDescription, + final DataHubViewDefinition newDefinition, + final long createdAtMs, + final long updatedAtMs) { + + DataHubViewInfo info = new DataHubViewInfo(); + info.setType(newType); + info.setName(newName); + info.setDescription(newDescription); + info.setDefinition(newDefinition); + info.setCreated(new AuditStamp().setActor(TEST_USER_URN).setTime(createdAtMs)); + info.setLastModified(new AuditStamp().setActor(TEST_USER_URN).setTime(updatedAtMs)); + + final MetadataChangeProposal mcp = new MetadataChangeProposal(); + mcp.setEntityUrn(urn); + mcp.setEntityType(DATAHUB_VIEW_ENTITY_NAME); + mcp.setAspectName(DATAHUB_VIEW_INFO_ASPECT_NAME); + mcp.setChangeType(ChangeType.UPSERT); + mcp.setAspect(GenericRecordUtils.serializeAspect(info)); + return mcp; + } + + private static EntityClient createViewMockEntityClient() throws Exception { + EntityClient mockClient = Mockito.mock(EntityClient.class); + Mockito.when(mockClient.ingestProposal( + Mockito.any(MetadataChangeProposal.class), + Mockito.any(Authentication.class), + Mockito.eq(false))).thenReturn(TEST_VIEW_URN.toString()); + return mockClient; + } + + private static void resetUpdateViewMockEntityClient( + final EntityClient mockClient, + final Urn viewUrn, + final DataHubViewType existingType, + final String existingName, + final String existingDescription, + final DataHubViewDefinition existingDefinition, + final Urn existingOwner, + final long existingCreatedAt, + final long existingUpdatedAt) throws Exception { + + Mockito.reset(mockClient); + + Mockito.when(mockClient.ingestProposal( + Mockito.any(MetadataChangeProposal.class), + Mockito.any(Authentication.class), + Mockito.eq(false))).thenReturn(viewUrn.toString()); + + final DataHubViewInfo existingInfo = new DataHubViewInfo() + .setType(existingType) + .setName(existingName) + .setDescription(existingDescription) + .setDefinition(existingDefinition) + .setCreated(new AuditStamp().setActor(existingOwner).setTime(existingCreatedAt)) + .setLastModified(new AuditStamp().setActor(existingOwner).setTime(existingUpdatedAt)); + + Mockito.when(mockClient.getV2( + Mockito.eq(DATAHUB_VIEW_ENTITY_NAME), + Mockito.eq(viewUrn), + Mockito.eq(ImmutableSet.of(DATAHUB_VIEW_INFO_ASPECT_NAME)), + Mockito.any(Authentication.class))) + .thenReturn( + new EntityResponse() + .setUrn(viewUrn) + .setEntityName(DATAHUB_VIEW_ENTITY_NAME) + .setAspects(new EnvelopedAspectMap(ImmutableMap.of( + DATAHUB_VIEW_INFO_ASPECT_NAME, + new EnvelopedAspect().setValue(new Aspect(existingInfo.data())) + )))); + } + + private static void resetGetViewInfoMockEntityClient( + final EntityClient mockClient, + final Urn viewUrn, + final DataHubViewType existingType, + final String existingName, + final String existingDescription, + final DataHubViewDefinition existingDefinition, + final Urn existingOwner, + final long existingCreatedAt, + final long existingUpdatedAt) throws Exception { + + Mockito.reset(mockClient); + + final DataHubViewInfo existingInfo = new DataHubViewInfo() + .setType(existingType) + .setName(existingName) + .setDescription(existingDescription) + .setDefinition(existingDefinition) + .setCreated(new AuditStamp().setActor(existingOwner).setTime(existingCreatedAt)) + .setLastModified(new AuditStamp().setActor(existingOwner).setTime(existingUpdatedAt)); + + Mockito.when(mockClient.getV2( + Mockito.eq(DATAHUB_VIEW_ENTITY_NAME), + Mockito.eq(viewUrn), + Mockito.eq(ImmutableSet.of(DATAHUB_VIEW_INFO_ASPECT_NAME)), + Mockito.any(Authentication.class))) + .thenReturn( + new EntityResponse() + .setUrn(viewUrn) + .setEntityName(DATAHUB_VIEW_ENTITY_NAME) + .setAspects(new EnvelopedAspectMap(ImmutableMap.of( + DATAHUB_VIEW_INFO_ASPECT_NAME, + new EnvelopedAspect().setValue(new Aspect(existingInfo.data())) + )))); + } + + private static Authentication mockAuthentication() { + Authentication mockAuth = Mockito.mock(Authentication.class); + Mockito.when(mockAuth.getActor()).thenReturn(new Actor(ActorType.USER, TEST_USER_URN.getId())); + return mockAuth; + } +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserSettings.pdl b/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserSettings.pdl index 4465b05d8519fe..4c36d00a6ad2fc 100644 --- a/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserSettings.pdl +++ b/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserSettings.pdl @@ -8,7 +8,12 @@ namespace com.linkedin.identity } record CorpUserSettings { /** - *Settings for a user around the appearance of their DataHub U + * Settings for a user around the appearance of their DataHub U */ appearance: CorpUserAppearanceSettings + + /** + * User preferences for the Views feature. + */ + views: optional CorpUserViewsSettings } diff --git a/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserViewsSettings.pdl b/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserViewsSettings.pdl new file mode 100644 index 00000000000000..a4705a689735a6 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/identity/CorpUserViewsSettings.pdl @@ -0,0 +1,14 @@ +namespace com.linkedin.identity + +import com.linkedin.common.Urn + +/** + * Settings related to the 'Views' feature. + */ +record CorpUserViewsSettings { + /** + * The default View which is selected for the user. + * If none is chosen, then this value will be left blank. + */ + defaultView: optional Urn +} diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/key/DataHubViewKey.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/key/DataHubViewKey.pdl new file mode 100644 index 00000000000000..9cedf0251802d0 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/key/DataHubViewKey.pdl @@ -0,0 +1,14 @@ +namespace com.linkedin.metadata.key + +/** + * Key for a DataHub View + */ +@Aspect = { + "name": "dataHubViewKey" +} +record DataHubViewKey { + /** + * A unique id for the View + */ + id: string +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/metadata/key/GlobalSettingsKey.pdl b/metadata-models/src/main/pegasus/com/linkedin/metadata/key/GlobalSettingsKey.pdl new file mode 100644 index 00000000000000..f94002edb98418 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/metadata/key/GlobalSettingsKey.pdl @@ -0,0 +1,17 @@ +namespace com.linkedin.metadata.key + +import com.linkedin.common.Urn +import com.linkedin.common.FabricType + +/** + * Key for a Global Settings + */ +@Aspect = { + "name": "globalSettingsKey" +} +record GlobalSettingsKey { + /** + * Id for the settings. There should be only 1 global settings urn: urn:li:globalSettings:0 + */ + id: string +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/settings/global/GlobalSettingsInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/settings/global/GlobalSettingsInfo.pdl new file mode 100644 index 00000000000000..7d83d333843cc6 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/settings/global/GlobalSettingsInfo.pdl @@ -0,0 +1,14 @@ +namespace com.linkedin.settings.global + +/** + * DataHub Global platform settings. Careful - these should not be modified by the outside world! + */ +@Aspect = { + "name": "globalSettingsInfo" +} +record GlobalSettingsInfo { + /** + * Settings related to the Views Feature + */ + views: optional GlobalViewsSettings +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/settings/global/GlobalViewsSettings.pdl b/metadata-models/src/main/pegasus/com/linkedin/settings/global/GlobalViewsSettings.pdl new file mode 100644 index 00000000000000..ff34dea5568553 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/settings/global/GlobalViewsSettings.pdl @@ -0,0 +1,13 @@ +namespace com.linkedin.settings.global + +import com.linkedin.common.Urn + +/** + * Settings for DataHub Views feature. + */ +record GlobalViewsSettings { + /** + * The default View for the instance, or organization. + */ + defaultView: optional Urn +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/view/DataHubViewDefinition.pdl b/metadata-models/src/main/pegasus/com/linkedin/view/DataHubViewDefinition.pdl new file mode 100644 index 00000000000000..33597aff37353a --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/view/DataHubViewDefinition.pdl @@ -0,0 +1,18 @@ +namespace com.linkedin.view + +import com.linkedin.metadata.query.filter.Filter + +/** + * A View definition. + */ +record DataHubViewDefinition { + /** + * The Entity Types in the scope of the View. + */ + entityTypes: array[string] + + /** + * The filter criteria, which represents the view itself + */ + filter: Filter +} \ No newline at end of file diff --git a/metadata-models/src/main/pegasus/com/linkedin/view/DataHubViewInfo.pdl b/metadata-models/src/main/pegasus/com/linkedin/view/DataHubViewInfo.pdl new file mode 100644 index 00000000000000..f92094c57b1864 --- /dev/null +++ b/metadata-models/src/main/pegasus/com/linkedin/view/DataHubViewInfo.pdl @@ -0,0 +1,73 @@ +namespace com.linkedin.view + +import com.linkedin.metadata.query.filter.Filter +import com.linkedin.common.Urn +import com.linkedin.common.AuditStamp + +/** + * Information about a DataHub View. -- TODO: Understand whether an entity type filter is required. + */ +@Aspect = { + "name": "dataHubViewInfo" +} +record DataHubViewInfo { + /** + * The name of the View + */ + @Searchable = { + "fieldType": "TEXT_PARTIAL" + } + name: string + + /** + * Description of the view + */ + description: optional string + + /** + * The type of View + */ + @Searchable = {} + type: enum DataHubViewType { + /** + * A view private for a specific person. + */ + PERSONAL + + /** + * A global view, which all users can see and use. + */ + GLOBAL + } + + /** + * The view itself + */ + definition: DataHubViewDefinition + + /** + * Audit stamp capturing the time and actor who created the View. + */ + @Searchable = { + "/time": { + "fieldType": "DATETIME", + "fieldName": "createdAt" + }, + "/actor": { + "fieldType": "URN", + "fieldName": "createdBy" + } + } + created: AuditStamp + + /** + * Audit stamp capturing the time and actor who last modified the View. + */ + @Searchable = { + "/time": { + "fieldType": "DATETIME", + "fieldName": "lastModifiedAt" + } + } + lastModified: AuditStamp +} \ No newline at end of file diff --git a/metadata-models/src/main/resources/entity-registry.yml b/metadata-models/src/main/resources/entity-registry.yml index 52ac991ce2fca2..71d6f8b20b6dab 100644 --- a/metadata-models/src/main/resources/entity-registry.yml +++ b/metadata-models/src/main/resources/entity-registry.yml @@ -261,6 +261,12 @@ entities: category: core keyAspect: schemaFieldKey aspects: [] + - name: globalSettings + doc: Global settings for an the platform + category: internal + keyAspect: globalSettingsKey + aspects: + - globalSettingsInfo - name: dataHubRole category: core keyAspect: dataHubRoleKey @@ -276,4 +282,9 @@ entities: keyAspect: dataHubStepStateKey aspects: - dataHubStepStateProperties + - name: dataHubView + category: core + keyAspect: dataHubViewKey + aspects: + - dataHubViewInfo events: diff --git a/metadata-service/auth-impl/src/main/java/com/datahub/telemetry/TrackingService.java b/metadata-service/auth-impl/src/main/java/com/datahub/telemetry/TrackingService.java index 0294c9e6d37445..d253c17a42f822 100644 --- a/metadata-service/auth-impl/src/main/java/com/datahub/telemetry/TrackingService.java +++ b/metadata-service/auth-impl/src/main/java/com/datahub/telemetry/TrackingService.java @@ -55,12 +55,13 @@ public class TrackingService { private static final String POLICY_URN_FIELD = "policyUrn"; private static final String SOURCE_TYPE_FIELD = "sourceType"; private static final String INTERVAL_FIELD = "interval"; + private static final String VIEW_TYPE_FIELD = "viewType"; private static final Set ALLOWED_EVENT_FIELDS = new HashSet<>( ImmutableList.of(EVENT_TYPE_FIELD, SIGN_UP_TITLE_FIELD, ENTITY_TYPE_FIELD, ENTITY_TYPE_FILTER_FIELD, PAGE_NUMBER_FIELD, PAGE_FIELD, TOTAL_FIELD, INDEX_FIELD, RESULT_TYPE_FIELD, RENDER_ID_FIELD, MODULE_ID_FIELD, RENDER_TYPE_FIELD, SCENARIO_TYPE_FIELD, SECTION_FIELD, ACCESS_TOKEN_TYPE_FIELD, DURATION_FIELD, - ROLE_URN_FIELD, POLICY_URN_FIELD, SOURCE_TYPE_FIELD, INTERVAL_FIELD)); + ROLE_URN_FIELD, POLICY_URN_FIELD, SOURCE_TYPE_FIELD, INTERVAL_FIELD, VIEW_TYPE_FIELD)); private static final String ACTOR_URN_FIELD = "actorUrn"; private static final String ORIGIN_FIELD = "origin"; diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/config/ConfigurationProvider.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/config/ConfigurationProvider.java index 77b0ffa65cee8e..7faec29ae592d5 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/config/ConfigurationProvider.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/config/ConfigurationProvider.java @@ -6,6 +6,7 @@ import com.linkedin.metadata.config.DataHubConfiguration; import com.linkedin.metadata.config.IngestionConfiguration; import com.linkedin.metadata.config.TestsConfiguration; +import com.linkedin.metadata.config.ViewsConfiguration; import com.linkedin.metadata.telemetry.TelemetryConfiguration; import com.linkedin.gms.factory.spring.YamlPropertySourceFactory; import com.linkedin.metadata.config.VisualConfiguration; @@ -48,7 +49,10 @@ public class ConfigurationProvider { * DataHub top-level server configurations */ private DataHubConfiguration datahub; - + /** + * Views feature related configs + */ + private ViewsConfiguration views; /** * Feature flags indicating what is turned on vs turned off */ diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java index d7613f276e1382..8383485fa5bed1 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/graphql/GraphQLEngineFactory.java @@ -26,6 +26,8 @@ import com.linkedin.metadata.models.registry.EntityRegistry; import com.linkedin.metadata.recommendation.RecommendationsService; import com.linkedin.metadata.secret.SecretService; +import com.linkedin.metadata.service.SettingsService; +import com.linkedin.metadata.service.ViewService; import com.linkedin.metadata.timeline.TimelineService; import com.linkedin.metadata.timeseries.TimeseriesAspectService; import com.linkedin.metadata.utils.elasticsearch.IndexConvention; @@ -128,6 +130,14 @@ public class GraphQLEngineFactory { @Qualifier("postService") private PostService _postService; + @Autowired + @Qualifier("viewService") + private ViewService _viewService; + + @Autowired + @Qualifier("settingsService") + private SettingsService _settingsService; + @Value("${platformAnalytics.enabled}") // TODO: Migrate to DATAHUB_ANALYTICS_ENABLED private Boolean isAnalyticsEnabled; @@ -158,11 +168,14 @@ protected GraphQLEngine getInstance() { _configProvider.getTelemetry(), _configProvider.getMetadataTests(), _configProvider.getDatahub(), + _configProvider.getViews(), _siblingGraphService, _groupService, _roleService, _inviteTokenService, _postService, + _viewService, + _settingsService, _configProvider.getFeatureFlags() ).builder().build(); } @@ -188,11 +201,14 @@ protected GraphQLEngine getInstance() { _configProvider.getTelemetry(), _configProvider.getMetadataTests(), _configProvider.getDatahub(), + _configProvider.getViews(), _siblingGraphService, _groupService, _roleService, _inviteTokenService, _postService, + _viewService, + _settingsService, _configProvider.getFeatureFlags() ).builder().build(); } diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/views/ViewServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/views/ViewServiceFactory.java new file mode 100644 index 00000000000000..006b992191cfa4 --- /dev/null +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/search/views/ViewServiceFactory.java @@ -0,0 +1,33 @@ +package com.linkedin.gms.factory.search.views; + +import com.datahub.authentication.Authentication; +import com.linkedin.gms.factory.spring.YamlPropertySourceFactory; +import com.linkedin.metadata.client.JavaEntityClient; +import com.linkedin.metadata.service.ViewService; +import javax.annotation.Nonnull; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; +import org.springframework.context.annotation.Scope; + + +@Configuration +@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) +public class ViewServiceFactory { + @Autowired + @Qualifier("javaEntityClient") + private JavaEntityClient _javaEntityClient; + + @Autowired + @Qualifier("systemAuthentication") + private Authentication _authentication; + + @Bean(name = "viewService") + @Scope("singleton") + @Nonnull + protected ViewService getInstance() throws Exception { + return new ViewService(_javaEntityClient, _authentication); + } +} diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/settings/SettingsServiceFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/settings/SettingsServiceFactory.java new file mode 100644 index 00000000000000..73ec79fa7ed088 --- /dev/null +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/settings/SettingsServiceFactory.java @@ -0,0 +1,33 @@ +package com.linkedin.gms.factory.settings; + +import com.datahub.authentication.Authentication; +import com.linkedin.gms.factory.spring.YamlPropertySourceFactory; +import com.linkedin.metadata.client.JavaEntityClient; +import com.linkedin.metadata.service.SettingsService; +import javax.annotation.Nonnull; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.context.annotation.PropertySource; +import org.springframework.context.annotation.Scope; + + +@Configuration +@PropertySource(value = "classpath:/application.yml", factory = YamlPropertySourceFactory.class) +public class SettingsServiceFactory { + @Autowired + @Qualifier("javaEntityClient") + private JavaEntityClient _javaEntityClient; + + @Autowired + @Qualifier("systemAuthentication") + private Authentication _authentication; + + @Bean(name = "settingsService") + @Scope("singleton") + @Nonnull + protected SettingsService getInstance() throws Exception { + return new SettingsService(_javaEntityClient, _authentication); + } +} diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java index a102e632e2e52d..077e1519c1f9c3 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/factories/BootstrapManagerFactory.java @@ -10,6 +10,7 @@ import com.linkedin.metadata.boot.steps.IndexDataPlatformsStep; import com.linkedin.metadata.boot.steps.IngestDataPlatformInstancesStep; import com.linkedin.metadata.boot.steps.IngestDataPlatformsStep; +import com.linkedin.metadata.boot.steps.IngestDefaultGlobalSettingsStep; import com.linkedin.metadata.boot.steps.IngestPoliciesStep; import com.linkedin.metadata.boot.steps.IngestRetentionPoliciesStep; import com.linkedin.metadata.boot.steps.IngestRolesStep; @@ -87,9 +88,10 @@ protected BootstrapManager createInstance() { new RestoreDbtSiblingsIndices(_entityService, _entityRegistry); final RemoveClientIdAspectStep removeClientIdAspectStep = new RemoveClientIdAspectStep(_entityService); final RestoreColumnLineageIndices restoreColumnLineageIndices = new RestoreColumnLineageIndices(_entityService, _entityRegistry); + final IngestDefaultGlobalSettingsStep ingestSettingsStep = new IngestDefaultGlobalSettingsStep(_entityService); final List finalSteps = new ArrayList<>(ImmutableList.of(ingestRootUserStep, ingestPoliciesStep, ingestRolesStep, - ingestDataPlatformsStep, ingestDataPlatformInstancesStep, _ingestRetentionPoliciesStep, restoreGlossaryIndicesStep, + ingestDataPlatformsStep, ingestDataPlatformInstancesStep, _ingestRetentionPoliciesStep, ingestSettingsStep, restoreGlossaryIndicesStep, removeClientIdAspectStep, restoreDbtSiblingsIndices, indexDataPlatformsStep, restoreColumnLineageIndices)); if (_upgradeDefaultBrowsePathsEnabled) { diff --git a/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDefaultGlobalSettingsStep.java b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDefaultGlobalSettingsStep.java new file mode 100644 index 00000000000000..8f87e5fe272e3a --- /dev/null +++ b/metadata-service/factories/src/main/java/com/linkedin/metadata/boot/steps/IngestDefaultGlobalSettingsStep.java @@ -0,0 +1,133 @@ +package com.linkedin.metadata.boot.steps; + +import com.datahub.util.RecordUtils; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.Urn; +import com.linkedin.data.DataMap; +import com.linkedin.data.schema.validation.CoercionMode; +import com.linkedin.data.schema.validation.RequiredMode; +import com.linkedin.data.schema.validation.UnrecognizedFieldMode; +import com.linkedin.data.schema.validation.ValidateDataAgainstSchema; +import com.linkedin.data.schema.validation.ValidationOptions; +import com.linkedin.data.schema.validation.ValidationResult; +import com.linkedin.data.template.RecordTemplate; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.Constants; +import com.linkedin.metadata.boot.BootstrapStep; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.settings.global.GlobalSettingsInfo; +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.Objects; +import javax.annotation.Nonnull; +import lombok.extern.slf4j.Slf4j; +import org.springframework.core.io.ClassPathResource; + +import static com.linkedin.metadata.Constants.*; + + +/** + * This bootstrap step is responsible for ingesting a default Global Settings object if it does not already exist. + * + * If settings already exist, we merge the defaults and the existing settings such that the container will also + * get new settings when they are added. + */ +@Slf4j +public class IngestDefaultGlobalSettingsStep implements BootstrapStep { + + private static final String DEFAULT_SETTINGS_RESOURCE_PATH = "./boot/global_settings.json"; + private final EntityService _entityService; + private final String _resourcePath; + + public IngestDefaultGlobalSettingsStep(@Nonnull final EntityService entityService) { + this(entityService, DEFAULT_SETTINGS_RESOURCE_PATH); + } + + public IngestDefaultGlobalSettingsStep( + @Nonnull final EntityService entityService, + @Nonnull final String resourcePath) { + _entityService = Objects.requireNonNull(entityService); + _resourcePath = Objects.requireNonNull(resourcePath); + } + + @Override + public String name() { + return getClass().getName(); + } + + @Override + public void execute() throws IOException, URISyntaxException { + + final ObjectMapper mapper = new ObjectMapper(); + + log.info("Ingesting default global settings..."); + + // 1. Read from the file into JSON. + JsonNode defaultSettingsObj; + try { + defaultSettingsObj = mapper.readTree(new ClassPathResource(_resourcePath).getFile()); + } catch (Exception e) { + throw new RuntimeException( + String.format("Failed to parse global settings file. Could not parse valid json at resource path %s", + _resourcePath), + e); + } + + if (!defaultSettingsObj.isObject()) { + throw new RuntimeException(String.format("Found malformed global settings info file, expected an Object but found %s", + defaultSettingsObj.getNodeType())); + } + + // 2. Bind the global settings json into a GlobalSettingsInfo aspect. + GlobalSettingsInfo defaultSettings; + defaultSettings = RecordUtils.toRecordTemplate(GlobalSettingsInfo.class, defaultSettingsObj.toString()); + ValidationResult result = ValidateDataAgainstSchema.validate( + defaultSettings, + new ValidationOptions( + RequiredMode.CAN_BE_ABSENT_IF_HAS_DEFAULT, + CoercionMode.NORMAL, + UnrecognizedFieldMode.DISALLOW + )); + + if (!result.isValid()) { + throw new RuntimeException(String.format( + "Failed to parse global settings file. Provided JSON does not match GlobalSettingsInfo.pdl model. %s", result.getMessages())); + } + + // 3. Get existing settings or empty settings object + final GlobalSettingsInfo existingSettings = getExistingGlobalSettingsOrEmpty(); + + // 4. Merge existing settings onto previous settings. Be careful - if we change the settings schema dramatically in future we may need to account for that. + final GlobalSettingsInfo newSettings = new GlobalSettingsInfo(mergeDataMaps(defaultSettings.data(), existingSettings.data())); + + // 5. Ingest into DataHub. + final MetadataChangeProposal proposal = new MetadataChangeProposal(); + proposal.setEntityUrn(GLOBAL_SETTINGS_URN); + proposal.setEntityType(GLOBAL_SETTINGS_ENTITY_NAME); + proposal.setAspectName(GLOBAL_SETTINGS_INFO_ASPECT_NAME); + proposal.setAspect(GenericRecordUtils.serializeAspect(newSettings)); + proposal.setChangeType(ChangeType.UPSERT); + + _entityService.ingestProposal( + proposal, + new AuditStamp().setActor(Urn.createFromString(Constants.SYSTEM_ACTOR)).setTime(System.currentTimeMillis()), + false); + } + + private GlobalSettingsInfo getExistingGlobalSettingsOrEmpty() { + RecordTemplate aspect = _entityService.getAspect(GLOBAL_SETTINGS_URN, GLOBAL_SETTINGS_INFO_ASPECT_NAME, 0); + return aspect != null ? (GlobalSettingsInfo) aspect : new GlobalSettingsInfo(); + } + + private DataMap mergeDataMaps(final DataMap map1, final DataMap map2) { + final DataMap result = new DataMap(); + // TODO: Replace with a nested merge. This only copies top level keys. + result.putAll(map1); + result.putAll(map2); + return result; + } +} diff --git a/metadata-service/factories/src/main/resources/application.yml b/metadata-service/factories/src/main/resources/application.yml index 01aa12fca0478c..116f134080f735 100644 --- a/metadata-service/factories/src/main/resources/application.yml +++ b/metadata-service/factories/src/main/resources/application.yml @@ -214,3 +214,6 @@ featureFlags: entityChangeEvents: enabled: ${ENABLE_ENTITY_CHANGE_EVENTS_HOOK:true} + +views: + enabled: ${VIEWS_ENABLED:true} diff --git a/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestDefaultGlobalSettingsStepTest.java b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestDefaultGlobalSettingsStepTest.java new file mode 100644 index 00000000000000..edd5da7006f4ab --- /dev/null +++ b/metadata-service/factories/src/test/java/com/linkedin/metadata/boot/steps/IngestDefaultGlobalSettingsStepTest.java @@ -0,0 +1,122 @@ +package com.linkedin.metadata.boot.steps; + +import com.linkedin.common.AuditStamp; +import com.linkedin.common.urn.UrnUtils; +import com.linkedin.events.metadata.ChangeType; +import com.linkedin.metadata.entity.EntityService; +import com.linkedin.metadata.utils.GenericRecordUtils; +import com.linkedin.mxe.MetadataChangeProposal; +import com.linkedin.settings.global.GlobalSettingsInfo; +import com.linkedin.settings.global.GlobalViewsSettings; +import org.mockito.Mockito; +import org.testng.Assert; +import org.testng.annotations.Test; + +import static com.linkedin.metadata.Constants.*; +import static org.mockito.Mockito.*; + + +/** + * Test the behavior of IngestDefaultGlobalSettingsStep. + * + * We expect it to ingest a JSON file, throwing if the JSON file + * is malformed or does not match the PDL model for GlobalSettings.pdl. + */ +public class IngestDefaultGlobalSettingsStepTest { + + @Test + public void testExecuteValidSettingsNoExistingSettings() throws Exception { + final EntityService entityService = mock(EntityService.class); + configureEntityServiceMock(entityService, null); + + final IngestDefaultGlobalSettingsStep step = new IngestDefaultGlobalSettingsStep( + entityService, + "./boot/test_global_settings_valid.json"); + + step.execute(); + + GlobalSettingsInfo expectedResult = new GlobalSettingsInfo(); + expectedResult.setViews(new GlobalViewsSettings().setDefaultView(UrnUtils.getUrn("urn:li:dataHubView:test"))); + + Mockito.verify(entityService, times(1)).ingestProposal( + Mockito.eq(buildUpdateSettingsProposal(expectedResult)), + Mockito.any(AuditStamp.class), + Mockito.eq(false) + ); + } + + @Test + public void testExecuteValidSettingsExistingSettings() throws Exception { + + // Verify that the user provided settings overrides are NOT overwritten. + final EntityService entityService = mock(EntityService.class); + final GlobalSettingsInfo existingSettings = new GlobalSettingsInfo() + .setViews(new GlobalViewsSettings() + .setDefaultView(UrnUtils.getUrn("urn:li:dataHubView:custom"))); + configureEntityServiceMock(entityService, existingSettings); + + final IngestDefaultGlobalSettingsStep step = new IngestDefaultGlobalSettingsStep( + entityService, + "./boot/test_global_settings_valid.json"); + + step.execute(); + + // Verify that the merge preserves the user settings. + GlobalSettingsInfo expectedResult = new GlobalSettingsInfo(); + expectedResult.setViews(new GlobalViewsSettings().setDefaultView(UrnUtils.getUrn("urn:li:dataHubView:custom"))); + + Mockito.verify(entityService, times(1)).ingestProposal( + Mockito.eq(buildUpdateSettingsProposal(expectedResult)), + Mockito.any(AuditStamp.class), + Mockito.eq(false) + ); + } + + @Test + public void testExecuteInvalidJsonSettings() throws Exception { + final EntityService entityService = mock(EntityService.class); + configureEntityServiceMock(entityService, null); + + final IngestDefaultGlobalSettingsStep step = new IngestDefaultGlobalSettingsStep( + entityService, + "./boot/test_global_settings_invalid_json.json"); + + Assert.assertThrows(RuntimeException.class, step::execute); + + // Verify no interactions + verifyZeroInteractions(entityService); + } + + @Test + public void testExecuteInvalidModelSettings() throws Exception { + final EntityService entityService = mock(EntityService.class); + configureEntityServiceMock(entityService, null); + + final IngestDefaultGlobalSettingsStep step = new IngestDefaultGlobalSettingsStep( + entityService, + "./boot/test_global_settings_invalid_model.json"); + + Assert.assertThrows(RuntimeException.class, step::execute); + + // Verify no interactions + verifyZeroInteractions(entityService); + } + + private static void configureEntityServiceMock(final EntityService mockService, final GlobalSettingsInfo settingsInfo) { + Mockito.when(mockService.getAspect( + Mockito.eq(GLOBAL_SETTINGS_URN), + Mockito.eq(GLOBAL_SETTINGS_INFO_ASPECT_NAME), + Mockito.eq(0L) + )).thenReturn(settingsInfo); + } + + private static MetadataChangeProposal buildUpdateSettingsProposal(final GlobalSettingsInfo settings) { + final MetadataChangeProposal mcp = new MetadataChangeProposal(); + mcp.setEntityUrn(GLOBAL_SETTINGS_URN); + mcp.setEntityType(GLOBAL_SETTINGS_ENTITY_NAME); + mcp.setAspectName(GLOBAL_SETTINGS_INFO_ASPECT_NAME); + mcp.setChangeType(ChangeType.UPSERT); + mcp.setAspect(GenericRecordUtils.serializeAspect(settings)); + return mcp; + } +} \ No newline at end of file diff --git a/metadata-service/factories/src/test/resources/boot/test_global_settings_invalid_json.json b/metadata-service/factories/src/test/resources/boot/test_global_settings_invalid_json.json new file mode 100644 index 00000000000000..461dbb9b531fbd --- /dev/null +++ b/metadata-service/factories/src/test/resources/boot/test_global_settings_invalid_json.json @@ -0,0 +1,4 @@ +{ + "views": { + "defaultView": "urn:li:dataHubView:test" +} \ No newline at end of file diff --git a/metadata-service/factories/src/test/resources/boot/test_global_settings_invalid_model.json b/metadata-service/factories/src/test/resources/boot/test_global_settings_invalid_model.json new file mode 100644 index 00000000000000..2215a61c9938af --- /dev/null +++ b/metadata-service/factories/src/test/resources/boot/test_global_settings_invalid_model.json @@ -0,0 +1,3 @@ +{ + "views": 10 +} \ No newline at end of file diff --git a/metadata-service/factories/src/test/resources/boot/test_global_settings_valid.json b/metadata-service/factories/src/test/resources/boot/test_global_settings_valid.json new file mode 100644 index 00000000000000..defcb2b289088e --- /dev/null +++ b/metadata-service/factories/src/test/resources/boot/test_global_settings_valid.json @@ -0,0 +1,5 @@ +{ + "views": { + "defaultView": "urn:li:dataHubView:test" + } +} \ No newline at end of file diff --git a/metadata-service/war/src/main/resources/boot/global_settings.json b/metadata-service/war/src/main/resources/boot/global_settings.json new file mode 100644 index 00000000000000..129783afd6df49 --- /dev/null +++ b/metadata-service/war/src/main/resources/boot/global_settings.json @@ -0,0 +1,4 @@ +{ + "views": { + } +} \ No newline at end of file diff --git a/metadata-service/war/src/main/resources/boot/policies.json b/metadata-service/war/src/main/resources/boot/policies.json index faf814ef43fe85..d7cd6e6319aca5 100644 --- a/metadata-service/war/src/main/resources/boot/policies.json +++ b/metadata-service/war/src/main/resources/boot/policies.json @@ -22,7 +22,8 @@ "MANAGE_TESTS", "MANAGE_GLOSSARIES", "MANAGE_USER_CREDENTIALS", - "MANAGE_TAGS" + "MANAGE_TAGS", + "MANAGE_GLOBAL_VIEWS" ], "displayName":"Root User - All Platform Privileges", "description":"Grants full platform privileges to root datahub super user.", @@ -167,7 +168,8 @@ "MANAGE_DOMAINS", "MANAGE_GLOSSARIES", "MANAGE_USER_CREDENTIALS", - "MANAGE_TAGS" + "MANAGE_TAGS", + "MANAGE_GLOBAL_VIEWS" ], "displayName":"Admins - Platform Policy", "description":"Admins have all platform privileges.", diff --git a/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java b/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java index 20f35bd83bb5c9..e5a1fdc6711c51 100644 --- a/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java +++ b/metadata-utils/src/main/java/com/linkedin/metadata/authorization/PoliciesConfig.java @@ -98,6 +98,11 @@ public class PoliciesConfig { "Create Global Announcements", "Create new Global Announcements."); + public static final Privilege MANAGE_GLOBAL_VIEWS = Privilege.of( + "MANAGE_GLOBAL_VIEWS", + "Manage Public Views", + "Create, update, and delete any Public (shared) Views."); + public static final List PLATFORM_PRIVILEGES = ImmutableList.of( MANAGE_POLICIES_PRIVILEGE, MANAGE_USERS_AND_GROUPS_PRIVILEGE, @@ -112,7 +117,9 @@ public class PoliciesConfig { MANAGE_USER_CREDENTIALS_PRIVILEGE, MANAGE_TAGS_PRIVILEGE, CREATE_TAGS_PRIVILEGE, - CREATE_DOMAINS_PRIVILEGE, CREATE_GLOBAL_ANNOUNCEMENTS_PRIVILEGE + CREATE_DOMAINS_PRIVILEGE, + CREATE_GLOBAL_ANNOUNCEMENTS_PRIVILEGE, + MANAGE_GLOBAL_VIEWS ); // Resource Privileges // diff --git a/smoke-test/tests/cypress/cypress/integration/views/manage_views.js b/smoke-test/tests/cypress/cypress/integration/views/manage_views.js new file mode 100644 index 00000000000000..dd63ddccfb43eb --- /dev/null +++ b/smoke-test/tests/cypress/cypress/integration/views/manage_views.js @@ -0,0 +1,36 @@ +describe("manage views", () => { + it("go to views settings page, create, edit, make default, delete a view", () => { + const viewName = "Test View" + + cy.login(); + cy.goToViewsSettings(); + + cy.clickOptionWithText("Create new View"); + cy.get(".ant-form-item-control-input-content > input[type='text']").first().type(viewName); + cy.clickOptionWithTestId("view-builder-save"); + + // Confirm that the test has been created. + cy.waitTextVisible("Test View"); + + // Now edit the View + cy.clickFirstOptionWithTestId("views-table-dropdown"); + cy.get('[data-testid="view-dropdown-edit"]').click({ force: true }); + cy.get(".ant-form-item-control-input-content > input[type='text']").first().clear().type("New View Name"); + cy.clickOptionWithTestId("view-builder-save"); + cy.waitTextVisible("New View Name"); + + // Now make the view the default + cy.clickFirstOptionWithTestId("views-table-dropdown"); + cy.get('[data-testid="view-dropdown-set-user-default"]').click({ force: true }); + + // Now unset as the default + cy.clickFirstOptionWithTestId("views-table-dropdown"); + cy.get('[data-testid="view-dropdown-remove-user-default"]').click({ force: true }); + + // Now delete the View + cy.clickFirstOptionWithTestId("views-table-dropdown"); + cy.get('[data-testid="view-dropdown-delete"]').click({ force: true }); + cy.clickOptionWithText("Yes"); + + }); +}); diff --git a/smoke-test/tests/cypress/cypress/integration/views/view_select.js b/smoke-test/tests/cypress/cypress/integration/views/view_select.js new file mode 100644 index 00000000000000..7475cf18f2d4e6 --- /dev/null +++ b/smoke-test/tests/cypress/cypress/integration/views/view_select.js @@ -0,0 +1,102 @@ +describe("view select", () => { + it("click view select, create view, clear view, make defaults, clear view", () => { + const viewName = "Test View" + const viewDescription = "View Description" + const newViewName = "New View Name" + + // Resize Observer Loop warning can be safely ignored - ref. https://github.com/cypress-io/cypress/issues/22113 + const resizeObserverLoopErrRe = "ResizeObserver loop limit exceeded"; + cy.on('uncaught:exception', (err) => { + if (err.message.includes(resizeObserverLoopErrRe)) { + return false; + } + }); + + cy.login(); + cy.visit("/search?page=1&query=%2A&unionType=0"); + + // Create a View from the select + cy.get('[data-testid="view-select"]').click(); + cy.clickOptionWithTestId("view-select-create"); + + cy.get(".ant-form-item-control-input-content > input[type='text']").first().type(viewName); + + // Add Column Glossary Term Filter + cy.contains("Add Filter").click(); + cy.contains("Column Glossary Term").click({ force: true }); + cy.get('[data-testid="tag-term-modal-input"]').type("CypressColumnInfo"); + cy.wait(2000); + cy.get('[data-testid="tag-term-option"]').click({ force: true }); + cy.get('[data-testid="add-tag-term-from-modal-btn"]').click({ + force: true, + }); + + cy.clickOptionWithTestId("view-builder-save"); + cy.waitTextVisible(viewName); + + cy.wait(2000); // Allow search to reload + + // Ensure the View filter has been applied correctly. + cy.contains("SampleCypressHdfsDataset"); + cy.contains("cypress_logging_events"); + cy.contains("of 2 results"); + + // Clear the selected view view + cy.get('[data-testid="view-select"]').click(); + cy.clickOptionWithTestId("view-select-clear"); + cy.get("input[data-testid=search-input]").click(); // unfocus + cy.contains(viewName).should("not.be.visible"); + + cy.wait(2000); // Allow search to reload + cy.ensureTextNotPresent("of 2 results"); + + // Now edit the view + cy.get('[data-testid="view-select"]').click(); + cy.get('[data-testid="view-select-item"]').first().trigger('mouseover') + cy.get('[data-testid="views-table-dropdown"]').first().trigger('mouseover'); + cy.get('[data-testid="view-dropdown-edit"]').click(); + cy.get(".ant-form-item-control-input-content > input[type='text']").first().clear().type(newViewName); + // Update the actual filters by adding another filter + cy.contains("Add Filter").click(); + cy.get('[data-testid="adv-search-add-filter-description"]').click({ + force: true, + }); + cy.get('[data-testid="edit-text-input"]').type("log event"); + cy.get('[data-testid="edit-text-done-btn"]').click(); + + // Save View + cy.clickOptionWithTestId("view-builder-save"); + + cy.wait(2000); // Allow search to reload + + cy.contains("cypress_logging_events"); + cy.contains("of 1 result"); + + // Now set the View as the personal Default + cy.get('[data-testid="view-select"]').click(); + cy.get('[data-testid="view-select-item"]').first().trigger('mouseover') + cy.get('[data-testid="views-table-dropdown"]').first().trigger('mouseover'); + cy.get('[data-testid="view-dropdown-set-user-default"]').click(); + cy.get("input[data-testid=search-input]").click(); // unfocus + + // Now unset as the personal default + cy.get('[data-testid="view-select"]').click(); + cy.get('[data-testid="view-select-item"]').first().trigger('mouseover') + cy.get('[data-testid="views-table-dropdown"]').first().trigger('mouseover'); + cy.get('[data-testid="view-dropdown-remove-user-default"]').click(); + cy.get("input[data-testid=search-input]").click(); // unfocus + + // Now delete the View + cy.get('[data-testid="view-select"]').click(); + cy.get('[data-testid="view-select-item"]').first().trigger('mouseover') + cy.get('[data-testid="views-table-dropdown"]').first().trigger('mouseover'); + cy.get('[data-testid="view-dropdown-delete"]').click(); + cy.clickOptionWithText("Yes"); + + // Ensure that the view was deleted. + cy.ensureTextNotPresent(viewName); + cy.wait(2000); // Allow search to reload + cy.ensureTextNotPresent("of 1 result"); + + }); +}); diff --git a/smoke-test/tests/cypress/cypress/support/commands.js b/smoke-test/tests/cypress/cypress/support/commands.js index 2abea209a4ec76..156c75573c816f 100644 --- a/smoke-test/tests/cypress/cypress/support/commands.js +++ b/smoke-test/tests/cypress/cypress/support/commands.js @@ -48,6 +48,11 @@ Cypress.Commands.add("goToDomainList", () => { cy.waitTextVisible("New Domain"); }); +Cypress.Commands.add("goToViewsSettings", () => { + cy.visit("/settings/views"); + cy.waitTextVisible("Manage Views"); +}); + Cypress.Commands.add("goToDataset", (urn, dataset_name) => { cy.visit( "/dataset/" + urn @@ -136,6 +141,12 @@ Cypress.Commands.add("clickOptionWithTestId", (id) => { }); }) +Cypress.Commands.add("clickFirstOptionWithTestId", (id) => { + cy.get('[data-testid="' + id +'"]').first().click({ + force: true, + }); +}) + Cypress.Commands.add("hideOnboardingTour", () => { cy.get('body').type("{ctrl} {meta} h"); }); diff --git a/smoke-test/tests/cypress/cypress/support/index.js b/smoke-test/tests/cypress/cypress/support/index.js index d68db96df2697e..63268c1f345c93 100644 --- a/smoke-test/tests/cypress/cypress/support/index.js +++ b/smoke-test/tests/cypress/cypress/support/index.js @@ -17,4 +17,4 @@ import './commands' // Alternatively you can use CommonJS syntax: -// require('./commands') +// require('./commands') \ No newline at end of file diff --git a/smoke-test/tests/views/__init__.py b/smoke-test/tests/views/__init__.py new file mode 100644 index 00000000000000..e69de29bb2d1d6 diff --git a/smoke-test/tests/views/views_test.py b/smoke-test/tests/views/views_test.py new file mode 100644 index 00000000000000..4da69750a167b1 --- /dev/null +++ b/smoke-test/tests/views/views_test.py @@ -0,0 +1,424 @@ +import pytest +import time +import tenacity +from tests.utils import ( + delete_urns_from_file, + get_frontend_url, + get_gms_url, + ingest_file_via_rest, + get_sleep_info, +) + +sleep_sec, sleep_times = get_sleep_info() + +@pytest.mark.dependency() +def test_healthchecks(wait_for_healthchecks): + # Call to wait_for_healthchecks fixture will do the actual functionality. + pass + + +@tenacity.retry( + stop=tenacity.stop_after_attempt(sleep_times), wait=tenacity.wait_fixed(sleep_sec) +) +def _ensure_more_views(frontend_session, list_views_json, query_name, before_count): + + # Get new count of Views + response = frontend_session.post( + f"{get_frontend_url()}/api/v2/graphql", json=list_views_json + ) + response.raise_for_status() + res_data = response.json() + + assert res_data + assert res_data["data"] + assert res_data["data"][query_name]["total"] is not None + assert "errors" not in res_data + + # Assert that there are more views now. + after_count = res_data["data"][query_name]["total"] + print(f"after_count is {after_count}") + assert after_count == before_count + 1 + return after_count + +@tenacity.retry( + stop=tenacity.stop_after_attempt(sleep_times), wait=tenacity.wait_fixed(sleep_sec) +) +def _ensure_less_views(frontend_session, list_views_json, query_name, before_count): + + # Get new count of Views + response = frontend_session.post( + f"{get_frontend_url()}/api/v2/graphql", json=list_views_json + ) + response.raise_for_status() + res_data = response.json() + + assert res_data + assert res_data["data"] + assert res_data["data"][query_name]["total"] is not None + assert "errors" not in res_data + + # Assert that there are more views now. + after_count = res_data["data"][query_name]["total"] + print(f"after_count is {after_count}") + assert after_count == before_count - 1 + + +@pytest.mark.dependency(depends=["test_healthchecks"]) +def test_create_list_delete_global_view(frontend_session): + + # Get count of existing views + list_global_views_json = { + "query": """query listGlobalViews($input: ListGlobalViewsInput!) {\n + listGlobalViews(input: $input) {\n + start\n + count\n + total\n + views {\n + urn\n + viewType\n + name\n + description\n + definition {\n + entityTypes\n + filter {\n + operator\n + filters {\n + field\n + values\n + condition\n + }\n + }\n + }\n + }\n + }\n + }""", + "variables": {"input": {"start": "0", "count": "20"}}, + } + + response = frontend_session.post( + f"{get_frontend_url()}/api/v2/graphql", json=list_global_views_json + ) + response.raise_for_status() + res_data = response.json() + + assert res_data + assert res_data["data"] + assert res_data["data"]["listGlobalViews"]["total"] is not None + assert "errors" not in res_data + + before_count = res_data["data"]["listGlobalViews"]["total"] + + new_view_name = "Test View" + new_view_description = "Test Description" + new_view_definition = { + "entityTypes": ["DATASET", "DASHBOARD"], + "filter": { + "operator": "AND", + "filters": [ + { + "field": "tags", + "values": ["urn:li:tag:test"], + "negated": False, + "condition": "EQUAL" + } + ] + } + } + + # Create new View + create_view_json = { + "query": """mutation createView($input: CreateViewInput!) {\n + createView(input: $input) {\n + urn\n + }\n + }""", + "variables": { + "input": { + "viewType": "GLOBAL", + "name": new_view_name, + "description": new_view_description, + "definition": new_view_definition + } + }, + } + + response = frontend_session.post( + f"{get_frontend_url()}/api/v2/graphql", json=create_view_json + ) + response.raise_for_status() + res_data = response.json() + + assert res_data + assert res_data["data"] + assert res_data["data"]["createView"] is not None + assert "errors" not in res_data + + view_urn = res_data["data"]["createView"]["urn"] + + new_count = _ensure_more_views( + frontend_session=frontend_session, + list_views_json=list_global_views_json, + query_name="listGlobalViews", + before_count=before_count, + ) + + delete_json = {"urn": view_urn} + + # Delete the View + delete_view_json = { + "query": """mutation deleteView($urn: String!) {\n + deleteView(urn: $urn) + }""", + "variables": { + "urn": view_urn + }, + } + + response = frontend_session.post( + f"{get_frontend_url()}/api/v2/graphql", json=delete_view_json + ) + response.raise_for_status() + res_data = response.json() + assert "errors" not in res_data + + _ensure_less_views( + frontend_session=frontend_session, + list_views_json=list_global_views_json, + query_name="listGlobalViews", + before_count=new_count, + ) + + +@pytest.mark.dependency(depends=["test_healthchecks", "test_create_list_delete_global_view"]) +def test_create_list_delete_personal_view(frontend_session): + + # Get count of existing views + list_my_views_json = { + "query": """query listMyViews($input: ListMyViewsInput!) {\n + listMyViews(input: $input) {\n + start\n + count\n + total\n + views {\n + urn\n + viewType\n + name\n + description\n + definition {\n + entityTypes\n + filter {\n + operator\n + filters {\n + field\n + values\n + condition\n + }\n + }\n + }\n + }\n + }\n + }""", + "variables": {"input": {"start": "0", "count": "20"}}, + } + + response = frontend_session.post( + f"{get_frontend_url()}/api/v2/graphql", json=list_my_views_json + ) + response.raise_for_status() + res_data = response.json() + + assert res_data + assert res_data["data"] + assert res_data["data"]["listMyViews"]["total"] is not None + assert "errors" not in res_data + + before_count = res_data["data"]["listMyViews"]["total"] + + new_view_name = "Test View" + new_view_description = "Test Description" + new_view_definition = { + "entityTypes": ["DATASET", "DASHBOARD"], + "filter": { + "operator": "AND", + "filters": [ + { + "field": "tags", + "values": ["urn:li:tag:test"], + "negated": False, + "condition": "EQUAL" + } + ] + } + } + + # Create new View + create_view_json = { + "query": """mutation createView($input: CreateViewInput!) {\n + createView(input: $input) {\n + urn\n + }\n + }""", + "variables": { + "input": { + "viewType": "PERSONAL", + "name": new_view_name, + "description": new_view_description, + "definition": new_view_definition + } + }, + } + + response = frontend_session.post( + f"{get_frontend_url()}/api/v2/graphql", json=create_view_json + ) + response.raise_for_status() + res_data = response.json() + + assert res_data + assert res_data["data"] + assert res_data["data"]["createView"] is not None + assert "errors" not in res_data + + view_urn = res_data["data"]["createView"]["urn"] + + new_count = _ensure_more_views( + frontend_session=frontend_session, + list_views_json=list_my_views_json, + query_name="listMyViews", + before_count=before_count, + ) + + # Delete the View + delete_view_json = { + "query": """mutation deleteView($urn: String!) {\n + deleteView(urn: $urn) + }""", + "variables": { + "urn": view_urn + }, + } + + response = frontend_session.post( + f"{get_frontend_url()}/api/v2/graphql", json=delete_view_json + ) + response.raise_for_status() + res_data = response.json() + assert "errors" not in res_data + + _ensure_less_views( + frontend_session=frontend_session, + list_views_json=list_my_views_json, + query_name="listMyViews", + before_count=new_count, + ) + +@pytest.mark.dependency(depends=["test_healthchecks", "test_create_list_delete_personal_view"]) +def test_update_global_view(frontend_session): + + # First create a view + new_view_name = "Test View" + new_view_description = "Test Description" + new_view_definition = { + "entityTypes": ["DATASET", "DASHBOARD"], + "filter": { + "operator": "AND", + "filters": [ + { + "field": "tags", + "values": ["urn:li:tag:test"], + "negated": False, + "condition": "EQUAL" + } + ] + } + } + + # Create new View + create_view_json = { + "query": """mutation createView($input: CreateViewInput!) {\n + createView(input: $input) {\n + urn\n + }\n + }""", + "variables": { + "input": { + "viewType": "PERSONAL", + "name": new_view_name, + "description": new_view_description, + "definition": new_view_definition + } + }, + } + + response = frontend_session.post( + f"{get_frontend_url()}/api/v2/graphql", json=create_view_json + ) + response.raise_for_status() + res_data = response.json() + + assert res_data + assert res_data["data"] + assert res_data["data"]["createView"] is not None + assert "errors" not in res_data + + view_urn = res_data["data"]["createView"]["urn"] + + new_view_name = "New Test View" + new_view_description = "New Test Description" + new_view_definition = { + "entityTypes": ["DATASET", "DASHBOARD", "CHART", "DATA_FLOW"], + "filter": { + "operator": "OR", + "filters": [ + { + "field": "glossaryTerms", + "values": ["urn:li:glossaryTerm:test"], + "negated": True, + "condition": "CONTAIN" + } + ] + } + } + + update_view_json = { + "query": """mutation updateView($urn: String!, $input: UpdateViewInput!) {\n + updateView(urn: $urn, input: $input) {\n + urn\n + }\n + }""", + "variables": { + "urn": view_urn, + "input": { + "name": new_view_name, + "description": new_view_description, + "definition": new_view_definition + } + }, + } + + response = frontend_session.post( + f"{get_frontend_url()}/api/v2/graphql", json=update_view_json + ) + response.raise_for_status() + res_data = response.json() + + assert res_data + assert res_data["data"]["updateView"] is not None + assert "errors" not in res_data + + # Delete the View + delete_view_json = { + "query": """mutation deleteView($urn: String!) {\n + deleteView(urn: $urn) + }""", + "variables": { + "urn": view_urn + }, + } + + response = frontend_session.post( + f"{get_frontend_url()}/api/v2/graphql", json=delete_view_json + ) + response.raise_for_status() + res_data = response.json() + assert "errors" not in res_data From 5658fd5a54ac32b9d9ca9882c8ce3275f21456bd Mon Sep 17 00:00:00 2001 From: Tamas Nemeth Date: Tue, 13 Dec 2022 01:25:32 +0100 Subject: [PATCH 10/19] feat(ingest): bigquery - external url support and a small profiling filter fix (#6714) --- .../ingestion/source/bigquery_v2/bigquery.py | 27 ++++++++++++++++--- .../source/bigquery_v2/bigquery_config.py | 24 +++++++++++++++++ .../ingestion/source/bigquery_v2/common.py | 3 +++ .../ingestion/source/bigquery_v2/profiler.py | 7 ++++- 4 files changed, 57 insertions(+), 4 deletions(-) diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py index 04d7f67daf73a2..7fe76e1b2b3b96 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery.py @@ -10,6 +10,7 @@ from google.cloud import bigquery from google.cloud.bigquery.table import TableListItem +from datahub.configuration.pattern_utils import is_schema_allowed from datahub.emitter.mce_builder import ( make_container_urn, make_data_platform_urn, @@ -54,7 +55,11 @@ BigqueryTable, BigqueryView, ) -from datahub.ingestion.source.bigquery_v2.common import get_bigquery_client +from datahub.ingestion.source.bigquery_v2.common import ( + BQ_EXTERNAL_DATASET_URL_TEMPLATE, + BQ_EXTERNAL_TABLE_URL_TEMPLATE, + get_bigquery_client, +) from datahub.ingestion.source.bigquery_v2.lineage import BigqueryLineageExtractor from datahub.ingestion.source.bigquery_v2.profiler import BigqueryProfiler from datahub.ingestion.source.bigquery_v2.usage import BigQueryUsageExtractor @@ -459,6 +464,11 @@ def gen_dataset_containers( dataset, ["Dataset"], database_container_key, + external_url=BQ_EXTERNAL_DATASET_URL_TEMPLATE.format( + project=project_id, dataset=dataset + ) + if self.config.include_external_url + else None, ) self.stale_entity_removal_handler.add_entity_to_state( @@ -570,8 +580,12 @@ def _process_project( bigquery_project.datasets ) for bigquery_dataset in bigquery_project.datasets: - - if not self.config.dataset_pattern.allowed(bigquery_dataset.name): + if not is_schema_allowed( + self.config.dataset_pattern, + bigquery_dataset.name, + project_id, + self.config.match_fully_qualified_names, + ): self.report.report_dropped(f"{bigquery_dataset.name}.*") continue try: @@ -854,6 +868,13 @@ def gen_dataset_workunits( else None, lastModified=TimeStamp(time=int(table.last_altered.timestamp() * 1000)) if table.last_altered is not None + else TimeStamp(time=int(table.created.timestamp() * 1000)) + if table.created is not None + else None, + externalUrl=BQ_EXTERNAL_TABLE_URL_TEMPLATE.format( + project=project_id, dataset=dataset_name, table=table.name + ) + if self.config.include_external_url else None, ) if custom_properties: diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py index 94117f26ff794d..c2518cd4fc4789 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/bigquery_config.py @@ -48,6 +48,16 @@ class BigQueryV2Config(BigQueryConfig, LineageConfig): description="Regex patterns for dataset to filter in ingestion. Specify regex to only match the schema name. e.g. to match all tables in schema analytics, use the regex 'analytics'", ) + match_fully_qualified_names: bool = Field( + default=False, + description="Whether `dataset_pattern` is matched against fully qualified dataset name `.`.", + ) + + include_external_url: bool = Field( + default=True, + description="Whether to populate BigQuery Console url to Datasets/Tables", + ) + debug_include_full_payloads: bool = Field( default=False, description="Include full payload into events. It is only for debugging and internal use.", @@ -128,6 +138,20 @@ def backward_compatibility_configs_set(cls, values: Dict) -> Dict: logging.warning( "schema_pattern will be ignored in favour of dataset_pattern. schema_pattern will be deprecated, please use dataset_pattern only." ) + + match_fully_qualified_names = values.get("match_fully_qualified_names") + + if ( + dataset_pattern is not None + and dataset_pattern != AllowDenyPattern.allow_all() + and match_fully_qualified_names is not None + and not match_fully_qualified_names + ): + logger.warning( + "Please update `dataset_pattern` to match against fully qualified schema name `.` and set config `match_fully_qualified_names : True`." + "Current default `match_fully_qualified_names: False` is only to maintain backward compatibility. " + "The config option `match_fully_qualified_names` will be deprecated in future and the default behavior will assume `match_fully_qualified_names: True`." + ) return values def get_table_pattern(self, pattern: List[str]) -> str: diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/common.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/common.py index 8a00f8f1d5fe43..4ff509858b87d0 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/common.py @@ -8,6 +8,9 @@ BQ_DATETIME_FORMAT = "%Y-%m-%dT%H:%M:%SZ" BQ_DATE_SHARD_FORMAT = "%Y%m%d" +BQ_EXTERNAL_TABLE_URL_TEMPLATE = "https://console.cloud.google.com/bigquery?project={project}&ws=!1m5!1m4!4m3!1s{project}!2s{dataset}!3s{table}" +BQ_EXTERNAL_DATASET_URL_TEMPLATE = "https://console.cloud.google.com/bigquery?project={project}&ws=!1m4!1m3!3m2!1s{project}!2s{dataset}" + def _make_gcp_logging_client( project_id: Optional[str] = None, extra_client_options: Dict[str, Any] = {} diff --git a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/profiler.py b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/profiler.py index a83787beb84d83..9e232993b7c5f4 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/profiler.py +++ b/metadata-ingestion/src/datahub/ingestion/source/bigquery_v2/profiler.py @@ -172,9 +172,14 @@ def get_workunits( word in column.data_type.lower() for word in ["array", "struct", "geography", "json"] ): + normalized_table_name = BigqueryTableIdentifier( + project_id=project, dataset=dataset, table=table.name + ).get_table_name() + self.config.profile_pattern.deny.append( - f"^{project}.{dataset}.{table.name}.{column.field_path}$" + f"^{normalized_table_name}.{column.field_path}$" ) + # Emit the profile work unit profile_request = self.get_bigquery_profile_request( project=project, dataset=dataset, table=table From 85bb1f503013bc3b45dfd412d7f00afbe427ff81 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Mon, 12 Dec 2022 21:02:52 -0500 Subject: [PATCH 11/19] test(ingest): make hive/trino test more reliable (#6741) --- .../tests/integration/hive/docker-compose.yml | 2 +- .../integration/presto-on-hive/docker-compose.yml | 2 +- .../tests/integration/trino/docker-compose.yml | 11 +++++------ .../tests/test_helpers/docker_helpers.py | 4 ++-- 4 files changed, 9 insertions(+), 10 deletions(-) diff --git a/metadata-ingestion/tests/integration/hive/docker-compose.yml b/metadata-ingestion/tests/integration/hive/docker-compose.yml index 6821d6dc679d8d..9f2f3a6a92d554 100644 --- a/metadata-ingestion/tests/integration/hive/docker-compose.yml +++ b/metadata-ingestion/tests/integration/hive/docker-compose.yml @@ -12,7 +12,7 @@ services: env_file: - ./hadoop-hive.env ports: - - "50070:50070" + - "50070" datanode: image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8 volumes: diff --git a/metadata-ingestion/tests/integration/presto-on-hive/docker-compose.yml b/metadata-ingestion/tests/integration/presto-on-hive/docker-compose.yml index 16eab23aa2b66c..bc52779d71cf8f 100644 --- a/metadata-ingestion/tests/integration/presto-on-hive/docker-compose.yml +++ b/metadata-ingestion/tests/integration/presto-on-hive/docker-compose.yml @@ -29,7 +29,7 @@ services: env_file: - ./setup/hadoop-hive.env ports: - - "50070:50070" + - "50070" datanode: image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8 volumes: diff --git a/metadata-ingestion/tests/integration/trino/docker-compose.yml b/metadata-ingestion/tests/integration/trino/docker-compose.yml index 7c521d777f1dba..59ba91e46d5433 100644 --- a/metadata-ingestion/tests/integration/trino/docker-compose.yml +++ b/metadata-ingestion/tests/integration/trino/docker-compose.yml @@ -3,7 +3,7 @@ version: "3" services: - + testtrino: image: trinodb/trino:369 container_name: "testtrino" @@ -14,13 +14,13 @@ services: depends_on: - "trinodb_postgres" - "hive-metastore" - + trinodb_postgres: image: postgres:alpine container_name: "trinodb_postgres" environment: POSTGRES_PASSWORD: datahub - volumes: + volumes: - ./setup/setup.sql:/docker-entrypoint-initdb.d/postgres_setup.sql ports: - "5432:5432" @@ -33,7 +33,7 @@ services: env_file: - ./setup/hadoop-hive.env ports: - - "50070:50070" + - "50070" datanode: image: bde2020/hadoop-datanode:2.0.0-hadoop2.7.4-java8 volumes: @@ -43,7 +43,7 @@ services: environment: SERVICE_PRECONDITION: "namenode:50070" ports: - - "50075:50075" + - "50075" hive-server: image: bde2020/hive:2.3.2-postgresql-metastore container_name: "testhiveserver2" @@ -58,7 +58,6 @@ services: - ./setup/hive_setup.sql:/hive_setup.sql hive-metastore: image: bde2020/hive:2.3.2-postgresql-metastore - container_name: "hive-metastore" env_file: - ./setup/hadoop-hive.env command: /opt/hive/bin/hive --service metastore diff --git a/metadata-ingestion/tests/test_helpers/docker_helpers.py b/metadata-ingestion/tests/test_helpers/docker_helpers.py index 0cbae4b2db104d..62654cf9f611ce 100644 --- a/metadata-ingestion/tests/test_helpers/docker_helpers.py +++ b/metadata-ingestion/tests/test_helpers/docker_helpers.py @@ -48,14 +48,14 @@ def docker_compose_runner( ): @contextlib.contextmanager def run( - compose_file_path: Union[str, list], key: str + compose_file_path: Union[str, list], key: str, cleanup: bool = True ) -> pytest_docker.plugin.Services: with pytest_docker.plugin.get_docker_services( docker_compose_command=docker_compose_command, docker_compose_file=compose_file_path, docker_compose_project_name=f"{docker_compose_project_name}-{key}", docker_setup=docker_setup, - docker_cleanup=docker_cleanup, + docker_cleanup=docker_cleanup if cleanup else False, ) as docker_services: yield docker_services From 43f95d2b5cd4df01f3a0c9df4056a64eeba8c5e1 Mon Sep 17 00:00:00 2001 From: Tamas Nemeth Date: Tue, 13 Dec 2022 03:43:43 +0100 Subject: [PATCH 12/19] Initial commit for bigquery ingestion guide (#6587) * Initial commit for bigquery ingestion guide * Addressing PR review comments * Fixing lint error * Shorten titles * Removing images * update copy on overview.md * update to setup steps with additional roles * update configuration.md * lowcasing overview.md filename * lowcasing setup.md * lowcasing configuration.md * update reference to setup.md * update reference to setup.md * update reference to configuration.md * lowcase bigquery ingestion guide filenames * Update location of ingestion guides in sidebar * renaming ingestion quickstart guide sidebar * remove old files * Update docs-website/sidebars.js * tweak Co-authored-by: Maggie Hays Co-authored-by: Harshal Sheth --- docs-website/sidebars.js | 11 ++ .../bigquery/configuration.md | 150 ++++++++++++++++++ .../bigquery/overview.md | 37 +++++ docs/quick-ingestion-guides/bigquery/setup.md | 60 +++++++ 4 files changed, 258 insertions(+) create mode 100644 docs/quick-ingestion-guides/bigquery/configuration.md create mode 100644 docs/quick-ingestion-guides/bigquery/overview.md create mode 100644 docs/quick-ingestion-guides/bigquery/setup.md diff --git a/docs-website/sidebars.js b/docs-website/sidebars.js index 57faeec74c6c70..a565dafcdd78d4 100644 --- a/docs-website/sidebars.js +++ b/docs-website/sidebars.js @@ -105,6 +105,17 @@ module.exports = { // "docs/wip/guide-enrich-your-metadata", ], }, + { + "Ingestion Quickstart Guides": [ + { + BigQuery: [ + "docs/quick-ingestion-guides/bigquery/overview", + "docs/quick-ingestion-guides/bigquery/setup", + "docs/quick-ingestion-guides/bigquery/configuration", + ], + }, + ], + }, ], "Ingest Metadata": [ // The purpose of this section is to provide a deeper understanding of how ingestion works. diff --git a/docs/quick-ingestion-guides/bigquery/configuration.md b/docs/quick-ingestion-guides/bigquery/configuration.md new file mode 100644 index 00000000000000..ed43f10ddf007a --- /dev/null +++ b/docs/quick-ingestion-guides/bigquery/configuration.md @@ -0,0 +1,150 @@ +--- +title: Configuration +--- +# Configuring Your BigQuery Connector to DataHub + +Now that you have created a Service Account and Service Account Key in BigQuery in [the prior step](setup.md), it's now time to set up a connection via the DataHub UI. + +## Configure Secrets + +1. Within DataHub, navigate to the **Ingestion** tab in the top, right corner of your screen + +

+ Navigate to the "Ingestion Tab" +

+ +:::note +If you do not see the Ingestion tab, please contact your DataHub admin to grant you the correct permissions +::: + +2. Navigate to the **Secrets** tab and click **Create new secret** + +

+ Secrets Tab +

+ + +3. Create a Private Key secret + +This will securely store your BigQuery Service Account Private Key within DataHub + + * Enter a name like `BIGQUERY_PRIVATE_KEY` - we will use this later to refer to the secret + * Copy and paste the `private_key` value from your Service Account Key + * Optionally add a description + * Click **Create** + +

+ Private Key Secret +

+ +4. Create a Private Key ID secret + +This will securely store your BigQuery Service Account Private Key ID within DataHub + + * Click **Create new secret** again + * Enter a name like `BIGQUERY_PRIVATE_KEY_ID` - we will use this later to refer to the secret + * Copy and paste the `private_key_id` value from your Service Account Key + * Optionally add a description + * Click **Create** + +

+ Private Key Id Secret +

+ +## Configure Recipe + +5. Navigate to the **Sources** tab and click **Create new source** + +

+ Click "Create new source" +

+ +6. Select BigQuery + +

+ Select BigQuery from the options +

+ +7. Fill out the BigQuery Recipe + +You can find the following details in your Service Account Key file: + +* Project ID +* Client Email +* Client ID + +Populate the Secret Fields by selecting the Primary Key and Primary Key ID secrets you created in steps 3 and 4. + +

+ Fill out the BigQuery Recipe +

+ +8. Click **Test Connection** + +This step will ensure you have configured your credentials accurately and confirm you have the required permissions to extract all relevant metadata. + +

+ Test BigQuery connection +

+ +After you have successfully tested your connection, click **Next**. + +## Schedule Execution + +Now it's time to schedule a recurring ingestion pipeline to regularly extract metadata from your BigQuery instance. + +9. Decide how regularly you want this ingestion to run-- day, month, year, hour, minute, etc. Select from the dropdown +

+ schedule selector +

+ +10. Ensure you've configured your correct timezone +

+ timezone_selector +

+ +11. Click **Next** when you are done + +## Finish Up + +12. Name your ingestion source, then click **Save and Run** +

+ Name your ingestion +

+ +You will now find your new ingestion source running + +

+ ingestion_running +

+ +## Validate Ingestion Runs + +13. View the latest status of ingestion runs on the Ingestion page + +

+ ingestion succeeded +

+ +14. Click the plus sign to expand the full list of historical runs and outcomes; click **Details** to see the outcomes of a specific run + +

+ ingestion_details +

+ +15. From the Ingestion Run Details page, pick **View All** to see which entities were ingested + +

+ ingestion_details_view_all +

+ +16. Pick an entity from the list to manually validate if it contains the detail you expected + +

+ ingestion_details_view_all +

+ + +**Congratulations!** You've successfully set up BigQuery as an ingestion source for DataHub! + +*Need more help? Join the conversation in [Slack](http://slack.datahubproject.io)!* diff --git a/docs/quick-ingestion-guides/bigquery/overview.md b/docs/quick-ingestion-guides/bigquery/overview.md new file mode 100644 index 00000000000000..8cd68798478987 --- /dev/null +++ b/docs/quick-ingestion-guides/bigquery/overview.md @@ -0,0 +1,37 @@ +--- +title: Overview +--- +# BigQuery Ingestion Guide: Overview + +## What You Will Get Out of This Guide + +This guide will help you set up the BigQuery connector through the DataHub UI to begin ingesting metadata into DataHub. + +Upon completing this guide, you will have a recurring ingestion pipeline that will extract metadata from BigQuery and load it into DataHub. This will include to following BigQuery asset types: + +* [Projects](https://cloud.google.com/bigquery/docs/resource-hierarchy#projects) +* [Datasets](https://cloud.google.com/bigquery/docs/datasets-intro) +* [Tables](https://cloud.google.com/bigquery/docs/tables-intro) +* [Views](https://cloud.google.com/bigquery/docs/views-intro) +* [Materialized Views](https://cloud.google.com/bigquery/docs/materialized-views-intro) + +This recurring ingestion pipeline will also extract: + +* **Usage statistics** to help you understand recent query activity +* **Table-level lineage** (where available) to automatically define interdependencies between datasets +* **Table- and column-level profile statistics** to help you understand the shape of the data + +:::caution +You will NOT have extracted [Routines](https://cloud.google.com/bigquery/docs/routines), [Search Indexes](https://cloud.google.com/bigquery/docs/search-intro) from BigQuery, as the connector does not support ingesting these assets +::: + +## Next Steps +If that all sounds like what you're looking for, navigate to the [next page](setup.md), where we'll talk about prerequisites + +## Advanced Guides and Reference +If you're looking to do something more in-depth, want to use CLI instead of the DataHub UI, or just need to look at the reference documentation for this connector, use these links: + +* Learn about CLI Ingestion in the [Introduction to Metadata Ingestion](../../../metadata-ingestion/README.md) +* [BigQuery Ingestion Reference Guide](https://datahubproject.io/docs/generated/ingestion/sources/bigquery/#module-bigquery) + +*Need more help? Join the conversation in [Slack](http://slack.datahubproject.io)!* diff --git a/docs/quick-ingestion-guides/bigquery/setup.md b/docs/quick-ingestion-guides/bigquery/setup.md new file mode 100644 index 00000000000000..3635ea66f74ac0 --- /dev/null +++ b/docs/quick-ingestion-guides/bigquery/setup.md @@ -0,0 +1,60 @@ +--- +title: Setup +--- +# BigQuery Ingestion Guide: Setup & Prerequisites + +To configure ingestion from BigQuery, you'll need a [Service Account](https://cloud.google.com/iam/docs/creating-managing-service-accounts) configured with the proper permission sets, and an associated [Service Account Key](https://cloud.google.com/iam/docs/creating-managing-service-account-keys). + +This setup guide will walk you through the steps you'll need to take via your Google Cloud Console. + +## BigQuery Prerequisites + +If you do not have an existing Service Account and Service Account Key, please work with your BigQuery Admin to ensure you have the appropriate permissions and/or roles to continue with this setup guide. + +When creating and managing new Service Accounts and Service Account Keys, we have found the following permissions and roles to be required: + +* Create a Service Account: `iam.serviceAccounts.create` permission +* Assign roles to a Service Account: `serviceusage.services.enable` permission +* Set permission policy to the project: `resourcemanager.projects.setIamPolicy` permission +* Generate Key for Service Account: Service Account Key Admin (`roles/iam.serviceAccountKeyAdmin`) IAM role + +:::note +Please refer to the BigQuery [Permissions](https://cloud.google.com/iam/docs/permissions-reference) and [IAM Roles](https://cloud.google.com/iam/docs/understanding-roles) references for details +::: + +## BigQuery Setup + +1. To set up a new Service Account follow [this guide](https://cloud.google.com/iam/docs/creating-managing-service-accounts) + +2. When you are creating a Service Account, assign the following predefined Roles: + - [BigQuery Job User](https://cloud.google.com/bigquery/docs/access-control#bigquery.jobUser) + - [BigQuery Metadata Viewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.metadataViewer) + - [BigQuery Resource Viewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.resourceViewer) -> This role is for Table-Level Lineage and Usage extraction + - [Logs View Accessor](https://cloud.google.com/bigquery/docs/access-control#bigquery.dataViewer) -> This role is for Table-Level Lineage and Usage extraction + - [BigQuery Data Viewer](https://cloud.google.com/bigquery/docs/access-control#bigquery.dataViewer) -> This role is for Profiling + +:::note +You can always add/remove roles to Service Accounts later on. Please refer to the BigQuery [Manage access to projects, folders, and organizations](https://cloud.google.com/iam/docs/granting-changing-revoking-access) guide for more details. +::: + +3. Create and download a [Service Account Key](https://cloud.google.com/iam/docs/creating-managing-service-account-keys). We will use this to set up authentication within DataHub. + +The key file looks like this: +```json +{ + "type": "service_account", + "project_id": "project-id-1234567", + "private_key_id": "d0121d0000882411234e11166c6aaa23ed5d74e0", + "private_key": "-----BEGIN PRIVATE KEY-----\nMIIyourkey\n-----END PRIVATE KEY-----", + "client_email": "test@suppproject-id-1234567.iam.gserviceaccount.com", + "client_id": "113545814931671546333", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test%suppproject-id-1234567.iam.gserviceaccount.com" +} +``` +## Next Steps +Once you've confirmed all of the above in BigQuery, it's time to [move on](configuration.md) to configure the actual ingestion source within the DataHub UI. + +*Need more help? Join the conversation in [Slack](http://slack.datahubproject.io)!* From 1da27edcbc97d64d5e88321c8727fabf92a576f8 Mon Sep 17 00:00:00 2001 From: Aseem Bansal Date: Tue, 13 Dec 2022 10:42:21 +0530 Subject: [PATCH 13/19] fix(ci): remove warnings due to deprecated action (#6735) --- .github/actions/docker-custom-build-and-push/action.yml | 7 +++---- .github/workflows/docker-ingestion.yml | 6 +++--- .github/workflows/docker-postgres-setup.yml | 6 +++--- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/actions/docker-custom-build-and-push/action.yml b/.github/actions/docker-custom-build-and-push/action.yml index 0cb553ca8cf03d..993daf467c5a7f 100644 --- a/.github/actions/docker-custom-build-and-push/action.yml +++ b/.github/actions/docker-custom-build-and-push/action.yml @@ -39,14 +39,13 @@ runs: steps: - name: Docker meta id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + uses: docker/metadata-action@v4 with: # list of Docker images to use as base name for tags images: ${{ inputs.images }} # add git short SHA as Docker tag - tag-custom: ${{ inputs.tags }} - tag-custom-only: true - + tags: | + type=raw,value=${{ inputs.tags }} # Code for testing the build when not pushing to Docker Hub. - name: Build and Load image for testing (if not publishing) uses: docker/build-push-action@v3 diff --git a/.github/workflows/docker-ingestion.yml b/.github/workflows/docker-ingestion.yml index e5bc725f69a6b0..4b4db7512d75d5 100644 --- a/.github/workflows/docker-ingestion.yml +++ b/.github/workflows/docker-ingestion.yml @@ -63,14 +63,14 @@ jobs: fetch-depth: 0 - name: Docker meta id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + uses: docker/metadata-action@v4 with: # list of Docker images to use as base name for tags images: | linkedin/datahub-ingestion # add git short SHA as Docker tag - tag-custom: ${{ needs.setup.outputs.tag }} - tag-custom-only: true + tags: | + type=raw,value=${{ needs.setup.outputs.tag }} - name: Set up QEMU uses: docker/setup-qemu-action@v2 - name: Set up Docker Buildx diff --git a/.github/workflows/docker-postgres-setup.yml b/.github/workflows/docker-postgres-setup.yml index ad0a49dd7718c5..ad745b07d0af9a 100644 --- a/.github/workflows/docker-postgres-setup.yml +++ b/.github/workflows/docker-postgres-setup.yml @@ -54,14 +54,14 @@ jobs: fetch-depth: 0 - name: Docker meta id: docker_meta - uses: crazy-max/ghaction-docker-meta@v1 + uses: docker/metadata-action@v4 with: # list of Docker images to use as base name for tags images: | acryldata/datahub-postgres-setup # add git short SHA as Docker tag - tag-custom: ${{ needs.setup.outputs.tag }} - tag-custom-only: true + tags: | + type=raw,value=${{ needs.setup.outputs.tag }} - name: Login to DockerHub if: ${{ needs.setup.outputs.publish == 'true' }} uses: docker/login-action@v2 From 551ef1b33595a3d370c4feb769ffc9bd01181a08 Mon Sep 17 00:00:00 2001 From: Dmitry Bryazgin <58312247+bda618@users.noreply.github.com> Date: Tue, 13 Dec 2022 01:13:39 -0500 Subject: [PATCH 14/19] feat(ingest): add stateful ingestion to the ldap source (#6127) Co-authored-by: Harshal Sheth --- .../src/datahub/ingestion/source/ldap.py | 80 ++++-- .../ingestion/source/state/ldap_state.py | 39 +++ .../integration/ldap/ldap_mces_golden.json | 234 ++++++++++++------ ...ap_mces_golden_deleted_group_stateful.json | 51 ++++ .../ldap_mces_golden_deleted_stateful.json | 54 ++++ .../ldap/ldap_mces_golden_group_stateful.json | 86 +++++++ .../ldap/ldap_mces_golden_stateful.json | 81 ++++++ .../ldap/ldap_memberof_mces_golden.json | 64 +++-- .../integration/ldap/setup/custom/sample.ldif | 10 +- .../integration/ldap/test_ldap_stateful.py | 207 ++++++++++++++++ .../tests/unit/test_ldap_state.py | 57 +++++ 11 files changed, 844 insertions(+), 119 deletions(-) create mode 100644 metadata-ingestion/src/datahub/ingestion/source/state/ldap_state.py create mode 100644 metadata-ingestion/tests/integration/ldap/ldap_mces_golden_deleted_group_stateful.json create mode 100644 metadata-ingestion/tests/integration/ldap/ldap_mces_golden_deleted_stateful.json create mode 100644 metadata-ingestion/tests/integration/ldap/ldap_mces_golden_group_stateful.json create mode 100644 metadata-ingestion/tests/integration/ldap/ldap_mces_golden_stateful.json create mode 100644 metadata-ingestion/tests/integration/ldap/test_ldap_stateful.py create mode 100644 metadata-ingestion/tests/unit/test_ldap_state.py diff --git a/metadata-ingestion/src/datahub/ingestion/source/ldap.py b/metadata-ingestion/src/datahub/ingestion/source/ldap.py index 58fcb11f959e8c..e03da4b4681fb7 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/ldap.py +++ b/metadata-ingestion/src/datahub/ingestion/source/ldap.py @@ -7,7 +7,7 @@ from ldap.controls import SimplePagedResultsControl from pydantic.fields import Field -from datahub.configuration.common import ConfigModel, ConfigurationError +from datahub.configuration.common import ConfigurationError from datahub.ingestion.api.common import PipelineContext from datahub.ingestion.api.decorators import ( SupportStatus, @@ -15,8 +15,17 @@ platform_name, support_status, ) -from datahub.ingestion.api.source import Source, SourceReport from datahub.ingestion.api.workunit import MetadataWorkUnit +from datahub.ingestion.source.state.ldap_state import LdapCheckpointState +from datahub.ingestion.source.state.stale_entity_removal_handler import ( + StaleEntityRemovalHandler, + StaleEntityRemovalSourceReport, + StatefulStaleMetadataRemovalConfig, +) +from datahub.ingestion.source.state.stateful_ingestion_base import ( + StatefulIngestionConfigBase, + StatefulIngestionSourceBase, +) from datahub.metadata.com.linkedin.pegasus2avro.mxe import MetadataChangeEvent from datahub.metadata.schema_classes import ( CorpGroupInfoClass, @@ -25,6 +34,10 @@ CorpUserSnapshotClass, GroupMembershipClass, ) +from datahub.utilities.source_helpers import ( + auto_stale_entity_removal, + auto_status_aspect, +) # default mapping for attrs user_attrs_map: Dict[str, Any] = {} @@ -86,7 +99,7 @@ def set_cookie( return bool(cookie) -class LDAPSourceConfig(ConfigModel): +class LDAPSourceConfig(StatefulIngestionConfigBase): """Config used by the LDAP Source.""" # Server configuration. @@ -94,6 +107,9 @@ class LDAPSourceConfig(ConfigModel): ldap_user: str = Field(description="LDAP user.") ldap_password: str = Field(description="LDAP password.") + # Custom Stateful Ingestion settings + stateful_ingestion: Optional[StatefulStaleMetadataRemovalConfig] = None + # Extraction configuration. base_dn: str = Field(description="LDAP DN.") filter: str = Field(default="(objectClass=*)", description="LDAP extractor filter.") @@ -117,7 +133,7 @@ class LDAPSourceConfig(ConfigModel): @dataclasses.dataclass -class LDAPSourceReport(SourceReport): +class LDAPSourceReport(StaleEntityRemovalSourceReport): dropped_dns: List[str] = dataclasses.field(default_factory=list) @@ -151,7 +167,7 @@ def guess_person_ldap( @config_class(LDAPSourceConfig) @support_status(SupportStatus.CERTIFIED) @dataclasses.dataclass -class LDAPSource(Source): +class LDAPSource(StatefulIngestionSourceBase): """ This plugin extracts the following: - People @@ -161,11 +177,21 @@ class LDAPSource(Source): config: LDAPSourceConfig report: LDAPSourceReport + platform: str = "ldap" def __init__(self, ctx: PipelineContext, config: LDAPSourceConfig): """Constructor.""" - super().__init__(ctx) + super(LDAPSource, self).__init__(config, ctx) self.config = config + + self.stale_entity_removal_handler = StaleEntityRemovalHandler( + source=self, + config=self.config, + state_type_class=LdapCheckpointState, + pipeline_name=self.ctx.pipeline_name, + run_id=self.ctx.run_id, + ) + # ensure prior defaults are in place for k in user_attrs_map: if k not in self.config.user_attrs_map: @@ -199,6 +225,12 @@ def create(cls, config_dict: Dict[str, Any], ctx: PipelineContext) -> "LDAPSourc return cls(ctx, config) def get_workunits(self) -> Iterable[MetadataWorkUnit]: + return auto_stale_entity_removal( + self.stale_entity_removal_handler, + auto_status_aspect(self.get_workunits_internal()), + ) + + def get_workunits_internal(self) -> Iterable[MetadataWorkUnit]: """Returns an Iterable containing the workunits to ingest LDAP users or groups.""" cookie = True while cookie: @@ -251,6 +283,13 @@ def get_workunits(self) -> Iterable[MetadataWorkUnit]: cookie = set_cookie(self.lc, pctrls) + def get_platform_instance_id(self) -> str: + """ + The source identifier such as the specific source host address required for stateful ingestion. + Individual subclasses need to override this method appropriately. + """ + return self.config.ldap_server + def handle_user(self, dn: str, attrs: Dict[str, Any]) -> Iterable[MetadataWorkUnit]: """ Handle a DN and attributes by adding manager info and constructing a @@ -358,7 +397,7 @@ def build_corp_user_mce( countryCode=country_code, title=title, managerUrn=manager_urn, - ) + ), ], ) @@ -389,21 +428,20 @@ def build_corp_group_mce(self, attrs: dict) -> Optional[MetadataChangeEvent]: if self.config.group_attrs_map["displayName"] in attrs else None ) - return MetadataChangeEvent( - proposedSnapshot=CorpGroupSnapshotClass( - urn=f"urn:li:corpGroup:{full_name}", - aspects=[ - CorpGroupInfoClass( - email=email, - admins=admins, - members=members, - groups=[], - description=description, - displayName=displayName, - ) - ], - ) + group_snapshot = CorpGroupSnapshotClass( + urn=f"urn:li:corpGroup:{full_name}", + aspects=[ + CorpGroupInfoClass( + email=email, + admins=admins, + members=members, + groups=[], + description=description, + displayName=displayName, + ), + ], ) + return MetadataChangeEvent(proposedSnapshot=group_snapshot) return None def get_report(self) -> LDAPSourceReport: diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/ldap_state.py b/metadata-ingestion/src/datahub/ingestion/source/state/ldap_state.py new file mode 100644 index 00000000000000..b30fc4c026cd34 --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/state/ldap_state.py @@ -0,0 +1,39 @@ +from typing import Iterable, List + +import pydantic + +from datahub.ingestion.source.state.stale_entity_removal_handler import ( + StaleEntityCheckpointStateBase, +) +from datahub.utilities.urns.urn import guess_entity_type + + +class LdapCheckpointState(StaleEntityCheckpointStateBase["LdapCheckpointState"]): + """ + Base class for representing the checkpoint state for all LDAP based sources. + Stores all corpuser and corpGroup and being ingested and is used to remove any stale entities. + """ + + urns: List[str] = pydantic.Field(default_factory=list) + + @classmethod + def get_supported_types(cls) -> List[str]: + return ["corpuser", "corpGroup"] + + def add_checkpoint_urn(self, type: str, urn: str) -> None: + assert type in self.get_supported_types() + self.urns.append(urn) + + def get_urns_not_in( + self, type: str, other_checkpoint_state: "LdapCheckpointState" + ) -> Iterable[str]: + assert type in self.get_supported_types() + diff = set(self.urns) - set(other_checkpoint_state.urns) + yield from (urn for urn in diff if guess_entity_type(urn) == type) + + def get_percent_entities_changed( + self, old_checkpoint_state: "LdapCheckpointState" + ) -> float: + return StaleEntityCheckpointStateBase.compute_percent_entities_changed( + [(self.urns, old_checkpoint_state.urns)] + ) diff --git a/metadata-ingestion/tests/integration/ldap/ldap_mces_golden.json b/metadata-ingestion/tests/integration/ldap/ldap_mces_golden.json index c88815bfba63f0..4cc3d7e7c7b5e4 100644 --- a/metadata-ingestion/tests/integration/ldap/ldap_mces_golden.json +++ b/metadata-ingestion/tests/integration/ldap/ldap_mces_golden.json @@ -1,35 +1,29 @@ [ { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.CorpGroupSnapshot": { "urn": "urn:li:corpGroup:simpons-group", "aspects": [ { "com.linkedin.pegasus2avro.identity.CorpGroupInfo": { - "displayName": null, "email": "simpons-group", "admins": [], "members": [ "urn:li:corpuser:hsimpson", "urn:li:corpuser:lsimpson" ], - "groups": [], - "description": null + "groups": [] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test", - "properties": null + "runId": "ldap-test" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": { "urn": "urn:li:corpuser:bsimpson", @@ -40,27 +34,20 @@ "displayName": "Bart Simpson", "email": "bsimpson", "title": "Mr. Boss", - "managerUrn": null, - "departmentId": null, - "departmentName": null, "firstName": "Bart", "lastName": "Simpson", - "fullName": "Bart Simpson", - "countryCode": null + "fullName": "Bart Simpson" } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test", - "properties": null + "runId": "ldap-test" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": { "urn": "urn:li:corpuser:hsimpson", @@ -76,22 +63,18 @@ "departmentName": "1001", "firstName": "Homer", "lastName": "Simpson", - "fullName": "Homer Simpson", - "countryCode": null + "fullName": "Homer Simpson" } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test", - "properties": null + "runId": "ldap-test" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": { "urn": "urn:li:corpuser:lsimpson", @@ -101,28 +84,20 @@ "active": true, "displayName": "Lisa Simpson", "email": "lsimpson", - "title": null, - "managerUrn": null, - "departmentId": null, - "departmentName": null, "firstName": "Lisa", "lastName": "Simpson", - "fullName": "Lisa Simpson", - "countryCode": null + "fullName": "Lisa Simpson" } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test", - "properties": null + "runId": "ldap-test" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": { "urn": "urn:li:corpuser:msimpson", @@ -132,28 +107,20 @@ "active": true, "displayName": "Maggie Simpson", "email": "msimpson", - "title": null, - "managerUrn": null, - "departmentId": null, - "departmentName": null, "firstName": "Maggie", "lastName": "Simpson", - "fullName": "Maggie Simpson", - "countryCode": null + "fullName": "Maggie Simpson" } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test", - "properties": null + "runId": "ldap-test" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": { "urn": "urn:li:corpuser:hbevan", @@ -163,28 +130,20 @@ "active": true, "displayName": "Hester Bevan", "email": "hbevan", - "title": null, - "managerUrn": null, - "departmentId": null, - "departmentName": null, "firstName": "Hester", "lastName": "Bevan", - "fullName": "Hester Bevan", - "countryCode": null + "fullName": "Hester Bevan" } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test", - "properties": null + "runId": "ldap-test" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": { "urn": "urn:li:corpuser:ehaas", @@ -194,50 +153,185 @@ "active": true, "displayName": "Evalyn Haas", "email": "ehaas", - "title": null, - "managerUrn": null, - "departmentId": null, - "departmentName": null, "firstName": "Evalyn", "lastName": "Haas", - "fullName": "Evalyn Haas", - "countryCode": null + "fullName": "Evalyn Haas" } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test", - "properties": null + "runId": "ldap-test" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.CorpGroupSnapshot": { "urn": "urn:li:corpGroup:HR Department", "aspects": [ { "com.linkedin.pegasus2avro.identity.CorpGroupInfo": { - "displayName": null, "email": "HR Department", "admins": [], "members": [], - "groups": [], - "description": null + "groups": [] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test", - "properties": null + "runId": "ldap-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.CorpGroupSnapshot": { + "urn": "urn:li:corpGroup:Finance Department", + "aspects": [ + { + "com.linkedin.pegasus2avro.identity.CorpGroupInfo": { + "email": "Finance Department", + "admins": [], + "members": [], + "groups": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "ldap-test" + } +}, +{ + "entityType": "corpGroup", + "entityUrn": "urn:li:corpGroup:Finance Department", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "ldap-test" + } +}, +{ + "entityType": "corpGroup", + "entityUrn": "urn:li:corpGroup:HR Department", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "ldap-test" + } +}, +{ + "entityType": "corpGroup", + "entityUrn": "urn:li:corpGroup:simpons-group", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "ldap-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:bsimpson", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "ldap-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:ehaas", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "ldap-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:hbevan", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "ldap-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:hsimpson", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "ldap-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:lsimpson", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "ldap-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:msimpson", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "ldap-test" } } -] \ No newline at end of file +] diff --git a/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_deleted_group_stateful.json b/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_deleted_group_stateful.json new file mode 100644 index 00000000000000..d4bafbd664a15e --- /dev/null +++ b/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_deleted_group_stateful.json @@ -0,0 +1,51 @@ +[ +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.CorpGroupSnapshot": { + "urn": "urn:li:corpGroup:HR Department", + "aspects": [ + { + "com.linkedin.pegasus2avro.identity.CorpGroupInfo": { + "email": "HR Department", + "admins": [], + "members": [], + "groups": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1660460400000, + "runId": "ldap-test" + } +}, +{ + "entityType": "corpGroup", + "entityUrn": "urn:li:corpGroup:HR Department", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1660460400000, + "runId": "ldap-test" + } +}, +{ + "entityType": "corpGroup", + "entityUrn": "urn:li:corpGroup:Finance Department", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": true}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1660460400000, + "runId": "ldap-test" + } +} +] diff --git a/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_deleted_stateful.json b/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_deleted_stateful.json new file mode 100644 index 00000000000000..5af7b9b951eb8c --- /dev/null +++ b/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_deleted_stateful.json @@ -0,0 +1,54 @@ +[ +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": { + "urn": "urn:li:corpuser:bsimpson", + "aspects": [ + { + "com.linkedin.pegasus2avro.identity.CorpUserInfo": { + "active": true, + "displayName": "Bart Simpson", + "email": "bsimpson", + "title": "Mr. Boss", + "firstName": "Bart", + "lastName": "Simpson", + "fullName": "Bart Simpson" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1660460400000, + "runId": "ldap-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:bsimpson", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1660460400000, + "runId": "ldap-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:hsimpson", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": true}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1660460400000, + "runId": "ldap-test" + } +} +] diff --git a/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_group_stateful.json b/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_group_stateful.json new file mode 100644 index 00000000000000..dc971469827958 --- /dev/null +++ b/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_group_stateful.json @@ -0,0 +1,86 @@ +[ +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.CorpGroupSnapshot": { + "urn": "urn:li:corpGroup:HR Department", + "aspects": [ + { + "com.linkedin.pegasus2avro.identity.CorpGroupInfo": { + "email": "HR Department", + "admins": [], + "members": [], + "groups": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1660460400000, + "runId": "ldap-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.CorpGroupSnapshot": { + "urn": "urn:li:corpGroup:Finance Department", + "aspects": [ + { + "com.linkedin.pegasus2avro.identity.CorpGroupInfo": { + "email": "Finance Department", + "admins": [], + "members": [], + "groups": [] + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1660460400000, + "runId": "ldap-test" + } +}, +{ + "entityType": "corpGroup", + "entityUrn": "urn:li:corpGroup:Finance Department", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1660460400000, + "runId": "ldap-test" + } +}, +{ + "entityType": "corpGroup", + "entityUrn": "urn:li:corpGroup:HR Department", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1660460400000, + "runId": "ldap-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:bsimpson", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": true}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1660460400000, + "runId": "ldap-test" + } +} +] diff --git a/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_stateful.json b/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_stateful.json new file mode 100644 index 00000000000000..2f5cf3992c21ca --- /dev/null +++ b/metadata-ingestion/tests/integration/ldap/ldap_mces_golden_stateful.json @@ -0,0 +1,81 @@ +[ +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": { + "urn": "urn:li:corpuser:bsimpson", + "aspects": [ + { + "com.linkedin.pegasus2avro.identity.CorpUserInfo": { + "active": true, + "displayName": "Bart Simpson", + "email": "bsimpson", + "title": "Mr. Boss", + "firstName": "Bart", + "lastName": "Simpson", + "fullName": "Bart Simpson" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1660460400000, + "runId": "ldap-test" + } +}, +{ + "proposedSnapshot": { + "com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": { + "urn": "urn:li:corpuser:hsimpson", + "aspects": [ + { + "com.linkedin.pegasus2avro.identity.CorpUserInfo": { + "active": true, + "displayName": "Homer Simpson", + "email": "hsimpson", + "title": "Mr. Everything", + "managerUrn": "urn:li:corpuser:bsimpson", + "departmentId": 1001, + "departmentName": "1001", + "firstName": "Homer", + "lastName": "Simpson", + "fullName": "Homer Simpson" + } + } + ] + } + }, + "systemMetadata": { + "lastObserved": 1660460400000, + "runId": "ldap-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:bsimpson", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1660460400000, + "runId": "ldap-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:hsimpson", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1660460400000, + "runId": "ldap-test" + } +} +] diff --git a/metadata-ingestion/tests/integration/ldap/ldap_memberof_mces_golden.json b/metadata-ingestion/tests/integration/ldap/ldap_memberof_mces_golden.json index 1ac0c691f3e908..a62f974fba24af 100644 --- a/metadata-ingestion/tests/integration/ldap/ldap_memberof_mces_golden.json +++ b/metadata-ingestion/tests/integration/ldap/ldap_memberof_mces_golden.json @@ -1,6 +1,5 @@ [ { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": { "urn": "urn:li:corpuser:hbevan", @@ -10,37 +9,28 @@ "active": true, "displayName": "Hester Bevan", "email": "hbevan", - "title": null, - "managerUrn": null, - "departmentId": null, - "departmentName": null, "firstName": "Hester", "lastName": "Bevan", - "fullName": "Hester Bevan", - "countryCode": null + "fullName": "Hester Bevan" } }, { "com.linkedin.pegasus2avro.identity.GroupMembership": { "groups": [ - "urn:li:corpGroup:HR Department" + "urn:li:corpGroup:HR Department", + "urn:li:corpGroup:Finance Department" ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "ldap-test" } }, { - "auditHeader": null, "proposedSnapshot": { "com.linkedin.pegasus2avro.metadata.snapshot.CorpUserSnapshot": { "urn": "urn:li:corpuser:ehaas", @@ -50,33 +40,53 @@ "active": true, "displayName": "Evalyn Haas", "email": "ehaas", - "title": null, - "managerUrn": null, - "departmentId": null, - "departmentName": null, "firstName": "Evalyn", "lastName": "Haas", - "fullName": "Evalyn Haas", - "countryCode": null + "fullName": "Evalyn Haas" } }, { "com.linkedin.pegasus2avro.identity.GroupMembership": { "groups": [ - "urn:li:corpGroup:HR Department" + "urn:li:corpGroup:HR Department", + "urn:li:corpGroup:Finance Department" ] } } ] } }, - "proposedDelta": null, "systemMetadata": { "lastObserved": 1615443388097, - "runId": "ldap-test", - "registryName": null, - "registryVersion": null, - "properties": null + "runId": "ldap-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:ehaas", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "ldap-test" + } +}, +{ + "entityType": "corpuser", + "entityUrn": "urn:li:corpuser:hbevan", + "changeType": "UPSERT", + "aspectName": "status", + "aspect": { + "value": "{\"removed\": false}", + "contentType": "application/json" + }, + "systemMetadata": { + "lastObserved": 1615443388097, + "runId": "ldap-test" } } -] \ No newline at end of file +] diff --git a/metadata-ingestion/tests/integration/ldap/setup/custom/sample.ldif b/metadata-ingestion/tests/integration/ldap/setup/custom/sample.ldif index ecb39d64ea6a77..15e90923cb49f6 100644 --- a/metadata-ingestion/tests/integration/ldap/setup/custom/sample.ldif +++ b/metadata-ingestion/tests/integration/ldap/setup/custom/sample.ldif @@ -161,4 +161,12 @@ cn: HR Department objectclass: groupOfNames objectclass: top member: cn=Hester Bevan,ou=people,dc=example,dc=org -member: cn=Evalyn Haas,ou=people,dc=example,dc=org \ No newline at end of file +member: cn=Evalyn Haas,ou=people,dc=example,dc=org + +# Entry 13: cn=Finance Department,ou=groups,dc=example,dc=org +dn: cn=Finance Department,dc=example,dc=org +cn: Finance Department +objectclass: groupOfNames +objectclass: top +member: cn=Hester Bevan,ou=people,dc=example,dc=org +member: cn=Evalyn Haas,ou=people,dc=example,dc=org diff --git a/metadata-ingestion/tests/integration/ldap/test_ldap_stateful.py b/metadata-ingestion/tests/integration/ldap/test_ldap_stateful.py new file mode 100644 index 00000000000000..00755f70dac526 --- /dev/null +++ b/metadata-ingestion/tests/integration/ldap/test_ldap_stateful.py @@ -0,0 +1,207 @@ +import pathlib +import time +from typing import Optional, cast +from unittest import mock + +from freezegun import freeze_time + +from datahub.ingestion.run.pipeline import Pipeline +from datahub.ingestion.source.ldap import LDAPSource +from datahub.ingestion.source.state.checkpoint import Checkpoint +from datahub.ingestion.source.state.ldap_state import LdapCheckpointState +from tests.test_helpers import mce_helpers +from tests.test_helpers.docker_helpers import wait_for_port +from tests.test_helpers.state_helpers import ( + validate_all_providers_have_committed_successfully, +) + +FROZEN_TIME = "2022-08-14 07:00:00" + +GMS_PORT = 8080 +GMS_SERVER = f"http://localhost:{GMS_PORT}" + + +def ldap_ingest_common( + docker_compose_runner, + pytestconfig, + tmp_path, + golden_file_name, + output_file_name, + filter, + mock_datahub_graph, +): + test_resources_dir = pathlib.Path(pytestconfig.rootpath / "tests/integration/ldap") + with docker_compose_runner( + test_resources_dir / "docker-compose.yml", "ldap" + ) as docker_services: + # The openldap container loads the sample data after exposing the port publicly. As such, + # we must wait a little bit extra to ensure that the sample data is loaded. + wait_for_port(docker_services, "openldap", 389) + time.sleep(5) + + with mock.patch( + "datahub.ingestion.source.state_provider.datahub_ingestion_checkpointing_provider.DataHubGraph", + mock_datahub_graph, + ) as mock_checkpoint: + mock_checkpoint.return_value = mock_datahub_graph + + pipeline = Pipeline.create( + { + "run_id": "ldap-test", + "pipeline_name": "ldap-test-pipeline", + "source": { + "type": "ldap", + "config": { + "ldap_server": "ldap://localhost", + "ldap_user": "cn=admin,dc=example,dc=org", + "ldap_password": "admin", + "base_dn": "dc=example,dc=org", + "filter": filter, + "attrs_list": ["+", "*"], + "stateful_ingestion": { + "enabled": True, + "remove_stale_metadata": True, + "fail_safe_threshold": 100.0, + "state_provider": { + "type": "datahub", + "config": {"datahub_api": {"server": GMS_SERVER}}, + }, + }, + }, + }, + "sink": { + "type": "file", + "config": { + "filename": f"{tmp_path}/{output_file_name}", + }, + }, + } + ) + pipeline.run() + pipeline.raise_from_status() + + mce_helpers.check_golden_file( + pytestconfig, + output_path=tmp_path / output_file_name, + golden_path=test_resources_dir / golden_file_name, + ignore_paths=mce_helpers.IGNORE_PATH_TIMESTAMPS, + ) + return pipeline + + +def get_current_checkpoint_from_pipeline( + pipeline: Pipeline, +) -> Optional[Checkpoint]: + ldap_source = cast(LDAPSource, pipeline.source) + return ldap_source.get_current_checkpoint( + ldap_source.stale_entity_removal_handler.job_id + ) + + +@freeze_time(FROZEN_TIME) +def test_ldap_stateful( + docker_compose_runner, pytestconfig, tmp_path, mock_time, mock_datahub_graph +): + golden_file_name: str = "ldap_mces_golden_stateful.json" + output_file_name: str = "ldap_mces_stateful.json" + + golden_file_deleted_name: str = "ldap_mces_golden_deleted_stateful.json" + output_file_deleted_name: str = "ldap_mces_deleted_stateful.json" + + golden_file_group: str = "ldap_mces_golden_group_stateful.json" + output_file_group: str = "ldap_mces_group_stateful.json" + + golden_file_delete_group: str = "ldap_mces_golden_deleted_group_stateful.json" + output_file_delete_group: str = "ldap_mces_deleted_group_stateful.json" + + filter1: str = "(|(cn=Bart Simpson)(cn=Homer Simpson))" + filter2: str = "(|(cn=Bart Simpson))" + filter3: str = "(&(objectClass=top)(|(cn=HR Department)(cn=Finance Department)))" + filter4: str = "(&(objectClass=top)(|(cn=HR Department)))" + + pipeline_run1 = ldap_ingest_common( + docker_compose_runner, + pytestconfig, + tmp_path, + golden_file_name, + output_file_name, + filter1, + mock_datahub_graph, + ) + + checkpoint1 = get_current_checkpoint_from_pipeline(pipeline_run1) + assert checkpoint1 + assert checkpoint1.state + + pipeline_run2 = ldap_ingest_common( + docker_compose_runner, + pytestconfig, + tmp_path, + golden_file_deleted_name, + output_file_deleted_name, + filter2, + mock_datahub_graph, + ) + + checkpoint2 = get_current_checkpoint_from_pipeline(pipeline_run2) + assert checkpoint2 + assert checkpoint2.state + + validate_all_providers_have_committed_successfully( + pipeline=pipeline_run1, expected_providers=1 + ) + validate_all_providers_have_committed_successfully( + pipeline=pipeline_run2, expected_providers=1 + ) + + state1 = cast(LdapCheckpointState, checkpoint1.state) + state2 = cast(LdapCheckpointState, checkpoint2.state) + + difference_dataset_urns = list( + state1.get_urns_not_in(type="corpuser", other_checkpoint_state=state2) + ) + assert len(difference_dataset_urns) == 1 + deleted_dataset_urns = [ + "urn:li:corpuser:hsimpson", + ] + assert sorted(deleted_dataset_urns) == sorted(difference_dataset_urns) + + pipeline_run3 = ldap_ingest_common( + docker_compose_runner, + pytestconfig, + tmp_path, + golden_file_group, + output_file_group, + filter3, + mock_datahub_graph, + ) + + checkpoint3 = get_current_checkpoint_from_pipeline(pipeline_run3) + assert checkpoint3 + assert checkpoint3.state + + pipeline_run4 = ldap_ingest_common( + docker_compose_runner, + pytestconfig, + tmp_path, + golden_file_delete_group, + output_file_delete_group, + filter4, + mock_datahub_graph, + ) + + checkpoint4 = get_current_checkpoint_from_pipeline(pipeline_run4) + assert checkpoint4 + assert checkpoint4.state + + state3 = cast(LdapCheckpointState, checkpoint3.state) + state4 = cast(LdapCheckpointState, checkpoint4.state) + + difference_dataset_urns = list( + state3.get_urns_not_in(type="corpGroup", other_checkpoint_state=state4) + ) + assert len(difference_dataset_urns) == 1 + deleted_dataset_urns = [ + "urn:li:corpGroup:Finance Department", + ] + assert sorted(deleted_dataset_urns) == sorted(difference_dataset_urns) diff --git a/metadata-ingestion/tests/unit/test_ldap_state.py b/metadata-ingestion/tests/unit/test_ldap_state.py new file mode 100644 index 00000000000000..f5b94cb53aaa07 --- /dev/null +++ b/metadata-ingestion/tests/unit/test_ldap_state.py @@ -0,0 +1,57 @@ +import pytest + +from datahub.ingestion.source.state.ldap_state import LdapCheckpointState + + +@pytest.fixture +def other_checkpoint_state(): + state = LdapCheckpointState() + state.add_checkpoint_urn("corpuser", "urn:li:corpuser:user1") + state.add_checkpoint_urn("corpuser", "urn:li:corpuser:user2") + state.add_checkpoint_urn("corpuser", "urn:li:corpuser:user3") + state.add_checkpoint_urn("corpuser", "urn:li:corpuser:user5") + state.add_checkpoint_urn("corpGroup", "urn:li:corpGroup:group1") + state.add_checkpoint_urn("corpGroup", "urn:li:corpGroup:group2") + state.add_checkpoint_urn("corpGroup", "urn:li:corpGroup:group3") + state.add_checkpoint_urn("corpGroup", "urn:li:corpGroup:group5") + return state + + +def test_add_checkpoint_urn(): + state = LdapCheckpointState() + assert len(state.urns) == 0 + state.add_checkpoint_urn("corpuser", "urn:li:corpuser:user1") + assert len(state.urns) == 1 + state.add_checkpoint_urn("corpGroup", "urn:li:corpGroup:group2") + assert len(state.urns) != 1 + + +def test_get_supported_types(): + assert LdapCheckpointState().get_supported_types() == ["corpuser", "corpGroup"] + + +def test_get_urns_not_in(other_checkpoint_state): + oldstate = LdapCheckpointState() + oldstate.add_checkpoint_urn("corpuser", "urn:li:corpuser:user1") + oldstate.add_checkpoint_urn("corpuser", "urn:li:corpuser:user2") + oldstate.add_checkpoint_urn("corpuser", "urn:li:corpuser:user4") + oldstate.add_checkpoint_urn("corpGroup", "urn:li:corpGroup:group1") + oldstate.add_checkpoint_urn("corpGroup", "urn:li:corpGroup:group2") + oldstate.add_checkpoint_urn("corpGroup", "urn:li:corpGroup:group4") + iterable = oldstate.get_urns_not_in("corpuser", other_checkpoint_state) + # urn:li:corpuser:user4 has been identified as a user to be deleted + for item in iterable: + assert item == "urn:li:corpuser:user4" + iterable = oldstate.get_urns_not_in("corpGroup", other_checkpoint_state) + for item in iterable: + assert item == "urn:li:corpGroup:group4" + + +def test_get_percent_entities_changed(other_checkpoint_state): + oldstate = LdapCheckpointState() + oldstate.add_checkpoint_urn("corpuser", "urn:li:corpuser:user1") + oldstate.add_checkpoint_urn("corpuser", "urn:li:corpuser:user2") + oldstate.add_checkpoint_urn("corpuser", "urn:li:corpuser:user4") + percent = oldstate.get_percent_entities_changed(other_checkpoint_state) + # two states have 75% of differences (total 8 elements, 6 elements are different (75%) and 2 (25%) are the same) + assert percent == 75.0 From 7d63399d00824a419cfd3b7a18038545fd4ecdec Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Tue, 13 Dec 2022 02:17:24 -0500 Subject: [PATCH 15/19] fix(ingest): fix serde for empty dicts in unions with null (#6745) The code changes in https://github.com/acryldata/avro_gen/pull/16, but tests are written here. --- .../scripts/install_editable_versions.sh | 4 ++-- metadata-ingestion/setup.py | 2 +- .../tests/unit/serde/test_serde.py | 20 +++++++++++++++++++ 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/metadata-ingestion/scripts/install_editable_versions.sh b/metadata-ingestion/scripts/install_editable_versions.sh index 3273b0e5244f08..578eac8158fa20 100755 --- a/metadata-ingestion/scripts/install_editable_versions.sh +++ b/metadata-ingestion/scripts/install_editable_versions.sh @@ -2,6 +2,6 @@ set -euxo pipefail -pip install -e 'git+https://github.com/hsheth2/avro_gen#egg=avro-gen3' -pip install -e 'git+https://github.com/hsheth2/PyHive#egg=acryl-pyhive[hive]' +pip install -e 'git+https://github.com/acryldata/avro_gen#egg=avro-gen3' +pip install -e 'git+https://github.com/acryldata/PyHive#egg=acryl-pyhive[hive]' pip install -e '.[dev]' diff --git a/metadata-ingestion/setup.py b/metadata-ingestion/setup.py index 1496dde1102ed4..440d9ee853f1bc 100644 --- a/metadata-ingestion/setup.py +++ b/metadata-ingestion/setup.py @@ -36,7 +36,7 @@ def get_long_description(): "entrypoints", "docker", "expandvars>=0.6.5", - "avro-gen3==0.7.7", + "avro-gen3==0.7.8", # "avro-gen3 @ git+https://github.com/acryldata/avro_gen@master#egg=avro-gen3", "avro>=1.10.2,<1.11", "python-dateutil>=2.8.0", diff --git a/metadata-ingestion/tests/unit/serde/test_serde.py b/metadata-ingestion/tests/unit/serde/test_serde.py index 6d1f630ea81be1..fd001b651250c9 100644 --- a/metadata-ingestion/tests/unit/serde/test_serde.py +++ b/metadata-ingestion/tests/unit/serde/test_serde.py @@ -309,3 +309,23 @@ def test_reserved_keywords() -> None: filter3 = models.FilterClass.from_obj(filter2.to_obj()) assert filter2 == filter3 + + +def test_read_empty_dict() -> None: + original = '{"type": "SUCCESS", "nativeResults": {}}' + + model = models.AssertionResultClass.from_obj(json.loads(original)) + assert model.nativeResults == {} + assert model == models.AssertionResultClass( + type=models.AssertionResultTypeClass.SUCCESS, nativeResults={} + ) + + +def test_write_optional_empty_dict() -> None: + model = models.AssertionResultClass( + type=models.AssertionResultTypeClass.SUCCESS, nativeResults={} + ) + assert model.nativeResults == {} + + out = json.dumps(model.to_obj()) + assert out == '{"type": "SUCCESS", "nativeResults": {}}' From cf3db168ac7c0c38f40be6e344abd00b8c6af50f Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Tue, 13 Dec 2022 04:05:57 -0500 Subject: [PATCH 16/19] feat(ingest): start simplifying stateful ingestion state (#6740) --- .../ingestion/source/dbt/dbt_common.py | 6 +- .../ingestion/source/looker/looker_source.py | 4 +- .../ingestion/source/looker/lookml_source.py | 4 +- .../ingestion/source/state/checkpoint.py | 12 +- .../ingestion/source/state/dbt_state.py | 101 ++---------- .../source/state/entity_removal_state.py | 76 +++++++++ .../ingestion/source/state/looker_state.py | 46 ------ .../ingestion/source/state/lookml_state.py | 46 ------ .../source/state/sql_common_state.py | 145 ++---------------- .../ingestion/source/state/tableau_state.py | 41 +---- .../__init__.py => datahub_provider/py.typed} | 0 .../tests/integration/dbt/test_dbt.py | 2 +- .../tests/integration/looker/test_looker.py | 6 +- .../tests/integration/lookml/test_lookml.py | 6 +- .../state/test_sql_common_state.py | 33 +++- .../tests/unit/test_glue_source.py | 2 +- 16 files changed, 151 insertions(+), 379 deletions(-) create mode 100644 metadata-ingestion/src/datahub/ingestion/source/state/entity_removal_state.py delete mode 100644 metadata-ingestion/src/datahub/ingestion/source/state/looker_state.py delete mode 100644 metadata-ingestion/src/datahub/ingestion/source/state/lookml_state.py rename metadata-ingestion/src/{datahub/ingestion/source/state_handler/__init__.py => datahub_provider/py.typed} (100%) diff --git a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py index 2660bb606fbf99..62c6c4c8b00cd5 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py +++ b/metadata-ingestion/src/datahub/ingestion/source/dbt/dbt_common.py @@ -713,11 +713,9 @@ def get_last_checkpoint( if last_checkpoint is not None and is_conversion_required: # Map the BaseSQLAlchemyCheckpointState to DbtCheckpointState dbt_checkpoint_state: DbtCheckpointState = DbtCheckpointState() - dbt_checkpoint_state.encoded_node_urns = ( + dbt_checkpoint_state.urns = ( cast(BaseSQLAlchemyCheckpointState, last_checkpoint.state) - ).encoded_table_urns - # Old dbt source was not supporting the assertion - dbt_checkpoint_state.encoded_assertion_urns = [] + ).urns last_checkpoint.state = dbt_checkpoint_state return last_checkpoint diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py index f82b1e68134971..d426cc1f0275bd 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/looker_source.py @@ -60,7 +60,7 @@ LookerAPI, LookerAPIConfig, ) -from datahub.ingestion.source.state.looker_state import LookerCheckpointState +from datahub.ingestion.source.state.entity_removal_state import GenericCheckpointState from datahub.ingestion.source.state.stale_entity_removal_handler import ( StaleEntityRemovalHandler, StatefulStaleMetadataRemovalConfig, @@ -234,7 +234,7 @@ def __init__(self, config: LookerDashboardSourceConfig, ctx: PipelineContext): self.stale_entity_removal_handler = StaleEntityRemovalHandler( source=self, config=self.source_config, - state_type_class=LookerCheckpointState, + state_type_class=GenericCheckpointState, pipeline_name=self.ctx.pipeline_name, run_id=self.ctx.run_id, ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py index 3283ad18d01a1c..42cd557df98e9e 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py +++ b/metadata-ingestion/src/datahub/ingestion/source/looker/lookml_source.py @@ -47,7 +47,7 @@ LookerAPIConfig, TransportOptionsConfig, ) -from datahub.ingestion.source.state.lookml_state import LookMLCheckpointState +from datahub.ingestion.source.state.entity_removal_state import GenericCheckpointState from datahub.ingestion.source.state.stale_entity_removal_handler import ( StaleEntityRemovalHandler, StaleEntityRemovalSourceReport, @@ -1089,7 +1089,7 @@ def __init__(self, config: LookMLSourceConfig, ctx: PipelineContext): self.stale_entity_removal_handler = StaleEntityRemovalHandler( source=self, config=self.source_config, - state_type_class=LookMLCheckpointState, + state_type_class=GenericCheckpointState, pipeline_name=self.ctx.pipeline_name, run_id=self.ctx.run_id, ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/checkpoint.py b/metadata-ingestion/src/datahub/ingestion/source/state/checkpoint.py index 0df8c9ddce99cd..b33d061e1a5b55 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/state/checkpoint.py +++ b/metadata-ingestion/src/datahub/ingestion/source/state/checkpoint.py @@ -1,5 +1,6 @@ import base64 import bz2 +import contextlib import functools import json import logging @@ -128,7 +129,9 @@ def create_from_checkpoint_aspect( ) elif checkpoint_aspect.state.serde == "base85": state_obj = Checkpoint._from_base85_bytes( - checkpoint_aspect, functools.partial(bz2.decompress) + checkpoint_aspect, + functools.partial(bz2.decompress), + state_class, ) elif checkpoint_aspect.state.serde == "base85-bz2-json": state_obj = Checkpoint._from_base85_json_bytes( @@ -177,11 +180,18 @@ def _from_utf8_bytes( def _from_base85_bytes( checkpoint_aspect: DatahubIngestionCheckpointClass, decompressor: Callable[[bytes], bytes], + state_class: Type[StateType], ) -> StateType: state: StateType = pickle.loads( decompressor(base64.b85decode(checkpoint_aspect.state.payload)) # type: ignore ) + with contextlib.suppress(Exception): + # When loading from pickle, the pydantic validators don't run. + # By re-serializing and re-parsing, we ensure that the state is valid. + # However, we also suppress any exceptions to make sure this doesn't blow up. + state = state_class.parse_obj(state.dict()) + # Because the base85 method is deprecated in favor of base85-bz2-json, # we will automatically switch the serde. state.serde = "base85-bz2-json" diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/dbt_state.py b/metadata-ingestion/src/datahub/ingestion/source/state/dbt_state.py index df9561ba11e1ce..00ab652eb97b0b 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/state/dbt_state.py +++ b/metadata-ingestion/src/datahub/ingestion/source/state/dbt_state.py @@ -1,102 +1,21 @@ -import logging -from typing import Callable, Dict, Iterable, List - -import pydantic - -from datahub.emitter.mce_builder import make_assertion_urn -from datahub.ingestion.source.state.stale_entity_removal_handler import ( - StaleEntityCheckpointStateBase, +from datahub.ingestion.source.state.entity_removal_state import ( + GenericCheckpointState, + pydantic_state_migrator, ) -from datahub.utilities.checkpoint_state_util import CheckpointStateUtil -from datahub.utilities.urns.urn import Urn -logger = logging.getLogger(__name__) - -class DbtCheckpointState(StaleEntityCheckpointStateBase["DbtCheckpointState"]): +class DbtCheckpointState(GenericCheckpointState): """ Class for representing the checkpoint state for DBT sources. Stores all nodes and assertions being ingested and is used to remove any stale entities. """ - encoded_node_urns: List[str] = pydantic.Field(default_factory=list) - encoded_assertion_urns: List[str] = pydantic.Field(default_factory=list) - - @classmethod - def get_supported_types(cls) -> List[str]: - return ["assertion", "dataset"] - - @staticmethod - def _get_assertion_lightweight_repr(assertion_urn: str) -> str: - """Reduces the amount of text in the URNs for smaller state footprint.""" - urn = Urn.create_from_string(assertion_urn) - key = urn.get_entity_id_as_string() - assert key is not None - return key - - def _add_assertion_urn(self, assertion_urn: str) -> None: - self.encoded_assertion_urns.append( - self._get_assertion_lightweight_repr(assertion_urn) - ) - - def _get_assertion_urns_not_in( - self, checkpoint: "DbtCheckpointState" - ) -> Iterable[str]: - """ - Dbt assertion are mapped to DataHub assertion concept - """ - difference = CheckpointStateUtil.get_encoded_urns_not_in( - self.encoded_assertion_urns, checkpoint.encoded_assertion_urns - ) - for key in difference: - yield make_assertion_urn(key) - - def _get_node_urns_not_in(self, checkpoint: "DbtCheckpointState") -> Iterable[str]: - """ - Dbt node are mapped to DataHub dataset concept - """ - yield from CheckpointStateUtil.get_dataset_urns_not_in( - self.encoded_node_urns, checkpoint.encoded_node_urns - ) - - def _add_node_urn(self, node_urn: str) -> None: - self.encoded_node_urns.append( - CheckpointStateUtil.get_dataset_lightweight_repr(node_urn) - ) - - def add_checkpoint_urn(self, type: str, urn: str) -> None: - supported_entities_add_handlers: Dict[str, Callable[[str], None]] = { - "dataset": self._add_node_urn, - "assertion": self._add_assertion_urn, + _migration = pydantic_state_migrator( + { + "encoded_node_urns": "dataset", + "encoded_assertion_urns": "assertion", } - - if type not in supported_entities_add_handlers: - logger.error(f"Can not save Unknown entity {type} to checkpoint.") - - supported_entities_add_handlers[type](urn) - - def get_urns_not_in( - self, type: str, other_checkpoint_state: "DbtCheckpointState" - ) -> Iterable[str]: - assert type in self.get_supported_types() - if type == "dataset": - yield from self._get_node_urns_not_in(other_checkpoint_state) - elif type == "assertion": - yield from self._get_assertion_urns_not_in(other_checkpoint_state) - - def get_percent_entities_changed( - self, old_checkpoint_state: "DbtCheckpointState" - ) -> float: - return StaleEntityCheckpointStateBase.compute_percent_entities_changed( - [ - (self.encoded_node_urns, old_checkpoint_state.encoded_node_urns), - ( - self.encoded_assertion_urns, - old_checkpoint_state.encoded_assertion_urns, - ), - ] - ) + ) def prepare_for_commit(self) -> None: - self.encoded_node_urns = list(set(self.encoded_node_urns)) - self.encoded_assertion_urns = list(set(self.encoded_assertion_urns)) + self.urns = list(set(self.urns)) diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/entity_removal_state.py b/metadata-ingestion/src/datahub/ingestion/source/state/entity_removal_state.py new file mode 100644 index 00000000000000..7caf9ee7db766c --- /dev/null +++ b/metadata-ingestion/src/datahub/ingestion/source/state/entity_removal_state.py @@ -0,0 +1,76 @@ +from typing import Dict, Iterable, List, Type + +import pydantic + +from datahub.emitter.mce_builder import make_assertion_urn, make_container_urn +from datahub.ingestion.source.state.stale_entity_removal_handler import ( + StaleEntityCheckpointStateBase, +) +from datahub.utilities.checkpoint_state_util import CheckpointStateUtil +from datahub.utilities.urns.urn import guess_entity_type + + +class GenericCheckpointState(StaleEntityCheckpointStateBase["GenericCheckpointState"]): + urns: List[str] = pydantic.Field(default_factory=list) + + @classmethod + def get_supported_types(cls) -> List[str]: + return ["*"] + + def add_checkpoint_urn(self, type: str, urn: str) -> None: + # TODO: dedup + self.urns.append(urn) + + def get_urns_not_in( + self, type: str, other_checkpoint_state: "GenericCheckpointState" + ) -> Iterable[str]: + diff = set(self.urns) - set(other_checkpoint_state.urns) + + # To maintain backwards compatibility, we provide this filtering mechanism. + if type == "*": + yield from diff + else: + yield from (urn for urn in diff if guess_entity_type(urn) == type) + + def get_percent_entities_changed( + self, old_checkpoint_state: "GenericCheckpointState" + ) -> float: + return StaleEntityCheckpointStateBase.compute_percent_entities_changed( + [(self.urns, old_checkpoint_state.urns)] + ) + + +def pydantic_state_migrator(mapping: Dict[str, str]) -> classmethod: + # mapping would be something like: + # { + # 'encoded_view_urns': 'dataset', + # 'encoded_container_urns': 'container', + # } + + SUPPORTED_TYPES = [ + "dataset", + "container", + "assertion", + ] + assert set(mapping.values()) <= set(SUPPORTED_TYPES) + + def _validate_field_rename(cls: Type, values: dict) -> dict: + values.setdefault("urns", []) + + for old_field, mapped_type in mapping.items(): + if old_field not in values: + continue + + value = values.pop(old_field) + if mapped_type == "dataset": + values["urns"] += CheckpointStateUtil.get_dataset_urns_not_in(value, []) + elif mapped_type == "container": + values["urns"] += [make_container_urn(guid) for guid in value] + elif mapped_type == "assertion": + values["urns"] += [make_assertion_urn(encoded) for encoded in value] + else: + raise ValueError(f"Unsupported type {mapped_type}") + + return values + + return pydantic.root_validator(pre=True, allow_reuse=True)(_validate_field_rename) diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/looker_state.py b/metadata-ingestion/src/datahub/ingestion/source/state/looker_state.py deleted file mode 100644 index dadca4a255cf35..00000000000000 --- a/metadata-ingestion/src/datahub/ingestion/source/state/looker_state.py +++ /dev/null @@ -1,46 +0,0 @@ -import logging -from typing import Iterable, List - -import pydantic - -from datahub.ingestion.source.state.stale_entity_removal_handler import ( - StaleEntityCheckpointStateBase, -) -from datahub.utilities.urns.urn import guess_entity_type - -logger = logging.getLogger(__name__) - - -class LookerCheckpointState(StaleEntityCheckpointStateBase["LookerCheckpointState"]): - """ - Class for representing the checkpoint state for Looker sources. - Stores all datasets, charts and dashboards being ingested and is - used to remove any stale entities. - """ - - urns: List[str] = pydantic.Field(default_factory=list) - - @classmethod - def get_supported_types(cls) -> List[str]: - return ["*"] - - def add_checkpoint_urn(self, type: str, urn: str) -> None: - self.urns.append(urn) - - def get_urns_not_in( - self, type: str, other_checkpoint_state: "LookerCheckpointState" - ) -> Iterable[str]: - diff = set(self.urns) - set(other_checkpoint_state.urns) - - # To maintain backwards compatibility, we provide this filtering mechanism. - if type == "*": - yield from diff - else: - yield from (urn for urn in diff if guess_entity_type(urn) == type) - - def get_percent_entities_changed( - self, old_checkpoint_state: "LookerCheckpointState" - ) -> float: - return StaleEntityCheckpointStateBase.compute_percent_entities_changed( - [(self.urns, old_checkpoint_state.urns)] - ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/lookml_state.py b/metadata-ingestion/src/datahub/ingestion/source/state/lookml_state.py deleted file mode 100644 index 5a013bbbbb9705..00000000000000 --- a/metadata-ingestion/src/datahub/ingestion/source/state/lookml_state.py +++ /dev/null @@ -1,46 +0,0 @@ -import logging -from typing import Iterable, List - -import pydantic - -from datahub.ingestion.source.state.stale_entity_removal_handler import ( - StaleEntityCheckpointStateBase, -) -from datahub.utilities.urns.urn import guess_entity_type - -logger = logging.getLogger(__name__) - - -class LookMLCheckpointState(StaleEntityCheckpointStateBase["LookMLCheckpointState"]): - """ - Class for representing the checkpoint state for Looker sources. - Stores all datasets, charts and dashboards being ingested and is - used to remove any stale entities. - """ - - urns: List[str] = pydantic.Field(default_factory=list) - - @classmethod - def get_supported_types(cls) -> List[str]: - return ["*"] - - def add_checkpoint_urn(self, type: str, urn: str) -> None: - self.urns.append(urn) - - def get_urns_not_in( - self, type: str, other_checkpoint_state: "LookMLCheckpointState" - ) -> Iterable[str]: - diff = set(self.urns) - set(other_checkpoint_state.urns) - - # To maintain backwards compatibility, we provide this filtering mechanism. - if type == "*": - yield from diff - else: - yield from (urn for urn in diff if guess_entity_type(urn) == type) - - def get_percent_entities_changed( - self, old_checkpoint_state: "LookMLCheckpointState" - ) -> float: - return StaleEntityCheckpointStateBase.compute_percent_entities_changed( - [(self.urns, old_checkpoint_state.urns)] - ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/sql_common_state.py b/metadata-ingestion/src/datahub/ingestion/source/state/sql_common_state.py index 13c3a6aeadd39a..c2645a761f7273 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/state/sql_common_state.py +++ b/metadata-ingestion/src/datahub/ingestion/source/state/sql_common_state.py @@ -1,142 +1,21 @@ -from typing import Iterable, List - -import pydantic - -from datahub.emitter.mce_builder import ( - assertion_urn_to_key, - container_urn_to_key, - make_assertion_urn, - make_container_urn, -) -from datahub.ingestion.source.state.stale_entity_removal_handler import ( - StaleEntityCheckpointStateBase, +from datahub.ingestion.source.state.entity_removal_state import ( + GenericCheckpointState, + pydantic_state_migrator, ) -from datahub.utilities.checkpoint_state_util import CheckpointStateUtil -class BaseSQLAlchemyCheckpointState( - StaleEntityCheckpointStateBase["BaseSQLAlchemyCheckpointState"] -): +class BaseSQLAlchemyCheckpointState(GenericCheckpointState): """ Base class for representing the checkpoint state for all SQLAlchemy based sources. Stores all tables and views being ingested and is used to remove any stale entities. Subclasses can define additional state as appropriate. """ - encoded_table_urns: List[str] = pydantic.Field(default_factory=list) - encoded_view_urns: List[str] = pydantic.Field(default_factory=list) - encoded_container_urns: List[str] = pydantic.Field(default_factory=list) - encoded_assertion_urns: List[str] = pydantic.Field(default_factory=list) - - @classmethod - def get_supported_types(cls) -> List[str]: - return ["assertion", "container", "table", "view"] - - @staticmethod - def _get_lightweight_repr(dataset_urn: str) -> str: - """Reduces the amount of text in the URNs for smaller state footprint.""" - return CheckpointStateUtil.get_dataset_lightweight_repr(dataset_urn) - - @staticmethod - def _get_container_lightweight_repr(container_urn: str) -> str: - """Reduces the amount of text in the URNs for smaller state footprint.""" - key = container_urn_to_key(container_urn) - assert key is not None - return f"{key.guid}" - - @staticmethod - def _get_container_urns_not_in( - encoded_urns_1: List[str], encoded_urns_2: List[str] - ) -> Iterable[str]: - difference = CheckpointStateUtil.get_encoded_urns_not_in( - encoded_urns_1, encoded_urns_2 - ) - for guid in difference: - yield make_container_urn(guid) - - def _get_table_urns_not_in( - self, checkpoint: "BaseSQLAlchemyCheckpointState" - ) -> Iterable[str]: - """Tables are mapped to DataHub dataset concept.""" - yield from CheckpointStateUtil.get_dataset_urns_not_in( - self.encoded_table_urns, checkpoint.encoded_table_urns - ) - - def _get_view_urns_not_in( - self, checkpoint: "BaseSQLAlchemyCheckpointState" - ) -> Iterable[str]: - """Views are mapped to DataHub dataset concept.""" - yield from CheckpointStateUtil.get_dataset_urns_not_in( - self.encoded_view_urns, checkpoint.encoded_view_urns - ) - - def _get_assertion_urns_not_in( - self, checkpoint: "BaseSQLAlchemyCheckpointState" - ) -> Iterable[str]: - """Tables are mapped to DataHub dataset concept.""" - diff = CheckpointStateUtil.get_encoded_urns_not_in( - self.encoded_assertion_urns, checkpoint.encoded_assertion_urns - ) - for assertion_id in diff: - yield make_assertion_urn(assertion_id) - - def _add_table_urn(self, table_urn: str) -> None: - self.encoded_table_urns.append(self._get_lightweight_repr(table_urn)) - - def _add_assertion_urn(self, assertion_urn: str) -> None: - key = assertion_urn_to_key(assertion_urn) - assert key is not None - self.encoded_assertion_urns.append(key.assertionId) - - def _add_view_urn(self, view_urn: str) -> None: - self.encoded_view_urns.append(self._get_lightweight_repr(view_urn)) - - def _add_container_guid(self, container_urn: str) -> None: - self.encoded_container_urns.append( - self._get_container_lightweight_repr(container_urn) - ) - - def add_checkpoint_urn(self, type: str, urn: str) -> None: - assert type in self.get_supported_types() - if type == "assertion": - self._add_assertion_urn(urn) - elif type == "container": - self._add_container_guid(urn) - elif type == "table": - self._add_table_urn(urn) - elif type == "view": - self._add_view_urn(urn) - - def get_urns_not_in( - self, type: str, other_checkpoint_state: "BaseSQLAlchemyCheckpointState" - ) -> Iterable[str]: - assert type in self.get_supported_types() - if type == "assertion": - yield from self._get_assertion_urns_not_in(other_checkpoint_state) - if type == "container": - yield from self._get_container_urns_not_in( - self.encoded_container_urns, - other_checkpoint_state.encoded_container_urns, - ) - elif type == "table": - yield from self._get_table_urns_not_in(other_checkpoint_state) - elif type == "view": - yield from self._get_view_urns_not_in(other_checkpoint_state) - - def get_percent_entities_changed( - self, old_checkpoint_state: "BaseSQLAlchemyCheckpointState" - ) -> float: - return StaleEntityCheckpointStateBase.compute_percent_entities_changed( - [ - ( - self.encoded_assertion_urns, - old_checkpoint_state.encoded_assertion_urns, - ), - ( - self.encoded_container_urns, - old_checkpoint_state.encoded_container_urns, - ), - (self.encoded_table_urns, old_checkpoint_state.encoded_table_urns), - (self.encoded_view_urns, old_checkpoint_state.encoded_view_urns), - ] - ) + _migration = pydantic_state_migrator( + { + "encoded_table_urns": "dataset", + "encoded_view_urns": "dataset", + "encoded_container_urns": "container", + "encoded_assertion_urns": "assertion", + } + ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/state/tableau_state.py b/metadata-ingestion/src/datahub/ingestion/source/state/tableau_state.py index 310c86fa50cbef..ae935b2d2cb09d 100644 --- a/metadata-ingestion/src/datahub/ingestion/source/state/tableau_state.py +++ b/metadata-ingestion/src/datahub/ingestion/source/state/tableau_state.py @@ -1,46 +1,9 @@ -import logging -from typing import Iterable, List +from datahub.ingestion.source.state.entity_removal_state import GenericCheckpointState -import pydantic -from datahub.ingestion.source.state.stale_entity_removal_handler import ( - StaleEntityCheckpointStateBase, -) -from datahub.utilities.urns.urn import guess_entity_type - -logger = logging.getLogger(__name__) - - -class TableauCheckpointState(StaleEntityCheckpointStateBase["TableauCheckpointState"]): +class TableauCheckpointState(GenericCheckpointState): """ Class for representing the checkpoint state for Tableau sources. Stores all datasets, charts and dashboards being ingested and is used to remove any stale entities. """ - - urns: List[str] = pydantic.Field(default_factory=list) - - @classmethod - def get_supported_types(cls) -> List[str]: - return ["*"] - - def add_checkpoint_urn(self, type: str, urn: str) -> None: - self.urns.append(urn) - - def get_urns_not_in( - self, type: str, other_checkpoint_state: "TableauCheckpointState" - ) -> Iterable[str]: - diff = set(self.urns) - set(other_checkpoint_state.urns) - - # To maintain backwards compatibility, we provide this filtering mechanism. - if type == "*": - yield from diff - else: - yield from (urn for urn in diff if guess_entity_type(urn) == type) - - def get_percent_entities_changed( - self, old_checkpoint_state: "TableauCheckpointState" - ) -> float: - return StaleEntityCheckpointStateBase.compute_percent_entities_changed( - [(self.urns, old_checkpoint_state.urns)] - ) diff --git a/metadata-ingestion/src/datahub/ingestion/source/state_handler/__init__.py b/metadata-ingestion/src/datahub_provider/py.typed similarity index 100% rename from metadata-ingestion/src/datahub/ingestion/source/state_handler/__init__.py rename to metadata-ingestion/src/datahub_provider/py.typed diff --git a/metadata-ingestion/tests/integration/dbt/test_dbt.py b/metadata-ingestion/tests/integration/dbt/test_dbt.py index 2ae4097dad2265..74477245b728fa 100644 --- a/metadata-ingestion/tests/integration/dbt/test_dbt.py +++ b/metadata-ingestion/tests/integration/dbt/test_dbt.py @@ -360,7 +360,7 @@ def test_dbt_stateful(pytestconfig, tmp_path, mock_time, mock_datahub_graph): state1 = cast(DbtCheckpointState, checkpoint1.state) state2 = cast(DbtCheckpointState, checkpoint2.state) difference_urns = list( - state1.get_urns_not_in(type="dataset", other_checkpoint_state=state2) + state1.get_urns_not_in(type="*", other_checkpoint_state=state2) ) assert len(difference_urns) == 2 diff --git a/metadata-ingestion/tests/integration/looker/test_looker.py b/metadata-ingestion/tests/integration/looker/test_looker.py index 537303aa4f32cb..1237b174e2e736 100644 --- a/metadata-ingestion/tests/integration/looker/test_looker.py +++ b/metadata-ingestion/tests/integration/looker/test_looker.py @@ -29,7 +29,7 @@ ) from datahub.ingestion.source.looker.looker_source import LookerDashboardSource from datahub.ingestion.source.state.checkpoint import Checkpoint -from datahub.ingestion.source.state.looker_state import LookerCheckpointState +from datahub.ingestion.source.state.entity_removal_state import GenericCheckpointState from tests.test_helpers import mce_helpers from tests.test_helpers.state_helpers import ( validate_all_providers_have_committed_successfully, @@ -689,8 +689,8 @@ def looker_source_config(sink_file_name): # Perform all assertions on the states. The deleted table should not be # part of the second state - state1 = cast(LookerCheckpointState, checkpoint1.state) - state2 = cast(LookerCheckpointState, checkpoint2.state) + state1 = cast(GenericCheckpointState, checkpoint1.state) + state2 = cast(GenericCheckpointState, checkpoint2.state) difference_dataset_urns = list( state1.get_urns_not_in(type="dataset", other_checkpoint_state=state2) diff --git a/metadata-ingestion/tests/integration/lookml/test_lookml.py b/metadata-ingestion/tests/integration/lookml/test_lookml.py index 33ddced1ddc530..4055ac1002e0d1 100644 --- a/metadata-ingestion/tests/integration/lookml/test_lookml.py +++ b/metadata-ingestion/tests/integration/lookml/test_lookml.py @@ -10,7 +10,7 @@ from datahub.ingestion.run.pipeline import Pipeline from datahub.ingestion.source.looker.lookml_source import LookMLSource from datahub.ingestion.source.state.checkpoint import Checkpoint -from datahub.ingestion.source.state.lookml_state import LookMLCheckpointState +from datahub.ingestion.source.state.entity_removal_state import GenericCheckpointState from datahub.metadata.schema_classes import ( DatasetSnapshotClass, MetadataChangeEventClass, @@ -624,8 +624,8 @@ def test_lookml_ingest_stateful(pytestconfig, tmp_path, mock_time, mock_datahub_ # Perform all assertions on the states. The deleted table should not be # part of the second state - state1 = cast(LookMLCheckpointState, checkpoint1.state) - state2 = cast(LookMLCheckpointState, checkpoint2.state) + state1 = cast(GenericCheckpointState, checkpoint1.state) + state2 = cast(GenericCheckpointState, checkpoint2.state) difference_dataset_urns = list( state1.get_urns_not_in(type="dataset", other_checkpoint_state=state2) diff --git a/metadata-ingestion/tests/unit/stateful_ingestion/state/test_sql_common_state.py b/metadata-ingestion/tests/unit/stateful_ingestion/state/test_sql_common_state.py index 001ed969002fa6..d91c80abe245d3 100644 --- a/metadata-ingestion/tests/unit/stateful_ingestion/state/test_sql_common_state.py +++ b/metadata-ingestion/tests/unit/stateful_ingestion/state/test_sql_common_state.py @@ -16,15 +16,15 @@ def test_sql_common_state() -> None: state2 = BaseSQLAlchemyCheckpointState() - table_urns_diff = list( - state1.get_urns_not_in(type="table", other_checkpoint_state=state2) + dataset_urns_diff = list( + state1.get_urns_not_in(type="dataset", other_checkpoint_state=state2) ) - assert len(table_urns_diff) == 1 and table_urns_diff[0] == test_table_urn - - view_urns_diff = list( - state1.get_urns_not_in(type="view", other_checkpoint_state=state2) + assert len(dataset_urns_diff) == 2 and sorted(dataset_urns_diff) == sorted( + [ + test_table_urn, + test_view_urn, + ] ) - assert len(view_urns_diff) == 1 and view_urns_diff[0] == test_view_urn container_urns_diff = list( state1.get_urns_not_in(type="container", other_checkpoint_state=state2) @@ -32,3 +32,22 @@ def test_sql_common_state() -> None: assert ( len(container_urns_diff) == 1 and container_urns_diff[0] == test_container_urn ) + + +def test_backward_compat() -> None: + state = BaseSQLAlchemyCheckpointState.parse_obj( + dict( + encoded_table_urns=["mysql||db1.t1||PROD"], + encoded_view_urns=["mysql||db1.v1||PROD"], + encoded_container_urns=["1154d1da73a95376c9f33f47694cf1de"], + encoded_assertion_urns=["815963e1332b46a203504ba46ebfab24"], + ) + ) + assert state == BaseSQLAlchemyCheckpointState( + urns=[ + "urn:li:dataset:(urn:li:dataPlatform:mysql,db1.t1,PROD)", + "urn:li:dataset:(urn:li:dataPlatform:mysql,db1.v1,PROD)", + "urn:li:container:1154d1da73a95376c9f33f47694cf1de", + "urn:li:assertion:815963e1332b46a203504ba46ebfab24", + ] + ) diff --git a/metadata-ingestion/tests/unit/test_glue_source.py b/metadata-ingestion/tests/unit/test_glue_source.py index 9d6e64f326e344..0acd744e75fba6 100644 --- a/metadata-ingestion/tests/unit/test_glue_source.py +++ b/metadata-ingestion/tests/unit/test_glue_source.py @@ -327,7 +327,7 @@ def test_glue_stateful(pytestconfig, tmp_path, mock_time, mock_datahub_graph): state1 = cast(BaseSQLAlchemyCheckpointState, checkpoint1.state) state2 = cast(BaseSQLAlchemyCheckpointState, checkpoint2.state) difference_urns = list( - state1.get_urns_not_in(type="table", other_checkpoint_state=state2) + state1.get_urns_not_in(type="*", other_checkpoint_state=state2) ) assert len(difference_urns) == 1 From b7a2ce5c00150d8f408b7eca562fa6b7af5ff0c2 Mon Sep 17 00:00:00 2001 From: mohdsiddique Date: Tue, 13 Dec 2022 22:52:52 +0530 Subject: [PATCH 17/19] fix(): Add auth-api as compileOnly dependency (#6747) Co-authored-by: MohdSiddique Bagwan --- docs/plugins.md | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/docs/plugins.md b/docs/plugins.md index d03ff92ed60ee3..772c877cff646e 100644 --- a/docs/plugins.md +++ b/docs/plugins.md @@ -23,13 +23,17 @@ Instead, custom authenticators are useful for authenticating API requests to Dat The sample authenticator implementation can be found at [Authenticator Sample](../metadata-service/plugin/src/test/sample-test-plugins) ### Implementing an Authentication Plugin -1. Add _datahub-auth-api_ as implementation dependency: Maven coordinates of _datahub-auth-api_ can be found at [Maven](https://mvnrepository.com/artifact/io.acryl/datahub-auth-api) +1. Add _datahub-auth-api_ as compileOnly dependency: Maven coordinates of _datahub-auth-api_ can be found at [Maven](https://mvnrepository.com/artifact/io.acryl/datahub-auth-api) Example of gradle dependency is given below. ```groovy dependencies { - implementation 'io.acryl:datahub-auth-api:0.8.45' + + def auth_api = 'io.acryl:datahub-auth-api:0.9.3-3rc3' + compileOnly "${auth_api}" + testImplementation "${auth_api}" + } ``` @@ -65,13 +69,13 @@ The sample authenticator implementation can be found at [Authenticator Sample](. To see an example of building an uber jar, check out the `build.gradle` file for the apache-ranger-plugin file of [Apache Ranger Plugin](https://github.com/acryldata/datahub-ranger-auth-plugin/tree/main/apache-ranger-plugin) for reference. - Exclude datahub plugin dependency and signature classes as shown in below `shadowJar` task. + Exclude signature files as shown in below `shadowJar` task. ```groovy apply plugin: 'com.github.johnrengelman.shadow'; shadowJar { // Exclude com.datahub.plugins package and files related to jar signature - exclude "com/linkedin/common/", "com/datahub/", "META-INF/*.RSA", "META-INF/*.SF","META-INF/*.DSA" + exclude "META-INF/*.RSA", "META-INF/*.SF","META-INF/*.DSA" } ``` 5. Refer section [Plugin Installation](#plugin-installation) for plugin installation in DataHub environment @@ -104,8 +108,20 @@ The sample authorizer implementation can be found at [Authorizer Sample](https:/ ### Implementing an Authorization Plugin -1. Add _datahub-auth-api_ as implementation dependency: Maven coordinates of _datahub-auth-api_ can be found at [Maven](https://mvnrepository.com/artifact/io.acryl/datahub-auth-api) +1. Add _datahub-auth-api_ as compileOnly dependency: Maven coordinates of _datahub-auth-api_ can be found at [Maven](https://mvnrepository.com/artifact/io.acryl/datahub-auth-api) + + Example of gradle dependency is given below. + + ```groovy + dependencies { + + def auth_api = 'io.acryl:datahub-auth-api:0.9.3-3rc3' + compileOnly "${auth_api}" + testImplementation "${auth_api}" + } + ``` + 2. Implement the Authorizer interface: [Authorizer Sample](https://github.com/acryldata/datahub-ranger-auth-plugin/tree/main/apache-ranger-plugin)
@@ -140,15 +156,16 @@ The sample authorizer implementation can be found at [Authorizer Sample](https:/ To see an example of building an uber jar, check out the `build.gradle` file for the apache-ranger-plugin file of [Apache Ranger Plugin](https://github.com/acryldata/datahub-ranger-auth-plugin/tree/main/apache-ranger-plugin) for reference. - Exclude datahub plugin dependency and signature classes as shown in below `shadowJar` task. + Exclude signature files as shown in below `shadowJar` task. ```groovy apply plugin: 'com.github.johnrengelman.shadow'; shadowJar { // Exclude com.datahub.plugins package and files related to jar signature - exclude "com/linkedin/common/", "com/datahub/", "META-INF/*.RSA", "META-INF/*.SF","META-INF/*.DSA" + exclude "META-INF/*.RSA", "META-INF/*.SF","META-INF/*.DSA" } ``` + 5. Install the Plugin: Refer to the section (Plugin Installation)[#plugin_installation] for plugin installation in DataHub environment From f85fd157e953535d2629b3612fe8222a74e36cdc Mon Sep 17 00:00:00 2001 From: RyanHolstien Date: Tue, 13 Dec 2022 18:11:02 -0600 Subject: [PATCH 18/19] fix(elasticsearch): build in resilience against IO exceptions on httpclient (#6680) * fix(elasticsearch): build in resilience against IO exceptions on http client --- .../common/RestHighLevelClientFactory.java | 154 ++++++++++-------- 1 file changed, 83 insertions(+), 71 deletions(-) diff --git a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/RestHighLevelClientFactory.java b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/RestHighLevelClientFactory.java index f5ec707f749a9a..1da66f3192f807 100644 --- a/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/RestHighLevelClientFactory.java +++ b/metadata-service/factories/src/main/java/com/linkedin/gms/factory/common/RestHighLevelClientFactory.java @@ -2,14 +2,28 @@ import com.linkedin.gms.factory.auth.AwsRequestSigningApacheInterceptor; import com.linkedin.gms.factory.spring.YamlPropertySourceFactory; +import java.io.IOException; import javax.annotation.Nonnull; +import javax.net.ssl.HostnameVerifier; import javax.net.ssl.SSLContext; import lombok.extern.slf4j.Slf4j; import org.apache.commons.lang3.StringUtils; import org.apache.http.HttpHost; import org.apache.http.HttpRequestInterceptor; +import org.apache.http.config.RegistryBuilder; +import org.apache.http.conn.ssl.DefaultHostnameVerifier; import org.apache.http.conn.ssl.NoopHostnameVerifier; +import org.apache.http.conn.util.PublicSuffixMatcherLoader; +import org.apache.http.impl.nio.conn.PoolingNHttpClientConnectionManager; +import org.apache.http.impl.nio.reactor.DefaultConnectingIOReactor; import org.apache.http.impl.nio.reactor.IOReactorConfig; +import org.apache.http.nio.conn.NHttpClientConnectionManager; +import org.apache.http.nio.conn.NoopIOSessionStrategy; +import org.apache.http.nio.conn.SchemeIOSessionStrategy; +import org.apache.http.nio.conn.ssl.SSLIOSessionStrategy; +import org.apache.http.nio.reactor.IOReactorException; +import org.apache.http.nio.reactor.IOReactorExceptionHandler; +import org.apache.http.ssl.SSLContexts; import org.elasticsearch.client.RestClient; import org.elasticsearch.client.RestClientBuilder; import org.elasticsearch.client.RestHighLevelClient; @@ -70,106 +84,104 @@ public class RestHighLevelClientFactory { @Bean(name = "elasticSearchRestHighLevelClient") @Nonnull - protected RestHighLevelClient createInstance() { - RestClientBuilder restClientBuilder; - if (useSSL) { - restClientBuilder = loadRestHttpsClient(host, port, pathPrefix, threadCount, connectionRequestTimeout, sslContext, username, - password, opensearchUseAwsIamAuth, region); - } else { - restClientBuilder = loadRestHttpClient(host, port, pathPrefix, threadCount, connectionRequestTimeout, username, - password, opensearchUseAwsIamAuth, region); - } + public RestHighLevelClient createInstance(RestClientBuilder restClientBuilder) { return new RestHighLevelClient(restClientBuilder); } - @Nonnull - private static RestClientBuilder loadRestHttpClient(@Nonnull String host, int port, String pathPrefix, int threadCount, - int connectionRequestTimeout) { - RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, "http")) - .setHttpClientConfigCallback(httpAsyncClientBuilder -> httpAsyncClientBuilder - .setDefaultIOReactorConfig(IOReactorConfig.custom().setIoThreadCount(threadCount).build())); + @Bean + public RestClientBuilder loadRestClient() { + final RestClientBuilder builder = createBuilder(useSSL ? "https" : "http"); - if (!StringUtils.isEmpty(pathPrefix)) { - builder.setPathPrefix(pathPrefix); - } - - builder.setRequestConfigCallback( - requestConfigBuilder -> requestConfigBuilder.setConnectionRequestTimeout(connectionRequestTimeout)); + builder.setHttpClientConfigCallback(httpAsyncClientBuilder -> { + if (useSSL) { + httpAsyncClientBuilder.setSSLContext(sslContext).setSSLHostnameVerifier(new NoopHostnameVerifier()); + } + try { + httpAsyncClientBuilder.setConnectionManager(createConnectionManager()); + } catch (IOReactorException e) { + throw new IllegalStateException("Unable to start ElasticSearch client. Please verify connection configuration."); + } + httpAsyncClientBuilder.setDefaultIOReactorConfig(IOReactorConfig.custom().setIoThreadCount(threadCount).build()); - return builder; - } + setCredentials(httpAsyncClientBuilder); - @Nonnull - private static RestClientBuilder loadRestHttpClient(@Nonnull String host, int port, String pathPrefix, int threadCount, - int connectionRequestTimeout, String username, String password, boolean opensearchUseAwsIamAuth, String region) { - RestClientBuilder builder = loadRestHttpClient(host, port, pathPrefix, threadCount, connectionRequestTimeout); - - builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { - public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) { - httpAsyncClientBuilder.setDefaultIOReactorConfig(IOReactorConfig.custom().setIoThreadCount(threadCount).build()); - - if (username != null && password != null) { - final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password)); - httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider); - } - if (opensearchUseAwsIamAuth) { - HttpRequestInterceptor interceptor = getAwsRequestSigningInterceptor(region); - httpAsyncClientBuilder.addInterceptorLast(interceptor); - } - - return httpAsyncClientBuilder; - } + return httpAsyncClientBuilder; }); return builder; } @Nonnull - private static RestClientBuilder loadRestHttpsClient(@Nonnull String host, int port, String pathPrefix, int threadCount, - int connectionRequestTimeout, @Nonnull SSLContext sslContext, String username, String password, - boolean opensearchUseAwsIamAuth, String region) { - - final RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, "https")); + private RestClientBuilder createBuilder(String scheme) { + final RestClientBuilder builder = RestClient.builder(new HttpHost(host, port, scheme)); if (!StringUtils.isEmpty(pathPrefix)) { builder.setPathPrefix(pathPrefix); } - builder.setHttpClientConfigCallback(new RestClientBuilder.HttpClientConfigCallback() { - public HttpAsyncClientBuilder customizeHttpClient(HttpAsyncClientBuilder httpAsyncClientBuilder) { - httpAsyncClientBuilder.setSSLContext(sslContext).setSSLHostnameVerifier(new NoopHostnameVerifier()) - .setDefaultIOReactorConfig(IOReactorConfig.custom().setIoThreadCount(threadCount).build()); - - if (username != null && password != null) { - final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); - credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password)); - httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider); - } else if (opensearchUseAwsIamAuth) { - HttpRequestInterceptor interceptor = getAwsRequestSigningInterceptor(region); - httpAsyncClientBuilder.addInterceptorLast(interceptor); - } - - return httpAsyncClientBuilder; - } - }); - builder.setRequestConfigCallback( requestConfigBuilder -> requestConfigBuilder.setConnectionRequestTimeout(connectionRequestTimeout)); return builder; } - private static HttpRequestInterceptor getAwsRequestSigningInterceptor(String region) { + /** + * Needed to override ExceptionHandler behavior for cases where IO error would have put client in unrecoverable state + * We don't utilize system properties in the client builder, so setting defaults pulled from + * {@link HttpAsyncClientBuilder#build()}. + * @return + */ + private NHttpClientConnectionManager createConnectionManager() throws IOReactorException { + SSLContext sslContext = SSLContexts.createDefault(); + HostnameVerifier hostnameVerifier = new DefaultHostnameVerifier(PublicSuffixMatcherLoader.getDefault()); + SchemeIOSessionStrategy sslStrategy = + new SSLIOSessionStrategy(sslContext, null, null, hostnameVerifier); + + IOReactorConfig ioReactorConfig = IOReactorConfig.custom().setIoThreadCount(threadCount).build(); + DefaultConnectingIOReactor ioReactor = new DefaultConnectingIOReactor(ioReactorConfig); + IOReactorExceptionHandler ioReactorExceptionHandler = new IOReactorExceptionHandler() { + @Override + public boolean handle(IOException ex) { + log.error("IO Exception caught during ElasticSearch connection.", ex); + return true; + } + + @Override + public boolean handle(RuntimeException ex) { + log.error("Runtime Exception caught during ElasticSearch connection.", ex); + return true; + } + }; + ioReactor.setExceptionHandler(ioReactorExceptionHandler); + + return new PoolingNHttpClientConnectionManager(ioReactor, + RegistryBuilder.create() + .register("http", NoopIOSessionStrategy.INSTANCE) + .register("https", sslStrategy) + .build()); + } + + private void setCredentials(HttpAsyncClientBuilder httpAsyncClientBuilder) { + if (username != null && password != null) { + final CredentialsProvider credentialsProvider = new BasicCredentialsProvider(); + credentialsProvider.setCredentials(AuthScope.ANY, new UsernamePasswordCredentials(username, password)); + httpAsyncClientBuilder.setDefaultCredentialsProvider(credentialsProvider); + } + if (opensearchUseAwsIamAuth) { + HttpRequestInterceptor interceptor = getAwsRequestSigningInterceptor(region); + httpAsyncClientBuilder.addInterceptorLast(interceptor); + } + } + + private HttpRequestInterceptor getAwsRequestSigningInterceptor(String region) { if (region == null) { - throw new NullPointerException("Region must not be null when opensearchUseAwsIamAuth is enabled"); + throw new IllegalArgumentException("Region must not be null when opensearchUseAwsIamAuth is enabled"); } Aws4Signer signer = Aws4Signer.create(); // Uses default AWS credentials - HttpRequestInterceptor interceptor = new AwsRequestSigningApacheInterceptor("es", signer, + return new AwsRequestSigningApacheInterceptor("es", signer, DefaultCredentialsProvider.create(), region); - return interceptor; } } From 8c14dfc617e476e752ac30068218d5e676fb63f7 Mon Sep 17 00:00:00 2001 From: Harshal Sheth Date: Tue, 13 Dec 2022 20:16:59 -0500 Subject: [PATCH 19/19] ci: fix ingestion gradle retry (#6752) --- docker/datahub-ingestion/Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docker/datahub-ingestion/Dockerfile b/docker/datahub-ingestion/Dockerfile index 09797f741a8255..86c8272d5aef55 100644 --- a/docker/datahub-ingestion/Dockerfile +++ b/docker/datahub-ingestion/Dockerfile @@ -16,7 +16,7 @@ COPY . /datahub-src # and https://unix.stackexchange.com/a/82610/378179. # This is a workaround for https://github.com/gradle/gradle/issues/18124. RUN cd /datahub-src && \ - (for attempt in 1 2 3 4 5; do ./gradlew --version && break ; echo "Failed to download gradle wrapper (attempt $attempt)" && sleep $((2*2**$attempt)) ; done ) && \ + (for attempt in 1 2 3 4 5; do ./gradlew --version && break ; echo "Failed to download gradle wrapper (attempt $attempt)" && sleep $((2<<$attempt)) ; done ) && \ ./gradlew :metadata-events:mxe-schemas:build FROM base as prod-codegen