diff --git a/bootstrap/sql/migrations/native/1.6.2/mysql/schemaChanges.sql b/bootstrap/sql/migrations/native/1.6.2/mysql/schemaChanges.sql index af8cd97bb514..eb23ec4a2ee0 100644 --- a/bootstrap/sql/migrations/native/1.6.2/mysql/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.6.2/mysql/schemaChanges.sql @@ -1,2 +1,26 @@ -- add timestamp index for test case result reindex performance -ALTER TABLE data_quality_data_time_series ADD INDEX `idx_timestamp_desc` (timestamp DESC); \ No newline at end of file +ALTER TABLE data_quality_data_time_series ADD INDEX `idx_timestamp_desc` (timestamp DESC); + +-- rename executable -> basic for test suites +UPDATE test_suite +SET json = JSON_INSERT( + JSON_REMOVE(json, '$.executable'), + '$.basic', + JSON_EXTRACT(json, '$.executable') +) +WHERE JSON_EXTRACT(json, '$.executable') IS NOT NULL; + +-- rename executableEntityReference -> basicEntityReference for test suites +UPDATE test_suite +SET json = JSON_INSERT( + JSON_REMOVE(json, '$.executableEntityReference'), + '$.basicEntityReference', + JSON_EXTRACT(json, '$.executableEntityReference') +) +WHERE JSON_EXTRACT(json, '$.executableEntityReference') IS NOT NULL; + +-- clean up the testSuites +UPDATE test_case SET json = json_remove(json, '$.testSuites'); + +-- clean up the testSuites in the version history too +UPDATE entity_extension SET json = json_remove(json, '$.testSuites') WHERE jsonSchema = 'testCase'; diff --git a/bootstrap/sql/migrations/native/1.6.2/postgres/schemaChanges.sql b/bootstrap/sql/migrations/native/1.6.2/postgres/schemaChanges.sql index 776cc2bd2ee0..a1ee9a6c6e70 100644 --- a/bootstrap/sql/migrations/native/1.6.2/postgres/schemaChanges.sql +++ b/bootstrap/sql/migrations/native/1.6.2/postgres/schemaChanges.sql @@ -1,2 +1,28 @@ -- add timestamp index for test case result reindex performance CREATE INDEX idx_timestamp_desc ON data_quality_data_time_series (timestamp DESC); + +-- rename executable -> basic for test suites +UPDATE test_suite +SET json = jsonb_set( + json::jsonb #- '{executable}', + '{basic}', + (json #> '{executable}')::jsonb, + true +) +WHERE json #>> '{executable}' IS NOT NULL; + +-- rename executableEntityReference -> basicEntityReference for test suites +UPDATE test_suite +SET json = jsonb_set( + json::jsonb #- '{executableEntityReference}', + '{basicEntityReference}', + (json #> '{executableEntityReference}')::jsonb, + true +) +WHERE json #>> '{executableEntityReference}' IS NOT NULL; + +-- clean up the testSuites +UPDATE test_case SET json = json::jsonb #- '{testSuites}'; + +-- clean up the testSuites in the version history too +UPDATE entity_extension SET json = json::jsonb #- '{testSuites}' WHERE jsonSchema = 'testCase'; diff --git a/ingestion/src/metadata/data_quality/api/models.py b/ingestion/src/metadata/data_quality/api/models.py index 82de6625cffb..39af90c06de6 100644 --- a/ingestion/src/metadata/data_quality/api/models.py +++ b/ingestion/src/metadata/data_quality/api/models.py @@ -23,6 +23,7 @@ from metadata.config.common import ConfigModel from metadata.generated.schema.api.tests.createTestSuite import CreateTestSuiteRequest from metadata.generated.schema.entity.data.table import Table +from metadata.generated.schema.entity.services.databaseService import DatabaseConnection from metadata.generated.schema.tests.basic import TestCaseResult from metadata.generated.schema.tests.testCase import TestCase, TestCaseParameterValue from metadata.ingestion.models.custom_pydantic import BaseModel @@ -63,6 +64,9 @@ class TableAndTests(BaseModel): executable_test_suite: Optional[CreateTestSuiteRequest] = Field( None, description="If no executable test suite is found, we'll create one" ) + service_connection: DatabaseConnection = Field( + ..., description="Service connection for the given table" + ) class TestCaseResults(BaseModel): diff --git a/ingestion/src/metadata/data_quality/processor/test_case_runner.py b/ingestion/src/metadata/data_quality/processor/test_case_runner.py index 696caaae0ce9..0e363eafb18d 100644 --- a/ingestion/src/metadata/data_quality/processor/test_case_runner.py +++ b/ingestion/src/metadata/data_quality/processor/test_case_runner.py @@ -27,6 +27,7 @@ from metadata.data_quality.runner.core import DataTestsRunner from metadata.generated.schema.api.tests.createTestCase import CreateTestCaseRequest from metadata.generated.schema.entity.data.table import Table +from metadata.generated.schema.entity.services.databaseService import DatabaseConnection from metadata.generated.schema.entity.services.ingestionPipelines.status import ( StackTraceError, ) @@ -93,7 +94,9 @@ def _run(self, record: TableAndTests) -> Either: record.table, openmetadata_test_cases ) - test_suite_runner = self.get_test_suite_runner(record.table) + test_suite_runner = self.get_test_suite_runner( + record.table, record.service_connection + ) logger.debug( f"Found {len(openmetadata_test_cases)} test cases for table {record.table.fullyQualifiedName.root}" @@ -351,9 +354,9 @@ def filter_incompatible_test_cases( result.append(tc) return result - def get_test_suite_runner(self, table: Table): + def get_test_suite_runner( + self, table: Table, service_connection: DatabaseConnection + ): return BaseTestSuiteRunner( - self.config, - self.metadata, - table, + self.config, self.metadata, table, service_connection ).get_data_quality_runner() diff --git a/ingestion/src/metadata/data_quality/runner/base_test_suite_source.py b/ingestion/src/metadata/data_quality/runner/base_test_suite_source.py index bf4897843a9f..dbbd7b71c64e 100644 --- a/ingestion/src/metadata/data_quality/runner/base_test_suite_source.py +++ b/ingestion/src/metadata/data_quality/runner/base_test_suite_source.py @@ -46,11 +46,12 @@ def __init__( config: OpenMetadataWorkflowConfig, ometa_client: OpenMetadata, entity: Table, + service_connection: DatabaseConnection, ): self.validator_builder_class = ValidatorBuilder self._interface = None self.entity = entity - self.service_conn_config = self._copy_service_config(config, self.entity.database) # type: ignore + self.service_conn_config = self._copy_service_config(service_connection, self.entity.database) # type: ignore self._interface_type: str = self.service_conn_config.type.value.lower() self.source_config = TestSuitePipeline.model_validate( @@ -67,7 +68,7 @@ def interface(self, interface): self._interface = interface def _copy_service_config( - self, config: OpenMetadataWorkflowConfig, database: EntityReference + self, service_connection: DatabaseConnection, database: EntityReference ) -> DatabaseConnection: """Make a copy of the service config and update the database name @@ -77,9 +78,7 @@ def _copy_service_config( Returns: DatabaseService.__config__ """ - config_copy = deepcopy( - config.source.serviceConnection.root.config # type: ignore - ) + config_copy = deepcopy(service_connection.config) # type: ignore if hasattr( config_copy, # type: ignore "supportsDatabase", diff --git a/ingestion/src/metadata/data_quality/source/test_suite.py b/ingestion/src/metadata/data_quality/source/test_suite.py index a2fb2cfae2ee..9558d36a1785 100644 --- a/ingestion/src/metadata/data_quality/source/test_suite.py +++ b/ingestion/src/metadata/data_quality/source/test_suite.py @@ -14,11 +14,17 @@ The main goal is to get the configured table from the API. """ -from typing import Iterable, List, Optional, cast +import itertools +import traceback +from typing import Dict, Iterable, List, Optional, cast from metadata.data_quality.api.models import TableAndTests from metadata.generated.schema.api.tests.createTestSuite import CreateTestSuiteRequest from metadata.generated.schema.entity.data.table import Table +from metadata.generated.schema.entity.services.databaseService import ( + DatabaseConnection, + DatabaseService, +) from metadata.generated.schema.entity.services.ingestionPipelines.status import ( StackTraceError, ) @@ -36,7 +42,7 @@ from metadata.ingestion.api.step import Step from metadata.ingestion.api.steps import Source from metadata.ingestion.ometa.ometa_api import OpenMetadata -from metadata.utils import fqn +from metadata.utils import entity_link, fqn from metadata.utils.constants import CUSTOM_CONNECTOR_PREFIX from metadata.utils.logger import test_suite_logger from metadata.utils.service_spec.service_spec import import_source_class @@ -61,18 +67,36 @@ def __init__( self.source_config: TestSuitePipeline = self.config.source.sourceConfig.config + # Build at runtime - if not informed in the yaml - the service connection map + self.service_connection_map: Dict[ + str, DatabaseConnection + ] = self._load_yaml_service_connections() + self.test_connection() @property def name(self) -> str: return "OpenMetadata" + def _load_yaml_service_connections(self) -> Dict[str, DatabaseConnection]: + """Load the service connections from the YAML file""" + service_connections = self.source_config.serviceConnections + if not service_connections: + return {} + return { + conn.serviceName: cast(DatabaseConnection, conn.serviceConnection.root) + for conn in service_connections + } + def _get_table_entity(self) -> Optional[Table]: """given an entity fqn return the table entity Args: entity_fqn: entity fqn for the test case """ + # Logical test suites don't have associated tables + if self.source_config.entityFullyQualifiedName is None: + return None table: Table = self.metadata.get_by_name( entity=Table, fqn=self.source_config.entityFullyQualifiedName.root, @@ -81,23 +105,51 @@ def _get_table_entity(self) -> Optional[Table]: return table - def _get_test_cases_from_test_suite( - self, test_suite: Optional[TestSuite] - ) -> List[TestCase]: + def _get_table_service_connection(self, table: Table) -> DatabaseConnection: + """Get the service connection for the table""" + service_name = table.service.name + + if service_name not in self.service_connection_map: + try: + service: DatabaseService = self.metadata.get_by_name( + DatabaseService, service_name + ) + if not service: + raise ConnectionError( + f"Could not retrieve service with name `{service_name}`. " + "Typically caused by the `entityFullyQualifiedName` does not exists in OpenMetadata " + "or the JWT Token is invalid." + ) + if not service.connection: + raise ConnectionError( + f"Service with name `{service_name}` does not have a connection. " + "If the connection is not stored in OpenMetadata, please provide it in the YAML file." + ) + self.service_connection_map[service_name] = service.connection + + except Exception as exc: + logger.debug(traceback.format_exc()) + logger.error( + f"Error getting service connection for service name [{service_name}]" + f" using the secrets manager provider [{self.metadata.config.secretsManagerProvider}]: {exc}" + ) + raise exc + + return self.service_connection_map[service_name] + + def _get_test_cases_from_test_suite(self, test_suite: TestSuite) -> List[TestCase]: """Return test cases if the test suite exists and has them""" - if test_suite: - test_cases = self.metadata.list_all_entities( - entity=TestCase, - fields=["testSuite", "entityLink", "testDefinition"], - params={"testSuiteId": test_suite.id.root}, - ) - test_cases = cast(List[TestCase], test_cases) # satisfy type checker - if self.source_config.testCases is not None: - test_cases = [ - t for t in test_cases if t.name in self.source_config.testCases - ] - return test_cases - return [] + test_cases = self.metadata.list_all_entities( + entity=TestCase, + fields=["testSuite", "entityLink", "testDefinition"], + params={"testSuiteId": test_suite.id.root}, + ) + test_cases = cast(List[TestCase], test_cases) # satisfy type checker + if self.source_config.testCases is not None: + test_cases = [ + t for t in test_cases if t.name in self.source_config.testCases + ] + return test_cases def prepare(self): """Nothing to prepare""" @@ -106,6 +158,7 @@ def test_connection(self) -> None: self.metadata.health_check() def _iter(self) -> Iterable[Either[TableAndTests]]: + # Basic tests suites will have a table informed table: Table = self._get_table_entity() if table: source_type = table.serviceType.value.lower() @@ -119,20 +172,29 @@ def _iter(self) -> Iterable[Either[TableAndTests]]: ) yield from self._process_table_suite(table) + # Logical test suites won't have a table, we'll need to group the execution by tests else: - yield Either( - left=StackTraceError( - name="Missing Table", - error=f"Could not retrieve table entity for {self.source_config.entityFullyQualifiedName.root}." - " Make sure the table exists in OpenMetadata and/or the JWT Token provided is valid.", - ) - ) + yield from self._process_logical_suite() def _process_table_suite(self, table: Table) -> Iterable[Either[TableAndTests]]: """ Check that the table has the proper test suite built in """ + try: + service_connection: DatabaseConnection = self._get_table_service_connection( + table + ) + except Exception as exc: + yield Either( + left=StackTraceError( + name="Error getting service connection", + error=f"Error getting the service connection for table {table.name.root}: {exc}", + stackTrace=traceback.format_exc(), + ) + ) + return # If there is no executable test suite yet for the table, we'll need to create one + # Then, the suite won't have yet any tests if not table.testSuite: executable_test_suite = CreateTestSuiteRequest( name=fqn.build( @@ -143,38 +205,80 @@ def _process_table_suite(self, table: Table) -> Iterable[Either[TableAndTests]]: displayName=f"{self.source_config.entityFullyQualifiedName.root} Test Suite", description="Test Suite created from YAML processor config file", owners=None, - executableEntityReference=self.source_config.entityFullyQualifiedName.root, + basicEntityReference=self.source_config.entityFullyQualifiedName.root, ) yield Either( right=TableAndTests( executable_test_suite=executable_test_suite, - service_type=self.config.source.serviceConnection.root.config.type.value, + service_type=service_connection.config.type.value, + service_connection=service_connection, ) ) + test_suite_cases = [] - test_suite: Optional[TestSuite] = None - if table.testSuite: - test_suite = self.metadata.get_by_id( + # Otherwise, we pick the tests already registered in the suite + else: + test_suite: Optional[TestSuite] = self.metadata.get_by_id( entity=TestSuite, entity_id=table.testSuite.id.root ) + test_suite_cases = self._get_test_cases_from_test_suite(test_suite) + + yield Either( + right=TableAndTests( + table=table, + test_cases=test_suite_cases, + service_type=service_connection.config.type.value, + service_connection=service_connection, + ) + ) - if test_suite and not test_suite.executable: + def _process_logical_suite(self): + """Process logical test suite, collect all test cases and yield them in batches by table""" + test_suite = self.metadata.get_by_name( + entity=TestSuite, fqn=self.config.source.serviceName + ) + if test_suite is None: yield Either( left=StackTraceError( - name="Non-executable Test Suite", - error=f"The table {self.source_config.entityFullyQualifiedName.root} " - "has a test suite that is not executable.", + name="Test Suite not found", + error=f"Test Suite with name {self.config.source.serviceName} not found", ) ) + test_cases: List[TestCase] = self._get_test_cases_from_test_suite(test_suite) + grouped_by_table = itertools.groupby( + test_cases, key=lambda t: entity_link.get_table_fqn(t.entityLink.root) + ) + for table_fqn, group in grouped_by_table: + table_entity: Table = self.metadata.get_by_name(Table, table_fqn) + if table_entity is None: + yield Either( + left=StackTraceError( + name="Table not found", + error=f"Table with fqn {table_fqn} not found for test suite {test_suite.name.root}", + ) + ) + continue - else: - test_suite_cases = self._get_test_cases_from_test_suite(test_suite) + try: + service_connection: DatabaseConnection = ( + self._get_table_service_connection(table_entity) + ) + except Exception as exc: + yield Either( + left=StackTraceError( + name="Error getting service connection", + error=f"Error getting the service connection for table {table_entity.name.root}: {exc}", + stackTrace=traceback.format_exc(), + ) + ) + continue yield Either( right=TableAndTests( - table=table, - test_cases=test_suite_cases, - service_type=self.config.source.serviceConnection.root.config.type.value, + table=table_entity, + test_cases=list(group), + service_type=service_connection.config.type.value, + service_connection=service_connection, ) ) diff --git a/ingestion/src/metadata/great_expectations/action.py b/ingestion/src/metadata/great_expectations/action.py index c51d146785c7..62fc07309a36 100644 --- a/ingestion/src/metadata/great_expectations/action.py +++ b/ingestion/src/metadata/great_expectations/action.py @@ -257,7 +257,7 @@ def _check_or_create_test_suite(self, table_entity: Table) -> TestSuite: create_test_suite = CreateTestSuiteRequest( name=f"{table_entity.fullyQualifiedName.root}.TestSuite", - executableEntityReference=table_entity.fullyQualifiedName.root, + basicEntityReference=table_entity.fullyQualifiedName.root, ) # type: ignore test_suite = self.ometa_conn.create_or_update_executable_test_suite( create_test_suite diff --git a/ingestion/src/metadata/ingestion/ometa/mixins/tests_mixin.py b/ingestion/src/metadata/ingestion/ometa/mixins/tests_mixin.py index c76ddc896318..d4eb2fd1b742 100644 --- a/ingestion/src/metadata/ingestion/ometa/mixins/tests_mixin.py +++ b/ingestion/src/metadata/ingestion/ometa/mixins/tests_mixin.py @@ -228,7 +228,7 @@ def get_or_create_executable_test_suite( create_test_suite = CreateTestSuiteRequest( name=f"{table_entity.fullyQualifiedName.root}.TestSuite", - executableEntityReference=table_entity.fullyQualifiedName.root, + basicEntityReference=table_entity.fullyQualifiedName.root, ) # type: ignore test_suite = self.create_or_update_executable_test_suite(create_test_suite) return test_suite diff --git a/ingestion/src/metadata/ingestion/source/database/sample_data.py b/ingestion/src/metadata/ingestion/source/database/sample_data.py index f809f045206b..69cdb59897e0 100644 --- a/ingestion/src/metadata/ingestion/source/database/sample_data.py +++ b/ingestion/src/metadata/ingestion/source/database/sample_data.py @@ -1548,9 +1548,7 @@ def ingest_test_suite(self) -> Iterable[Either[OMetaTestSuiteSample]]: test_suite=CreateTestSuiteRequest( name=test_suite["testSuiteName"], description=test_suite["testSuiteDescription"], - executableEntityReference=test_suite[ - "executableEntityReference" - ], + basicEntityReference=test_suite["executableEntityReference"], ) ) ) diff --git a/ingestion/src/metadata/workflow/data_quality.py b/ingestion/src/metadata/workflow/data_quality.py index b2b5b878b3b9..ea08be6107c4 100644 --- a/ingestion/src/metadata/workflow/data_quality.py +++ b/ingestion/src/metadata/workflow/data_quality.py @@ -11,15 +11,10 @@ """ Workflow definition for the Data Quality """ -import traceback from typing import Optional from metadata.data_quality.processor.test_case_runner import TestCaseRunner from metadata.data_quality.source.test_suite import TestSuiteSource -from metadata.generated.schema.entity.services.connections.serviceConnection import ( - ServiceConnection, -) -from metadata.generated.schema.entity.services.databaseService import DatabaseService from metadata.generated.schema.tests.testSuite import ServiceType, TestSuite from metadata.ingestion.api.steps import Processor, Sink from metadata.ingestion.ometa.utils import model_str @@ -63,44 +58,11 @@ def _get_sink(self) -> Sink: def _get_test_runner_processor(self) -> Processor: return TestCaseRunner.create(self.config.model_dump(), self.metadata) - def _retrieve_service_connection_if_needed(self, service_type: ServiceType) -> None: - """Get service object from source config `entityFullyQualifiedName`""" - if ( - not self.config.source.serviceConnection - and not self.metadata.config.forceEntityOverwriting - ): - fully_qualified_name = ( - self.config.source.sourceConfig.config.entityFullyQualifiedName.root - ) - try: - service_name = fqn.split(fully_qualified_name)[0] - except IndexError as exc: - logger.debug(traceback.format_exc()) - raise IndexError( - f"Could not retrieve service name from entity fully qualified name {fully_qualified_name}: {exc}" - ) - try: - service: DatabaseService = self.metadata.get_by_name( - DatabaseService, service_name - ) - if not service: - raise ConnectionError( - f"Could not retrieve service with name `{service_name}`. " - "Typically caused by the `entityFullyQualifiedName` does not exists in OpenMetadata " - "or the JWT Token is invalid." - ) - - self.config.source.serviceConnection = ServiceConnection( - service.connection - ) - - except Exception as exc: - logger.debug(traceback.format_exc()) - logger.error( - f"Error getting service connection for service name [{service_name}]" - f" using the secrets manager provider [{self.metadata.config.secretsManagerProvider}]: {exc}" - ) - raise exc + def _retrieve_service_connection_if_needed(self, _: ServiceType) -> None: + """A test suite might require multiple connections (e.g., for logical test suites) + We'll skip this step and get the connections at runtime if they are not informed + in the YAML already. + """ def _get_ingestion_pipeline_service(self) -> Optional[T]: """ diff --git a/ingestion/tests/integration/datalake/conftest.py b/ingestion/tests/integration/datalake/conftest.py index febbc3fbb1f6..491606a58718 100644 --- a/ingestion/tests/integration/datalake/conftest.py +++ b/ingestion/tests/integration/datalake/conftest.py @@ -209,8 +209,11 @@ def run_ingestion(metadata, ingestion_config): @pytest.fixture(scope="class") def run_test_suite_workflow(run_ingestion, ingestion_config): workflow_config = deepcopy(DATA_QUALITY_CONFIG) - workflow_config["source"]["serviceConnection"] = ingestion_config["source"][ - "serviceConnection" + workflow_config["source"]["sourceConfig"]["config"]["serviceConnections"] = [ + { + "serviceName": ingestion_config["source"]["serviceName"], + "serviceConnection": ingestion_config["source"]["serviceConnection"], + } ] ingestion_workflow = TestSuiteWorkflow.create(workflow_config) ingestion_workflow.execute() @@ -229,8 +232,11 @@ def run_sampled_test_suite_workflow(metadata, run_ingestion, ingestion_config): ), ) workflow_config = deepcopy(DATA_QUALITY_CONFIG) - workflow_config["source"]["serviceConnection"] = ingestion_config["source"][ - "serviceConnection" + workflow_config["source"]["sourceConfig"]["config"]["serviceConnections"] = [ + { + "serviceName": ingestion_config["source"]["serviceName"], + "serviceConnection": ingestion_config["source"]["serviceConnection"], + } ] ingestion_workflow = TestSuiteWorkflow.create(workflow_config) ingestion_workflow.execute() @@ -259,8 +265,11 @@ def run_partitioned_test_suite_workflow(metadata, run_ingestion, ingestion_confi ), ) workflow_config = deepcopy(DATA_QUALITY_CONFIG) - workflow_config["source"]["serviceConnection"] = ingestion_config["source"][ - "serviceConnection" + workflow_config["source"]["sourceConfig"]["config"]["serviceConnections"] = [ + { + "serviceName": ingestion_config["source"]["serviceName"], + "serviceConnection": ingestion_config["source"]["serviceConnection"], + } ] ingestion_workflow = TestSuiteWorkflow.create(workflow_config) ingestion_workflow.execute() diff --git a/ingestion/tests/integration/integration_base.py b/ingestion/tests/integration/integration_base.py index 11fe6ee449aa..bed6db07c0e1 100644 --- a/ingestion/tests/integration/integration_base.py +++ b/ingestion/tests/integration/integration_base.py @@ -371,7 +371,7 @@ def get_create_test_suite( return CreateTestSuiteRequest( name=TestSuiteEntityName(name), description=Markdown(description), - executableEntityReference=FullyQualifiedEntityName(executable_entity_reference), + basicEntityReference=FullyQualifiedEntityName(executable_entity_reference), ) diff --git a/ingestion/tests/integration/ometa/test_ometa_test_suite.py b/ingestion/tests/integration/ometa/test_ometa_test_suite.py index c9391066a7bb..5e4f7711cad3 100644 --- a/ingestion/tests/integration/ometa/test_ometa_test_suite.py +++ b/ingestion/tests/integration/ometa/test_ometa_test_suite.py @@ -100,7 +100,7 @@ def setUpClass(cls) -> None: description=Markdown( root="This is a test suite for the integration tests" ), - executableEntityReference=FullyQualifiedEntityName( + basicEntityReference=FullyQualifiedEntityName( "sample_data.ecommerce_db.shopify.dim_address" ), ) diff --git a/ingestion/tests/integration/postgres/test_data_quality.py b/ingestion/tests/integration/postgres/test_data_quality.py index f37c9560ae75..36b1cb9a0ccc 100644 --- a/ingestion/tests/integration/postgres/test_data_quality.py +++ b/ingestion/tests/integration/postgres/test_data_quality.py @@ -9,6 +9,7 @@ from metadata.data_quality.api.models import TestCaseDefinition from metadata.generated.schema.entity.services.databaseService import DatabaseService from metadata.generated.schema.metadataIngestion.testSuitePipeline import ( + ServiceConnections, TestSuiteConfigType, TestSuitePipeline, ) @@ -55,9 +56,14 @@ def run_data_quality_workflow( config=TestSuitePipeline( type=TestSuiteConfigType.TestSuite, entityFullyQualifiedName=f"{db_service.fullyQualifiedName.root}.dvdrental.public.customer", + serviceConnections=[ + ServiceConnections( + serviceName=db_service.name.root, + serviceConnection=db_service.connection, + ) + ], ) ), - serviceConnection=db_service.connection, ), processor=Processor( type="orm-test-runner", diff --git a/ingestion/tests/integration/test_suite/test_workflow.py b/ingestion/tests/integration/test_suite/test_workflow.py index 240493bb104d..42e45887e3cc 100644 --- a/ingestion/tests/integration/test_suite/test_workflow.py +++ b/ingestion/tests/integration/test_suite/test_workflow.py @@ -134,7 +134,7 @@ def setUpClass(cls) -> None: cls.test_suite = cls.metadata.create_or_update_executable_test_suite( data=CreateTestSuiteRequest( name="test-suite", - executableEntityReference=cls.table_with_suite.fullyQualifiedName.root, + basicEntityReference=cls.table_with_suite.fullyQualifiedName.root, ) ) diff --git a/ingestion/tests/unit/data_quality/source/test_test_suite.py b/ingestion/tests/unit/data_quality/source/test_test_suite.py index 9359a6e3b1e0..b153162e689d 100644 --- a/ingestion/tests/unit/data_quality/source/test_test_suite.py +++ b/ingestion/tests/unit/data_quality/source/test_test_suite.py @@ -5,7 +5,11 @@ from metadata.data_quality.source.test_suite import TestSuiteSource from metadata.generated.schema.entity.data.table import Table +from metadata.generated.schema.entity.services.connections.database.postgresConnection import ( + PostgresConnection, +) from metadata.generated.schema.entity.services.databaseService import ( + DatabaseConnection, DatabaseServiceType, ) from metadata.generated.schema.metadataIngestion.workflow import ( @@ -72,6 +76,19 @@ def test_source_config(parameters, expected, monkeypatch): }, } monkeypatch.setattr(TestSuiteSource, "test_connection", Mock()) + monkeypatch.setattr( + TestSuiteSource, + "_get_table_service_connection", + Mock( + return_value=DatabaseConnection( + config=PostgresConnection( + username="foo", + hostPort="localhost:5432", + database="postgres", + ) + ) + ), + ) mock_metadata = Mock(spec=OpenMetadata) mock_metadata.get_by_name.return_value = Table( @@ -98,7 +115,7 @@ def test_source_config(parameters, expected, monkeypatch): ), ] mock_metadata.get_by_id.return_value = TestSuite( - name="test_suite", executable=True, id=UUID(int=0) + name="test_suite", basic=True, id=UUID(int=0) ) source = TestSuiteSource( diff --git a/ingestion/tests/unit/metadata/utils/test_entity_link.py b/ingestion/tests/unit/metadata/utils/test_entity_link.py index 840b22f56c3f..bd0bb95a9437 100644 --- a/ingestion/tests/unit/metadata/utils/test_entity_link.py +++ b/ingestion/tests/unit/metadata/utils/test_entity_link.py @@ -16,7 +16,11 @@ import pytest from antlr4.error.Errors import ParseCancellationException -from metadata.utils.entity_link import get_decoded_column, get_table_or_column_fqn +from metadata.utils.entity_link import ( + get_decoded_column, + get_table_fqn, + get_table_or_column_fqn, +) @pytest.mark.parametrize( @@ -154,3 +158,16 @@ def test_invalid_get_table_or_column_fqn(entity_link, error): """ with pytest.raises(error): get_table_or_column_fqn(entity_link) + + +@pytest.mark.parametrize( + "entity_link,expected", + [ + ( + "<#E::table::red.dev.dbt_jaffle.customers::columns::a>", + "red.dev.dbt_jaffle.customers", + ), + ], +) +def test_get_table_fqn(entity_link, expected): + assert get_table_fqn(entity_link) == expected diff --git a/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_deploy.py b/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_deploy.py index 57d14b33c91f..329e7b3cf936 100644 --- a/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_deploy.py +++ b/openmetadata-airflow-apis/tests/unit/ingestion_pipeline/test_deploy.py @@ -15,8 +15,6 @@ import uuid from unittest.mock import patch -from openmetadata_managed_apis.operations.deploy import dump_with_safe_jwt - from metadata.generated.schema.entity.services.connections.metadata.openMetadataConnection import ( AuthProvider, OpenMetadataConnection, @@ -55,9 +53,11 @@ ) -@patch.dict(os.environ, {"AWS_DEFAULT_REGION": "us-east-2"}) +@patch.dict(os.environ, {"AWS_DEFAULT_REGION": "us-east-2", "AIRFLOW_HOME": "/tmp"}) def test_deploy_ingestion_pipeline(): """We can dump an ingestion pipeline to a file without exposing secrets""" + from openmetadata_managed_apis.operations.deploy import dump_with_safe_jwt + # Instantiate the Secrets Manager SecretsManagerFactory.clear_all() with patch.object(AWSSecretsManager, "get_string_value", return_value=SECRET_VALUE): diff --git a/openmetadata-docs/content/partials/v1.6/deployment/upgrade/upgrade-prerequisites.md b/openmetadata-docs/content/partials/v1.6/deployment/upgrade/upgrade-prerequisites.md index 465e14bb70fb..78def9dcdd72 100644 --- a/openmetadata-docs/content/partials/v1.6/deployment/upgrade/upgrade-prerequisites.md +++ b/openmetadata-docs/content/partials/v1.6/deployment/upgrade/upgrade-prerequisites.md @@ -86,6 +86,122 @@ After the migration is finished, you can revert this changes. # Backward Incompatible Changes +## 1.6.2 + +### Executable Logical Test Suites + +We are introducing a new feature that allows users to execute logical test suites. This feature will allow users to run +groups of Data Quality tests, even if they belong to different tables (or even services!). Note that before, you could +only schedule and execute the tests for each of the tables. + +From the UI, you can now create a new Test Suite, add any tests you want and create and schedule the run. + +This change, however, requires some adjustments if you are directly interacting with the OpenMetadata API or if you +are running the ingestions externally: + +#### `/executable` endpoints Changes + +CRUD operations around "executable" Test Suites - the ones directly related to a single table - were managed by the +`/executable` endpoints, e.g., `POST /v1/dataQuality/testSuites/executable`. We'll keep this endpoints until the next release, +but users should update their operations to use the new `/base` endpoints, e.g., `POST /v1/dataQuality/testSuites/base`. + +This is to adjust the naming convention since all Test Suites are executable, so we're differentiating between "base" and +"logical" Test Suites. + +In the meantime, you can use the `/executable` endpoints to create and manage the Test Suites, but you'll get deprecation +headers in the response. We recommend migrating to the new endpoints as soon as possible to avoid any issues when the `/executable` +endpoints get completely removed. + +#### YAML Changes + +If you're running the DQ Workflows externally AND YOU ARE NOT STORING THE SERVICE INFORMATION IN OPENMETADATA, this is how they'll change: + +A YAML file for 1.5.x would look like this: + +```yaml +source: + type: testsuite + serviceName: red # Test Suite Name + serviceConnection: + config: + hostPort: + username: + password: + database: + type: Redshift + sourceConfig: + config: + type: TestSuite + entityFullyQualifiedName: red.dev.dbt_jaffle.customers + profileSampleType: PERCENTAGE +processor: + type: "orm-test-runner" + config: {} +sink: + type: metadata-rest + config: {} +workflowConfig: + openMetadataServerConfig: + hostPort: http://localhost:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "..." +``` + +Basically, if you are not storing the service connection in OpenMetadata, you could leverage the `source.serviceConnection` +entry to pass that information. + +However, with the ability to execute Logical Test Suites, you can now have multiple tests from different services! This means, +that the connection information needs to be placed differently. The new YAML file would look like this: + +```yaml +source: + type: testsuite + serviceName: Logical # Test Suite Name + sourceConfig: + config: + type: TestSuite + serviceConnections: + - serviceName: red + serviceConnection: + config: + hostPort: + username: + password: + database: + type: Redshift + - serviceName: snowflake + serviceConnection: + config: + hostPort: + username: + password: + database: + type: Snowflake +processor: + type: "orm-test-runner" + config: {} +sink: + type: metadata-rest + config: {} +workflowConfig: + openMetadataServerConfig: + hostPort: http://localhost:8585/api + authProvider: openmetadata + securityConfig: + jwtToken: "..." +``` + +As you can see, you can pass multiple `serviceConnections` to the `sourceConfig` entry, each one with the connection information +and the `serviceName` they are linked to. + +{% note noteType="Warning" %} + +If you are already storing the service connection information in OpenMetadata (e.g., because you have created the services via the UI), +there's nothing you need to do. The ingestion will automatically pick up the connection information from the service. + +{% /note %} + ## 1.6.0 ### Ingestion Workflow Status diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ListFilter.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ListFilter.java index 1722181a18a7..a9a9a0875a6d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ListFilter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/ListFilter.java @@ -272,22 +272,22 @@ private String getTestSuiteTypeCondition(String tableName) { } return switch (testSuiteType) { - case ("executable") -> { + // We'll clean up the executable when we deprecate the /executable endpoints + case "basic", "executable" -> { if (Boolean.TRUE.equals(DatasourceConfig.getInstance().isMySQL())) { yield String.format( - "(JSON_UNQUOTE(JSON_EXTRACT(%s.json, '$.executable')) = 'true')", tableName); + "(JSON_UNQUOTE(JSON_EXTRACT(%s.json, '$.basic')) = 'true')", tableName); } - yield String.format("(%s.json->>'executable' = 'true')", tableName); + yield String.format("(%s.json->>'basic' = 'true')", tableName); } - case ("logical") -> { + case "logical" -> { if (Boolean.TRUE.equals(DatasourceConfig.getInstance().isMySQL())) { yield String.format( - "(JSON_UNQUOTE(JSON_EXTRACT(%s.json, '$.executable')) = 'false' OR JSON_UNQUOTE(JSON_EXTRACT(%s.json, '$.executable')) IS NULL)", + "(JSON_UNQUOTE(JSON_EXTRACT(%s.json, '$.basic')) = 'false' OR JSON_UNQUOTE(JSON_EXTRACT(%s.json, '$.basic')) IS NULL)", tableName, tableName); } yield String.format( - "(%s.json->>'executable' = 'false' or %s.json -> 'executable' is null)", - tableName, tableName); + "(%s.json->>'basic' = 'false' or %s.json -> 'basic' is null)", tableName, tableName); } default -> ""; }; diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java index 741d407dc356..e986c2381651 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestCaseRepository.java @@ -184,7 +184,7 @@ private EntityReference getTestSuite(TestCase test) throws EntityNotFoundExcepti findFromRecords(test.getId(), entityType, Relationship.CONTAINS, TEST_SUITE); for (CollectionDAO.EntityRelationshipRecord testSuiteId : records) { TestSuite testSuite = Entity.getEntity(TEST_SUITE, testSuiteId.getId(), "", Include.ALL); - if (Boolean.TRUE.equals(testSuite.getExecutable())) { + if (Boolean.TRUE.equals(testSuite.getBasic())) { return testSuite.getEntityReference(); } } @@ -244,14 +244,16 @@ public void storeEntity(TestCase test, boolean update) { EntityReference testSuite = test.getTestSuite(); EntityReference testDefinition = test.getTestDefinition(); TestCaseResult testCaseResult = test.getTestCaseResult(); + List testSuites = test.getTestSuites(); // Don't store testCaseResult, owner, database, href and tags as JSON. // Build it on the fly based on relationships - test.withTestSuite(null).withTestDefinition(null).withTestCaseResult(null); + test.withTestSuite(null).withTestSuites(null).withTestDefinition(null).withTestCaseResult(null); store(test, update); // Restore the relationships test.withTestSuite(testSuite) + .withTestSuites(testSuites) .withTestDefinition(testDefinition) .withTestCaseResult(testCaseResult); } @@ -427,13 +429,13 @@ public int getTestCaseCount(List testCaseIds) { return daoCollection.testCaseDAO().countOfTestCases(testCaseIds); } - public void isTestSuiteExecutable(String testSuiteFqn) { + public void isTestSuiteBasic(String testSuiteFqn) { TestSuite testSuite = Entity.getEntityByName(Entity.TEST_SUITE, testSuiteFqn, null, null); - if (Boolean.FALSE.equals(testSuite.getExecutable())) { + if (Boolean.FALSE.equals(testSuite.getBasic())) { throw new IllegalArgumentException( "Test suite " + testSuite.getName() - + " is not executable. Cannot create test cases for non-executable test suites."); + + " is not basic. Cannot create test cases for non-basic test suites."); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java index 7d4b486e8f08..edf54543dd63 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TestSuiteRepository.java @@ -127,10 +127,10 @@ public void setFields(TestSuite entity, EntityUtil.Fields fields) { @Override public void setInheritedFields(TestSuite testSuite, EntityUtil.Fields fields) { - if (Boolean.TRUE.equals(testSuite.getExecutable())) { + if (Boolean.TRUE.equals(testSuite.getBasic())) { Table table = Entity.getEntity( - TABLE, testSuite.getExecutableEntityReference().getId(), "owners,domain", ALL); + TABLE, testSuite.getBasicEntityReference().getId(), "owners,domain", ALL); inheritOwners(testSuite, fields, table); inheritDomain(testSuite, fields, table); } @@ -145,10 +145,10 @@ public void clearFields(TestSuite entity, EntityUtil.Fields fields) { @Override public void setFullyQualifiedName(TestSuite testSuite) { - if (testSuite.getExecutableEntityReference() != null) { + if (testSuite.getBasicEntityReference() != null) { testSuite.setFullyQualifiedName( FullyQualifiedName.add( - testSuite.getExecutableEntityReference().getFullyQualifiedName(), "testSuite")); + testSuite.getBasicEntityReference().getFullyQualifiedName(), "testSuite")); } else { testSuite.setFullyQualifiedName(quoteName(testSuite.getName())); } @@ -328,22 +328,21 @@ public TestSummary getTestSummary(UUID testSuiteId) { @Override protected void postCreate(TestSuite entity) { super.postCreate(entity); - if (Boolean.TRUE.equals(entity.getExecutable()) - && entity.getExecutableEntityReference() != null) { + if (Boolean.TRUE.equals(entity.getBasic()) && entity.getBasicEntityReference() != null) { // Update table index with test suite field EntityInterface entityInterface = - getEntity(entity.getExecutableEntityReference(), "testSuite", ALL); + getEntity(entity.getBasicEntityReference(), "testSuite", ALL); IndexMapping indexMapping = - searchRepository.getIndexMapping(entity.getExecutableEntityReference().getType()); + searchRepository.getIndexMapping(entity.getBasicEntityReference().getType()); SearchClient searchClient = searchRepository.getSearchClient(); SearchIndex index = searchRepository .getSearchIndexFactory() - .buildIndex(entity.getExecutableEntityReference().getType(), entityInterface); + .buildIndex(entity.getBasicEntityReference().getType(), entityInterface); Map doc = index.buildSearchIndexDoc(); searchClient.updateEntity( indexMapping.getIndexName(searchRepository.getClusterAlias()), - entity.getExecutableEntityReference().getId().toString(), + entity.getBasicEntityReference().getId().toString(), doc, "ctx._source.testSuite = params.testSuite;"); } @@ -413,7 +412,7 @@ public void storeEntity(TestSuite entity, boolean update) { @Override public void storeRelationships(TestSuite entity) { - if (Boolean.TRUE.equals(entity.getExecutable())) { + if (Boolean.TRUE.equals(entity.getBasic())) { storeExecutableRelationship(entity); } } @@ -421,10 +420,7 @@ public void storeRelationships(TestSuite entity) { public void storeExecutableRelationship(TestSuite testSuite) { Table table = Entity.getEntityByName( - Entity.TABLE, - testSuite.getExecutableEntityReference().getFullyQualifiedName(), - null, - null); + Entity.TABLE, testSuite.getBasicEntityReference().getFullyQualifiedName(), null, null); addRelationship( table.getId(), testSuite.getId(), Entity.TABLE, TEST_SUITE, Relationship.CONTAINS); } @@ -492,8 +488,8 @@ public static TestSuite copyTestSuite(TestSuite testSuite) { .withHref(testSuite.getHref()) .withId(testSuite.getId()) .withName(testSuite.getName()) - .withExecutable(testSuite.getExecutable()) - .withExecutableEntityReference(testSuite.getExecutableEntityReference()) + .withBasic(testSuite.getBasic()) + .withBasicEntityReference(testSuite.getBasicEntityReference()) .withServiceType(testSuite.getServiceType()) .withOwners(testSuite.getOwners()) .withUpdatedBy(testSuite.getUpdatedBy()) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/V112/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/V112/MigrationUtil.java index 1c4c0cd0065f..a97b3e9978a7 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/V112/MigrationUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/V112/MigrationUtil.java @@ -25,9 +25,8 @@ public static void fixExecutableTestSuiteFQN(CollectionDAO collectionDAO) { testSuiteRepository.listAll( new EntityUtil.Fields(Set.of("id")), new ListFilter(Include.ALL)); for (TestSuite suite : testSuites) { - if (Boolean.TRUE.equals(suite.getExecutable()) - && suite.getExecutableEntityReference() != null) { - String tableFQN = suite.getExecutableEntityReference().getFullyQualifiedName(); + if (Boolean.TRUE.equals(suite.getBasic()) && suite.getBasicEntityReference() != null) { + String tableFQN = suite.getBasicEntityReference().getFullyQualifiedName(); String suiteFQN = tableFQN + ".testSuite"; suite.setName(suiteFQN); suite.setFullyQualifiedName(suiteFQN); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/V114/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/V114/MigrationUtil.java index a9a4c74769f1..06b0d6b80039 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/V114/MigrationUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/V114/MigrationUtil.java @@ -48,13 +48,13 @@ public static void fixTestSuites(CollectionDAO collectionDAO) { testSuiteRepository.listAll( new EntityUtil.Fields(Set.of("id")), new ListFilter(Include.ALL)); for (TestSuite suite : testSuites) { - if (suite.getExecutableEntityReference() != null - && (!suite.getExecutable() || !suite.getFullyQualifiedName().contains("testSuite"))) { - String tableFQN = suite.getExecutableEntityReference().getFullyQualifiedName(); + if (suite.getBasicEntityReference() != null + && (!suite.getBasic() || !suite.getFullyQualifiedName().contains("testSuite"))) { + String tableFQN = suite.getBasicEntityReference().getFullyQualifiedName(); String suiteFQN = tableFQN + ".testSuite"; suite.setName(suiteFQN); suite.setFullyQualifiedName(suiteFQN); - suite.setExecutable(true); + suite.setBasic(true); collectionDAO.testSuiteDAO().update(suite); } } @@ -80,7 +80,7 @@ public static void fixTestSuites(CollectionDAO collectionDAO) { try { TestSuite existingTestSuite = testSuiteRepository.getDao().findEntityById(existingTestSuiteRel.getId()); - if (Boolean.TRUE.equals(existingTestSuite.getExecutable()) + if (Boolean.TRUE.equals(existingTestSuite.getBasic()) && existingTestSuite.getFullyQualifiedName().equals(executableTestSuiteFQN)) { // There is a native test suite associated with this testCase. relationWithExecutableTestSuiteExists = true; @@ -111,7 +111,7 @@ public static void fixTestSuites(CollectionDAO collectionDAO) { // check from table -> nativeTestSuite there should only one relation List testSuiteRels = testSuiteRepository.findToRecords( - executableTestSuite.getExecutableEntityReference().getId(), + executableTestSuite.getBasicEntityReference().getId(), TABLE, Relationship.CONTAINS, TEST_SUITE); @@ -122,7 +122,7 @@ public static void fixTestSuites(CollectionDAO collectionDAO) { // if testsuite cannot be retrieved but the relation exists, then this is orphaned // relation, we will delete the relation testSuiteRepository.deleteRelationship( - executableTestSuite.getExecutableEntityReference().getId(), + executableTestSuite.getBasicEntityReference().getId(), TABLE, testSuiteRel.getId(), TEST_SUITE, @@ -158,9 +158,9 @@ private static TestSuite getOrCreateExecutableTestSuite( new CreateTestSuite() .withName(FullyQualifiedName.buildHash(executableTestSuiteFQN)) .withDisplayName(executableTestSuiteFQN) - .withExecutableEntityReference(entityLink.getEntityFQN()), + .withBasicEntityReference(entityLink.getEntityFQN()), "ingestion-bot") - .withExecutable(true) + .withBasic(true) .withFullyQualifiedName(executableTestSuiteFQN); testSuiteRepository.prepareInternal(newExecutableTestSuite, false); testSuiteRepository @@ -169,7 +169,7 @@ private static TestSuite getOrCreateExecutableTestSuite( "fqnHash", newExecutableTestSuite, newExecutableTestSuite.getFullyQualifiedName()); // add relationship between executable TestSuite with Table testSuiteRepository.addRelationship( - newExecutableTestSuite.getExecutableEntityReference().getId(), + newExecutableTestSuite.getBasicEntityReference().getId(), newExecutableTestSuite.getId(), Entity.TABLE, TEST_SUITE, diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v110/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v110/MigrationUtil.java index fd815283029a..e719728ab85e 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v110/MigrationUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v110/MigrationUtil.java @@ -460,17 +460,16 @@ public static TestSuite getTestSuite(CollectionDAO dao, CreateTestSuite create, .withDescription(create.getDescription()) .withDisplayName(create.getDisplayName()) .withName(create.getName()); - if (create.getExecutableEntityReference() != null) { + if (create.getBasicEntityReference() != null) { Table table = - Entity.getEntityByName( - Entity.TABLE, create.getExecutableEntityReference(), "", Include.ALL); + Entity.getEntityByName(Entity.TABLE, create.getBasicEntityReference(), "", Include.ALL); EntityReference entityReference = new EntityReference() .withId(table.getId()) .withFullyQualifiedName(table.getFullyQualifiedName()) .withName(table.getName()) .withType(Entity.TABLE); - testSuite.setExecutableEntityReference(entityReference); + testSuite.setBasicEntityReference(entityReference); } return testSuite; } @@ -517,9 +516,9 @@ public static void testSuitesMigration(CollectionDAO collectionDAO) { new CreateTestSuite() .withName(FullyQualifiedName.buildHash(nativeTestSuiteFqn)) .withDisplayName(nativeTestSuiteFqn) - .withExecutableEntityReference(entityLink.getEntityFQN()), + .withBasicEntityReference(entityLink.getEntityFQN()), "ingestion-bot") - .withExecutable(true) + .withBasic(true) .withFullyQualifiedName(nativeTestSuiteFqn); testSuiteRepository.prepareInternal(newExecutableTestSuite, false); try { @@ -534,7 +533,7 @@ public static void testSuitesMigration(CollectionDAO collectionDAO) { } // add relationship between executable TestSuite with Table testSuiteRepository.addRelationship( - newExecutableTestSuite.getExecutableEntityReference().getId(), + newExecutableTestSuite.getBasicEntityReference().getId(), newExecutableTestSuite.getId(), Entity.TABLE, TEST_SUITE, @@ -565,7 +564,7 @@ private static void migrateExistingTestSuitesToLogical(CollectionDAO collectionD ListFilter filter = new ListFilter(Include.ALL); List testSuites = testSuiteRepository.listAll(new Fields(Set.of("id")), filter); for (TestSuite testSuite : testSuites) { - testSuite.setExecutable(false); + testSuite.setBasic(false); List ingestionPipelineRecords = collectionDAO .relationshipDAO() diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v111/MigrationUtilV111.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v111/MigrationUtilV111.java index 0bffc8871e82..a2a368d22bcf 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v111/MigrationUtilV111.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v111/MigrationUtilV111.java @@ -106,16 +106,16 @@ public static void runTestSuiteMigration( suite = JsonUtils.readValue(pgObject.getValue(), TestSuite.class); } // Only Test Suite which are executable needs to be updated - if (Boolean.TRUE.equals(suite.getExecutable())) { - if (suite.getExecutableEntityReference() != null) { + if (Boolean.TRUE.equals(suite.getBasic())) { + if (suite.getBasicEntityReference() != null) { updateTestSuite(handle, suite, updateSql); } else { String entityName = StringUtils.replaceOnce(suite.getDisplayName(), ".testSuite", ""); try { Table table = collectionDAO.tableDAO().findEntityByName(entityName, Include.ALL); // Update Test Suite - suite.setExecutable(true); - suite.setExecutableEntityReference(table.getEntityReference()); + suite.setBasic(true); + suite.setBasicEntityReference(table.getEntityReference()); updateTestSuite(handle, suite, updateSql); removeDuplicateTestCases(collectionDAO, handle, getSql); } catch (Exception ex) { @@ -133,9 +133,9 @@ public static void runTestSuiteMigration( } public static void updateTestSuite(Handle handle, TestSuite suite, String updateSql) { - if (suite.getExecutableEntityReference() != null) { + if (suite.getBasicEntityReference() != null) { try { - EntityReference executableEntityRef = suite.getExecutableEntityReference(); + EntityReference executableEntityRef = suite.getBasicEntityReference(); // Run new Migrations suite.setName(String.format("%s.testSuite", executableEntityRef.getName())); suite.setFullyQualifiedName( diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java index 48ad9ccd2768..9f277a249949 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestCaseResource.java @@ -130,8 +130,8 @@ public static class TestCaseResultList extends ResultList { "Get a list of test. Use `fields` " + "parameter to get only necessary fields. Use cursor-based pagination to limit the number " + "entries in the list using `limit` and `before` or `after` query params." - + "Use the `testSuite` field to get the executable Test Suite linked to this test case " - + "or use the `testSuites` field to list test suites (executable and logical) linked.", + + "Use the `testSuite` field to get the Basic Test Suite linked to this test case " + + "or use the `testSuites` field to list test suites (Basic and Logical) linked.", responses = { @ApiResponse( responseCode = "200", @@ -242,8 +242,8 @@ public ResultList list( "Get a list of test cases using the search service. Use `fields` " + "parameter to get only necessary fields. Use offset/limit pagination to limit the number " + "entries in the list using `limit` and `offset` query params." - + "Use the `testSuite` field to get the executable Test Suite linked to this test case " - + "or use the `testSuites` field to list test suites (executable and logical) linked.", + + "Use the `testSuite` field to get the Basic Test Suite linked to this test case " + + "or use the `testSuites` field to list test suites (Basic and Logical) linked.", responses = { @ApiResponse( responseCode = "200", @@ -647,7 +647,7 @@ public Response create( new CreateResourceContext<>(entityType, test), new OperationContext(Entity.TEST_CASE, MetadataOperation.EDIT_TESTS)); authorizer.authorize(securityContext, operationContext, resourceContext); - repository.isTestSuiteExecutable(create.getTestSuite()); + repository.isTestSuiteBasic(create.getTestSuite()); test = addHref(uriInfo, repository.create(uriInfo, test)); return Response.created(test.getHref()).entity(test).build(); } @@ -762,7 +762,7 @@ public Response createOrUpdate( new OperationContext(Entity.TABLE, MetadataOperation.EDIT_TESTS); authorizer.authorize(securityContext, operationContext, resourceContext); TestCase test = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); - repository.isTestSuiteExecutable(create.getTestSuite()); + repository.isTestSuiteBasic(create.getTestSuite()); repository.prepareInternal(test, true); PutResponse response = repository.createOrUpdate(uriInfo, test); addHref(uriInfo, response.getEntity()); @@ -1167,9 +1167,8 @@ public Response addTestCasesToLogicalTestSuite( ResourceContextInterface resourceContext = TestCaseResourceContext.builder().entity(testSuite).build(); authorizer.authorize(securityContext, operationContext, resourceContext); - if (Boolean.TRUE.equals(testSuite.getExecutable())) { - throw new IllegalArgumentException( - "You are trying to add test cases to an executable test suite."); + if (Boolean.TRUE.equals(testSuite.getBasic())) { + throw new IllegalArgumentException("You are trying to add test cases to a basic test suite."); } List testCaseIds = createLogicalTestCases.getTestCaseIds(); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteMapper.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteMapper.java index d0d364158057..f5bb30c2f6e2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteMapper.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteMapper.java @@ -15,16 +15,16 @@ public TestSuite createToEntity(CreateTestSuite create, String user) { .withDescription(create.getDescription()) .withDisplayName(create.getDisplayName()) .withName(create.getName()); - if (create.getExecutableEntityReference() != null) { + if (create.getBasicEntityReference() != null) { Table table = - Entity.getEntityByName(Entity.TABLE, create.getExecutableEntityReference(), null, null); + Entity.getEntityByName(Entity.TABLE, create.getBasicEntityReference(), null, null); EntityReference entityReference = new EntityReference() .withId(table.getId()) .withFullyQualifiedName(table.getFullyQualifiedName()) .withName(table.getName()) .withType(Entity.TABLE); - testSuite.setExecutableEntityReference(entityReference); + testSuite.setBasicEntityReference(entityReference); } return testSuite; } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java index 2b5585a9797f..ce570d4346ac 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/resources/dqtests/TestSuiteResource.java @@ -72,10 +72,10 @@ public class TestSuiteResource extends EntityResource { public static final String COLLECTION_PATH = "/v1/dataQuality/testSuites"; private final TestSuiteMapper mapper = new TestSuiteMapper(); - public static final String EXECUTABLE_TEST_SUITE_DELETION_ERROR = + public static final String BASIC_TEST_SUITE_DELETION_ERROR = "Cannot delete logical test suite. To delete logical test suite, use DELETE /v1/dataQuality/testSuites/<...>"; - public static final String NON_EXECUTABLE_TEST_SUITE_DELETION_ERROR = - "Cannot delete executable test suite. To delete executable test suite, use DELETE /v1/dataQuality/testSuites/executable/<...>"; + public static final String NON_BASIC_TEST_SUITE_DELETION_ERROR = + "Cannot delete executable test suite. To delete executable test suite, use DELETE /v1/dataQuality/testSuites/basic/<...>"; static final String FIELDS = "owners,tests,summary"; static final String SEARCH_FIELDS_EXCLUDE = "table,database,databaseSchema,service"; @@ -130,7 +130,7 @@ public ResultList list( @Parameter( description = "Returns executable or logical test suites. If omitted, returns all test suites.", - schema = @Schema(type = "string", example = "executable")) + schema = @Schema(type = "string", example = "basic")) @QueryParam("testSuiteType") String testSuiteType, @Parameter( @@ -553,11 +553,11 @@ public Response create( @Context SecurityContext securityContext, @Valid CreateTestSuite create) { create = - create.withExecutableEntityReference( + create.withBasicEntityReference( null); // entity reference is not applicable for logical test suites TestSuite testSuite = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); - testSuite.setExecutable(false); + testSuite.setBasic(false); return create(uriInfo, securityContext, testSuite); } @@ -580,10 +580,42 @@ public Response create( public Response createExecutable( @Context UriInfo uriInfo, @Context SecurityContext securityContext, + @Context HttpServletResponse response, @Valid CreateTestSuite create) { TestSuite testSuite = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); - testSuite.setExecutable(true); + testSuite.setBasic(true); + // Set the deprecation header based on draft specification from IETF + // https://datatracker.ietf.org/doc/html/draft-ietf-httpapi-deprecation-header-02 + response.setHeader("Deprecation", "Monday, March 24, 2025"); + response.setHeader("Link", "api/v1/dataQuality/testSuites/basic; rel=\"alternate\""); + return create(uriInfo, securityContext, testSuite); + } + + @POST + @Path("/basic") + @Operation( + operationId = "createBasicTestSuite", + summary = "Create a basic test suite", + description = "Create a basic test suite.", + responses = { + @ApiResponse( + responseCode = "200", + description = "Basic test suite", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = TestSuite.class))), + @ApiResponse(responseCode = "400", description = "Bad request") + }) + public Response createBasic( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Context HttpServletResponse response, + @Valid CreateTestSuite create) { + TestSuite testSuite = + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); + testSuite.setBasic(true); return create(uriInfo, securityContext, testSuite); } @@ -636,11 +668,11 @@ public Response createOrUpdate( @Context SecurityContext securityContext, @Valid CreateTestSuite create) { create = - create.withExecutableEntityReference( + create.withBasicEntityReference( null); // entity reference is not applicable for logical test suites TestSuite testSuite = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); - testSuite.setExecutable(false); + testSuite.setBasic(false); return createOrUpdate(uriInfo, securityContext, testSuite); } @@ -661,12 +693,42 @@ public Response createOrUpdate( schema = @Schema(implementation = TestSuite.class))) }) public Response createOrUpdateExecutable( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Context HttpServletResponse response, + @Valid CreateTestSuite create) { + TestSuite testSuite = + mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); + testSuite.setBasic(true); + // Set the deprecation header based on draft specification from IETF + // https://datatracker.ietf.org/doc/html/draft-ietf-httpapi-deprecation-header-02 + response.setHeader("Deprecation", "Monday, March 24, 2025"); + response.setHeader("Link", "api/v1/dataQuality/testSuites/basic; rel=\"alternate\""); + return createOrUpdate(uriInfo, securityContext, testSuite); + } + + @PUT + @Path("/basic") + @Operation( + operationId = "createOrUpdateBasicTestSuite", + summary = "Create or Update Basic test suite", + description = "Create a Basic TestSuite if it does not exist or update an existing one.", + responses = { + @ApiResponse( + responseCode = "200", + description = "The updated test definition ", + content = + @Content( + mediaType = "application/json", + schema = @Schema(implementation = TestSuite.class))) + }) + public Response createOrUpdateBasic( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Valid CreateTestSuite create) { TestSuite testSuite = mapper.createToEntity(create, securityContext.getUserPrincipal().getName()); - testSuite.setExecutable(true); + testSuite.setBasic(true); return createOrUpdate(uriInfo, securityContext, testSuite); } @@ -695,8 +757,8 @@ public Response delete( OperationContext operationContext = new OperationContext(entityType, MetadataOperation.DELETE); authorizer.authorize(securityContext, operationContext, getResourceContextById(id)); TestSuite testSuite = Entity.getEntity(Entity.TEST_SUITE, id, "*", ALL); - if (Boolean.TRUE.equals(testSuite.getExecutable())) { - throw new IllegalArgumentException(NON_EXECUTABLE_TEST_SUITE_DELETION_ERROR); + if (Boolean.TRUE.equals(testSuite.getBasic())) { + throw new IllegalArgumentException(NON_BASIC_TEST_SUITE_DELETION_ERROR); } RestUtil.DeleteResponse response = repository.deleteLogicalTestSuite(securityContext, testSuite, hardDelete); @@ -730,8 +792,8 @@ public Response delete( OperationContext operationContext = new OperationContext(entityType, MetadataOperation.DELETE); authorizer.authorize(securityContext, operationContext, getResourceContextByName(name)); TestSuite testSuite = Entity.getEntityByName(Entity.TEST_SUITE, name, "*", ALL); - if (Boolean.TRUE.equals(testSuite.getExecutable())) { - throw new IllegalArgumentException(NON_EXECUTABLE_TEST_SUITE_DELETION_ERROR); + if (Boolean.TRUE.equals(testSuite.getBasic())) { + throw new IllegalArgumentException(NON_BASIC_TEST_SUITE_DELETION_ERROR); } RestUtil.DeleteResponse response = repository.deleteLogicalTestSuite(securityContext, testSuite, hardDelete); @@ -752,6 +814,47 @@ public Response delete( description = "Test suite for instance {name} is not found") }) public Response deleteExecutable( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Context HttpServletResponse response, + @Parameter( + description = "Recursively delete this entity and it's children. (Default `false`)") + @DefaultValue("false") + @QueryParam("recursive") + boolean recursive, + @Parameter(description = "Hard delete the entity. (Default = `false`)") + @QueryParam("hardDelete") + @DefaultValue("false") + boolean hardDelete, + @Parameter(description = "Name of the test suite", schema = @Schema(type = "string")) + @PathParam("name") + String name) { + OperationContext operationContext = new OperationContext(entityType, MetadataOperation.DELETE); + authorizer.authorize(securityContext, operationContext, getResourceContextByName(name)); + TestSuite testSuite = Entity.getEntityByName(Entity.TEST_SUITE, name, "*", ALL); + if (Boolean.FALSE.equals(testSuite.getBasic())) { + throw new IllegalArgumentException(BASIC_TEST_SUITE_DELETION_ERROR); + } + // Set the deprecation header based on draft specification from IETF + // https://datatracker.ietf.org/doc/html/draft-ietf-httpapi-deprecation-header-02 + response.setHeader("Deprecation", "Monday, March 24, 2025"); + response.setHeader("Link", "api/v1/dataQuality/testSuites/basic; rel=\"alternate\""); + return deleteByName(uriInfo, securityContext, name, recursive, hardDelete); + } + + @DELETE + @Path("/basic/name/{name}") + @Operation( + operationId = "deleteTestSuiteByName", + summary = "Delete a test suite", + description = "Delete a test suite by `name`.", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse( + responseCode = "404", + description = "Test suite for instance {name} is not found") + }) + public Response deleteBasic( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Parameter( @@ -769,8 +872,8 @@ public Response deleteExecutable( OperationContext operationContext = new OperationContext(entityType, MetadataOperation.DELETE); authorizer.authorize(securityContext, operationContext, getResourceContextByName(name)); TestSuite testSuite = Entity.getEntityByName(Entity.TEST_SUITE, name, "*", ALL); - if (Boolean.FALSE.equals(testSuite.getExecutable())) { - throw new IllegalArgumentException(EXECUTABLE_TEST_SUITE_DELETION_ERROR); + if (Boolean.FALSE.equals(testSuite.getBasic())) { + throw new IllegalArgumentException(BASIC_TEST_SUITE_DELETION_ERROR); } return deleteByName(uriInfo, securityContext, name, recursive, hardDelete); } @@ -788,6 +891,47 @@ public Response deleteExecutable( description = "Test suite for instance {id} is not found") }) public Response deleteExecutable( + @Context UriInfo uriInfo, + @Context SecurityContext securityContext, + @Context HttpServletResponse response, + @Parameter( + description = "Recursively delete this entity and it's children. (Default `false`)") + @DefaultValue("false") + @QueryParam("recursive") + boolean recursive, + @Parameter(description = "Hard delete the entity. (Default = `false`)") + @QueryParam("hardDelete") + @DefaultValue("false") + boolean hardDelete, + @Parameter(description = "Id of the test suite", schema = @Schema(type = "UUID")) + @PathParam("id") + UUID id) { + OperationContext operationContext = new OperationContext(entityType, MetadataOperation.DELETE); + authorizer.authorize(securityContext, operationContext, getResourceContextById(id)); + TestSuite testSuite = Entity.getEntity(Entity.TEST_SUITE, id, "*", ALL); + if (Boolean.FALSE.equals(testSuite.getBasic())) { + throw new IllegalArgumentException(BASIC_TEST_SUITE_DELETION_ERROR); + } + // Set the deprecation header based on draft specification from IETF + // https://datatracker.ietf.org/doc/html/draft-ietf-httpapi-deprecation-header-02 + response.setHeader("Deprecation", "Monday, March 24, 2025"); + response.setHeader("Link", "api/v1/dataQuality/testSuites/basic; rel=\"alternate\""); + return delete(uriInfo, securityContext, id, recursive, hardDelete); + } + + @DELETE + @Path("/basic/{id}") + @Operation( + operationId = "deleteTestSuite", + summary = "Delete a test suite", + description = "Delete a test suite by `Id`.", + responses = { + @ApiResponse(responseCode = "200", description = "OK"), + @ApiResponse( + responseCode = "404", + description = "Test suite for instance {id} is not found") + }) + public Response deleteBasic( @Context UriInfo uriInfo, @Context SecurityContext securityContext, @Parameter( @@ -805,8 +949,8 @@ public Response deleteExecutable( OperationContext operationContext = new OperationContext(entityType, MetadataOperation.DELETE); authorizer.authorize(securityContext, operationContext, getResourceContextById(id)); TestSuite testSuite = Entity.getEntity(Entity.TEST_SUITE, id, "*", ALL); - if (Boolean.FALSE.equals(testSuite.getExecutable())) { - throw new IllegalArgumentException(EXECUTABLE_TEST_SUITE_DELETION_ERROR); + if (Boolean.FALSE.equals(testSuite.getBasic())) { + throw new IllegalArgumentException(BASIC_TEST_SUITE_DELETION_ERROR); } return delete(uriInfo, securityContext, id, recursive, hardDelete); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java index bf7580aaf878..be8b447c1c72 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchListFilter.java @@ -257,8 +257,8 @@ private String getTestSuiteCondition() { boolean includeEmptyTestSuites = Boolean.parseBoolean(getQueryParam("includeEmptyTestSuites")); if (testSuiteType != null) { - boolean executable = !testSuiteType.equals("logical"); - conditions.add(String.format("{\"term\": {\"executable\": \"%s\"}}", executable)); + boolean basic = !testSuiteType.equals("logical"); + conditions.add(String.format("{\"term\": {\"basic\": \"%s\"}}", basic)); } if (!includeEmptyTestSuites) { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java index d1e368fdc463..3cfe3b04dfb5 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/SearchRepository.java @@ -745,7 +745,7 @@ public void deleteOrUpdateChildren(EntityInterface entity, IndexMapping indexMap } case Entity.TEST_SUITE -> { TestSuite testSuite = (TestSuite) entity; - if (Boolean.TRUE.equals(testSuite.getExecutable())) { + if (Boolean.TRUE.equals(testSuite.getBasic())) { searchClient.deleteEntityByFields( indexMapping.getChildAliases(clusterAlias), List.of(new ImmutablePair<>("testSuite.id", docId))); diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java index 92e82f5f4269..46d67218d746 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseIndex.java @@ -69,7 +69,7 @@ private void setParentRelationships(Map doc, TestCase testCase) return; } TestSuite testSuite = Entity.getEntityOrNull(testSuiteEntityReference, "", Include.ALL); - EntityReference entityReference = testSuite.getExecutableEntityReference(); + EntityReference entityReference = testSuite.getBasicEntityReference(); TestSuiteIndex.addTestSuiteParentEntityRelations(entityReference, doc); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResolutionStatusIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResolutionStatusIndex.java index 4f9a512bd0c3..b63d3940db8d 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResolutionStatusIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestCaseResolutionStatusIndex.java @@ -65,7 +65,7 @@ private void setParentRelationships(Map doc) { TestSuite testSuite = Entity.getEntityOrNull(testCase.getTestSuite(), "", Include.ALL); if (testSuite == null) return; doc.put("testSuite", testSuite.getEntityReference()); - TestSuiteIndex.addTestSuiteParentEntityRelations(testSuite.getExecutableEntityReference(), doc); + TestSuiteIndex.addTestSuiteParentEntityRelations(testSuite.getBasicEntityReference(), doc); } public static Map getFields() { diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestSuiteIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestSuiteIndex.java index 62c52fc3c2b3..cffe87bef2d6 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestSuiteIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/TestSuiteIndex.java @@ -39,7 +39,7 @@ public Map buildSearchIndexDocInternal(Map doc) private void setParentRelationships(Map doc, TestSuite testSuite) { // denormalize the parent relationships for search - EntityReference entityReference = testSuite.getExecutableEntityReference(); + EntityReference entityReference = testSuite.getBasicEntityReference(); if (entityReference == null) return; addTestSuiteParentEntityRelations(entityReference, doc); } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java index a6c51064a08f..9d12cdb0b9d0 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java @@ -2225,7 +2225,7 @@ void test_ownershipInheritance(TestInfo test) throws IOException { CreateTestSuite createTestSuite = testSuiteResourceTest.createRequest(table.getFullyQualifiedName()); TestSuite testSuite = - testSuiteResourceTest.createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + testSuiteResourceTest.createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); CreateTestCase createTestCase = testCaseResourceTest @@ -2293,7 +2293,7 @@ void test_domainInheritance(TestInfo test) throws IOException { CreateTestSuite createTestSuite = testSuiteResourceTest.createRequest(table.getFullyQualifiedName()); TestSuite testSuite = - testSuiteResourceTest.createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + testSuiteResourceTest.createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); CreateTestCase createTestCase = testCaseResourceTest @@ -2433,8 +2433,7 @@ void get_tablesWithTestCases(TestInfo test) throws IOException { CreateTestSuite createExecutableTestSuite = testSuiteResourceTest.createRequest(table1.getFullyQualifiedName()); TestSuite executableTestSuite = - testSuiteResourceTest.createExecutableTestSuite( - createExecutableTestSuite, ADMIN_AUTH_HEADERS); + testSuiteResourceTest.createBasicTestSuite(createExecutableTestSuite, ADMIN_AUTH_HEADERS); HashMap queryParams = new HashMap<>(); queryParams.put("includeEmptyTestSuite", "false"); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java index d3cf7b2675c2..0b8a2c3d4a6f 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestCaseResourceTest.java @@ -248,7 +248,7 @@ void post_testWithInvalidEntityTestSuite_4xx(TestInfo test) throws IOException { CreateTestSuite createTestSuite = testSuiteResourceTest.createRequest(test).withName(TEST_TABLE1.getFullyQualifiedName()); TestSuite testSuite = - testSuiteResourceTest.createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + testSuiteResourceTest.createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); create.withEntityLink(INVALID_LINK1).withTestSuite(testSuite.getFullyQualifiedName()); assertResponseContains( @@ -585,7 +585,7 @@ void get_listTestCasesFromSearchWithPagination(TestInfo testInfo) CreateTestSuite createTestSuite = testSuiteResourceTest.createRequest(table.getFullyQualifiedName()); TestSuite testSuite = - testSuiteResourceTest.createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + testSuiteResourceTest.createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); testSuites.put(table.getFullyQualifiedName(), testSuite); } @@ -676,7 +676,7 @@ void test_getSimpleListFromSearch(TestInfo testInfo) throws IOException, ParseEx CreateTestSuite createTestSuite = testSuiteResourceTest.createRequest(table.getFullyQualifiedName()); TestSuite testSuite = - testSuiteResourceTest.createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + testSuiteResourceTest.createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); testSuites.put(table.getFullyQualifiedName(), testSuite); } @@ -899,7 +899,7 @@ void test_testCaseInheritedFields(TestInfo testInfo) throws IOException { CreateTestSuite createTestSuite = testSuiteResourceTest.createRequest(table.getFullyQualifiedName()); TestSuite testSuite = - testSuiteResourceTest.createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + testSuiteResourceTest.createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); CreateTestCase create = createRequest(testInfo) @@ -1133,7 +1133,7 @@ void list_allTestSuitesFromTestCase_200(TestInfo test) throws IOException { CreateTestSuite createTestSuite = testSuiteResourceTest.createRequest(test).withName(TEST_TABLE2.getFullyQualifiedName()); TestSuite executableTestSuite = - testSuiteResourceTest.createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + testSuiteResourceTest.createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); // Create the test cases (need to be created against an executable test suite) CreateTestCase create = @@ -2171,8 +2171,7 @@ void test_testCaseResultState(TestInfo test) throws IOException, ParseException .withDescription(test.getDisplayName()) .withEntityLink( String.format( - "<#E::table::%s>", - testSuite.getExecutableEntityReference().getFullyQualifiedName())) + "<#E::table::%s>", testSuite.getBasicEntityReference().getFullyQualifiedName())) .withTestSuite(testSuite.getFullyQualifiedName()) .withTestDefinition(TEST_DEFINITION1.getFullyQualifiedName()); TestCase testCase = createAndCheckEntity(createTestCase, ADMIN_AUTH_HEADERS); @@ -2634,7 +2633,7 @@ private TestSuite createExecutableTestSuite(TestInfo test) throws IOException { Table table = tableResourceTest.createAndCheckEntity(tableReq, ADMIN_AUTH_HEADERS); CreateTestSuite createExecutableTestSuite = testSuiteResourceTest.createRequest(table.getFullyQualifiedName()); - return testSuiteResourceTest.createExecutableTestSuite( + return testSuiteResourceTest.createBasicTestSuite( createExecutableTestSuite, ADMIN_AUTH_HEADERS); } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java index ceb09aedca82..3289b67bf21f 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/dqtests/TestSuiteResourceTest.java @@ -113,10 +113,10 @@ public void setupTestSuites(TestInfo test) throws IOException { TEST_SUITE_TABLE2 = tableResourceTest.createAndCheckEntity(tableReq, ADMIN_AUTH_HEADERS); CREATE_TEST_SUITE1 = createRequest(DATABASE_SCHEMA.getFullyQualifiedName() + "." + TEST_SUITE_TABLE_NAME1); - TEST_SUITE1 = createExecutableTestSuite(CREATE_TEST_SUITE1, ADMIN_AUTH_HEADERS); + TEST_SUITE1 = createBasicTestSuite(CREATE_TEST_SUITE1, ADMIN_AUTH_HEADERS); CREATE_TEST_SUITE2 = createRequest(DATABASE_SCHEMA.getFullyQualifiedName() + "." + TEST_SUITE_TABLE_NAME2); - TEST_SUITE2 = createExecutableTestSuite(CREATE_TEST_SUITE2, ADMIN_AUTH_HEADERS); + TEST_SUITE2 = createBasicTestSuite(CREATE_TEST_SUITE2, ADMIN_AUTH_HEADERS); } @Test @@ -169,7 +169,7 @@ void put_testCaseResults_200() throws IOException, ParseException { verifyTestCases(testSuite.getTests(), testCases1); } } - deleteExecutableTestSuite(TEST_SUITE1.getId(), true, false, ADMIN_AUTH_HEADERS); + deleteBasicTestSuite(TEST_SUITE1.getId(), true, false, ADMIN_AUTH_HEADERS); assertResponse( () -> getEntity(TEST_SUITE1.getId(), ADMIN_AUTH_HEADERS), NOT_FOUND, @@ -200,7 +200,7 @@ void list_testSuitesIncludeEmpty_200(TestInfo test) throws IOException { .withDataLength(10))); Table table = tableResourceTest.createEntity(tableReq, ADMIN_AUTH_HEADERS); CreateTestSuite createTestSuite = createRequest(table.getFullyQualifiedName()); - TestSuite testSuite = createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + TestSuite testSuite = createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); for (int j = 0; j < 3; j++) { CreateTestCase createTestCase = testCaseResourceTest @@ -224,7 +224,7 @@ void list_testSuitesIncludeEmpty_200(TestInfo test) throws IOException { .withDataLength(10))); Table table = tableResourceTest.createEntity(tableReq, ADMIN_AUTH_HEADERS); CreateTestSuite createTestSuite = createRequest(table.getFullyQualifiedName()); - createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); testSuites.add(createTestSuite); ResultList actualTestSuites = @@ -276,7 +276,7 @@ void test_inheritOwnerFromTable(TestInfo test) throws IOException { table = tableResourceTest.getEntity(table.getId(), "*", ADMIN_AUTH_HEADERS); CreateTestSuite createExecutableTestSuite = createRequest(table.getFullyQualifiedName()); TestSuite executableTestSuite = - createExecutableTestSuite(createExecutableTestSuite, ADMIN_AUTH_HEADERS); + createBasicTestSuite(createExecutableTestSuite, ADMIN_AUTH_HEADERS); TestSuite testSuite = getEntity(executableTestSuite.getId(), "*", ADMIN_AUTH_HEADERS); assertOwners(testSuite.getOwners(), table.getOwners()); Table updateTableOwner = table; @@ -306,7 +306,7 @@ void test_inheritDomainFromTable(TestInfo test) throws IOException { table = tableResourceTest.getEntity(table.getId(), "*", ADMIN_AUTH_HEADERS); CreateTestSuite createExecutableTestSuite = createRequest(table.getFullyQualifiedName()); TestSuite executableTestSuite = - createExecutableTestSuite(createExecutableTestSuite, ADMIN_AUTH_HEADERS); + createBasicTestSuite(createExecutableTestSuite, ADMIN_AUTH_HEADERS); TestSuite testSuite = getEntity(executableTestSuite.getId(), "domain", ADMIN_AUTH_HEADERS); assertEquals(DOMAIN1.getId(), testSuite.getDomain().getId()); ResultList testSuites = @@ -349,7 +349,7 @@ void post_createLogicalTestSuiteAndAddTests_200(TestInfo test) throws IOExceptio CreateTestSuite createExecutableTestSuite = createRequest(table.getFullyQualifiedName()); createExecutableTestSuite.withOwners(List.of(user1Ref)); TestSuite executableTestSuite = - createExecutableTestSuite(createExecutableTestSuite, ADMIN_AUTH_HEADERS); + createBasicTestSuite(createExecutableTestSuite, ADMIN_AUTH_HEADERS); List testCases1 = new ArrayList<>(); // We'll create tests cases for testSuite1 @@ -399,7 +399,7 @@ void post_createLogicalTestSuiteAndAddTests_200(TestInfo test) throws IOExceptio allEntities.getData().stream() .anyMatch(ts -> ts.getId().equals(finalExecutableTestSuite.getId()))); // 2. List only executable test suites - queryParams.put("testSuiteType", "executable"); + queryParams.put("testSuiteType", "basic"); queryParams.put("fields", "tests"); ResultList executableTestSuites = listEntitiesFromSearch(queryParams, 100, 0, ADMIN_AUTH_HEADERS); @@ -489,7 +489,7 @@ void addTestCaseWithLogicalEndPoint(TestInfo test) throws IOException { .withDataLength(10))); Table table = tableResourceTest.createEntity(tableReq, ADMIN_AUTH_HEADERS); CreateTestSuite createTestSuite = createRequest(table.getFullyQualifiedName()); - TestSuite testSuite = createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + TestSuite testSuite = createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); List testCases1 = new ArrayList<>(); // We'll create tests cases for testSuite1 @@ -511,14 +511,14 @@ void addTestCaseWithLogicalEndPoint(TestInfo test) throws IOException { executableTestSuite, testCases1.stream().map(EntityReference::getId).collect(Collectors.toList())), BAD_REQUEST, - "You are trying to add test cases to an executable test suite."); + "You are trying to add test cases to a basic test suite."); } @Test void post_createExecTestSuiteNonExistingEntity_400(TestInfo test) { CreateTestSuite createTestSuite = createRequest(test); assertResponse( - () -> createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS), + () -> createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS), NOT_FOUND, String.format("table instance for %s not found", createTestSuite.getName())); } @@ -540,7 +540,7 @@ void get_execTestSuiteFromTable_200(TestInfo test) throws IOException { .withDataLength(10))); Table table = tableResourceTest.createEntity(tableReq, ADMIN_AUTH_HEADERS); CreateTestSuite createTestSuite = createRequest(table.getFullyQualifiedName()); - TestSuite testSuite = createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + TestSuite testSuite = createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); // We'll create tests cases for testSuite for (int i = 0; i < 5; i++) { @@ -563,7 +563,7 @@ void get_execTestSuiteFromTable_200(TestInfo test) throws IOException { assertEquals(5, tableTestSuite.getTests().size()); // Soft delete entity - deleteExecutableTestSuite(tableTestSuite.getId(), true, false, ADMIN_AUTH_HEADERS); + deleteBasicTestSuite(tableTestSuite.getId(), true, false, ADMIN_AUTH_HEADERS); actualTable = tableResourceTest.getEntity( actualTable.getId(), queryParams, "testSuite", ADMIN_AUTH_HEADERS); @@ -574,7 +574,7 @@ void get_execTestSuiteFromTable_200(TestInfo test) throws IOException { assertEquals(true, tableTestSuite.getDeleted()); // Hard delete entity - deleteExecutableTestSuite(tableTestSuite.getId(), true, true, ADMIN_AUTH_HEADERS); + deleteBasicTestSuite(tableTestSuite.getId(), true, true, ADMIN_AUTH_HEADERS); actualTable = tableResourceTest.getEntity(table.getId(), "testSuite", ADMIN_AUTH_HEADERS); assertNull(actualTable.getTestSuite()); } @@ -596,7 +596,7 @@ void get_execTestSuiteDeletedOnTableDeletion(TestInfo test) throws IOException { .withDataLength(10))); Table table = tableResourceTest.createEntity(tableReq, ADMIN_AUTH_HEADERS); CreateTestSuite createTestSuite = createRequest(table.getFullyQualifiedName()); - TestSuite testSuite = createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + TestSuite testSuite = createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); HashMap queryParams = new HashMap<>(); queryParams.put("include", Include.ALL.value()); @@ -635,13 +635,13 @@ void get_filterTestSuiteType_200(TestInfo test) throws IOException { ResultList testSuiteResultList = listEntities(queryParams, ADMIN_AUTH_HEADERS); assertEquals(10, testSuiteResultList.getData().size()); - queryParams.put("testSuiteType", "executable"); + queryParams.put("testSuiteType", "basic"); testSuiteResultList = listEntities(queryParams, ADMIN_AUTH_HEADERS); - testSuiteResultList.getData().forEach(ts -> assertEquals(true, ts.getExecutable())); + testSuiteResultList.getData().forEach(ts -> assertEquals(true, ts.getBasic())); queryParams.put("testSuiteType", "logical"); testSuiteResultList = listEntities(queryParams, ADMIN_AUTH_HEADERS); - testSuiteResultList.getData().forEach(ts -> assertEquals(false, ts.getExecutable())); + testSuiteResultList.getData().forEach(ts -> assertEquals(false, ts.getBasic())); queryParams.put("includeEmptyTestSuites", "false"); testSuiteResultList = listEntities(queryParams, ADMIN_AUTH_HEADERS); @@ -691,7 +691,7 @@ void delete_LogicalTestSuite_200(TestInfo test) throws IOException { Table table = tableResourceTest.createEntity(tableReq, ADMIN_AUTH_HEADERS); CreateTestSuite createExecutableTestSuite = createRequest(table.getFullyQualifiedName()); TestSuite executableTestSuite = - createExecutableTestSuite(createExecutableTestSuite, ADMIN_AUTH_HEADERS); + createBasicTestSuite(createExecutableTestSuite, ADMIN_AUTH_HEADERS); List testCases = new ArrayList<>(); // We'll create tests cases for testSuite1 @@ -750,7 +750,7 @@ void get_listTestSuiteFromSearchWithPagination(TestInfo testInfo) throws IOExcep CreateTestSuite createTestSuite = testSuiteResourceTest.createRequest(table.getFullyQualifiedName()); TestSuite testSuite = - testSuiteResourceTest.createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + testSuiteResourceTest.createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); testSuites.put(table.getFullyQualifiedName(), testSuite); } validateEntityListFromSearchWithPagination(new HashMap<>(), testSuites.size()); @@ -772,7 +772,7 @@ void create_executableTestSuiteAndCheckSearchClient(TestInfo test) throws IOExce .withDataLength(10))); Table table = tableResourceTest.createEntity(tableReq, ADMIN_AUTH_HEADERS); CreateTestSuite createTestSuite = createRequest(table.getFullyQualifiedName()); - TestSuite testSuite = createExecutableTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); + TestSuite testSuite = createBasicTestSuite(createTestSuite, ADMIN_AUTH_HEADERS); RestClient searchClient = getSearchClient(); IndexMapping index = Entity.getSearchRepository().getIndexMapping(Entity.TABLE); es.org.elasticsearch.client.Response response; @@ -827,10 +827,10 @@ public ResultList getTestSuites( return TestUtils.get(target, TestSuiteResource.TestSuiteList.class, authHeaders); } - public TestSuite createExecutableTestSuite( + public TestSuite createBasicTestSuite( CreateTestSuite createTestSuite, Map authHeaders) throws IOException { - WebTarget target = getResource("dataQuality/testSuites/executable"); - createTestSuite.setExecutableEntityReference(createTestSuite.getName()); + WebTarget target = getResource("dataQuality/testSuites/basic"); + createTestSuite.setBasicEntityReference(createTestSuite.getName()); return TestUtils.post(target, createTestSuite, TestSuite.class, authHeaders); } @@ -844,11 +844,10 @@ public void addTestCasesToLogicalTestSuite(TestSuite testSuite, List testC TestUtils.put(target, createLogicalTestCases, Response.Status.OK, ADMIN_AUTH_HEADERS); } - public void deleteExecutableTestSuite( + public void deleteBasicTestSuite( UUID id, boolean recursive, boolean hardDelete, Map authHeaders) throws IOException { - WebTarget target = - getResource(String.format("dataQuality/testSuites/executable/%s", id.toString())); + WebTarget target = getResource(String.format("dataQuality/testSuites/basic/%s", id.toString())); target = recursive ? target.queryParam("recursive", true) : target; target = hardDelete ? target.queryParam("hardDelete", true) : target; TestUtils.delete(target, TestSuite.class, authHeaders); diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/lineage/LineageResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/lineage/LineageResourceTest.java index 2e5af9e3ba1b..4c794113bb71 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/lineage/LineageResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/lineage/LineageResourceTest.java @@ -504,9 +504,9 @@ void get_dataQualityLineage(TestInfo test) CreateTestSuite createTestSuite6 = testSuiteResourceTest.createRequest(test).withName(TABLES.get(6).getFullyQualifiedName()); TestSuite testSuite4 = - testSuiteResourceTest.createExecutableTestSuite(createTestSuite4, ADMIN_AUTH_HEADERS); + testSuiteResourceTest.createBasicTestSuite(createTestSuite4, ADMIN_AUTH_HEADERS); TestSuite testSuite6 = - testSuiteResourceTest.createExecutableTestSuite(createTestSuite6, ADMIN_AUTH_HEADERS); + testSuiteResourceTest.createBasicTestSuite(createTestSuite6, ADMIN_AUTH_HEADERS); MessageParser.EntityLink TABLE4_LINK = new MessageParser.EntityLink(Entity.TABLE, TABLES.get(4).getFullyQualifiedName()); diff --git a/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestSuite.json b/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestSuite.json index e3b549e6317f..d20860176143 100644 --- a/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestSuite.json +++ b/openmetadata-spec/src/main/resources/json/schema/api/tests/createTestSuite.json @@ -33,8 +33,8 @@ "description": "Owners of this test suite", "$ref": "../../type/entityReferenceList.json" }, - "executableEntityReference": { - "description": "FQN of the entity the test suite is executed against. Only applicable for executable test suites.", + "basicEntityReference": { + "description": "Entity reference the test suite needs to execute the test against. Only applicable if the test suite is basic.", "$ref": "../../type/basic.json#/definitions/fullyQualifiedEntityName" }, "domain": { diff --git a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/testSuitePipeline.json b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/testSuitePipeline.json index 9ca076b5f734..f43ca42a1d30 100644 --- a/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/testSuitePipeline.json +++ b/openmetadata-spec/src/main/resources/json/schema/metadataIngestion/testSuitePipeline.json @@ -10,6 +10,21 @@ "type": "string", "enum": ["TestSuite"], "default": "TestSuite" + }, + "serviceConnections": { + "description": "Service connections available for the logical test suite.", + "type": "object", + "properties": { + "serviceName": { + "type": "string" + }, + "serviceConnection": { + "description": "Connection configuration for the source. ex: mysql , tableau connection.", + "$ref": "../entity/services/connections/serviceConnection.json#/definitions/serviceConnection" + } + }, + "additionalProperties": false, + "required": ["serviceName", "serviceConnection"] } }, "properties": { @@ -19,9 +34,17 @@ "default": "TestSuite" }, "entityFullyQualifiedName": { - "description": "Fully qualified name of the entity to be tested.", + "description": "Fully qualified name of the entity to be tested, if we're working with a basic suite.", "$ref": "../type/basic.json#/definitions/fullyQualifiedEntityName" }, + "serviceConnections": { + "description": "Service connections to be used for the logical test suite.", + "type": "array", + "items": { + "$ref": "#/definitions/serviceConnections" + }, + "default": null + }, "profileSample": { "description": "Percentage of data or no. of rows we want to execute the profiler and tests on", "type": "number", @@ -45,6 +68,6 @@ "default": null } }, - "required": ["type", "entityFullyQualifiedName"], + "required": ["type"], "additionalProperties": false } diff --git a/openmetadata-spec/src/main/resources/json/schema/tests/testCase.json b/openmetadata-spec/src/main/resources/json/schema/tests/testCase.json index 9de6c5b776c9..7d79359093a5 100644 --- a/openmetadata-spec/src/main/resources/json/schema/tests/testCase.json +++ b/openmetadata-spec/src/main/resources/json/schema/tests/testCase.json @@ -56,11 +56,12 @@ "type": "string" }, "testSuite": { - "description": "Test Suite that this test case belongs to.", + "description": "Basic Test Suite that this test case belongs to.", "$ref": "../type/entityReference.json" }, "testSuites": { "type": "array", + "description": "Basic and Logical Test Suites this test case belongs to", "items": { "$ref": "./testSuite.json" } diff --git a/openmetadata-spec/src/main/resources/json/schema/tests/testSuite.json b/openmetadata-spec/src/main/resources/json/schema/tests/testSuite.json index 51d46d165325..fc4d2c0efb24 100644 --- a/openmetadata-spec/src/main/resources/json/schema/tests/testSuite.json +++ b/openmetadata-spec/src/main/resources/json/schema/tests/testSuite.json @@ -75,7 +75,7 @@ "$ref": "../entity/services/connections/testConnectionResult.json" }, "pipelines": { - "description": "References to pipelines deployed for this database service to extract metadata, usage, lineage etc..", + "description": "References to pipelines deployed for this Test Suite to execute the tests.", "$ref": "../type/entityReferenceList.json", "default": null }, @@ -120,13 +120,13 @@ "type": "boolean", "default": false }, - "executable": { - "description": "Indicates if the test suite is executable. Set on the backend.", + "basic": { + "description": "Indicates if the test suite is basic, i.e., the parent suite of a test and linked to an entity. Set on the backend.", "type": "boolean", "default": false }, - "executableEntityReference": { - "description": "Entity reference the test suite is executed against. Only applicable if the test suite is executable.", + "basicEntityReference": { + "description": "Entity reference the test suite needs to execute the test against. Only applicable if the test suite is basic.", "$ref": "../type/entityReference.json" }, "summary": { diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/DataQualityAndProfiler.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/DataQualityAndProfiler.spec.ts index 41d4f38993f2..6ceaa212c3e1 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/DataQualityAndProfiler.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/DataQualityAndProfiler.spec.ts @@ -637,7 +637,7 @@ test('TestCase filters', PLAYWRIGHT_INGESTION_TAG_OBJ, async ({ page }) => { await filterTable1.createTestSuiteAndPipelines(apiContext); const { testSuiteData: testSuite2Response } = await filterTable1.createTestSuiteAndPipelines(apiContext, { - executableEntityReference: filterTable2Response?.['fullyQualifiedName'], + basicEntityReference: filterTable2Response?.['fullyQualifiedName'], }); const testCaseResult = { diff --git a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/TestSuite.spec.ts b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/TestSuite.spec.ts index 14cde3d2309c..99842af0227a 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/TestSuite.spec.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/e2e/Pages/TestSuite.spec.ts @@ -77,7 +77,7 @@ test('Logical TestSuite', async ({ page }) => { await page.click('[data-testid="submit-button"]'); const getTestCase = page.waitForResponse( - '/api/v1/search/query?q=*&index=test_case_search_index*' + `/api/v1/dataQuality/testCases/search/list?*${testCaseName1}*` ); await page.fill('[data-testid="searchbar"]', testCaseName1); await getTestCase; @@ -127,13 +127,13 @@ test('Logical TestSuite', async ({ page }) => { await test.step('Add test case to logical test suite', async () => { const testCaseResponse = page.waitForResponse( - '/api/v1/search/query?q=*&index=test_case_search_index*' + '/api/v1/dataQuality/testCases/search/list*' ); await page.click('[data-testid="add-test-case-btn"]'); await testCaseResponse; const getTestCase = page.waitForResponse( - `/api/v1/search/query?q=*${testCaseName2}*&index=test_case_search_index*` + `/api/v1/dataQuality/testCases/search/list?*${testCaseName2}*` ); await page.fill('[data-testid="searchbar"]', testCaseName2); await getTestCase; @@ -149,6 +149,31 @@ test('Logical TestSuite', async ({ page }) => { }); }); + await test.step('Add test suite pipeline', async () => { + await page.getByRole('tab', { name: 'Pipeline' }).click(); + + await expect(page.getByTestId('add-placeholder-button')).toBeVisible(); + + await page.getByTestId('add-placeholder-button').click(); + await page.getByTestId('select-all-test-cases').click(); + + await expect(page.getByTestId('cron-type').getByText('Day')).toBeAttached(); + + await page.getByTestId('deploy-button').click(); + + await expect(page.getByTestId('view-service-button')).toBeVisible(); + + await page.waitForSelector('[data-testid="body-text"]', { + state: 'detached', + }); + + await expect(page.getByTestId('success-line')).toContainText( + /has been created and deployed successfully/ + ); + + await page.getByTestId('view-service-button').click(); + }); + await test.step('Remove test case from logical test suite', async () => { await page.click(`[data-testid="remove-${testCaseName1}"]`); const removeTestCase1 = page.waitForResponse( diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/Entity.interface.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/Entity.interface.ts index 51eece88b0c7..691a7ac1201d 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/Entity.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/Entity.interface.ts @@ -88,7 +88,7 @@ export type TestCaseData = { export type TestSuiteData = { name?: string; - executableEntityReference?: string; + basicEntityReference?: string; description?: string; }; diff --git a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/TableClass.ts b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/TableClass.ts index bd40908dceb3..51d07d7ec0ee 100644 --- a/openmetadata-ui/src/main/resources/ui/playwright/support/entity/TableClass.ts +++ b/openmetadata-ui/src/main/resources/ui/playwright/support/entity/TableClass.ts @@ -235,11 +235,10 @@ export class TableClass extends EntityClass { } const testSuiteData = await apiContext - .post('/api/v1/dataQuality/testSuites/executable', { + .post('/api/v1/dataQuality/testSuites/basic', { data: { name: `pw-test-suite-${uuid()}`, - executableEntityReference: - this.entityResponseData?.['fullyQualifiedName'], + basicEntityReference: this.entityResponseData?.['fullyQualifiedName'], description: 'Playwright test suite for table', ...testSuite, }, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/FeedCardBody/FeedCardBodyV1.tsx b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/FeedCardBody/FeedCardBodyV1.tsx index 0e03ff70b1f1..843f3c0c569f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/FeedCardBody/FeedCardBodyV1.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/ActivityFeed/ActivityFeedCard/FeedCardBody/FeedCardBodyV1.tsx @@ -83,7 +83,7 @@ const FeedCardBodyV1 = ({ if (ASSET_CARD_STYLES.includes(cardStyle as CardStyle)) { const entityInfo = feed.feedInfo?.entitySpecificInfo?.entity; const isExecutableTestSuite = - entityType === EntityType.TEST_SUITE && entityInfo.executable; + entityType === EntityType.TEST_SUITE && entityInfo.basic; const isObservabilityAlert = entityType === EntityType.EVENT_SUBSCRIPTION && (entityInfo as EventSubscription).alertType === diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddDataQualityTest/AddDataQualityTest.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddDataQualityTest/AddDataQualityTest.interface.ts index d4572f996a8d..d7524c23279a 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddDataQualityTest/AddDataQualityTest.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddDataQualityTest/AddDataQualityTest.interface.ts @@ -18,6 +18,7 @@ import { IngestionPipeline } from '../../../generated/entity/services/ingestionP import { TestCase } from '../../../generated/tests/testCase'; import { TestDefinition } from '../../../generated/tests/testDefinition'; import { TestSuite } from '../../../generated/tests/testSuite'; +import { ListTestCaseParamsBySearch } from '../../../rest/testAPI'; export interface AddDataQualityTestProps { table: Table; @@ -48,10 +49,11 @@ export type TestSuiteIngestionDataType = { export interface AddTestSuitePipelineProps { initialData?: Partial; isLoading: boolean; - testSuiteFQN?: string; + testSuite?: TestSuite; onSubmit: (data: TestSuiteIngestionDataType) => void; includePeriodOptions?: string[]; onCancel?: () => void; + testCaseParams?: ListTestCaseParamsBySearch; } export interface RightPanelProps { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddDataQualityTest/AddDataQualityTestV1.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddDataQualityTest/AddDataQualityTestV1.tsx index a39d2f882316..770adc7cddba 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddDataQualityTest/AddDataQualityTestV1.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddDataQualityTest/AddDataQualityTestV1.tsx @@ -37,6 +37,7 @@ import { FormSubmitType } from '../../../enums/form.enum'; import { ProfilerDashboardType } from '../../../enums/table.enum'; import { OwnerType } from '../../../enums/user.enum'; import { CreateTestCase } from '../../../generated/api/tests/createTestCase'; +import { CreateTestSuite } from '../../../generated/api/tests/createTestSuite'; import { TestCase } from '../../../generated/tests/testCase'; import { TestSuite } from '../../../generated/tests/testSuite'; import { useApplicationStore } from '../../../hooks/useApplicationStore'; @@ -127,9 +128,9 @@ const AddDataQualityTestV1: React.FC = ({ }; const createTestSuite = async () => { - const testSuite = { + const testSuite: CreateTestSuite = { name: `${table.fullyQualifiedName}.testSuite`, - executableEntityReference: table.fullyQualifiedName, + basicEntityReference: table.fullyQualifiedName, owners, }; const response = await createExecutableTestSuite(testSuite); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddDataQualityTest/TestSuiteIngestion.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddDataQualityTest/TestSuiteIngestion.tsx index 573e714455ca..a8671fcfd83f 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddDataQualityTest/TestSuiteIngestion.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddDataQualityTest/TestSuiteIngestion.tsx @@ -157,7 +157,9 @@ const TestSuiteIngestion: React.FC = ({ const createIngestionPipeline = async (data: TestSuiteIngestionDataType) => { const tableName = replaceAllSpacialCharWith_( getNameFromFQN( - testSuite.executableEntityReference?.fullyQualifiedName ?? '' + (testSuite.basic + ? testSuite.basicEntityReference?.fullyQualifiedName + : testSuite.fullyQualifiedName) ?? '' ) ); const updatedName = @@ -179,7 +181,7 @@ const TestSuiteIngestion: React.FC = ({ config: { type: ConfigType.TestSuite, entityFullyQualifiedName: - testSuite.executableEntityReference?.fullyQualifiedName, + testSuite.basicEntityReference?.fullyQualifiedName, testCases: data?.testCases, }, }, @@ -290,7 +292,7 @@ const TestSuiteIngestion: React.FC = ({ includePeriodOptions={schedulerOptions} initialData={initialFormData} isLoading={isLoading} - testSuiteFQN={testSuite?.fullyQualifiedName} + testSuite={testSuite} onCancel={onCancel} onSubmit={handleIngestionSubmit} /> diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddDataQualityTest/components/AddTestSuitePipeline.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddDataQualityTest/components/AddTestSuitePipeline.test.tsx index a97ecd31ace4..310692ccad40 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddDataQualityTest/components/AddTestSuitePipeline.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddDataQualityTest/components/AddTestSuitePipeline.test.tsx @@ -18,6 +18,11 @@ import AddTestSuitePipeline from './AddTestSuitePipeline'; const mockUseHistory = { goBack: jest.fn(), }; +jest.mock('../../../../hooks/useCustomLocation/useCustomLocation', () => { + return jest.fn().mockImplementation(() => ({ + search: `?testSuiteId=test-suite-id`, + })); +}); jest.mock('../../../../hooks/useFqn', () => ({ useFqn: jest.fn().mockReturnValue({ fqn: 'test-suite-fqn' }), })); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddDataQualityTest/components/AddTestSuitePipeline.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddDataQualityTest/components/AddTestSuitePipeline.tsx index ce65038745dc..dd6030292d2c 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddDataQualityTest/components/AddTestSuitePipeline.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddDataQualityTest/components/AddTestSuitePipeline.tsx @@ -13,11 +13,13 @@ import { Col, Form, Row } from 'antd'; import { FormProviderProps } from 'antd/lib/form/context'; import { isEmpty, isString } from 'lodash'; -import React, { useState } from 'react'; +import QueryString from 'qs'; +import React, { useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { useHistory } from 'react-router-dom'; import { DEFAULT_SCHEDULE_CRON_DAILY } from '../../../../constants/Schedular.constants'; import { TestCase } from '../../../../generated/tests/testCase'; +import useCustomLocation from '../../../../hooks/useCustomLocation/useCustomLocation'; import { useFqn } from '../../../../hooks/useFqn'; import { FieldProp, @@ -41,11 +43,24 @@ const AddTestSuitePipeline = ({ onSubmit, onCancel, includePeriodOptions, - testSuiteFQN, + testSuite, }: AddTestSuitePipelineProps) => { const { t } = useTranslation(); const history = useHistory(); const { fqn, ingestionFQN } = useFqn(); + const location = useCustomLocation(); + + const testSuiteId = useMemo(() => { + const param = location.search; + const searchData = QueryString.parse( + param.startsWith('?') ? param.substring(1) : param + ); + const testSuiteIdData = + testSuite?.id ?? (searchData as { testSuiteId: string }).testSuiteId; + + return testSuite?.basic ? undefined : testSuiteIdData; + }, [location.search]); + const [selectAllTestCases, setSelectAllTestCases] = useState( initialData?.selectAllTestCases ); @@ -153,10 +168,15 @@ const AddTestSuitePipeline = ({ ]} valuePropName="selectedTest"> diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddTestCaseList/AddTestCaseList.component.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddTestCaseList/AddTestCaseList.component.test.tsx index 0b7b53d728b2..d7ccd5addaeb 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddTestCaseList/AddTestCaseList.component.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddTestCaseList/AddTestCaseList.component.test.tsx @@ -52,14 +52,12 @@ jest.mock('../../../utils/CommonUtils', () => { getNameFromFQN: jest.fn().mockImplementation((fqn) => fqn), }; }); -jest.mock('../../../rest/searchAPI', () => { +jest.mock('../../../rest/testAPI', () => { return { - searchQuery: jest.fn().mockResolvedValue({ - hits: { - hits: [], - total: { - value: 0, - }, + getListTestCaseBySearch: jest.fn().mockResolvedValue({ + data: [], + paging: { + total: 0, }, }), }; diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddTestCaseList/AddTestCaseList.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddTestCaseList/AddTestCaseList.component.tsx index e811e57204c6..3c409794a2b0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddTestCaseList/AddTestCaseList.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddTestCaseList/AddTestCaseList.component.tsx @@ -29,9 +29,8 @@ import { } from '../../../constants/constants'; import { ERROR_PLACEHOLDER_TYPE } from '../../../enums/common.enum'; import { EntityTabs, EntityType } from '../../../enums/entity.enum'; -import { SearchIndex } from '../../../enums/search.enum'; import { TestCase } from '../../../generated/tests/testCase'; -import { searchQuery } from '../../../rest/searchAPI'; +import { getListTestCaseBySearch } from '../../../rest/testAPI'; import { getNameFromFQN } from '../../../utils/CommonUtils'; import { getColumnNameFromEntityLink, @@ -53,6 +52,7 @@ export const AddTestCaseList = ({ selectedTest, onChange, showButton = true, + testCaseParams, }: AddTestCaseModalProps) => { const { t } = useTranslation(); const [searchTerm, setSearchTerm] = useState(); @@ -70,20 +70,20 @@ export const AddTestCaseList = ({ async ({ searchText = WILD_CARD_CHAR, page = 1 }) => { try { setIsLoading(true); - const res = await searchQuery({ - pageNumber: page, - pageSize: PAGE_SIZE_MEDIUM, - searchIndex: SearchIndex.TEST_CASE, - query: searchText, - filters, + + const testCaseResponse = await getListTestCaseBySearch({ + q: filters ? `${searchText} && ${filters}` : searchText, + limit: PAGE_SIZE_MEDIUM, + offset: (page - 1) * PAGE_SIZE_MEDIUM, + ...(testCaseParams ?? {}), }); - const hits = res.hits.hits.map((hit) => hit._source as TestCase); - setTotalCount(res.hits.total.value ?? 0); + + setTotalCount(testCaseResponse.paging.total ?? 0); if (selectedTest) { setSelectedItems((pre) => { const selectedItemsMap = new Map(); pre?.forEach((item) => selectedItemsMap.set(item.id, item)); - hits.forEach((hit) => { + testCaseResponse.data.forEach((hit) => { if (selectedTest.find((test) => hit.name === test)) { selectedItemsMap.set(hit.id ?? '', hit); } @@ -92,7 +92,11 @@ export const AddTestCaseList = ({ return selectedItemsMap; }); } - setItems(page === 1 ? hits : (prevItems) => [...prevItems, ...hits]); + setItems( + page === 1 + ? testCaseResponse.data + : (prevItems) => [...prevItems, ...testCaseResponse.data] + ); setPageNumber(page); } catch (_) { // Nothing here diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddTestCaseList/AddTestCaseList.interface.ts b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddTestCaseList/AddTestCaseList.interface.ts index 421b5054d108..8a792411fa85 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddTestCaseList/AddTestCaseList.interface.ts +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/AddTestCaseList/AddTestCaseList.interface.ts @@ -11,6 +11,7 @@ * limitations under the License. */ import { EntityReference, TestCase } from '../../../generated/tests/testCase'; +import { ListTestCaseParamsBySearch } from '../../../rest/testAPI'; export interface AddTestCaseModalProps { onCancel?: () => void; @@ -20,6 +21,7 @@ export interface AddTestCaseModalProps { cancelText?: string; submitText?: string; filters?: string; + testCaseParams?: ListTestCaseParamsBySearch; selectedTest?: string[]; showButton?: boolean; } diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteList/TestSuites.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteList/TestSuites.component.tsx index 6345a58abe08..ab7bbf4b6a06 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteList/TestSuites.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteList/TestSuites.component.tsx @@ -116,11 +116,11 @@ export const TestSuites = () => { dataIndex: 'name', key: 'name', sorter: (a, b) => { - if (a.executable) { - // Sort for executable test suites + if (a.basic) { + // Sort for basic test suites return ( - a.executableEntityReference?.fullyQualifiedName?.localeCompare( - b.executableEntityReference?.fullyQualifiedName ?? '' + a.basicEntityReference?.fullyQualifiedName?.localeCompare( + b.basicEntityReference?.fullyQualifiedName ?? '' ) ?? 0 ); } else { @@ -133,21 +133,21 @@ export const TestSuites = () => { }, sortDirections: ['ascend', 'descend'], render: (name, record) => { - return record.executable ? ( + return record.basic ? ( - {record.executableEntityReference?.fullyQualifiedName ?? - record.executableEntityReference?.name} + {record.basicEntityReference?.fullyQualifiedName ?? + record.basicEntityReference?.name} ) : ( { includeEmptyTestSuites: tab !== DataQualityPageTabs.TABLES, testSuiteType: tab === DataQualityPageTabs.TABLES - ? TestSuiteType.executable + ? TestSuiteType.basic : TestSuiteType.logical, sortField: 'testCaseResultSummary.timestamp', sortType: SORT_ORDER.DESC, diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteList/TestSuites.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteList/TestSuites.test.tsx index 82effefc8043..29ff886a4b21 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteList/TestSuites.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteList/TestSuites.test.tsx @@ -40,12 +40,12 @@ const mockList = { name: 'sample_data.ecommerce_db.shopify.dim_address.testSuite', fullyQualifiedName: 'sample_data.ecommerce_db.shopify.dim_address.testSuite', - description: 'This is an executable test suite linked to an entity', + description: 'This is an basic test suite linked to an entity', serviceType: 'TestSuite', href: 'href', deleted: false, - executable: true, - executableEntityReference: { + basic: true, + basicEntityReference: { id: 'id1', type: 'table', name: 'dim_address', @@ -161,7 +161,7 @@ describe('TestSuites component', () => { ).toBeInTheDocument(); }); - it('should send testSuiteType executable in api, if active tab is tables', async () => { + it('should send testSuiteType basic in api, if active tab is tables', async () => { const mockGetListTestSuites = getListTestSuitesBySearch as jest.Mock; render(); @@ -180,7 +180,7 @@ describe('TestSuites component', () => { sortNestedMode: ['max'], sortNestedPath: 'testCaseResultSummary', sortType: 'desc', - testSuiteType: 'executable', + testSuiteType: 'basic', }); }); @@ -202,7 +202,7 @@ describe('TestSuites component', () => { sortNestedMode: ['max'], sortNestedPath: 'testCaseResultSummary', sortType: 'desc', - testSuiteType: 'executable', + testSuiteType: 'basic', }); }); diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuitePipelineTab/TestSuitePipelineTab.component.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuitePipelineTab/TestSuitePipelineTab.component.tsx index bbfae7d4cee4..14a32abe86e8 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuitePipelineTab/TestSuitePipelineTab.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuitePipelineTab/TestSuitePipelineTab.component.tsx @@ -15,6 +15,7 @@ import { PlusOutlined } from '@ant-design/icons'; import { Button, Col, Row } from 'antd'; import { AxiosError } from 'axios'; import { sortBy } from 'lodash'; +import QueryString from 'qs'; import React, { useCallback, useEffect, useMemo, useState } from 'react'; import { useTranslation } from 'react-i18next'; import { useHistory } from 'react-router-dom'; @@ -27,6 +28,7 @@ import { PipelineType } from '../../../../generated/api/services/ingestionPipeli import { Table as TableType } from '../../../../generated/entity/data/table'; import { Operation } from '../../../../generated/entity/policies/policy'; import { IngestionPipeline } from '../../../../generated/entity/services/ingestionPipelines/ingestionPipeline'; +import { TestSuite } from '../../../../generated/tests/testCase'; import { useAirflowStatus } from '../../../../hooks/useAirflowStatus'; import { deployIngestionPipelineById, @@ -43,10 +45,14 @@ import ErrorPlaceHolderIngestion from '../../../common/ErrorWithPlaceholder/Erro import IngestionListTable from '../../../Settings/Services/Ingestion/IngestionListTable/IngestionListTable'; interface Props { - testSuite: TableType['testSuite']; + testSuite: TableType['testSuite'] | TestSuite; + isLogicalTestSuite?: boolean; } -const TestSuitePipelineTab = ({ testSuite }: Props) => { +const TestSuitePipelineTab = ({ + testSuite, + isLogicalTestSuite = false, +}: Props) => { const airflowInformation = useAirflowStatus(); const { t } = useTranslation(); const testSuiteFQN = testSuite?.fullyQualifiedName ?? testSuite?.name ?? ''; @@ -102,6 +108,15 @@ const TestSuitePipelineTab = ({ testSuite }: Props) => { } }, [testSuiteFQN]); + const handleAddPipelineRedirection = () => { + history.push({ + pathname: getTestSuiteIngestionPath(testSuiteFQN), + search: isLogicalTestSuite + ? QueryString.stringify({ testSuiteId: testSuite?.id }) + : undefined, + }); + }; + const handleEnableDisableIngestion = useCallback( async (id: string) => { try { @@ -194,9 +209,7 @@ const TestSuitePipelineTab = ({ testSuite }: Props) => { data-testid="add-placeholder-button" icon={} type="primary" - onClick={() => { - history.push(getTestSuiteIngestionPath(testSuiteFQN)); - }}> + onClick={handleAddPipelineRedirection}> {t('label.add')} } @@ -225,9 +238,7 @@ const TestSuitePipelineTab = ({ testSuite }: Props) => { diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuitePipelineTab/TestSuitePipelineTab.test.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuitePipelineTab/TestSuitePipelineTab.test.tsx index 72c7303512a8..d1cb54014a4b 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuitePipelineTab/TestSuitePipelineTab.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuitePipelineTab/TestSuitePipelineTab.test.tsx @@ -38,8 +38,8 @@ const mockTestSuite = { updatedAt: 1692766701920, updatedBy: 'admin', deleted: false, - executable: true, - executableEntityReference: { + basic: true, + basicEntityReference: { id: 'e926d275-441e-49ee-a073-ad509f625a14', type: 'table', name: 'web_analytic_event', diff --git a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteStepper/TestSuiteStepper.tsx b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteStepper/TestSuiteStepper.tsx index e7c0fc86efce..68c6d66bc1dc 100644 --- a/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteStepper/TestSuiteStepper.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/components/DataQuality/TestSuite/TestSuiteStepper/TestSuiteStepper.tsx @@ -38,6 +38,7 @@ import TitleBreadcrumb from '../../../common/TitleBreadcrumb/TitleBreadcrumb.com import IngestionStepper from '../../../Settings/Services/Ingestion/IngestionStepper/IngestionStepper.component'; import RightPanel from '../../AddDataQualityTest/components/RightPanel'; import { getRightPanelForAddTestSuitePage } from '../../AddDataQualityTest/rightPanelData'; +import TestSuiteIngestion from '../../AddDataQualityTest/TestSuiteIngestion'; import { AddTestCaseList } from '../../AddTestCaseList/AddTestCaseList.component'; import AddTestSuiteForm from '../AddTestSuiteForm/AddTestSuiteForm'; @@ -47,6 +48,7 @@ const TestSuiteStepper = () => { const { currentUser } = useApplicationStore(); const [activeServiceStep, setActiveServiceStep] = useState(1); const [testSuiteResponse, setTestSuiteResponse] = useState(); + const [addIngestion, setAddIngestion] = useState(false); const handleViewTestSuiteClick = () => { history.push(getTestSuitePath(testSuiteResponse?.fullyQualifiedName ?? '')); @@ -114,9 +116,10 @@ const TestSuiteStepper = () => { } else if (activeServiceStep === 3) { return ( setAddIngestion(true)} handleViewServiceClick={handleViewTestSuiteClick} name={testSuiteResponse?.name || ''} - showIngestionButton={false} state={FormSubmitType.ADD} viewServiceText="View Test Suite" /> @@ -142,25 +145,33 @@ const TestSuiteStepper = () => { data-testid="test-suite-stepper-container"> - - - - {t('label.add-entity', { - entity: t('label.test-suite'), - })} - - - - - - {RenderSelectedTab()} - + {addIngestion ? ( + setAddIngestion(false)} + onViewServiceClick={handleViewTestSuiteClick} + /> + ) : ( + + + + {t('label.add-entity', { + entity: t('label.test-suite'), + })} + + + + + + {RenderSelectedTab()} + + )} ), diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/api/tests/createTestSuite.ts b/openmetadata-ui/src/main/resources/ui/src/generated/api/tests/createTestSuite.ts index 2b393503b01c..a62d69b4dc13 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/api/tests/createTestSuite.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/api/tests/createTestSuite.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 Collate. + * Copyright 2025 Collate. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -10,12 +10,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - - /** +/** * Schema corresponding to a Test Suite */ export interface CreateTestSuite { + /** + * Entity reference the test suite needs to execute the test against. Only applicable if the + * test suite is basic. + */ + basicEntityReference?: string; /** * Description of the test suite. */ @@ -28,11 +31,6 @@ export interface CreateTestSuite { * Fully qualified name of the domain the Table belongs to. */ domain?: string; - /** - * FQN of the entity the test suite is executed against. Only applicable for executable test - * suites. - */ - executableEntityReference?: string; /** * Name that identifies this test suite. */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/testSuitePipeline.ts b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/testSuitePipeline.ts index edb969120ae7..5114f709c318 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/testSuitePipeline.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/metadataIngestion/testSuitePipeline.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 Collate. + * Copyright 2025 Collate. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -10,16 +10,14 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - - /** +/** * TestSuite Pipeline Configuration. */ export interface TestSuitePipeline { /** * Fully qualified name of the entity to be tested. */ - entityFullyQualifiedName: string; + entityFullyQualifiedName?: string; /** * Percentage of data or no. of rows we want to execute the profiler and tests on */ diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/tests/testCase.ts b/openmetadata-ui/src/main/resources/ui/src/generated/tests/testCase.ts index 7c26892b2a45..2150e10481ee 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/tests/testCase.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/tests/testCase.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 Collate. + * Copyright 2025 Collate. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -10,9 +10,7 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - - /** +/** * Test case is a test definition to capture data quality tests against tables, columns, and * other data assets. */ @@ -98,9 +96,12 @@ export interface TestCase { */ testDefinition: EntityReference; /** - * Test Suite that this test case belongs to. + * Basic Test Suite that this test case belongs to. + */ + testSuite: EntityReference; + /** + * Basic and Logical Test Suites this test case belongs to */ - testSuite: EntityReference; testSuites?: TestSuite[]; /** * Last update time corresponding to the new version of the entity in Unix epoch time @@ -184,13 +185,13 @@ export interface FieldChange { * * Test definition that this test case is based on. * - * Test Suite that this test case belongs to. + * Basic Test Suite that this test case belongs to. + * + * Entity reference the test suite needs to execute the test against. Only applicable if the + * test suite is basic. * * Domain the test Suite belongs to. When not set, the test Suite inherits the domain from * the table it belongs to. - * - * Entity reference the test suite is executed against. Only applicable if the test suite is - * executable. */ export interface EntityReference { /** @@ -461,6 +462,16 @@ export interface TestResultValue { * data entities. */ export interface TestSuite { + /** + * Indicates if the test suite is basic, i.e., the parent suite of a test and linked to an + * entity. Set on the backend. + */ + basic?: boolean; + /** + * Entity reference the test suite needs to execute the test against. Only applicable if the + * test suite is basic. + */ + basicEntityReference?: EntityReference; /** * Change that lead to this version of the entity. */ @@ -486,15 +497,6 @@ export interface TestSuite { * the table it belongs to. */ domain?: EntityReference; - /** - * Indicates if the test suite is executable. Set on the backend. - */ - executable?: boolean; - /** - * Entity reference the test suite is executed against. Only applicable if the test suite is - * executable. - */ - executableEntityReference?: EntityReference; /** * FullyQualifiedName same as `name`. */ @@ -520,8 +522,7 @@ export interface TestSuite { */ owners?: EntityReference[]; /** - * References to pipelines deployed for this database service to extract metadata, usage, - * lineage etc.. + * References to pipelines deployed for this Test Suite to execute the tests. */ pipelines?: EntityReference[]; /** diff --git a/openmetadata-ui/src/main/resources/ui/src/generated/tests/testSuite.ts b/openmetadata-ui/src/main/resources/ui/src/generated/tests/testSuite.ts index c80982499b21..ab6d7ed391d3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/generated/tests/testSuite.ts +++ b/openmetadata-ui/src/main/resources/ui/src/generated/tests/testSuite.ts @@ -1,5 +1,5 @@ /* - * Copyright 2024 Collate. + * Copyright 2025 Collate. * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at @@ -10,13 +10,21 @@ * See the License for the specific language governing permissions and * limitations under the License. */ - - - /** +/** * TestSuite is a set of test cases grouped together to capture data quality tests against * data entities. */ export interface TestSuite { + /** + * Indicates if the test suite is basic, i.e., the parent suite of a test and linked to an + * entity. Set on the backend. + */ + basic?: boolean; + /** + * Entity reference the test suite needs to execute the test against. Only applicable if the + * test suite is basic. + */ + basicEntityReference?: EntityReference; /** * Change that lead to this version of the entity. */ @@ -42,15 +50,6 @@ export interface TestSuite { * the table it belongs to. */ domain?: EntityReference; - /** - * Indicates if the test suite is executable. Set on the backend. - */ - executable?: boolean; - /** - * Entity reference the test suite is executed against. Only applicable if the test suite is - * executable. - */ - executableEntityReference?: EntityReference; /** * FullyQualifiedName same as `name`. */ @@ -76,8 +75,7 @@ export interface TestSuite { */ owners?: EntityReference[]; /** - * References to pipelines deployed for this database service to extract metadata, usage, - * lineage etc.. + * References to pipelines deployed for this Test Suite to execute the tests. */ pipelines?: EntityReference[]; /** @@ -118,65 +116,16 @@ export interface TestSuite { } /** - * Change that lead to this version of the entity. - * - * Description of the change. - */ -export interface ChangeDescription { - /** - * Names of fields added during the version changes. - */ - fieldsAdded?: FieldChange[]; - /** - * Fields deleted during the version changes with old value before deleted. - */ - fieldsDeleted?: FieldChange[]; - /** - * Fields modified during the version changes with old and new values. - */ - fieldsUpdated?: FieldChange[]; - /** - * When a change did not result in change, this could be same as the current version. - */ - previousVersion?: number; -} - -export interface FieldChange { - /** - * Name of the entity field that changed. - */ - name?: string; - /** - * New value of the field. Note that this is a JSON string and use the corresponding field - * type to deserialize it. - */ - newValue?: any; - /** - * Previous value of the field. Note that this is a JSON string and use the corresponding - * field type to deserialize it. - */ - oldValue?: any; -} - -/** - * TestSuite mock connection, since it needs to implement a Service. - */ -export interface TestSuiteConnection { - config?: null; - [property: string]: any; -} - -/** - * Domain the test Suite belongs to. When not set, the test Suite inherits the domain from - * the table it belongs to. + * Entity reference the test suite needs to execute the test against. Only applicable if the + * test suite is basic. * * This schema defines the EntityReference type used for referencing an entity. * EntityReference is used for capturing relationships from one entity to another. For * example, a table has an attribute called database of type EntityReference that captures * the relationship of a table `belongs to a` database. * - * Entity reference the test suite is executed against. Only applicable if the test suite is - * executable. + * Domain the test Suite belongs to. When not set, the test Suite inherits the domain from + * the table it belongs to. * * Owners of this TestCase definition. * @@ -228,6 +177,55 @@ export interface EntityReference { type: string; } +/** + * Change that lead to this version of the entity. + * + * Description of the change. + */ +export interface ChangeDescription { + /** + * Names of fields added during the version changes. + */ + fieldsAdded?: FieldChange[]; + /** + * Fields deleted during the version changes with old value before deleted. + */ + fieldsDeleted?: FieldChange[]; + /** + * Fields modified during the version changes with old and new values. + */ + fieldsUpdated?: FieldChange[]; + /** + * When a change did not result in change, this could be same as the current version. + */ + previousVersion?: number; +} + +export interface FieldChange { + /** + * Name of the entity field that changed. + */ + name?: string; + /** + * New value of the field. Note that this is a JSON string and use the corresponding field + * type to deserialize it. + */ + newValue?: any; + /** + * Previous value of the field. Note that this is a JSON string and use the corresponding + * field type to deserialize it. + */ + oldValue?: any; +} + +/** + * TestSuite mock connection, since it needs to implement a Service. + */ +export interface TestSuiteConnection { + config?: null; + [property: string]: any; +} + /** * Type of database service such as MySQL, BigQuery, Snowflake, Redshift, Postgres... */ diff --git a/openmetadata-ui/src/main/resources/ui/src/mocks/TestCase.mock.ts b/openmetadata-ui/src/main/resources/ui/src/mocks/TestCase.mock.ts index eed16b7475d3..ade3de89f141 100644 --- a/openmetadata-ui/src/main/resources/ui/src/mocks/TestCase.mock.ts +++ b/openmetadata-ui/src/main/resources/ui/src/mocks/TestCase.mock.ts @@ -39,7 +39,7 @@ export const MOCK_TEST_CASE_DATA = { name: 'sample_data.ecommerce_db.shopify.dim_address.testSuite', fullyQualifiedName: 'sample_data.ecommerce_db.shopify.dim_address.testSuite', - description: 'This is an executable test suite linked to an entity', + description: 'This is an basic test suite linked to an entity', deleted: false, href: 'http://localhost:8585/api/v1/dataQuality/testSuites/bce9b69f-125a-42a8-b06a-13bdbc049f8d', }, diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/IncidentManager/IncidentManagerDetailPage/IncidentManagerDetailPage.test.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/IncidentManager/IncidentManagerDetailPage/IncidentManagerDetailPage.test.tsx index 64c67e9a374e..2f6aed44f2f0 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/IncidentManager/IncidentManagerDetailPage/IncidentManagerDetailPage.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/IncidentManager/IncidentManagerDetailPage/IncidentManagerDetailPage.test.tsx @@ -45,7 +45,7 @@ const mockTestCaseData = { name: 'sample_data.ecommerce_db.shopify.dim_address.testSuite', fullyQualifiedName: 'sample_data.ecommerce_db.shopify.dim_address.testSuite', - description: 'This is an executable test suite linked to an entity', + description: 'This is an basic test suite linked to an entity', deleted: false, href: 'http://localhost:8585/api/v1/dataQuality/testSuites/fe44ef1a-1b83-4872-bef6-fbd1885986b8', }, diff --git a/openmetadata-ui/src/main/resources/ui/src/pages/TestSuiteDetailsPage/TestSuiteDetailsPage.component.tsx b/openmetadata-ui/src/main/resources/ui/src/pages/TestSuiteDetailsPage/TestSuiteDetailsPage.component.tsx index 2c1cbec1e395..eae0a14d747e 100644 --- a/openmetadata-ui/src/main/resources/ui/src/pages/TestSuiteDetailsPage/TestSuiteDetailsPage.component.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/pages/TestSuiteDetailsPage/TestSuiteDetailsPage.component.tsx @@ -11,7 +11,7 @@ * limitations under the License. */ -import { Button, Col, Modal, Row, Space } from 'antd'; +import { Button, Col, Modal, Row, Space, Tabs } from 'antd'; import { AxiosError } from 'axios'; import { compare } from 'fast-json-patch'; import React, { useCallback, useEffect, useMemo, useState } from 'react'; @@ -30,6 +30,7 @@ import TitleBreadcrumb from '../../components/common/TitleBreadcrumb/TitleBreadc import { TitleBreadcrumbProps } from '../../components/common/TitleBreadcrumb/TitleBreadcrumb.interface'; import DataQualityTab from '../../components/Database/Profiler/DataQualityTab/DataQualityTab'; import { AddTestCaseList } from '../../components/DataQuality/AddTestCaseList/AddTestCaseList.component'; +import TestSuitePipelineTab from '../../components/DataQuality/TestSuite/TestSuitePipelineTab/TestSuitePipelineTab.component'; import PageLayoutV1 from '../../components/PageLayoutV1/PageLayoutV1'; import { INITIAL_PAGING_VALUE } from '../../constants/constants'; import { DEFAULT_SORT_ORDER } from '../../constants/profiler.constant'; @@ -39,7 +40,11 @@ import { ResourceEntity, } from '../../context/PermissionProvider/PermissionProvider.interface'; import { ACTION_TYPE, ERROR_PLACEHOLDER_TYPE } from '../../enums/common.enum'; -import { EntityType, TabSpecificField } from '../../enums/entity.enum'; +import { + EntityTabs, + EntityType, + TabSpecificField, +} from '../../enums/entity.enum'; import { TestCase } from '../../generated/tests/testCase'; import { TestSuite } from '../../generated/tests/testSuite'; import { Include } from '../../generated/type/include'; @@ -329,6 +334,48 @@ const TestSuiteDetailsPage = () => { [currentPage, paging, pageSize, handlePageSizeChange, handleTestCasePaging] ); + const tabs = useMemo( + () => [ + { + label: t('label.test-case-plural'), + key: EntityTabs.TEST_CASES, + children: ( + + ), + }, + { + label: t('label.pipeline-plural'), + key: EntityTabs.PIPELINE, + children: ( + + ), + }, + ], + [ + testSuite, + incidentUrlState, + isLoading, + isTestCaseLoading, + pagingData, + showPagination, + testCaseResult, + handleTestSuiteUpdate, + handleSortTestCase, + fetchTestCases, + ] + ); + if (isLoading) { return ; } @@ -398,18 +445,7 @@ const TestSuiteDetailsPage = () => { - + { @@ -78,12 +74,15 @@ const TestSuiteIngestionPage = () => { url: getDataQualityPagePath(), }, { - name: getEntityName(response.executableEntityReference), - url: getEntityDetailsPath( - EntityType.TABLE, - response.executableEntityReference?.fullyQualifiedName ?? '', - EntityTabs.PROFILER + name: getEntityName( + response.basic ? response.basicEntityReference : response ), + url: getTestSuiteDetailsPath({ + isExecutableTestSuite: response.basic, + fullyQualifiedName: response.basic + ? response.basicEntityReference?.fullyQualifiedName ?? '' + : response.fullyQualifiedName ?? '', + }), }, { name: `${ingestionFQN ? t('label.edit') : t('label.add')} ${t( diff --git a/openmetadata-ui/src/main/resources/ui/src/rest/testAPI.ts b/openmetadata-ui/src/main/resources/ui/src/rest/testAPI.ts index 4d424269d087..7985e990cfa3 100644 --- a/openmetadata-ui/src/main/resources/ui/src/rest/testAPI.ts +++ b/openmetadata-ui/src/main/resources/ui/src/rest/testAPI.ts @@ -36,7 +36,7 @@ import { getEncodedFqn } from '../utils/StringsUtils'; import APIClient from './index'; export enum TestSuiteType { - executable = 'executable', + basic = 'basic', logical = 'logical', } export enum TestCaseType { @@ -282,7 +282,7 @@ export const createExecutableTestSuite = async (data: CreateTestSuite) => { const response = await APIClient.post< CreateTestSuite, AxiosResponse - >(`${testSuiteUrl}/executable`, data); + >(`${testSuiteUrl}/basic`, data); return response.data; }; diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/EntityUtils.test.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/EntityUtils.test.tsx index b0e77f33a64c..a697153d6979 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/EntityUtils.test.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/EntityUtils.test.tsx @@ -92,17 +92,17 @@ describe('EntityUtils unit tests', () => { describe('getBreadcrumbForTestSuite', () => { const testSuiteData: TestSuite = { name: 'testSuite', - executableEntityReference: { + basicEntityReference: { fullyQualifiedName: 'test/testSuite', id: '123', type: 'testType', }, }; - it('should get breadcrumb if data is executable', () => { + it('should get breadcrumb if data is basic', () => { const result = getBreadcrumbForTestSuite({ ...testSuiteData, - executable: true, + basic: true, }); expect(result).toEqual([ @@ -111,7 +111,7 @@ describe('EntityUtils unit tests', () => { ]); }); - it('should get breadcrumb if data is not executable', () => { + it('should get breadcrumb if data is not basic', () => { const result = getBreadcrumbForTestSuite(testSuiteData); expect(result).toEqual([ diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/EntityUtils.tsx b/openmetadata-ui/src/main/resources/ui/src/utils/EntityUtils.tsx index cbd2d8ee75fc..c66960e873b9 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/EntityUtils.tsx +++ b/openmetadata-ui/src/main/resources/ui/src/utils/EntityUtils.tsx @@ -1777,12 +1777,12 @@ export const getBreadcrumbForTestCase = (entity: TestCase): TitleLink[] => [ ]; export const getBreadcrumbForTestSuite = (entity: TestSuite) => { - return entity.executable + return entity.basic ? [ { - name: getEntityName(entity.executableEntityReference), + name: getEntityName(entity.basicEntityReference), url: getEntityLinkFromType( - entity.executableEntityReference?.fullyQualifiedName ?? '', + entity.basicEntityReference?.fullyQualifiedName ?? '', EntityType.TABLE ), }, diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/LogsClassBase.ts b/openmetadata-ui/src/main/resources/ui/src/utils/LogsClassBase.ts index 6f975072204a..a738b29959bc 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/LogsClassBase.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/LogsClassBase.ts @@ -12,11 +12,8 @@ */ import { isUndefined, startCase } from 'lodash'; -import { TableProfilerTab } from '../components/Database/Profiler/ProfilerDashboard/profilerDashboard.interface'; -import { getEntityDetailsPath } from '../constants/constants'; import { GlobalSettingOptions } from '../constants/GlobalSettings.constants'; import { OPEN_METADATA } from '../constants/service-guide.constant'; -import { EntityTabs, EntityType } from '../enums/entity.enum'; import { Pipeline } from '../generated/api/services/ingestionPipelines/createIngestionPipeline'; import { IngestionPipeline } from '../generated/entity/services/ingestionPipelines/ingestionPipeline'; import { DataQualityPageTabs } from '../pages/DataQuality/DataQualityPage.interface'; @@ -30,6 +27,7 @@ import { getLogEntityPath, getSettingPath, } from './RouterUtils'; +import { getTestSuiteDetailsPath } from './TestSuiteUtils'; class LogsClassBase { /** @@ -81,6 +79,11 @@ class LogsClassBase { } if (serviceType === 'testSuite') { + const isExecutableTestSuite = !isUndefined( + (ingestionDetails.sourceConfig.config as Pipeline) + ?.entityFullyQualifiedName + ); + return [ { name: startCase(serviceType), @@ -88,13 +91,14 @@ class LogsClassBase { }, { name: ingestionDetails.name, - url: - getEntityDetailsPath( - EntityType.TABLE, - (ingestionDetails.sourceConfig.config as Pipeline) - ?.entityFullyQualifiedName ?? '', - EntityTabs.PROFILER - ) + `?activeTab=${TableProfilerTab.DATA_QUALITY}`, + url: getTestSuiteDetailsPath({ + isExecutableTestSuite, + fullyQualifiedName: + (isExecutableTestSuite + ? (ingestionDetails.sourceConfig.config as Pipeline) + ?.entityFullyQualifiedName + : ingestionDetails.service?.fullyQualifiedName) ?? '', + }), }, { name: i18n.t('label.log-plural'), diff --git a/openmetadata-ui/src/main/resources/ui/src/utils/SearchClassBase.ts b/openmetadata-ui/src/main/resources/ui/src/utils/SearchClassBase.ts index d14b26de5fd3..370a41f1edfc 100644 --- a/openmetadata-ui/src/main/resources/ui/src/utils/SearchClassBase.ts +++ b/openmetadata-ui/src/main/resources/ui/src/utils/SearchClassBase.ts @@ -557,7 +557,7 @@ class SearchClassBase { ): string | { pathname: string } { if (entity.entityType === EntityType.TEST_SUITE) { return getTestSuiteDetailsPath({ - isExecutableTestSuite: (entity as TestSuite).executable, + isExecutableTestSuite: (entity as TestSuite).basic, fullyQualifiedName: entity.fullyQualifiedName ?? '', }); }