From 0a0f248774ebd64d007389bfd87c3534ebe79ea4 Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Mon, 2 Dec 2024 15:45:23 -0800 Subject: [PATCH 1/8] Create a relationship to manage ER based on tableConstraints --- .../service/jdbi3/TableRepository.java | 16 ++++++++++++++++ .../json/schema/type/entityRelationship.json | 6 ++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java index bb428941ce55..afd5fe9290b1 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java @@ -1157,6 +1157,22 @@ private void updateConstraints(Table origTable, Table updatedTable) { added, deleted, EntityUtil.tableConstraintMatch); + for (TableConstraint constraint : added) { + for (String column : constraint.getReferredColumns()) { + String toParent = FullyQualifiedName.getParentFQN(column); + EntityReference toTable = EntityUtil.getEntityReference(TABLE, toParent); + addRelationship( + updatedTable.getId(), toTable.getId(), TABLE, TABLE, Relationship.RELATED_TO); + } + } + for (TableConstraint constraint : deleted) { + for (String column : constraint.getReferredColumns()) { + String toParent = FullyQualifiedName.getParentFQN(column); + EntityReference toTable = EntityUtil.getEntityReference(TABLE, toParent); + deleteRelationship( + updatedTable.getId(), TABLE, toTable.getId(), TABLE, Relationship.RELATED_TO); + } + } } } diff --git a/openmetadata-spec/src/main/resources/json/schema/type/entityRelationship.json b/openmetadata-spec/src/main/resources/json/schema/type/entityRelationship.json index 74e12bc3b772..e27c088d7af9 100644 --- a/openmetadata-spec/src/main/resources/json/schema/type/entityRelationship.json +++ b/openmetadata-spec/src/main/resources/json/schema/type/entityRelationship.json @@ -33,7 +33,8 @@ "voted", "expert", "editedBy", - "defaultsTo" + "defaultsTo", + "relatesTo" ], "javaEnums": [ { "name": "CONTAINS" }, @@ -57,7 +58,8 @@ { "name": "VOTED" }, { "name": "EXPERT" }, { "name": "EDITED_BY" }, - { "name": "DEFAULTS_TO" } + { "name": "DEFAULTS_TO" }, + { "name": "RELATES_TO" } ] } }, From bfaad28dc0a3da0eb71377c08c0c95d4f0946ee4 Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Mon, 2 Dec 2024 19:09:41 -0800 Subject: [PATCH 2/8] Add relationship/remove relationship from entity_relationship table whenever table constriants are updated/added , validate table constraints --- .../exception/EntityNotFoundException.java | 2 +- .../service/jdbi3/TableRepository.java | 121 +++++++++++++++--- .../databases/TableResourceTest.java | 42 ++++++ .../EntitySpecViolationException.java | 30 +++++ .../sdk/exception/EntityUpdateException.java | 30 +++++ 5 files changed, 209 insertions(+), 16 deletions(-) create mode 100644 openmetadata-spec/src/main/java/org/openmetadata/sdk/exception/EntitySpecViolationException.java create mode 100644 openmetadata-spec/src/main/java/org/openmetadata/sdk/exception/EntityUpdateException.java diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/exception/EntityNotFoundException.java b/openmetadata-service/src/main/java/org/openmetadata/service/exception/EntityNotFoundException.java index ee2271f0f8fa..cf0c82e44659 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/exception/EntityNotFoundException.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/exception/EntityNotFoundException.java @@ -31,7 +31,7 @@ public EntityNotFoundException(String message) { super(Response.Status.NOT_FOUND, ERROR_TYPE, message); } - private EntityNotFoundException(String message, Throwable cause) { + public EntityNotFoundException(String message, Throwable cause) { super(Response.Status.NOT_FOUND, ERROR_TYPE, message, cause); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java index afd5fe9290b1..2190465017b9 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java @@ -40,6 +40,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Optional; @@ -89,6 +90,7 @@ import org.openmetadata.schema.type.csv.CsvFile; import org.openmetadata.schema.type.csv.CsvHeader; import org.openmetadata.schema.type.csv.CsvImportResult; +import org.openmetadata.sdk.exception.EntitySpecViolationException; import org.openmetadata.sdk.exception.SuggestionException; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.CatalogExceptionMessage; @@ -676,6 +678,7 @@ public void prepare(Table table, boolean update) { .withDatabase(schema.getDatabase()) .withService(schema.getService()) .withServiceType(schema.getServiceType()); + validateTableConstraints(table); } @Override @@ -693,6 +696,8 @@ public void storeEntity(Table table, boolean update) { // Restore the relationships table.withColumns(columnWithTags).withService(service); + // Store ER relationships based on table constraints + addConstraintRelationship(table, table.getTableConstraints()); } @Override @@ -1117,6 +1122,32 @@ private List getCustomMetrics(Table table, String columnName) { return customMetrics; } + private void validateTableConstraints(Table table) { + if (!nullOrEmpty(table.getTableConstraints())) { + Set constraintSet = new HashSet<>(); + for (TableConstraint constraint : table.getTableConstraints()) { + if (!constraintSet.add(constraint)) { + throw new EntitySpecViolationException( + "Duplicate constraint found in request: " + constraint); + } + for (String column : constraint.getColumns()) { + validateColumn(table, column); + } + if (!nullOrEmpty(constraint.getReferredColumns())) { + for (String column : constraint.getReferredColumns()) { + String toParent = FullyQualifiedName.getParentFQN(column); + try { + Table toTable = findByName(toParent, NON_DELETED); + validateColumn(toTable, column); + } catch (EntityNotFoundException e) { + throw new EntitySpecViolationException("Table not found: " + toParent); + } + } + } + } + } + } + /** Handles entity updated from PUT and POST operation. */ public class TableUpdater extends ColumnEntityUpdater { public TableUpdater(Table original, Table updated, Operation operation) { @@ -1129,7 +1160,7 @@ public void entitySpecificUpdate() { Table updatedTable = updated; DatabaseUtil.validateColumns(updatedTable.getColumns()); recordChange("tableType", origTable.getTableType(), updatedTable.getTableType()); - updateConstraints(origTable, updatedTable); + updateTableConstraints(origTable, updatedTable, operation); updateColumns( COLUMN_FIELD, origTable.getColumns(), updated.getColumns(), EntityUtil.columnMatch); recordChange("sourceUrl", original.getSourceUrl(), updated.getSourceUrl()); @@ -1138,10 +1169,26 @@ public void entitySpecificUpdate() { recordChange("locationPath", original.getLocationPath(), updated.getLocationPath()); } - private void updateConstraints(Table origTable, Table updatedTable) { + private void updateTableConstraints(Table origTable, Table updatedTable, Operation operation) { + validateTableConstraints(updatedTable); + if (operation.isPatch() + && !nullOrEmpty(updatedTable.getTableConstraints()) + && !nullOrEmpty(origTable.getTableConstraints())) { + List newConstraints = new ArrayList<>(); + for (TableConstraint constraint : updatedTable.getTableConstraints()) { + TableConstraint existing = + origTable.getTableConstraints().stream() + .filter(c -> EntityUtil.tableConstraintMatch.test(c, constraint)) + .findAny() + .orElse(null); + if (existing == null) { + newConstraints.add(constraint); + } + } + checkDuplicateTableConstraints(origTable, newConstraints); + } List origConstraints = listOrEmpty(origTable.getTableConstraints()); List updatedConstraints = listOrEmpty(updatedTable.getTableConstraints()); - origConstraints.sort(EntityUtil.compareTableConstraint); origConstraints.stream().map(TableConstraint::getColumns).forEach(Collections::sort); @@ -1157,20 +1204,64 @@ private void updateConstraints(Table origTable, Table updatedTable) { added, deleted, EntityUtil.tableConstraintMatch); - for (TableConstraint constraint : added) { - for (String column : constraint.getReferredColumns()) { - String toParent = FullyQualifiedName.getParentFQN(column); - EntityReference toTable = EntityUtil.getEntityReference(TABLE, toParent); - addRelationship( - updatedTable.getId(), toTable.getId(), TABLE, TABLE, Relationship.RELATED_TO); + + // manage table ER relationship based on table constraints + addConstraintRelationship(origTable, added); + deleteConstraintRelationship(origTable, deleted); + } + } + + private void checkDuplicateTableConstraints( + Table origTable, List newConstraints) { + if (!nullOrEmpty(origTable.getTableConstraints()) && !nullOrEmpty(newConstraints)) { + Set origConstraints = + new HashSet<>(listOrEmpty(origTable.getTableConstraints())); + for (TableConstraint constraint : newConstraints) { + if (!origConstraints.add(constraint)) { + throw new EntitySpecViolationException("Table Constraint is Duplicate: " + constraint); } } - for (TableConstraint constraint : deleted) { - for (String column : constraint.getReferredColumns()) { - String toParent = FullyQualifiedName.getParentFQN(column); - EntityReference toTable = EntityUtil.getEntityReference(TABLE, toParent); - deleteRelationship( - updatedTable.getId(), TABLE, toTable.getId(), TABLE, Relationship.RELATED_TO); + } + } + + private void addConstraintRelationship(Table table, List constraints) { + if (!nullOrEmpty(constraints)) { + for (TableConstraint constraint : constraints) { + if (!nullOrEmpty(constraint.getReferredColumns())) { + for (String column : constraint.getReferredColumns()) { + String toParent = FullyQualifiedName.getParentFQN(column); + try { + EntityReference toTable = + Entity.getEntityReferenceByName(TABLE, toParent, NON_DELETED); + addRelationship( + table.getId(), toTable.getId(), TABLE, TABLE, Relationship.RELATED_TO); + } catch (EntityNotFoundException e) { + throw EntityNotFoundException.byName( + String.format( + "Failed to add table constraint due to missing table %s", toParent)); + } + } + } + } + } + } + + private void deleteConstraintRelationship(Table table, List constraints) { + if (!nullOrEmpty(constraints)) { + for (TableConstraint constraint : constraints) { + if (!nullOrEmpty(constraint.getReferredColumns())) { + for (String column : constraint.getReferredColumns()) { + String toParent = FullyQualifiedName.getParentFQN(column); + try { + EntityReference toTable = Entity.getEntityReferenceByName(TABLE, toParent, ALL); + deleteRelationship( + table.getId(), TABLE, toTable.getId(), TABLE, Relationship.RELATED_TO); + } catch (EntityNotFoundException e) { + throw EntityNotFoundException.byName( + String.format( + "Failed to add table constraint due to missing table %s", toParent)); + } + } } } } diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java index 629fba50de37..a6c51064a08f 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/resources/databases/TableResourceTest.java @@ -2954,6 +2954,48 @@ void get_TablesWithPagination_200(TestInfo test) throws IOException { ADMIN_AUTH_HEADERS); } + @Test + void put_tableTableConstraintDuplicate_400(TestInfo test) throws IOException { + // Create table with a constraint + CreateTable request = + createRequest(test) + .withColumns(List.of(getColumn(C1, BIGINT, USER_ADDRESS_TAG_LABEL))) + .withTableConstraints(null); + Table table = createAndCheckEntity(request, ADMIN_AUTH_HEADERS); + + // Attempt to add duplicate constraints + TableConstraint constraint = + new TableConstraint().withConstraintType(ConstraintType.UNIQUE).withColumns(List.of(C1)); + + request = request.withTableConstraints(List.of(constraint, constraint)); // Duplicate constraint + CreateTable finalRequest = request; + assertResponseContains( + () -> updateEntity(finalRequest, OK, ADMIN_AUTH_HEADERS), + BAD_REQUEST, + "Duplicate constraint found in request: "); + } + + @Test + void put_tableTableConstraintInvalidColumn_400(TestInfo test) throws IOException { + CreateTable request = + createRequest(test) + .withColumns(List.of(getColumn(C1, BIGINT, USER_ADDRESS_TAG_LABEL))) + .withTableConstraints(null); + Table table = createAndCheckEntity(request, ADMIN_AUTH_HEADERS); + + TableConstraint constraint = + new TableConstraint() + .withConstraintType(ConstraintType.UNIQUE) + .withColumns(List.of("invalid_column")); // Non-existent column + + request = request.withTableConstraints(List.of(constraint)); + CreateTable finalRequest = request; + assertResponseContains( + () -> updateEntity(finalRequest, OK, ADMIN_AUTH_HEADERS), + BAD_REQUEST, + "Invalid column name found in table constraint"); + } + void assertFields(List tableList, String fieldsParam) { tableList.forEach(t -> assertFields(t, fieldsParam)); } diff --git a/openmetadata-spec/src/main/java/org/openmetadata/sdk/exception/EntitySpecViolationException.java b/openmetadata-spec/src/main/java/org/openmetadata/sdk/exception/EntitySpecViolationException.java new file mode 100644 index 000000000000..0644a38059d3 --- /dev/null +++ b/openmetadata-spec/src/main/java/org/openmetadata/sdk/exception/EntitySpecViolationException.java @@ -0,0 +1,30 @@ +package org.openmetadata.sdk.exception; + +import javax.ws.rs.core.Response; + +public class EntitySpecViolationException extends WebServiceException { + private static final String BY_NAME_MESSAGE = "Entity Spec Violation [%s] due to [%s]."; + private static final String ERROR_TYPE = "ENTITY_SPEC_VIOLATION"; + + public EntitySpecViolationException(String message) { + super(Response.Status.BAD_REQUEST, ERROR_TYPE, message); + } + + public EntitySpecViolationException(Response.Status status, String message) { + super(status, ERROR_TYPE, message); + } + + public static EntitySpecViolationException byMessage( + String name, String errorMessage, Response.Status status) { + return new EntitySpecViolationException(status, buildMessageByName(name, errorMessage)); + } + + public static EntitySpecViolationException byMessage(String name, String errorMessage) { + return new EntitySpecViolationException( + Response.Status.BAD_REQUEST, buildMessageByName(name, errorMessage)); + } + + private static String buildMessageByName(String name, String errorMessage) { + return String.format(BY_NAME_MESSAGE, name, errorMessage); + } +} diff --git a/openmetadata-spec/src/main/java/org/openmetadata/sdk/exception/EntityUpdateException.java b/openmetadata-spec/src/main/java/org/openmetadata/sdk/exception/EntityUpdateException.java new file mode 100644 index 000000000000..306851e8be59 --- /dev/null +++ b/openmetadata-spec/src/main/java/org/openmetadata/sdk/exception/EntityUpdateException.java @@ -0,0 +1,30 @@ +package org.openmetadata.sdk.exception; + +import javax.ws.rs.core.Response; + +public class EntityUpdateException extends WebServiceException { + private static final String BY_NAME_MESSAGE = "Entity Update Exception [%s] due to [%s]."; + private static final String ERROR_TYPE = "ENTITY_UPDATE_EXCEPTION"; + + public EntityUpdateException(String message) { + super(Response.Status.BAD_REQUEST, ERROR_TYPE, message); + } + + public EntityUpdateException(Response.Status status, String message) { + super(status, ERROR_TYPE, message); + } + + public static EntityUpdateException byMessage( + String name, String errorMessage, Response.Status status) { + return new EntityUpdateException(status, buildMessageByName(name, errorMessage)); + } + + public static EntityUpdateException byMessage(String name, String errorMessage) { + return new EntityUpdateException( + Response.Status.BAD_REQUEST, buildMessageByName(name, errorMessage)); + } + + private static String buildMessageByName(String name, String errorMessage) { + return String.format(BY_NAME_MESSAGE, name, errorMessage); + } +} From 2806f822ee38fcaebb5ca9d79f73ff5ad5d3fff1 Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Mon, 2 Dec 2024 20:09:16 -0800 Subject: [PATCH 3/8] remove findRelatedTables queries --- .../service/search/indexes/SearchIndex.java | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java index 1e93848c05f3..e75cc7262b61 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/search/indexes/SearchIndex.java @@ -247,12 +247,15 @@ static List> populateEntityRelationshipData(Table entity) { // We need to query the table_entity table to find the references this current table // has with other tables. We pick this info from the ES however in case of re-indexing this info // needs to be picked from the db - CollectionDAO dao = Entity.getCollectionDAO(); - List json_array = - dao.tableDAO().findRelatedTables(entity.getFullyQualifiedName() + "%"); - for (String json : json_array) { - Table foreign_table = JsonUtils.readValue(json, Table.class); - processConstraints(foreign_table, entity, constraints, false); + List relatedTables = + Entity.getCollectionDAO() + .relationshipDAO() + .findFrom(entity.getId(), Entity.TABLE, Relationship.RELATED_TO.ordinal()); + + for (CollectionDAO.EntityRelationshipRecord table : relatedTables) { + Table foreignTable = + Entity.getEntity(Entity.TABLE, table.getId(), "tableConstraints", NON_DELETED); + processConstraints(foreignTable, entity, constraints, false); } return constraints; } From 8d28d88f027dcb428a1ab88fa3c0edb0750bc2fa Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Mon, 2 Dec 2024 21:33:29 -0800 Subject: [PATCH 4/8] Add Migrations to add constrait relationship --- .../service/jdbi3/TableRepository.java | 3 +- .../migration/mysql/v160/Migration.java | 2 + .../migration/postgres/v160/Migration.java | 2 + .../migration/utils/v160/MigrationUtil.java | 94 +++++++++++++++++++ 4 files changed, 100 insertions(+), 1 deletion(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java index 2190465017b9..a975cdcc34a2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/TableRepository.java @@ -1136,9 +1136,10 @@ private void validateTableConstraints(Table table) { if (!nullOrEmpty(constraint.getReferredColumns())) { for (String column : constraint.getReferredColumns()) { String toParent = FullyQualifiedName.getParentFQN(column); + String columnName = FullyQualifiedName.getColumnName(column); try { Table toTable = findByName(toParent, NON_DELETED); - validateColumn(toTable, column); + validateColumn(toTable, columnName); } catch (EntityNotFoundException e) { throw new EntitySpecViolationException("Table not found: " + toParent); } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v160/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v160/Migration.java index 130ec376b093..02e1e4532a89 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v160/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/mysql/v160/Migration.java @@ -2,6 +2,7 @@ import static org.openmetadata.service.migration.utils.v160.MigrationUtil.addDisplayNameToCustomProperty; import static org.openmetadata.service.migration.utils.v160.MigrationUtil.addEditGlossaryTermsToDataConsumerPolicy; +import static org.openmetadata.service.migration.utils.v160.MigrationUtil.addRelationsForTableConstraints; import static org.openmetadata.service.migration.utils.v160.MigrationUtil.addViewAllRuleToOrgPolicy; import static org.openmetadata.service.migration.utils.v160.MigrationUtil.migrateServiceTypesAndConnections; @@ -22,5 +23,6 @@ public void runDataMigration() { addViewAllRuleToOrgPolicy(collectionDAO); addEditGlossaryTermsToDataConsumerPolicy(collectionDAO); addDisplayNameToCustomProperty(handle, false); + addRelationsForTableConstraints(handle, false); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v160/Migration.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v160/Migration.java index eaf7f15fbf76..936a337f4888 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v160/Migration.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/postgres/v160/Migration.java @@ -2,6 +2,7 @@ import static org.openmetadata.service.migration.utils.v160.MigrationUtil.addDisplayNameToCustomProperty; import static org.openmetadata.service.migration.utils.v160.MigrationUtil.addEditGlossaryTermsToDataConsumerPolicy; +import static org.openmetadata.service.migration.utils.v160.MigrationUtil.addRelationsForTableConstraints; import static org.openmetadata.service.migration.utils.v160.MigrationUtil.addViewAllRuleToOrgPolicy; import static org.openmetadata.service.migration.utils.v160.MigrationUtil.migrateServiceTypesAndConnections; @@ -22,5 +23,6 @@ public void runDataMigration() { addViewAllRuleToOrgPolicy(collectionDAO); addEditGlossaryTermsToDataConsumerPolicy(collectionDAO); addDisplayNameToCustomProperty(handle, true); + addRelationsForTableConstraints(handle, true); } } diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v160/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v160/MigrationUtil.java index 9f928948952d..5143bb15d257 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v160/MigrationUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v160/MigrationUtil.java @@ -1,20 +1,31 @@ package org.openmetadata.service.migration.utils.v160; import static org.openmetadata.common.utils.CommonUtil.listOf; +import static org.openmetadata.common.utils.CommonUtil.nullOrEmpty; +import static org.openmetadata.schema.type.Include.NON_DELETED; +import static org.openmetadata.service.Entity.TABLE; import java.util.Arrays; +import java.util.HashMap; import java.util.List; +import java.util.Map; +import java.util.UUID; import lombok.extern.slf4j.Slf4j; import org.jdbi.v3.core.Handle; +import org.openmetadata.schema.entity.data.Table; import org.openmetadata.schema.entity.policies.Policy; import org.openmetadata.schema.entity.policies.accessControl.Rule; +import org.openmetadata.schema.type.EntityReference; import org.openmetadata.schema.type.Include; import org.openmetadata.schema.type.MetadataOperation; import org.openmetadata.schema.type.Relationship; +import org.openmetadata.schema.type.TableConstraint; import org.openmetadata.service.Entity; import org.openmetadata.service.exception.EntityNotFoundException; import org.openmetadata.service.jdbi3.CollectionDAO; import org.openmetadata.service.jdbi3.PolicyRepository; +import org.openmetadata.service.jdbi3.TableRepository; +import org.openmetadata.service.util.FullyQualifiedName; import org.openmetadata.service.util.JsonUtils; @Slf4j @@ -114,6 +125,89 @@ public static void addOperationsToPolicyRule( } } + public static void addRelationsForTableConstraints(Handle handle, boolean postgresql) { + LOG.info("Starting table constraint relationship migration"); + final int batchSize = 1000; + int offset = 0; + String fetchQuery = + "SELECT id, json FROM table_entity " + + "WHERE JSON_LENGTH(JSON_EXTRACT(json, '$.tableConstraints')) > 0 " + + "AND JSON_LENGTH(JSON_EXTRACT(json, '$.tableConstraints[*].referredColumns')) > 0 " + + "LIMIT :limit OFFSET :offset"; + + if (postgresql) { + fetchQuery = + "SELECT id, json FROM table_entity " + + "WHERE jsonb_typeof(json->'tableConstraints') = 'array' " + + "AND jsonb_array_length(json->'tableConstraints') > 0 " + + "AND EXISTS (" + + " SELECT 1 FROM jsonb_array_elements(json->'tableConstraints') AS constraint " + + " WHERE jsonb_typeof(constraint->'referredColumns') = 'array' " + + " AND jsonb_array_length(constraint->'referredColumns') > 0" + + ") " + + "LIMIT :limit OFFSET :offset"; + } + + TableRepository tableRepository = (TableRepository) Entity.getEntityRepository(TABLE); + + while (true) { + List> tables = + handle + .createQuery(fetchQuery) + .bind("limit", batchSize) + .bind("offset", offset) + .mapToMap() + .list(); + + if (tables.isEmpty()) { + break; + } + + for (Map tableRow : tables) { + String tableId = (String) tableRow.get("id"); + String json = (String) tableRow.get("json"); + try { + Table table = JsonUtils.readValue(json, Table.class); + addConstraintRelationship(table, table.getTableConstraints(), tableRepository); + } catch (Exception e) { + LOG.error("Error processing table ID '{}': {}", tableId, e.getMessage()); + } + } + + offset += batchSize; + LOG.debug("Processed of table constraint up to offset {}", offset); + } + } + + private static void addConstraintRelationship( + Table table, List constraints, TableRepository tableRepository) { + if (!nullOrEmpty(constraints)) { + for (TableConstraint constraint : constraints) { + if (!nullOrEmpty(constraint.getReferredColumns())) { + List relationships = + tableRepository.findTo(table.getId(), TABLE, Relationship.RELATED_TO, TABLE); + Map relatedTables = new HashMap<>(); + relationships.forEach(r -> relatedTables.put(r.getId(), r)); + for (String column : constraint.getReferredColumns()) { + String toParent = FullyQualifiedName.getParentFQN(column); + try { + EntityReference toTable = + Entity.getEntityReferenceByName(TABLE, toParent, NON_DELETED); + if (!relatedTables.containsKey(toTable.getId())) { + tableRepository.addRelationship( + table.getId(), toTable.getId(), TABLE, TABLE, Relationship.RELATED_TO); + } + } catch (EntityNotFoundException e) { + throw EntityNotFoundException.byName( + String.format( + "Failed to add table constraint due to missing table %s", toParent)); + } + } + } + } + } + } + public static void migrateServiceTypesAndConnections(Handle handle, boolean postgresql) { LOG.info("Starting service type and connection type migrations"); try { From 67dbefd6c12529e6e508804f5a9f8510dd3ebc73 Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Mon, 2 Dec 2024 21:59:06 -0800 Subject: [PATCH 5/8] remove findRelatedTables code --- .../service/jdbi3/CollectionDAO.java | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java index 7515277c1183..a5938ea9f4a2 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/jdbi3/CollectionDAO.java @@ -2741,25 +2741,6 @@ default List listAfter(ListFilter filter, int limit, String afterName, S return listAfter( getTableName(), filter.getQueryParams(), condition, condition, limit, afterName, afterId); } - - @ConnectionAwareSqlQuery( - value = - "SELECT json FROM table_entity " - + "WHERE JSON_SEARCH(JSON_EXTRACT(json, '$.tableConstraints[*].referredColumns'), " - + "'one', :fqn) IS NOT NULL", - connectionType = MYSQL) - @ConnectionAwareSqlQuery( - value = - "SELECT json " - + "FROM table_entity " - + "WHERE EXISTS (" - + " SELECT 1" - + " FROM jsonb_array_elements(json->'tableConstraints') AS constraints" - + " CROSS JOIN jsonb_array_elements_text(constraints->'referredColumns') AS referredColumn " - + " WHERE referredColumn LIKE :fqn" - + ")", - connectionType = POSTGRES) - List findRelatedTables(@Bind("fqn") String fqn); } interface StoredProcedureDAO extends EntityDAO { From 1e8f1b19198054e5a3f1dc872d840ee9a4d845aa Mon Sep 17 00:00:00 2001 From: Sriharsha Chintalapani Date: Mon, 2 Dec 2024 23:41:13 -0800 Subject: [PATCH 6/8] Fix postgres migration query --- .../service/migration/utils/v160/MigrationUtil.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v160/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v160/MigrationUtil.java index 5143bb15d257..b98408d0fd48 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v160/MigrationUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v160/MigrationUtil.java @@ -141,9 +141,9 @@ public static void addRelationsForTableConstraints(Handle handle, boolean postgr + "WHERE jsonb_typeof(json->'tableConstraints') = 'array' " + "AND jsonb_array_length(json->'tableConstraints') > 0 " + "AND EXISTS (" - + " SELECT 1 FROM jsonb_array_elements(json->'tableConstraints') AS constraint " - + " WHERE jsonb_typeof(constraint->'referredColumns') = 'array' " - + " AND jsonb_array_length(constraint->'referredColumns') > 0" + + " SELECT 1 FROM jsonb_array_elements(json->'tableConstraints') AS tc " + + " WHERE jsonb_typeof(tc->'referredColumns') = 'array' " + + " AND jsonb_array_length(tc->'referredColumns') > 0" + ") " + "LIMIT :limit OFFSET :offset"; } From 7c9084605fcb6f1b93c22c0ec91e5040bcf8772f Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Tue, 3 Dec 2024 10:48:40 +0100 Subject: [PATCH 7/8] fix pg migration --- .../service/migration/utils/v160/MigrationUtil.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v160/MigrationUtil.java b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v160/MigrationUtil.java index b98408d0fd48..ac5ac327c04c 100644 --- a/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v160/MigrationUtil.java +++ b/openmetadata-service/src/main/java/org/openmetadata/service/migration/utils/v160/MigrationUtil.java @@ -165,7 +165,7 @@ public static void addRelationsForTableConstraints(Handle handle, boolean postgr for (Map tableRow : tables) { String tableId = (String) tableRow.get("id"); - String json = (String) tableRow.get("json"); + String json = tableRow.get("json").toString(); try { Table table = JsonUtils.readValue(json, Table.class); addConstraintRelationship(table, table.getTableConstraints(), tableRepository); From 6c2f584b6c2e0059b6e77496784cc0761b626fe6 Mon Sep 17 00:00:00 2001 From: Pere Miquel Brull Date: Tue, 3 Dec 2024 12:46:11 +0100 Subject: [PATCH 8/8] fix test --- .../org/openmetadata/service/EnumBackwardCompatibilityTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openmetadata-service/src/test/java/org/openmetadata/service/EnumBackwardCompatibilityTest.java b/openmetadata-service/src/test/java/org/openmetadata/service/EnumBackwardCompatibilityTest.java index cb89874a2bdf..9d72e098019c 100644 --- a/openmetadata-service/src/test/java/org/openmetadata/service/EnumBackwardCompatibilityTest.java +++ b/openmetadata-service/src/test/java/org/openmetadata/service/EnumBackwardCompatibilityTest.java @@ -34,7 +34,7 @@ class EnumBackwardCompatibilityTest { /** */ @Test void testRelationshipEnumBackwardCompatible() { - assertEquals(22, Relationship.values().length); + assertEquals(23, Relationship.values().length); assertEquals(21, Relationship.DEFAULTS_TO.ordinal()); assertEquals(20, Relationship.EDITED_BY.ordinal()); assertEquals(19, Relationship.EXPERT.ordinal());