Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

DE OMAS, AL OMAS, OLS - simplify lineage creation by removing the port aliases #7660

Merged
merged 19 commits into from
May 24, 2023
Merged
Show file tree
Hide file tree
Changes from 14 commits
Commits
Show all changes
19 commits
Select commit Hold shift + click to select a range
519b0ec
OLS - query graph based on ProcessHierarchies instead of PortAliases
popa-raluca Apr 11, 2023
5a8f7ac
AL OMAS - remove PortAliases from Process context
popa-raluca Apr 11, 2023
08d9818
AL OMAS - remove PortAliases from Process context
popa-raluca Apr 11, 2023
9a14ad7
DE OMAS - remove PortAlias
popa-raluca Apr 11, 2023
bea449c
Update DE OMAS FVTs to use ProcessHierarchies instead of PortAliases
popa-raluca Apr 18, 2023
a62c1d4
Update DE OMAS FVTs
popa-raluca Apr 18, 2023
ac7b6bf
Merge branch 'main' of https://github.com/odpi/egeria into port-aliases
popa-raluca Apr 19, 2023
40e7219
Merge branch 'main' of https://github.com/odpi/egeria into port-aliases
popa-raluca May 1, 2023
f3b3540
DE OMAS - cleanup delete endpoint, service & clients for PortAliases
popa-raluca May 3, 2023
26814b6
DE OMAS - update postman collections & extract minimal flows example …
popa-raluca May 3, 2023
6ed3cf7
DE OMAS - cleanup delete endpoint, service & clients for PortAliases
popa-raluca May 3, 2023
67db945
DE OMAS - update doc
popa-raluca May 3, 2023
d07afb7
OLS - allow adding of new verticess for Processes on updateEntityEvents
popa-raluca May 3, 2023
ef373b4
Fix formatting
popa-raluca May 9, 2023
c9ef54c
DE OMAS - update documentation
popa-raluca May 9, 2023
e2335fb
Fix sonar warn
popa-raluca May 9, 2023
5a690a8
Merge branch 'main' into port-aliases
popa-raluca May 11, 2023
201df7e
Merge branch 'main' into port-aliases
lcpopa May 18, 2023
f676f48
Merge branch 'main' into port-aliases
popa-raluca May 24, 2023
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,6 @@
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.COLLECTION_MEMBERSHIP;
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.DATA_FLOW;
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.LINEAGE_MAPPING;
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.PORT_ALIAS;
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.PORT_DELEGATION;
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.PORT_IMPLEMENTATION;
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.PORT_SCHEMA;
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.PROCESS;
Expand Down Expand Up @@ -57,7 +55,7 @@ public ProcessContextHandler(AssetContextHandler assetContextHandler, HandlerHel

/**
* Retrieves the full context for a Process.
* This context contains the full description for the Port Aliases, Port Implementations, SchemaTypes and Tabular Columns related to the process.
* This context contains the full description for the Port Implementations, SchemaTypes and Tabular Columns related to the process.
*
* @param userId userId of user making request.
* @param process the process entity for which the context is built
Expand Down Expand Up @@ -151,9 +149,6 @@ private void addContextForColumn(String userId, Multimap<String, RelationshipsCo
private void addContextForPort(String userId, EntityDetail port, Set<GraphContext> relationshipsContext) throws OCFCheckedExceptionBase {
String portType = port.getType().getTypeDefName();

if (PORT_ALIAS.equals(portType)) {
handlerHelper.addContextForRelationships(userId, port, PORT_DELEGATION, relationshipsContext);
}
if (PORT_IMPLEMENTATION.equals(portType)) {
EntityDetail tabularSchemaType = handlerHelper.addContextForRelationships(userId, port, PORT_SCHEMA, relationshipsContext);

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,6 @@ public final class AssetLineageConstants {
public static final String DATABASE = "Database";
public static final String FILE_FOLDER = "FileFolder";
public static final String PROCESS = "Process";
public static final String PORT_ALIAS = "PortAlias";
public static final String PORT_IMPLEMENTATION = "PortImplementation";
public static final String TABULAR_SCHEMA_TYPE = "TabularSchemaType";
public static final String TABULAR_COLUMN = "TabularColumn";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.CONNECTION_ENDPOINT;
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.CONNECTION_TO_ASSET;
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.DATA_CONTENT_FOR_DATA_SET;
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.DATA_FLOW;
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.ENDPOINT;
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.FILE_FOLDER;
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.FOLDER_HIERARCHY;
Expand All @@ -39,7 +40,6 @@
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.NESTED_FILE;
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.NESTED_SCHEMA_ATTRIBUTE;
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.PORT;
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.PORT_DELEGATION;
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.PORT_SCHEMA;
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.PROCESS_HIERARCHY;
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.PROCESS_PORT;
Expand Down Expand Up @@ -78,8 +78,7 @@ public AssetLineageTypesValidator(OMRSRepositoryHelper repositoryHelper, Map<Str
final Set<String> defaultTopicRelationships =
Set.of(ATTRIBUTE_FOR_SCHEMA, SCHEMA_TYPE_OPTION, ASSET_SCHEMA_TYPE);
final Set<String> defaultProcessRelationships =
Set.of(ATTRIBUTE_FOR_SCHEMA, ASSET_SCHEMA_TYPE, PORT_SCHEMA, PORT_DELEGATION, PROCESS_PORT,
PROCESS_HIERARCHY, DATA_FLOW, LINEAGE_MAPPING);
Set.of(ATTRIBUTE_FOR_SCHEMA, ASSET_SCHEMA_TYPE, PORT_SCHEMA, PROCESS_PORT, PROCESS_HIERARCHY, DATA_FLOW, LINEAGE_MAPPING);
final Set<String> defaultGlossaryTermRelationships =
Set.of(SEMANTIC_ASSIGNMENT, TERM_CATEGORIZATION);

Expand Down Expand Up @@ -112,7 +111,7 @@ public boolean hasValidClassificationTypes(EntityDetail entityDetail) {

List<String> classificationNames = entityDetail.getClassifications().stream()
.map(classification -> classification.getType().getTypeDefName())
.collect(Collectors.toList());
.toList();
return !Collections.disjoint(lineageClassificationTypes, classificationNames);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,8 +31,6 @@
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.ATTRIBUTE_FOR_SCHEMA;
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.COLLECTION_MEMBERSHIP;
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.DATA_FLOW;
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.PORT_ALIAS;
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.PORT_DELEGATION;
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.PORT_IMPLEMENTATION;
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.PORT_SCHEMA;
import static org.odpi.openmetadata.accessservices.assetlineage.util.AssetLineageConstants.PROCESS;
Expand Down Expand Up @@ -67,27 +65,6 @@ void buildProcessContext_noPorts() throws OCFCheckedExceptionBase {
verify(handlerHelper, times(1)).addContextForRelationships(USER, collectionEntity, COLLECTION_MEMBERSHIP, context);
}

@Test
void buildProcessContext_withPortAlias() throws OCFCheckedExceptionBase {
List<Relationship> collection = mockGetRelationships(COLLECTION_MEMBERSHIP, PROCESS);
EntityDetail collectionEntity = mockEntityAtTheEnd(collection.get(0));
Set<GraphContext> collectionContext = mockGraphContext(collection, COLLECTION_MEMBERSHIP);

EntityDetail process = mockProcess();

List<Relationship> port = mockGetRelationships(PROCESS_PORT, PROCESS);
EntityDetail portEntity = mockEntityAtTheEnd(port.get(0));
InstanceType instanceType = mock(InstanceType.class);
when(instanceType.getTypeDefName()).thenReturn(PORT_ALIAS);
when(portEntity.getType()).thenReturn(instanceType);
Set<GraphContext> portContext = mockGraphContext(port, PORT_DELEGATION);

processContextHandler.buildProcessContext(USER, process);
verify(handlerHelper, times(1)).addContextForRelationships(USER, collectionEntity, COLLECTION_MEMBERSHIP, collectionContext);
verify(handlerHelper, times(1)).addContextForRelationships(USER, portEntity, PORT_DELEGATION, portContext);
}


@Test
void buildProcessContext_withPortImplementation() throws OCFCheckedExceptionBase {
List<Relationship> collection = mockGetRelationships(COLLECTION_MEMBERSHIP, PROCESS);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@
@JsonSubTypes({
@JsonSubTypes.Type(value = DataEngineRegistrationEvent.class, name = "DataEngineRegistrationEvent"),
@JsonSubTypes.Type(value = DataFlowsEvent.class, name = "DataFlowsEvent"),
@JsonSubTypes.Type(value = PortAliasEvent.class, name = "PortAliasEvent"),
@JsonSubTypes.Type(value = PortImplementationEvent.class, name = "PortImplementationEvent"),
@JsonSubTypes.Type(value = ProcessEvent.class, name = "ProcessEvent"),
@JsonSubTypes.Type(value = ProcessListEvent.class, name = "ProcessListEvent"),
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -31,14 +31,12 @@ public enum DataEngineEventType implements Serializable {
UNKNOWN_DATA_ENGINE_EVENT(0, "Unknown Data Engine Job Event", "An event that is not recognized by the local server."),
DATA_ENGINE_REGISTRATION_EVENT(1, "DataEngineRegistrationEvent", "An event that register a data engine as external source."),
DATA_FLOWS_EVENT(2, "DataFlowsEvent", "An event that adds or update data flows."),
PORT_ALIAS_EVENT(3, "PortAliasEvent", "An event that add or update port alias"),
PORT_IMPLEMENTATION_EVENT(4, "PortImplementationEvent", "An event that creates or updates port implementations."),
SCHEMA_TYPE_EVENT(5, "SchemaTypeEvent", "An event that creates or updates schema types."),
PROCESS_HIERARCHY_EVENT(6, "ProcessHierarchyEvent", "An event to setup a process hierarchy."),
DELETE_DATA_ENGINE_EVENT(7, "DeleteDataEngineEvent", "An event that deletes an external data engine."),
DELETE_SCHEMA_TYPE_EVENT(8, "DeleteSchemaTypeEvent", "An event that deletes a schema type."),
DELETE_PORT_IMPLEMENTATION_EVENT(9, "DeletePortImplementationEvent", "An event that deletes a port implementation."),
DELETE_PORT_ALIAS_EVENT(10, "DeletePortAliasesEvent", "An event that deletes a port alias."),
DATABASE_EVENT(11, "DatabaseEvent", "An event that creates or updates databases."),
DATABASE_SCHEMA_EVENT(12, "DatabaseSchemaEvent", "An event that creates or updates database schemas."),
RELATIONAL_TABLE_EVENT(13, "RelationalTableEvent", "An event that creates or updates relational tables."),
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,7 @@
@JsonInclude(JsonInclude.Include.NON_NULL)
@JsonIgnoreProperties(ignoreUnknown = true)
@JsonSubTypes({
@JsonSubTypes.Type(value = PortImplementation.class, name = "PortImplementation"),
@JsonSubTypes.Type(value = PortAlias.class, name = "PortAlias")
@JsonSubTypes.Type(value = PortImplementation.class, name = "PortImplementation")
})
@Getter
@Setter
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -73,17 +73,6 @@ public class Process extends Asset {
*/
private List<PortImplementation> portImplementations;

/**
* The port aliases
* -- GETTER --
* Gets port aliases.
* @return the port aliases
* -- SETTER --
* Sets port aliases.
* @param portAliases the port aliases
*/
private List<PortAlias> portAliases;

/**
* The data flows
* -- GETTER --
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -28,7 +28,6 @@
{
@JsonSubTypes.Type(value = DataEngineRegistrationRequestBody.class, name = "dataEngine"),
@JsonSubTypes.Type(value = PortImplementationRequestBody.class, name = "port"),
@JsonSubTypes.Type(value = PortAliasRequestBody.class, name = "portAlias"),
@JsonSubTypes.Type(value = ProcessRequestBody.class, name = "process"),
@JsonSubTypes.Type(value = SchemaTypeRequestBody.class, name = "schema"),
@JsonSubTypes.Type(value = DatabaseRequestBody.class, name = "database"),
Expand Down

This file was deleted.

Original file line number Diff line number Diff line change
Expand Up @@ -61,8 +61,7 @@ Once you have an instance of the client, you can use it to create, update or del

* Create/Update/Delete a schema type
* Create/Update/Delete a port implementation, with a schema type and the corresponding PortSchema relationship
* Create/Update/Delete a port alias, with a PortDelegation relationship
* Create/Update/Delete a process, with a corresponding port implementations and port aliases involved in a transformation
* Create/Update/Delete a process, with a corresponding port implementations involved in a transformation
* Create/Update/Delete a database, with a connection, endpoint, connector type and corresponding schema type attached
* Create/Update/Delete a database schema
* Create/Update/Delete a relational table with columns
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
import org.odpi.openmetadata.accessservices.dataengine.model.DatabaseSchema;
import org.odpi.openmetadata.accessservices.dataengine.model.EventType;
import org.odpi.openmetadata.accessservices.dataengine.model.DataFlow;
import org.odpi.openmetadata.accessservices.dataengine.model.PortAlias;
import org.odpi.openmetadata.accessservices.dataengine.model.PortImplementation;
import org.odpi.openmetadata.accessservices.dataengine.model.Process;
import org.odpi.openmetadata.accessservices.dataengine.model.ProcessHierarchy;
Expand Down Expand Up @@ -175,43 +174,6 @@ void deletePortImplementation(String userId, String qualifiedName, String guid)
PropertyServerException,
UserNotAuthorizedException,
ConnectorCheckedException;

/**
* Create or update the port alias entity with a PortDelegation relationship. It attaches the port alias to the provided process
*
* @param userId the name of the calling user
* @param portAlias the port alias bean
* @param processQualifiedName the process qualified name
*
* @return unique identifier of the port alias in the repository
*
* @throws InvalidParameterException the bean properties are invalid
* @throws UserNotAuthorizedException user not authorized to issue this request
* @throws PropertyServerException problem accessing the property server
* @throws ConnectorCheckedException internal problem with the connector
*/
String createOrUpdatePortAlias(String userId, PortAlias portAlias, String processQualifiedName) throws InvalidParameterException,
UserNotAuthorizedException,
PropertyServerException,
ConnectorCheckedException;

/**
* Delete the port alias
*
* @param userId the name of the calling user
* @param qualifiedName the qualified name of the port alias
* @param guid the unique identifier of the port alias
*
* @throws InvalidParameterException the bean properties are invalid
* @throws UserNotAuthorizedException user not authorized to issue this request
* @throws PropertyServerException problem accessing the property server
* @throws ConnectorCheckedException problem with the underlying connector (if used)
*/
void deletePortAlias(String userId, String qualifiedName, String guid) throws InvalidParameterException,
PropertyServerException,
UserNotAuthorizedException,
ConnectorCheckedException;

/**
* Add a ProcessHierarchy relationship to a process
*
Expand Down
Loading