From 469654ced75c3340276028068a6ca201eadc0cdf Mon Sep 17 00:00:00 2001 From: David Leifker Date: Fri, 9 Aug 2024 12:40:34 -0500 Subject: [PATCH] Revert "fix(openapi): fix openapi v2 endpoints & v3 documentation update" This reverts commit 573c1cb8407c2a5d152e5abb6b7d9f012eea75cb. --- docs/api/tutorials/structured-properties.md | 204 ++++++++---------- .../controller/GenericEntitiesController.java | 23 +- .../v2/controller/EntityController.java | 25 --- .../v3/controller/EntityController.java | 27 --- 4 files changed, 110 insertions(+), 169 deletions(-) diff --git a/docs/api/tutorials/structured-properties.md b/docs/api/tutorials/structured-properties.md index 00e992f2bd0bb..6f6c6541554d9 100644 --- a/docs/api/tutorials/structured-properties.md +++ b/docs/api/tutorials/structured-properties.md @@ -158,37 +158,29 @@ curl -X 'POST' -v \ -H 'accept: application/json' \ -H 'Content-Type: application/json' \ -d '{ - "value": { "qualifiedName": "io.acryl.privacy.retentionTime", - "valueType": "urn:li:dataType:datahub.number", - "description": "Retention Time is used to figure out how long to retain records in a dataset", - "displayName": "Retention Time", - "cardinality": "MULTIPLE", - "entityTypes": [ - "urn:li:entityType:datahub.dataset", - "urn:li:entityType:datahub.dataFlow" - ], - "allowedValues": [ - { - "value": { - "double": 30 - }, - "description": "30 days, usually reserved for datasets that are ephemeral and contain pii" - }, - { - "value": { - "double": 60 - }, - "description": "Use this for datasets that drive monthly reporting but contain pii" - }, - { - "value": { - "double": 365 - }, - "description": "Use this for non-sensitive data that can be retained for longer" - } - ] - } + "valueType": "urn:li:dataType:datahub.number", + "description": "Retention Time is used to figure out how long to retain records in a dataset", + "displayName": "Retention Time", + "cardinality": "MULTIPLE", + "entityTypes": [ + "urn:li:entityType:datahub.dataset", + "urn:li:entityType:datahub.dataFlow" + ], + "allowedValues": [ + { + "value": {"double": 30}, + "description": "30 days, usually reserved for datasets that are ephemeral and contain pii" + }, + { + "value": {"double": 60}, + "description": "Use this for datasets that drive monthly reporting but contain pii" + }, + { + "value": {"double": 365}, + "description": "Use this for non-sensitive data that can be retained for longer" + } + ] }' | jq ``` @@ -482,16 +474,14 @@ curl -X 'POST' -v \ -H 'accept: application/json' \ -H 'Content-Type: application/json' \ -d '{ - "value": { - "properties": [ - { - "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", - "values": [ - {"double": 60.0} - ] - } - ] - } + "properties": [ + { + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "values": [ + {"double": 60.0} + ] + } + ] }' | jq ``` Example Response: @@ -637,25 +627,23 @@ curl -X 'POST' -v \ -H 'accept: application/json' \ -H 'Content-Type: application/json' \ -d '{ - "value": { - "qualifiedName": "io.acryl.privacy.retentionTime02", - "displayName": "Retention Time 02", - "valueType": "urn:li:dataType:datahub.string", - "allowedValues": [ - { - "value": {"string": "foo2"}, - "description": "test foo2 value" - }, - { - "value": {"string": "bar2"}, - "description": "test bar2 value" - } - ], - "cardinality": "SINGLE", - "entityTypes": [ - "urn:li:entityType:datahub.dataset" - ] - } + "qualifiedName": "io.acryl.privacy.retentionTime02", + "displayName": "Retention Time 02", + "valueType": "urn:li:dataType:datahub.string", + "allowedValues": [ + { + "value": {"string": "foo2"}, + "description": "test foo2 value" + }, + { + "value": {"string": "bar2"}, + "description": "test bar2 value" + } + ], + "cardinality": "SINGLE", + "entityTypes": [ + "urn:li:entityType:datahub.dataset" + ] }' | jq ``` @@ -698,26 +686,24 @@ Specically, this will set `io.acryl.privacy.retentionTime` as `60.0` and `io.acr ```shell curl -X 'POST' -v \ - 'http://localhost:8080/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CSampleHiveDataset%2CPROD%29/structuredProperties?createIfNotExists=false' \ + 'http://localhost:8080/openapi/v3/entity/dataset/urn%3Ali%3Adataset%3A%28urn%3Ali%3AdataPlatform%3Ahive%2CSampleHiveDataset%2CPROD%29/structuredProperties' \ -H 'accept: application/json' \ -H 'Content-Type: application/json' \ -d '{ - "value": { - "properties": [ - { - "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", - "values": [ - {"double": 60.0} - ] - }, - { - "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime02", - "values": [ - {"string": "bar2"} - ] - } - ] - } + "properties": [ + { + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime", + "values": [ + {"double": 60.0} + ] + }, + { + "propertyUrn": "urn:li:structuredProperty:io.acryl.privacy.retentionTime02", + "values": [ + {"string": "bar2"} + ] + } + ] }' | jq ``` @@ -1125,9 +1111,7 @@ curl -X 'POST' \ -H 'accept: application/json' \ -H 'Content-Type: application/json' \ -d '{ - "value": { - "removed": true - } +"removed": true }' | jq ``` @@ -1148,13 +1132,11 @@ If you want to **remove the soft delete**, you can do so by either hard deleting ```shell curl -X 'POST' \ - 'http://localhost:8080/openapi/v3/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/status?systemMetadata=false&createIfNotExists=false' \ + 'http://localhost:8080/openapi/v3/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/status?systemMetadata=false' \ -H 'accept: application/json' \ -H 'Content-Type: application/json' \ -d '{ - "value": { - "removed": true - } +"removed": false }' | jq ``` @@ -1289,42 +1271,34 @@ Change the cardinality to `SINGLE` and add a `version`. ```shell curl -X 'POST' -v \ - 'http://localhost:8080/openapi/v3/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/propertyDefinition?createIfNotExists=false' \ + 'http://localhost:8080/openapi/v3/entity/structuredProperty/urn%3Ali%3AstructuredProperty%3Aio.acryl.privacy.retentionTime/propertyDefinition' \ -H 'accept: application/json' \ -H 'Content-Type: application/json' \ -d '{ - "value": { "qualifiedName": "io.acryl.privacy.retentionTime", - "valueType": "urn:li:dataType:datahub.number", - "description": "Retention Time is used to figure out how long to retain records in a dataset", - "displayName": "Retention Time", - "cardinality": "SINGLE", - "version": "20240614080000", - "entityTypes": [ - "urn:li:entityType:datahub.dataset", - "urn:li:entityType:datahub.dataFlow" - ], - "allowedValues": [ - { - "value": { - "double": 30 - }, - "description": "30 days, usually reserved for datasets that are ephemeral and contain pii" - }, - { - "value": { - "double": 60 - }, - "description": "Use this for datasets that drive monthly reporting but contain pii" - }, - { - "value": { - "double": 365 - }, - "description": "Use this for non-sensitive data that can be retained for longer" - } - ] - } + "valueType": "urn:li:dataType:datahub.number", + "description": "Retention Time is used to figure out how long to retain records in a dataset", + "displayName": "Retention Time", + "cardinality": "SINGLE", + "version": "20240614080000", + "entityTypes": [ + "urn:li:entityType:datahub.dataset", + "urn:li:entityType:datahub.dataFlow" + ], + "allowedValues": [ + { + "value": {"double": 30}, + "description": "30 days, usually reserved for datasets that are ephemeral and contain pii" + }, + { + "value": {"double": 60}, + "description": "Use this for datasets that drive monthly reporting but contain pii" + }, + { + "value": {"double": 365}, + "description": "Use this for non-sensitive data that can be retained for longer" + } + ] }' | jq ``` diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java index f415a4f47c9dc..de5d2ae1118d4 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/controller/GenericEntitiesController.java @@ -13,11 +13,14 @@ import com.datahub.authorization.AuthorizerChain; import com.datahub.util.RecordUtils; import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.JsonNode; import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableSet; import com.linkedin.common.urn.Urn; +import com.linkedin.data.ByteString; import com.linkedin.data.template.RecordTemplate; import com.linkedin.entity.EnvelopedAspect; +import com.linkedin.events.metadata.ChangeType; import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.aspect.batch.AspectsBatch; import com.linkedin.metadata.aspect.batch.ChangeMCP; @@ -38,6 +41,7 @@ import com.linkedin.metadata.search.SearchEntityArray; import com.linkedin.metadata.search.SearchService; import com.linkedin.metadata.utils.AuditStampUtils; +import com.linkedin.metadata.utils.GenericRecordUtils; import com.linkedin.metadata.utils.SearchUtil; import com.linkedin.mxe.SystemMetadata; import com.linkedin.util.Pair; @@ -53,6 +57,7 @@ import jakarta.servlet.http.HttpServletRequest; import java.lang.reflect.InvocationTargetException; import java.net.URISyntaxException; +import java.nio.charset.StandardCharsets; import java.util.*; import java.util.stream.Collectors; import javax.annotation.Nonnull; @@ -721,14 +726,28 @@ protected RecordTemplate toRecordTemplate( aspectSpec.getDataTemplateClass(), envelopedAspect.getValue().data()); } - protected abstract ChangeMCP toUpsertItem( + protected ChangeMCP toUpsertItem( @Nonnull AspectRetriever aspectRetriever, Urn entityUrn, AspectSpec aspectSpec, Boolean createIfNotExists, String jsonAspect, Actor actor) - throws URISyntaxException, JsonProcessingException; + throws JsonProcessingException { + JsonNode jsonNode = objectMapper.readTree(jsonAspect); + String aspectJson = jsonNode.get("value").toString(); + return ChangeItemImpl.builder() + .urn(entityUrn) + .aspectName(aspectSpec.getName()) + .changeType(Boolean.TRUE.equals(createIfNotExists) ? ChangeType.CREATE : ChangeType.UPSERT) + .auditStamp(AuditStampUtils.createAuditStamp(actor.toUrnStr())) + .recordTemplate( + GenericRecordUtils.deserializeAspect( + ByteString.copyString(aspectJson, StandardCharsets.UTF_8), + GenericRecordUtils.JSON, + aspectSpec)) + .build(aspectRetriever); + } protected ChangeMCP toUpsertItem( @Nonnull AspectRetriever aspectRetriever, diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java index 1207eb331b795..54a7724cadd34 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v2/controller/EntityController.java @@ -13,11 +13,8 @@ import com.linkedin.data.ByteString; import com.linkedin.data.template.RecordTemplate; import com.linkedin.entity.EnvelopedAspect; -import com.linkedin.events.metadata.ChangeType; -import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.aspect.batch.AspectsBatch; import com.linkedin.metadata.aspect.batch.BatchItem; -import com.linkedin.metadata.aspect.batch.ChangeMCP; import com.linkedin.metadata.entity.EntityApiUtils; import com.linkedin.metadata.entity.IngestResult; import com.linkedin.metadata.entity.UpdateAspectResult; @@ -263,26 +260,4 @@ protected List buildEntityList( } return responseList; } - - @Override - protected ChangeMCP toUpsertItem( - @Nonnull AspectRetriever aspectRetriever, - Urn entityUrn, - AspectSpec aspectSpec, - Boolean createIfNotExists, - String jsonAspect, - Actor actor) - throws URISyntaxException { - return ChangeItemImpl.builder() - .urn(entityUrn) - .aspectName(aspectSpec.getName()) - .changeType(Boolean.TRUE.equals(createIfNotExists) ? ChangeType.CREATE : ChangeType.UPSERT) - .auditStamp(AuditStampUtils.createAuditStamp(actor.toUrnStr())) - .recordTemplate( - GenericRecordUtils.deserializeAspect( - ByteString.copyString(jsonAspect, StandardCharsets.UTF_8), - GenericRecordUtils.JSON, - aspectSpec)) - .build(aspectRetriever); - } } diff --git a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java index fbc9bf2956cfd..a0478c9af1609 100644 --- a/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java +++ b/metadata-service/openapi-servlet/src/main/java/io/datahubproject/openapi/v3/controller/EntityController.java @@ -14,11 +14,8 @@ import com.linkedin.common.urn.Urn; import com.linkedin.data.ByteString; import com.linkedin.entity.EnvelopedAspect; -import com.linkedin.events.metadata.ChangeType; -import com.linkedin.metadata.aspect.AspectRetriever; import com.linkedin.metadata.aspect.batch.AspectsBatch; import com.linkedin.metadata.aspect.batch.BatchItem; -import com.linkedin.metadata.aspect.batch.ChangeMCP; import com.linkedin.metadata.entity.EntityApiUtils; import com.linkedin.metadata.entity.IngestResult; import com.linkedin.metadata.entity.UpdateAspectResult; @@ -351,28 +348,4 @@ protected AspectsBatch toMCPBatch( .retrieverContext(opContext.getRetrieverContext().get()) .build(); } - - @Override - protected ChangeMCP toUpsertItem( - @Nonnull AspectRetriever aspectRetriever, - Urn entityUrn, - AspectSpec aspectSpec, - Boolean createIfNotExists, - String jsonAspect, - Actor actor) - throws JsonProcessingException { - JsonNode jsonNode = objectMapper.readTree(jsonAspect); - String aspectJson = jsonNode.get("value").toString(); - return ChangeItemImpl.builder() - .urn(entityUrn) - .aspectName(aspectSpec.getName()) - .changeType(Boolean.TRUE.equals(createIfNotExists) ? ChangeType.CREATE : ChangeType.UPSERT) - .auditStamp(AuditStampUtils.createAuditStamp(actor.toUrnStr())) - .recordTemplate( - GenericRecordUtils.deserializeAspect( - ByteString.copyString(aspectJson, StandardCharsets.UTF_8), - GenericRecordUtils.JSON, - aspectSpec)) - .build(aspectRetriever); - } }