Merge branch 'master' into remove-verbose-log-airflow-provider

datahub-project · Nov 26, 2022 · 6227813 · 6227813
2 parents 4e26016 + d424edd
commit 6227813
Show file tree

Hide file tree

Showing 146 changed files with 3,076 additions and 1,115 deletions.
diff --git a/.github/actions/docker-custom-build-and-push/action.yml b/.github/actions/docker-custom-build-and-push/action.yml
@@ -49,7 +49,7 @@ runs:
 
     # Code for testing the build when not pushing to Docker Hub.
     - name: Build and Load image for testing (if not publishing)
-      uses: docker/build-push-action@v2
+      uses: docker/build-push-action@v3
       if: ${{ inputs.publish != 'true' }}
       with:
         context: ${{ inputs.context }}
@@ -80,7 +80,7 @@ runs:
         username: ${{ inputs.username }}
         password: ${{ inputs.password }}
     - name: Build and Push Multi-Platform image
-      uses: docker/build-push-action@v2
+      uses: docker/build-push-action@v3
       if: ${{ inputs.publish == 'true' }}
       with:
         context: ${{ inputs.context }}

diff --git a/.github/workflows/docker-feast-source.yml b/.github/workflows/docker-feast-source.yml
@@ -35,15 +35,14 @@ jobs:
           SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
           echo "SHORT_SHA: $SHORT_SHA"
           TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,latest\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
-          echo "tag=$TAG"
-          echo "::set-output name=tag::$TAG"
+          echo "tag=$TAG" >> $GITHUB_OUTPUT
       - name: Check whether publishing enabled
         id: publish
         env:
           ENABLE_PUBLISH: ${{ secrets.ORG_DOCKER_PASSWORD }}
         run: |
           echo "Enable publish: ${{ env.ENABLE_PUBLISH != '' }}"
-          echo "::set-output name=publish::${{ env.ENABLE_PUBLISH != '' }}"
+          echo "publish=${{ env.ENABLE_PUBLISH != '' }}" >> $GITHUB_OUTPUT
   push_to_registries:
     name: Build and Push Docker Image to Docker Hub
     runs-on: ubuntu-latest
@@ -70,7 +69,7 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.ORG_DOCKER_PASSWORD }}
       - name: Build and Push image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           context: ./metadata-ingestion/src/datahub/ingestion/source/feast_image
           tags: ${{ steps.docker_meta.outputs.tags }}

diff --git a/.github/workflows/docker-ingestion-base.yml b/.github/workflows/docker-ingestion-base.yml
@@ -33,7 +33,7 @@ jobs:
           username: ${{ secrets.ACRYL_DOCKER_USERNAME }}
           password: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
       - name: Build and Push image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           context: ./docker/datahub-ingestion
           file: ./docker/datahub-ingestion/base.Dockerfile

diff --git a/.github/workflows/docker-ingestion-smoke.yml b/.github/workflows/docker-ingestion-smoke.yml
@@ -35,17 +35,15 @@ jobs:
           SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
           TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,head\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
           UNIQUE_TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
-          echo "tag=$TAG"
-          echo "unique_tag=$UNIQUE_TAG"
-          echo "::set-output name=tag::$TAG"
-          echo "::set-output name=unique_tag::$UNIQUE_TAG"
+          echo "tag=$TAG" >> $GITHUB_OUTPUT
+          echo "unique_tag=$UNIQUE_TAG" >> $GITHUB_OUTPUT
       - name: Check whether publishing enabled
         id: publish
         env:
           ENABLE_PUBLISH: ${{ secrets.ACRYL_DOCKER_PASSWORD }}
         run: |
           echo "Enable publish: ${{ env.ENABLE_PUBLISH != '' }}"
-          echo "::set-output name=publish::${{ env.ENABLE_PUBLISH != '' }}"
+          echo "publish=${{ env.ENABLE_PUBLISH != '' }}" >> $GITHUB_OUTPUT
   build-smoke:
     name: Build and Push Docker Image to Docker Hub
     runs-on: ubuntu-latest

diff --git a/.github/workflows/docker-ingestion.yml b/.github/workflows/docker-ingestion.yml
@@ -37,23 +37,21 @@ jobs:
           echo "GITHUB_REF: $GITHUB_REF"
           SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
           TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,head\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
-          echo "tag=$TAG"
-          echo "::set-output name=tag::$TAG"
+          echo "tag=$TAG" >> $GITHUB_OUTPUT
       - name: Compute Python Release Version
         id: python_release_version
         run: |
           echo "GITHUB_REF: $GITHUB_REF"
           SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
           RELEASE_VERSION=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,0.0.0+docker.${SHORT_SHA},g" -e 's,refs/tags/v\(.*\),\1+docker,g' -e 's,refs/pull/\([0-9]*\).*,0.0.0+docker.pr\1,g')
-          echo "release_version=$RELEASE_VERSION"
-          echo "::set-output name=release_version::$RELEASE_VERSION"
+          echo "release_version=$RELEASE_VERSION" >> $GITHUB_OUTPUT
       - name: Check whether publishing enabled
         id: publish
         env:
           ENABLE_PUBLISH: ${{ secrets.DOCKER_PASSWORD }}
         run: |
           echo "Enable publish: ${{ env.ENABLE_PUBLISH != '' }}"
-          echo "::set-output name=publish::${{ env.ENABLE_PUBLISH != '' }}"
+          echo "publish=${{ env.ENABLE_PUBLISH != '' }}" >> $GITHUB_OUTPUT
   push_to_registries:
     name: Build and Push Docker Image to Docker Hub
     runs-on: ubuntu-latest
@@ -84,7 +82,7 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.DOCKER_PASSWORD }}
       - name: Build and Push image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           context: .
           file: ./docker/datahub-ingestion/Dockerfile

diff --git a/.github/workflows/docker-postgres-setup.yml b/.github/workflows/docker-postgres-setup.yml
@@ -35,15 +35,14 @@ jobs:
           SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
           echo "SHORT_SHA: $SHORT_SHA"
           TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/.*$,head\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
-          echo "tag=$TAG"
-          echo "::set-output name=tag::$TAG"
+          echo "tag=$TAG" >> $GITHUB_OUTPUT
       - name: Check whether publishing enabled
         id: publish
         env:
           ENABLE_PUBLISH: ${{ secrets.ORG_DOCKER_PASSWORD }}
         run: |
           echo "Enable publish: ${{ env.ENABLE_PUBLISH != '' }}"
-          echo "::set-output name=publish::${{ env.ENABLE_PUBLISH != '' }}"
+          echo "publish=${{ env.ENABLE_PUBLISH != '' }}" >> $GITHUB_OUTPUT
   push_to_registries:
     name: Build and Push Docker Image to Docker Hub
     runs-on: ubuntu-latest
@@ -70,7 +69,7 @@ jobs:
           username: ${{ secrets.DOCKER_USERNAME }}
           password: ${{ secrets.ORG_DOCKER_PASSWORD }}
       - name: Build and Push image
-        uses: docker/build-push-action@v2
+        uses: docker/build-push-action@v3
         with:
           context: .
           file: ./docker/postgres-setup/Dockerfile

diff --git a/.github/workflows/docker-unified.yml b/.github/workflows/docker-unified.yml
@@ -49,17 +49,15 @@ jobs:
           SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
           TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,head\,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
           UNIQUE_TAG=$(echo ${GITHUB_REF} | sed -e "s,refs/heads/master,${SHORT_SHA},g" -e 's,refs/tags/,,g' -e 's,refs/pull/\([0-9]*\).*,pr\1,g')
-          echo "tag=$TAG"
-          echo "unique_tag=$UNIQUE_TAG"
-          echo "::set-output name=tag::$TAG"
-          echo "::set-output name=unique_tag::$UNIQUE_TAG"
+          echo "tag=$TAG" >> $GITHUB_OUTPUT
+          echo "unique_tag=$UNIQUE_TAG" >> $GITHUB_OUTPUT
       - name: Check whether publishing enabled
         id: publish
         env:
           ENABLE_PUBLISH: ${{ secrets.DOCKER_PASSWORD }}
         run: |
           echo "Enable publish: ${{ env.ENABLE_PUBLISH != '' }}"
-          echo "::set-output name=publish::${{ env.ENABLE_PUBLISH != '' }}"
+          echo "publish=${{ env.ENABLE_PUBLISH != '' }}" >> $GITHUB_OUTPUT
 
   gms_build:
     name: Build and Push DataHub GMS Docker Image

diff --git a/.github/workflows/metadata-model.yml b/.github/workflows/metadata-model.yml
@@ -25,7 +25,7 @@ jobs:
           ENABLE_PUBLISH: ${{ secrets.DataHubToken }}
         run: |
           echo "Enable publish: ${{ env.ENABLE_PUBLISH != '' }}"
-          echo "::set-output name=publish::${{ env.ENABLE_PUBLISH != '' }}"
+          echo "publish=${{ env.ENABLE_PUBLISH != '' }}" >> $GITHUB_OUTPUT
   metadata-ingestion-docgen:
     runs-on: ubuntu-latest
     needs: setup

diff --git a/.github/workflows/publish-datahub-jars.yml b/.github/workflows/publish-datahub-jars.yml
@@ -28,7 +28,7 @@ jobs:
           SIGNING_KEY: ${{ secrets.SIGNING_KEY }}
         run: |
           echo "Enable publish: ${{ env.SIGNING_KEY != '' }}"
-          echo "::set-output name=publish::${{ env.SIGNING_KEY != '' }}"
+          echo "publish=${{ env.SIGNING_KEY != '' }}" >> $GITHUB_OUTPUT
   setup:
     if: startsWith(github.ref, 'refs/tags/v')
     runs-on: ubuntu-latest
@@ -43,8 +43,7 @@ jobs:
           echo "GITHUB_REF: $GITHUB_REF"
           SHORT_SHA=$(git rev-parse --short "$GITHUB_SHA")
           TAG=$(echo ${GITHUB_REF} | sed -e 's,refs/tags/v,,g')
-          echo "tag=$TAG"
-          echo "::set-output name=tag::$TAG"
+          echo "tag=$TAG" >> $GITHUB_OUTPUT>> $GITHUB_OUTPUT
   publish:
     runs-on: ubuntu-latest
     needs: ["check-secret", "setup"]

diff --git a/...ore/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java b/...ore/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolver.java
@@ -1,5 +1,7 @@
 package com.linkedin.datahub.graphql.resolvers.domain;
 
+import com.linkedin.common.AuditStamp;
+import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.data.template.SetMode;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.authorization.AuthorizationUtils;
@@ -65,7 +67,7 @@ public CompletableFuture<String> get(DataFetchingEnvironment environment) throws
         proposal.setEntityKeyAspect(GenericRecordUtils.serializeAspect(key));
         proposal.setEntityType(Constants.DOMAIN_ENTITY_NAME);
         proposal.setAspectName(Constants.DOMAIN_PROPERTIES_ASPECT_NAME);
-        proposal.setAspect(GenericRecordUtils.serializeAspect(mapDomainProperties(input)));
+        proposal.setAspect(GenericRecordUtils.serializeAspect(mapDomainProperties(input, context)));
         proposal.setChangeType(ChangeType.UPSERT);
 
         String domainUrn = _entityClient.ingestProposal(proposal, context.getAuthentication());
@@ -78,10 +80,11 @@ public CompletableFuture<String> get(DataFetchingEnvironment environment) throws
     });
   }
 
-  private DomainProperties mapDomainProperties(final CreateDomainInput input) {
+  private DomainProperties mapDomainProperties(final CreateDomainInput input, final QueryContext context) {
     final DomainProperties result = new DomainProperties();
     result.setName(input.getName());
     result.setDescription(input.getDescription(), SetMode.IGNORE_NULL);
+    result.setCreated(new AuditStamp().setActor(UrnUtils.getUrn(context.getActorUrn())).setTime(System.currentTimeMillis()));
     return result;
   }
 }
diff --git a/...core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java b/...core/src/main/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolver.java
@@ -10,17 +10,19 @@
 import com.linkedin.datahub.graphql.generated.ListDomainsResult;
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.metadata.Constants;
+import com.linkedin.metadata.query.filter.SortCriterion;
+import com.linkedin.metadata.query.filter.SortOrder;
 import com.linkedin.metadata.search.SearchEntity;
 import com.linkedin.metadata.search.SearchResult;
 import graphql.schema.DataFetcher;
 import graphql.schema.DataFetchingEnvironment;
 import java.util.ArrayList;
-import java.util.Collections;
 import java.util.List;
 import java.util.concurrent.CompletableFuture;
 import java.util.stream.Collectors;
 
 import static com.linkedin.datahub.graphql.resolvers.ResolverUtils.*;
+import static com.linkedin.metadata.Constants.*;
 
 
 /**
@@ -56,7 +58,8 @@ public CompletableFuture<ListDomainsResult> get(final DataFetchingEnvironment en
           final SearchResult gmsResult = _entityClient.search(
                   Constants.DOMAIN_ENTITY_NAME,
                   query,
-                  Collections.emptyMap(),
+                  null,
+                  new SortCriterion().setField(DOMAIN_CREATED_TIME_INDEX_FIELD_NAME).setOrder(SortOrder.DESCENDING),
                   start,
                   count,
                   context.getAuthentication());

diff --git a/.../test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainProposalMatcher.java b/.../test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainProposalMatcher.java
@@ -0,0 +1,44 @@
+package com.linkedin.datahub.graphql.resolvers.domain;
+
+import com.linkedin.domain.DomainProperties;
+import com.linkedin.metadata.utils.GenericRecordUtils;
+import com.linkedin.mxe.GenericAspect;
+import com.linkedin.mxe.MetadataChangeProposal;
+import org.mockito.ArgumentMatcher;
+
+
+public class CreateDomainProposalMatcher implements ArgumentMatcher<MetadataChangeProposal> {
+
+  private MetadataChangeProposal left;
+
+  public CreateDomainProposalMatcher(MetadataChangeProposal left) {
+      this.left = left;
+  }
+
+  @Override
+  public boolean matches(MetadataChangeProposal right) {
+    return left.getEntityType().equals(right.getEntityType())
+        && left.getAspectName().equals(right.getAspectName())
+        && left.getChangeType().equals(right.getChangeType())
+        && domainPropertiesMatch(left.getAspect(), right.getAspect());
+  }
+
+  private boolean domainPropertiesMatch(GenericAspect left, GenericAspect right) {
+    DomainProperties leftProps = GenericRecordUtils.deserializeAspect(
+        left.getValue(),
+        "application/json",
+        DomainProperties.class
+    );
+
+    DomainProperties rightProps = GenericRecordUtils.deserializeAspect(
+        right.getValue(),
+        "application/json",
+        DomainProperties.class
+    );
+
+    // Omit timestamp comparison.
+    return leftProps.getName().equals(rightProps.getName())
+        && leftProps.getDescription().equals(rightProps.getDescription())
+        && leftProps.getCreated().getActor().equals(rightProps.getCreated().getActor());
+  }
+}
diff --git a/...src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java b/...src/test/java/com/linkedin/datahub/graphql/resolvers/domain/CreateDomainResolverTest.java
@@ -1,6 +1,9 @@
 package com.linkedin.datahub.graphql.resolvers.domain;
 
 import com.datahub.authentication.Authentication;
+import com.linkedin.common.AuditStamp;
+import com.linkedin.common.urn.Urn;
+import com.linkedin.common.urn.UrnUtils;
 import com.linkedin.datahub.graphql.QueryContext;
 import com.linkedin.datahub.graphql.generated.CreateDomainInput;
 import com.linkedin.domain.DomainProperties;
@@ -28,6 +31,7 @@ public class CreateDomainResolverTest {
       "test-name",
       "test-description"
   );
+  private static final Urn TEST_ACTOR_URN = UrnUtils.getUrn("urn:li:corpuser:test");
   private static final String TEST_ENTITY_URN = "urn:li:dataset:(urn:li:dataPlatform:mysql,my-test,PROD)";
   private static final String TEST_TAG_1_URN = "urn:li:tag:test-id-1";
   private static final String TEST_TAG_2_URN = "urn:li:tag:test-id-2";
@@ -55,13 +59,14 @@ public void testGetSuccess() throws Exception {
     DomainProperties props = new DomainProperties();
     props.setDescription("test-description");
     props.setName("test-name");
+    props.setCreated(new AuditStamp().setActor(TEST_ACTOR_URN).setTime(0L));
     proposal.setAspectName(Constants.DOMAIN_PROPERTIES_ASPECT_NAME);
     proposal.setAspect(GenericRecordUtils.serializeAspect(props));
     proposal.setChangeType(ChangeType.UPSERT);
 
     // Not ideal to match against "any", but we don't know the auto-generated execution request id
     Mockito.verify(mockClient, Mockito.times(1)).ingestProposal(
-        Mockito.eq(proposal),
+        Mockito.argThat(new CreateDomainProposalMatcher(proposal)),
         Mockito.any(Authentication.class)
     );
   }

diff --git a/.../src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolverTest.java b/.../src/test/java/com/linkedin/datahub/graphql/resolvers/domain/ListDomainsResolverTest.java
@@ -7,17 +7,19 @@
 import com.linkedin.datahub.graphql.generated.ListDomainsInput;
 import com.linkedin.entity.client.EntityClient;
 import com.linkedin.metadata.Constants;
+import com.linkedin.metadata.query.filter.SortCriterion;
+import com.linkedin.metadata.query.filter.SortOrder;
 import com.linkedin.metadata.search.SearchEntity;
 import com.linkedin.metadata.search.SearchEntityArray;
 import com.linkedin.metadata.search.SearchResult;
 import com.linkedin.r2.RemoteInvocationException;
 import graphql.schema.DataFetchingEnvironment;
-import java.util.Collections;
 import java.util.concurrent.CompletionException;
 import org.mockito.Mockito;
 import org.testng.annotations.Test;
 
 import static com.linkedin.datahub.graphql.TestUtils.*;
+import static com.linkedin.metadata.Constants.*;
 import static org.testng.Assert.*;
 
 
@@ -37,7 +39,8 @@ public void testGetSuccess() throws Exception {
     Mockito.when(mockClient.search(
         Mockito.eq(Constants.DOMAIN_ENTITY_NAME),
         Mockito.eq(""),
-        Mockito.eq(Collections.emptyMap()),
+        Mockito.eq(null),
+        Mockito.eq(new SortCriterion().setField(DOMAIN_CREATED_TIME_INDEX_FIELD_NAME).setOrder(SortOrder.DESCENDING)),
         Mockito.eq(0),
         Mockito.eq(20),
         Mockito.any(Authentication.class)

diff --git a/datahub-ranger-plugin/build.gradle b/datahub-ranger-plugin/build.gradle
@@ -24,7 +24,7 @@ compileJava {
 }
 
 dependencies {
-    implementation 'org.apache.ranger:ranger-plugins-common:2.2.0'
+    implementation 'org.apache.ranger:ranger-plugins-common:2.3.0'
     compile 'org.apache.logging.log4j:log4j-1.2-api:2.17.1'
 
     testCompile externalDependency.testng

diff --git a/datahub-upgrade/build.gradle b/datahub-upgrade/build.gradle
@@ -27,6 +27,8 @@ dependencies {
   runtime externalDependency.mysqlConnector
   runtime externalDependency.postgresql
 
+  implementation externalDependency.awsMskIamAuth
+
   annotationProcessor externalDependency.lombok
   annotationProcessor externalDependency.picocli
 }