feat(models) : Joins (Datasets) Schema

li-utils/src/main/java/com/linkedin/metadata/Constants.java

@@ -45,6 +45,7 @@ public class Constants {
   public static final String TAG_ENTITY_NAME = "tag";
   public static final String CONTAINER_ENTITY_NAME = "container";
   public static final String DOMAIN_ENTITY_NAME = "domain";
+  public static final String JOIN_ENTITY_NAME = "join";
   public static final String ASSERTION_ENTITY_NAME = "assertion";
   public static final String INGESTION_SOURCE_ENTITY_NAME = "dataHubIngestionSource";
   public static final String SECRETS_ENTITY_NAME = "dataHubSecret";
@@ -216,6 +217,11 @@ public class Constants {
   public static final String DOMAINS_ASPECT_NAME = "domains";
   public static final String DOMAIN_CREATED_TIME_INDEX_FIELD_NAME = "createdTime";
 
+  // Join
+  public static final String JOIN_KEY_ASPECT_NAME = "joinKey";
+  public static final String JOIN_PROPERTIES_ASPECT_NAME = "joinProperties";
+  public static final String EDITABLE_JOIN_PROPERTIES_ASPECT_NAME = "editableJoinProperties";
+
   // Assertion
   public static final String ASSERTION_KEY_ASPECT_NAME = "assertionKey";
   public static final String ASSERTION_INFO_ASPECT_NAME = "assertionInfo";

li-utils/src/main/javaPegasus/com/linkedin/common/urn/JoinUrn.java

@@ -0,0 +1,66 @@
+package com.linkedin.common.urn;
+
+import com.linkedin.data.template.Custom;
+import com.linkedin.data.template.DirectCoercer;
+import com.linkedin.data.template.TemplateOutputCastException;
+import java.net.URISyntaxException;
+
+
+public class JoinUrn extends Urn {
+  public static final String ENTITY_TYPE = "join";
+
+  private final String _joinId;
+
+  public JoinUrn(String name) {
+    super(ENTITY_TYPE, TupleKey.create(name));
+    this._joinId = name;
+  }
+
+  public String getName() {
+    return _joinId;
+  }
+
+  public static JoinUrn createFromString(String rawUrn) throws URISyntaxException {
+    return createFromUrn(Urn.createFromString(rawUrn));
+  }
+
+  public static JoinUrn createFromUrn(Urn urn) throws URISyntaxException {
+    if (!"li".equals(urn.getNamespace())) {
+      throw new URISyntaxException(urn.toString(), "Urn namespace type should be 'li'.");
+    } else if (!ENTITY_TYPE.equals(urn.getEntityType())) {
+      throw new URISyntaxException(urn.toString(), "Urn entity type should be 'join'.");
+    } else {
+      TupleKey key = urn.getEntityKey();
+      if (key.size() != 1) {
+        throw new URISyntaxException(urn.toString(), "Invalid number of keys.");
+      } else {
+        try {
+          return new JoinUrn((String) key.getAs(0, String.class));
+        } catch (Exception var3) {
+          throw new URISyntaxException(urn.toString(), "Invalid URN Parameter: '" + var3.getMessage());
+        }
+      }
+    }
+  }
+
+  public static JoinUrn deserialize(String rawUrn) throws URISyntaxException {
+    return createFromString(rawUrn);
+  }
+
+  static {
+    Custom.initializeCustomClass(JoinUrn.class);
+    Custom.registerCoercer(new DirectCoercer<JoinUrn>() {
+      public Object coerceInput(JoinUrn object) throws ClassCastException {
+        return object.toString();
+      }
+
+      public JoinUrn coerceOutput(Object object) throws TemplateOutputCastException {
+        try {
+          return JoinUrn.createFromString((String) object);
+        } catch (URISyntaxException e) {
+          throw new TemplateOutputCastException("Invalid URN syntax: " + e.getMessage(), e);
+        }
+      }
+    }, JoinUrn.class);
+  }
+}

li-utils/src/main/pegasus/com/linkedin/common/JoinUrn.pdl

@@ -0,0 +1,24 @@
+namespace com.linkedin.common
+
+/**
+ * Standardized join identifier.
+ */
+@java.class = "com.linkedin.common.urn.JoinUrn"
+@validate.`com.linkedin.common.validator.TypedUrnValidator` = {
+  "accessible" : true,
+  "owningTeam" : "urn:li:internalTeam:datahub",
+  "entityType" : "join",
+  "constructable" : true,
+  "namespace" : "li",
+  "name" : "Join",
+  "doc" : "Standardized Join identifier.",
+  "owners" : [ "urn:li:corpuser:fbar", "urn:li:corpuser:bfoo" ],
+  "fields" : [ {
+    "name" : "joinId",
+    "doc" : "Join native name e.g. <db>.<table>, /dir/subdir/<name>, or <name>",
+    "type" : "string",
+    "maxLength" : 284
+  }],
+  "maxLength" : 284
+}
+typeref JoinUrn = string

metadata-models/src/main/pegasus/com/linkedin/join/EditableJoinProperties.pdl

@@ -0,0 +1,31 @@
+namespace com.linkedin.join
+
+import com.linkedin.common.ChangeAuditStamps
+
+
+/**
+ * EditableJoinProperties stores editable changes made to join properties. This separates changes made from
+ * ingestion pipelines and edits in the UI to avoid accidental overwrites of user-provided data by ingestion pipelines
+ */
+@Aspect = {
+  "name": "editableJoinProperties"
+}
+record EditableJoinProperties includes ChangeAuditStamps {
+  /**
+   * Documentation of the join
+   */
+  @Searchable = {
+    "fieldType": "TEXT",
+    "fieldName": "editedDescription",
+  }
+  description: optional string
+
+  /**
+   * Display name of the Join
+   */
+  @Searchable = {
+    "fieldType": "TEXT_PARTIAL",
+    "fieldName": "editedName",
+  }
+  name: optional string
+}

metadata-models/src/main/pegasus/com/linkedin/join/FieldMap.pdl

@@ -0,0 +1,19 @@
+namespace com.linkedin.join
+
+import com.linkedin.dataset.SchemaFieldPath
+
+/**
+ * Field Mapping of 1:1 field
+ */
+record FieldMap {
+  /**
+   * All fields from dataset A that are required for the join, maps to bFields 1:1
+   */
+  afield: SchemaFieldPath
+
+  /**
+   * All fields from dataset B that are required for the join, maps to aFields 1:1
+   */
+  bfield: SchemaFieldPath
+
+}

metadata-models/src/main/pegasus/com/linkedin/join/JoinFieldMapping.pdl

@@ -0,0 +1,18 @@
+namespace com.linkedin.join
+
+import com.linkedin.dataset.SchemaFieldPath
+
+/**
+ * Field Mapping about a join between two datasets
+ */
+record JoinFieldMapping {
+  /**
+   * All fields from dataset A that are required for the join to dataset B
+   */
+  fieldMapping: array[FieldMap]
+
+  /**
+   *  Any transformation logic or notes pertaining to this specific join
+   */
+  details: string
+}

metadata-models/src/main/pegasus/com/linkedin/join/JoinProperties.pdl

@@ -0,0 +1,69 @@
+namespace com.linkedin.join
+
+import com.linkedin.common.TimeStamp
+import com.linkedin.common.DatasetUrn
+import com.linkedin.common.CustomProperties
+
+/**
+ * Properties associated with a Join
+ */
+@Aspect = {
+  "name": "joinProperties"
+}
+record JoinProperties includes CustomProperties {
+
+  /**
+   * Display name of the Join
+   */
+  @Searchable = {
+    "fieldType": "TEXT_PARTIAL",
+    "enableAutocomplete": true,
+    "boostScore": 10.0
+  }
+  name: optional string
+
+  /**
+  * First dataset in the join (no directionality)
+  */
+   @Relationship = {
+     "name": "joinA",
+     "entityTypes": [ "dataset" ]
+   }
+  datasetA: DatasetUrn
+
+  /**
+  * Second dataset in the join (no directionality)
+  */
+   @Relationship = {
+     "name": "joinB",
+     "entityTypes": [ "dataset" ]
+   }
+  datasetB: DatasetUrn
+
+  /**
+   * Array of JoinFieldMapping
+   */
+  joinFieldMappings: array[JoinFieldMapping]
+
+  /**
+   * A timestamp documenting when the asset was created in the source Data Platform (not on DataHub)
+   */
+  @Searchable = {
+    "/time": {
+      "fieldName": "createdAt",
+      "fieldType": "DATETIME"
+    }
+  }
+  created: optional TimeStamp
+
+  /**
+   * A timestamp documenting when the asset was last modified in the source Data Platform (not on DataHub)
+   */
+  @Searchable = {
+    "/time": {
+      "fieldName": "lastModifiedAt",
+      "fieldType": "DATETIME"
+    }
+  }
+  lastModified: optional TimeStamp
+}

metadata-models/src/main/pegasus/com/linkedin/join/Joins.pdl

@@ -0,0 +1,15 @@
+namespace com.linkedin.join
+import com.linkedin.common.JoinUrn
+
+/**
+ * Joins information of an entity.
+ */
+@Aspect = {
+  "name": "joins"
+}
+record Joins {
+  /**
+   * Join
+   */
+  joins: array[JoinUrn]
+}

metadata-models/src/main/pegasus/com/linkedin/metadata/aspect/JoinAspect.pdl

@@ -0,0 +1,30 @@
+namespace com.linkedin.metadata.aspect
+
+import com.linkedin.metadata.key.JoinKey
+import com.linkedin.join.JoinProperties
+import com.linkedin.join.EditableJoinProperties
+
+import com.linkedin.common.InstitutionalMemory
+import com.linkedin.common.Ownership
+import com.linkedin.common.Status
+import com.linkedin.container.Container
+import com.linkedin.common.GlobalTags
+import com.linkedin.common.GlossaryTerms
+import com.linkedin.common.BrowsePaths
+
+
+/**
+ * A union of all supported metadata aspects for a Join
+ */
+typeref JoinAspect = union[
+  JoinKey,
+  JoinProperties,
+  EditableJoinProperties,
+  InstitutionalMemory,
+  Ownership,
+  Status,
+  Container,
+  GlobalTags,
+  GlossaryTerms,
+  BrowsePaths
+]

metadata-models/src/main/pegasus/com/linkedin/metadata/key/JoinKey.pdl

@@ -0,0 +1,18 @@
+namespace com.linkedin.metadata.key
+
+/**
+ * Key for a Join
+ */
+@Aspect = {
+  "name": "joinKey"
+}
+record JoinKey {
+  /*
+  * Unique guid for Join
+  */
+  @Searchable = {
+    "fieldType": "TEXT",
+  }
+  joinId: string
+
+}

metadata-models/src/main/pegasus/com/linkedin/metadata/snapshot/JoinSnapshot.pdl

@@ -0,0 +1,24 @@
+namespace com.linkedin.metadata.snapshot
+
+import com.linkedin.common.JoinUrn
+import com.linkedin.metadata.aspect.JoinAspect
+
+/**
+ * A metadata snapshot for a specific join entity.
+ */
+@Entity = {
+  "name": "join",
+  "keyAspect": "joinKey"
+}
+record JoinSnapshot {
+
+  /**
+   * URN for the entity the metadata snapshot is associated with.
+   */
+  urn: JoinUrn
+
+  /**
+   * The list of metadata aspects associated with the Join. Depending on the use case, this can either be all, or a selection, of supported aspects.
+   */
+  aspects: array[JoinAspect]
+}

metadata-models/src/main/resources/entity-registry.yml

@@ -419,6 +419,19 @@ entities:
     keyAspect: dataHubViewKey
     aspects:
       - dataHubViewInfo
+  - name: join
+    category: core
+    doc: Join Dataset Fields
+    keyAspect: joinKey
+    aspects:
+      - joinProperties
+      - editableJoinProperties
+      - ownership
+      - status
+      - container
+      - globalTags
+      - glossaryTerms
+      - browsePaths
   - name: query
     category: core
     keyAspect: queryKey