Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat(model): add created, lastModified auditstamps to SchemaField #4943

Merged
merged 2 commits into from
May 25, 2022
+147 −27
Merged

feat(model): add created, lastModified auditstamps to SchemaField #4943

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6a3e7dd
feat(model): add optional created, lastModified auditstamps to Schema…
mayurinehate May 19, 2022
b4deafe
feat(model): add ALTER operation type
mayurinehate May 19, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 8 additions & 2 deletions datahub-graphql-core/src/main/resources/entity.graphql
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5744,14 +5744,20 @@ enum OperationType {
DELETE

"""
When data is created.
When table is created.
"""
CREATE

"""
When data is dropped
When table is altered
"""
ALTER

"""
When table is dropped
"""
DROP

"""
Unknown operation
"""
Expand Down
16 changes: 16 additions & 0 deletions metadata-ingestion/examples/library/dataset_schema.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@

# Imports for metadata model classes
from datahub.metadata.schema_classes import (
AuditStampClass,
ChangeTypeClass,
DateTypeClass,
OtherSchemaClass,
Expand All @@ -25,24 +26,39 @@
version=0, # when the source system has a notion of versioning of schemas, insert this in, otherwise leave as 0
hash="", # when the source system has a notion of unique schemas identified via hash, include a hash, else leave it as empty string
platformSchema=OtherSchemaClass(rawSchema="__insert raw schema here__"),
lastModified=AuditStampClass(
time=1640692800000, actor="urn:li:corpuser:ingestion"
),
fields=[
SchemaFieldClass(
fieldPath="address.zipcode",
type=SchemaFieldDataTypeClass(type=StringTypeClass()),
nativeDataType="VARCHAR(50)", # use this to provide the type of the field in the source system's vernacular
description="This is the zipcode of the address. Specified using extended form and limited to addresses in the United States",
lastModified=AuditStampClass(
time=1640692800000, actor="urn:li:corpuser:ingestion"
),
),
SchemaFieldClass(
fieldPath="address.street",
type=SchemaFieldDataTypeClass(type=StringTypeClass()),
nativeDataType="VARCHAR(100)",
description="Street corresponding to the address",
lastModified=AuditStampClass(
time=1640692800000, actor="urn:li:corpuser:ingestion"
),
),
SchemaFieldClass(
fieldPath="last_sold_date",
type=SchemaFieldDataTypeClass(type=DateTypeClass()),
nativeDataType="Date",
description="Date of the last sale date for this property",
created=AuditStampClass(
time=1640692800000, actor="urn:li:corpuser:ingestion"
),
lastModified=AuditStampClass(
time=1640692800000, actor="urn:li:corpuser:ingestion"
),
),
],
),
Expand Down
1 change: 1 addition & 0 deletions metadata-models/src/main/pegasus/com/linkedin/common/OperationType.pdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ enum OperationType {
UPDATE
DELETE
CREATE
ALTER
DROP
UNKNOWN
}
11 changes: 11 additions & 0 deletions metadata-models/src/main/pegasus/com/linkedin/schema/SchemaField.pdl
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ namespace com.linkedin.schema
import com.linkedin.dataset.SchemaFieldPath
import com.linkedin.common.GlobalTags
import com.linkedin.common.GlossaryTerms
import com.linkedin.common.AuditStamp

/**
* SchemaField to describe metadata related to dataset schema.
Expand Down Expand Up @@ -39,6 +40,16 @@ record SchemaField {
}
description: optional string

/**
* An AuditStamp corresponding to the creation of this schema field.
*/
created: optional AuditStamp

/**
* An AuditStamp corresponding to the last modification of this schema field.
*/
lastModified: optional AuditStamp

/**
* Platform independent field type of the field.
*/
Expand Down
30 changes: 25 additions & 5 deletions metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -702,8 +702,10 @@
"doc" : "Instance of the data platform (e.g. db instance)",
"optional" : true,
"Searchable" : {
"addToFilters" : true,
"fieldName" : "platformInstance",
"fieldType" : "URN"
"fieldType" : "URN",
"filterNameOverride" : "Platform Instance"
}
} ],
"Aspect" : {
Expand Down Expand Up @@ -804,6 +806,10 @@
}
},
"doc" : "Urn of the applied tag",
"Relationship" : {
"entityTypes" : [ "tag" ],
"name" : "TaggedWith"
},
"Searchable" : {
"addToFilters" : true,
"fieldName" : "tags",
Expand Down Expand Up @@ -881,6 +887,10 @@
}
},
"doc" : "Urn of the applied glossary term",
"Relationship" : {
"entityTypes" : [ "glossaryTerm" ],
"name" : "TermedWith"
},
"Searchable" : {
"addToFilters" : true,
"fieldName" : "glossaryTerms",
Expand Down Expand Up @@ -2362,6 +2372,16 @@
"fieldName" : "fieldDescriptions",
"fieldType" : "TEXT"
}
}, {
"name" : "created",
"type" : "com.linkedin.common.AuditStamp",
"doc" : "An AuditStamp corresponding to the creation of this schema field.",
"optional" : true
}, {
"name" : "lastModified",
"type" : "com.linkedin.common.AuditStamp",
"doc" : "An AuditStamp corresponding to the last modification of this schema field.",
"optional" : true
}, {
"name" : "type",
"type" : {
Expand Down Expand Up @@ -2483,7 +2503,7 @@
"Relationship" : {
"/tags/*/tag" : {
"entityTypes" : [ "tag" ],
"name" : "FieldTaggedWith"
"name" : "SchemaFieldTaggedWith"
}
},
"Searchable" : {
Expand All @@ -2501,7 +2521,7 @@
"Relationship" : {
"/terms/*/urn" : {
"entityTypes" : [ "glossaryTerm" ],
"name" : "FieldWithGlossaryTerm"
"name" : "SchemaFieldWithGlossaryTerm"
}
},
"Searchable" : {
Expand Down Expand Up @@ -2670,7 +2690,7 @@
"Relationship" : {
"/tags/*/tag" : {
"entityTypes" : [ "tag" ],
"name" : "EditableFieldTaggedWith"
"name" : "EditableSchemaFieldTaggedWith"
}
},
"Searchable" : {
Expand All @@ -2688,7 +2708,7 @@
"Relationship" : {
"/terms/*/urn" : {
"entityTypes" : [ "glossaryTerm" ],
"name" : "EditableFieldWithGlossaryTerm"
"name" : "EditableSchemaFieldWithGlossaryTerm"
}
},
"Searchable" : {
Expand Down
39 changes: 31 additions & 8 deletions metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -581,8 +581,10 @@
"doc" : "Instance of the data platform (e.g. db instance)",
"optional" : true,
"Searchable" : {
"addToFilters" : true,
"fieldName" : "platformInstance",
"fieldType" : "URN"
"fieldType" : "URN",
"filterNameOverride" : "Platform Instance"
}
} ],
"Aspect" : {
Expand Down Expand Up @@ -785,6 +787,10 @@
}
},
"doc" : "Urn of the applied tag",
"Relationship" : {
"entityTypes" : [ "tag" ],
"name" : "TaggedWith"
},
"Searchable" : {
"addToFilters" : true,
"fieldName" : "tags",
Expand Down Expand Up @@ -862,6 +868,10 @@
}
},
"doc" : "Urn of the applied glossary term",
"Relationship" : {
"entityTypes" : [ "glossaryTerm" ],
"name" : "TermedWith"
},
"Searchable" : {
"addToFilters" : true,
"fieldName" : "glossaryTerms",
Expand Down Expand Up @@ -2755,6 +2765,16 @@
"fieldName" : "fieldDescriptions",
"fieldType" : "TEXT"
}
}, {
"name" : "created",
"type" : "com.linkedin.common.AuditStamp",
"doc" : "An AuditStamp corresponding to the creation of this schema field.",
"optional" : true
}, {
"name" : "lastModified",
"type" : "com.linkedin.common.AuditStamp",
"doc" : "An AuditStamp corresponding to the last modification of this schema field.",
"optional" : true
}, {
"name" : "type",
"type" : {
Expand Down Expand Up @@ -2876,7 +2896,7 @@
"Relationship" : {
"/tags/*/tag" : {
"entityTypes" : [ "tag" ],
"name" : "FieldTaggedWith"
"name" : "SchemaFieldTaggedWith"
}
},
"Searchable" : {
Expand All @@ -2894,7 +2914,7 @@
"Relationship" : {
"/terms/*/urn" : {
"entityTypes" : [ "glossaryTerm" ],
"name" : "FieldWithGlossaryTerm"
"name" : "SchemaFieldWithGlossaryTerm"
}
},
"Searchable" : {
Expand Down Expand Up @@ -3063,7 +3083,7 @@
"Relationship" : {
"/tags/*/tag" : {
"entityTypes" : [ "tag" ],
"name" : "EditableFieldTaggedWith"
"name" : "EditableSchemaFieldTaggedWith"
}
},
"Searchable" : {
Expand All @@ -3081,7 +3101,7 @@
"Relationship" : {
"/terms/*/urn" : {
"entityTypes" : [ "glossaryTerm" ],
"name" : "EditableFieldWithGlossaryTerm"
"name" : "EditableSchemaFieldWithGlossaryTerm"
}
},
"Searchable" : {
Expand Down Expand Up @@ -4602,12 +4622,15 @@
"type" : "string",
"doc" : "Display name of the Policy",
"Searchable" : {
"fieldType" : "KEYWORD"
"fieldType" : "TEXT_PARTIAL"
}
}, {
"name" : "description",
"type" : "string",
"doc" : "Description of the Policy"
"doc" : "Description of the Policy",
"Searchable" : {
"fieldType" : "TEXT"
}
}, {
"name" : "type",
"type" : "string",
Expand Down Expand Up @@ -5223,7 +5246,7 @@
"fields" : [ {
"name" : "entity",
"type" : "com.linkedin.common.Urn",
"doc" : " Urn of the entity containing a related aspect"
"doc" : " Urn of the entity that is referenced by the aspect."
}, {
"name" : "aspect",
"type" : "string"
Expand Down
30 changes: 25 additions & 5 deletions metadata-service/restli-api/src/main/snapshot/com.linkedin.entity.runs.snapshot.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -462,8 +462,10 @@
"doc" : "Instance of the data platform (e.g. db instance)",
"optional" : true,
"Searchable" : {
"addToFilters" : true,
"fieldName" : "platformInstance",
"fieldType" : "URN"
"fieldType" : "URN",
"filterNameOverride" : "Platform Instance"
}
} ],
"Aspect" : {
Expand Down Expand Up @@ -564,6 +566,10 @@
}
},
"doc" : "Urn of the applied tag",
"Relationship" : {
"entityTypes" : [ "tag" ],
"name" : "TaggedWith"
},
"Searchable" : {
"addToFilters" : true,
"fieldName" : "tags",
Expand Down Expand Up @@ -641,6 +647,10 @@
}
},
"doc" : "Urn of the applied glossary term",
"Relationship" : {
"entityTypes" : [ "glossaryTerm" ],
"name" : "TermedWith"
},
"Searchable" : {
"addToFilters" : true,
"fieldName" : "glossaryTerms",
Expand Down Expand Up @@ -2109,6 +2119,16 @@
"fieldName" : "fieldDescriptions",
"fieldType" : "TEXT"
}
}, {
"name" : "created",
"type" : "com.linkedin.common.AuditStamp",
"doc" : "An AuditStamp corresponding to the creation of this schema field.",
"optional" : true
}, {
"name" : "lastModified",
"type" : "com.linkedin.common.AuditStamp",
"doc" : "An AuditStamp corresponding to the last modification of this schema field.",
"optional" : true
}, {
"name" : "type",
"type" : {
Expand Down Expand Up @@ -2230,7 +2250,7 @@
"Relationship" : {
"/tags/*/tag" : {
"entityTypes" : [ "tag" ],
"name" : "FieldTaggedWith"
"name" : "SchemaFieldTaggedWith"
}
},
"Searchable" : {
Expand All @@ -2248,7 +2268,7 @@
"Relationship" : {
"/terms/*/urn" : {
"entityTypes" : [ "glossaryTerm" ],
"name" : "FieldWithGlossaryTerm"
"name" : "SchemaFieldWithGlossaryTerm"
}
},
"Searchable" : {
Expand Down Expand Up @@ -2417,7 +2437,7 @@
"Relationship" : {
"/tags/*/tag" : {
"entityTypes" : [ "tag" ],
"name" : "EditableFieldTaggedWith"
"name" : "EditableSchemaFieldTaggedWith"
}
},
"Searchable" : {
Expand All @@ -2435,7 +2455,7 @@
"Relationship" : {
"/terms/*/urn" : {
"entityTypes" : [ "glossaryTerm" ],
"name" : "EditableFieldWithGlossaryTerm"
"name" : "EditableSchemaFieldWithGlossaryTerm"
}
},
"Searchable" : {
Expand Down
Loading