feat(datajob): Backend implementation

docker/elasticsearch-setup/create-indices.sh

@@ -14,6 +14,8 @@ function create_index {
 create_index chartdocument chart/settings.json chart/mappings.json
 create_index corpuserinfodocument corp-user/settings.json corp-user/mappings.json
 create_index dashboarddocument dashboard/settings.json dashboard/mappings.json
+create_index datajobdocument datajob/settings.json datajob/mappings.json
+create_index dataflowdocument dataflow/settings.json dataflow/mappings.json
 create_index dataprocessdocument data-process/settings.json data-process/mappings.json
 create_index datasetdocument dataset/settings.json dataset/mappings.json
 create_index mlmodeldocument ml-model/settings.json ml-model/mappings.json

gms/api/src/main/idl/com.linkedin.dataflow.dataFlows.restspec.json

@@ -0,0 +1,141 @@
+{
+  "name" : "dataFlows",
+  "namespace" : "com.linkedin.dataflow",
+  "path" : "/dataFlows",
+  "schema" : "com.linkedin.datajob.DataFlow",
+  "doc" : "generated from: com.linkedin.metadata.resources.datajob.DataFlows",
+  "collection" : {
+    "identifier" : {
+      "name" : "key",
+      "type" : "com.linkedin.datajob.DataFlowKey",
+      "params" : "com.linkedin.restli.common.EmptyRecord"
+    },
+    "supports" : [ "batch_get", "get", "get_all" ],
+    "methods" : [ {
+      "method" : "get",
+      "parameters" : [ {
+        "name" : "aspects",
+        "type" : "{ \"type\" : \"array\", \"items\" : \"string\" }",
+        "optional" : true
+      } ]
+    }, {
+      "method" : "batch_get",
+      "parameters" : [ {
+        "name" : "aspects",
+        "type" : "{ \"type\" : \"array\", \"items\" : \"string\" }",
+        "optional" : true
+      } ]
+    }, {
+      "method" : "get_all",
+      "parameters" : [ {
+        "name" : "aspects",
+        "type" : "{ \"type\" : \"array\", \"items\" : \"string\" }",
+        "optional" : true
+      }, {
+        "name" : "filter",
+        "type" : "com.linkedin.metadata.query.Filter",
+        "optional" : true
+      }, {
+        "name" : "sort",
+        "type" : "com.linkedin.metadata.query.SortCriterion",
+        "optional" : true
+      } ],
+      "pagingSupported" : true
+    } ],
+    "finders" : [ {
+      "name" : "search",
+      "parameters" : [ {
+        "name" : "input",
+        "type" : "string"
+      }, {
+        "name" : "aspects",
+        "type" : "{ \"type\" : \"array\", \"items\" : \"string\" }",
+        "optional" : true
+      }, {
+        "name" : "filter",
+        "type" : "com.linkedin.metadata.query.Filter",
+        "optional" : true
+      }, {
+        "name" : "sort",
+        "type" : "com.linkedin.metadata.query.SortCriterion",
+        "optional" : true
+      } ],
+      "metadata" : {
+        "type" : "com.linkedin.metadata.query.SearchResultMetadata"
+      },
+      "pagingSupported" : true
+    } ],
+    "actions" : [ {
+      "name" : "autocomplete",
+      "parameters" : [ {
+        "name" : "query",
+        "type" : "string"
+      }, {
+        "name" : "field",
+        "type" : "string"
+      }, {
+        "name" : "filter",
+        "type" : "com.linkedin.metadata.query.Filter"
+      }, {
+        "name" : "limit",
+        "type" : "int"
+      } ],
+      "returns" : "com.linkedin.metadata.query.AutoCompleteResult"
+    }, {
+      "name" : "backfillWithUrns",
+      "parameters" : [ {
+        "name" : "urns",
+        "type" : "{ \"type\" : \"array\", \"items\" : \"string\" }"
+      }, {
+        "name" : "aspects",
+        "type" : "{ \"type\" : \"array\", \"items\" : \"string\" }",
+        "optional" : true
+      } ],
+      "returns" : "com.linkedin.metadata.restli.BackfillResult"
+    }, {
+      "name" : "browse",
+      "parameters" : [ {
+        "name" : "path",
+        "type" : "string"
+      }, {
+        "name" : "filter",
+        "type" : "com.linkedin.metadata.query.Filter",
+        "optional" : true
+      }, {
+        "name" : "start",
+        "type" : "int"
+      }, {
+        "name" : "limit",
+        "type" : "int"
+      } ],
+      "returns" : "com.linkedin.metadata.query.BrowseResult"
+    }, {
+      "name" : "getBrowsePaths",
+      "parameters" : [ {
+        "name" : "urn",
+        "type" : "com.linkedin.common.Urn"
+      } ],
+      "returns" : "{ \"type\" : \"array\", \"items\" : \"string\" }"
+    }, {
+      "name" : "getSnapshot",
+      "parameters" : [ {
+        "name" : "urn",
+        "type" : "string"
+      }, {
+        "name" : "aspects",
+        "type" : "{ \"type\" : \"array\", \"items\" : \"string\" }",
+        "optional" : true
+      } ],
+      "returns" : "com.linkedin.metadata.snapshot.DataFlowSnapshot"
+    }, {
+      "name" : "ingest",
+      "parameters" : [ {
+        "name" : "snapshot",
+        "type" : "com.linkedin.metadata.snapshot.DataFlowSnapshot"
+      } ]
+    } ],
+    "entity" : {
+      "path" : "/dataFlows/{key}"
+    }
+  }
+}

gms/api/src/main/idl/com.linkedin.datajob.dataJobs.restspec.json

@@ -0,0 +1,141 @@
+{
+  "name" : "dataJobs",
+  "namespace" : "com.linkedin.datajob",
+  "path" : "/dataJobs",
+  "schema" : "com.linkedin.datajob.DataJob",
+  "doc" : "generated from: com.linkedin.metadata.resources.datajob.DataJobs",
+  "collection" : {
+    "identifier" : {
+      "name" : "key",
+      "type" : "com.linkedin.datajob.DataJobKey",
+      "params" : "com.linkedin.restli.common.EmptyRecord"
+    },
+    "supports" : [ "batch_get", "get", "get_all" ],
+    "methods" : [ {
+      "method" : "get",
+      "parameters" : [ {
+        "name" : "aspects",
+        "type" : "{ \"type\" : \"array\", \"items\" : \"string\" }",
+        "optional" : true
+      } ]
+    }, {
+      "method" : "batch_get",
+      "parameters" : [ {
+        "name" : "aspects",
+        "type" : "{ \"type\" : \"array\", \"items\" : \"string\" }",
+        "optional" : true
+      } ]
+    }, {
+      "method" : "get_all",
+      "parameters" : [ {
+        "name" : "aspects",
+        "type" : "{ \"type\" : \"array\", \"items\" : \"string\" }",
+        "optional" : true
+      }, {
+        "name" : "filter",
+        "type" : "com.linkedin.metadata.query.Filter",
+        "optional" : true
+      }, {
+        "name" : "sort",
+        "type" : "com.linkedin.metadata.query.SortCriterion",
+        "optional" : true
+      } ],
+      "pagingSupported" : true
+    } ],
+    "finders" : [ {
+      "name" : "search",
+      "parameters" : [ {
+        "name" : "input",
+        "type" : "string"
+      }, {
+        "name" : "aspects",
+        "type" : "{ \"type\" : \"array\", \"items\" : \"string\" }",
+        "optional" : true
+      }, {
+        "name" : "filter",
+        "type" : "com.linkedin.metadata.query.Filter",
+        "optional" : true
+      }, {
+        "name" : "sort",
+        "type" : "com.linkedin.metadata.query.SortCriterion",
+        "optional" : true
+      } ],
+      "metadata" : {
+        "type" : "com.linkedin.metadata.query.SearchResultMetadata"
+      },
+      "pagingSupported" : true
+    } ],
+    "actions" : [ {
+      "name" : "autocomplete",
+      "parameters" : [ {
+        "name" : "query",
+        "type" : "string"
+      }, {
+        "name" : "field",
+        "type" : "string"
+      }, {
+        "name" : "filter",
+        "type" : "com.linkedin.metadata.query.Filter"
+      }, {
+        "name" : "limit",
+        "type" : "int"
+      } ],
+      "returns" : "com.linkedin.metadata.query.AutoCompleteResult"
+    }, {
+      "name" : "backfillWithUrns",
+      "parameters" : [ {
+        "name" : "urns",
+        "type" : "{ \"type\" : \"array\", \"items\" : \"string\" }"
+      }, {
+        "name" : "aspects",
+        "type" : "{ \"type\" : \"array\", \"items\" : \"string\" }",
+        "optional" : true
+      } ],
+      "returns" : "com.linkedin.metadata.restli.BackfillResult"
+    }, {
+      "name" : "browse",
+      "parameters" : [ {
+        "name" : "path",
+        "type" : "string"
+      }, {
+        "name" : "filter",
+        "type" : "com.linkedin.metadata.query.Filter",
+        "optional" : true
+      }, {
+        "name" : "start",
+        "type" : "int"
+      }, {
+        "name" : "limit",
+        "type" : "int"
+      } ],
+      "returns" : "com.linkedin.metadata.query.BrowseResult"
+    }, {
+      "name" : "getBrowsePaths",
+      "parameters" : [ {
+        "name" : "urn",
+        "type" : "com.linkedin.common.Urn"
+      } ],
+      "returns" : "{ \"type\" : \"array\", \"items\" : \"string\" }"
+    }, {
+      "name" : "getSnapshot",
+      "parameters" : [ {
+        "name" : "urn",
+        "type" : "string"
+      }, {
+        "name" : "aspects",
+        "type" : "{ \"type\" : \"array\", \"items\" : \"string\" }",
+        "optional" : true
+      } ],
+      "returns" : "com.linkedin.metadata.snapshot.DataJobSnapshot"
+    }, {
+      "name" : "ingest",
+      "parameters" : [ {
+        "name" : "snapshot",
+        "type" : "com.linkedin.metadata.snapshot.DataJobSnapshot"
+      } ]
+    } ],
+    "entity" : {
+      "path" : "/dataJobs/{key}"
+    }
+  }
+}

gms/api/src/main/pegasus/com/linkedin/datajob/DataFlow.pdl

@@ -0,0 +1,26 @@
+namespace com.linkedin.datajob
+
+import com.linkedin.common.ChangeAuditStamps
+import com.linkedin.common.DataFlowUrn
+import com.linkedin.common.Ownership
+
+/**
+ * Metadata for DataFlow
+ */
+record DataFlow includes DataFlowKey, ChangeAuditStamps {
+  /**
+   * DataFlow urn
+   */
+  urn: DataFlowUrn
+
+  /**
+   * Ownership Info
+   */
+  ownership: optional Ownership
+
+  /**
+   * Optional additional metadata about the data flow
+   */
+   info: optional DataFlowInfo
+
+}

gms/api/src/main/pegasus/com/linkedin/datajob/DataFlowKey.pdl

@@ -0,0 +1,36 @@
+namespace com.linkedin.datajob
+
+
+/**
+ * Key for data flow resource
+ */
+record DataFlowKey {
+
+  /**
+   * The workflow orchestrator, ex: Azkaban, Airflow
+   */
+  @validate.strlen = {
+    "max" : 50,
+    "min" : 1
+  }
+  orchestrator: string
+
+  /**
+   * Id of the flow
+   */
+  @validate.strlen = {
+    "max" : 200,
+    "min" : 1
+  }
+  flowId: string
+
+  /**
+   * Cluster of the flow
+   */
+  @validate.strlen = {
+    "max" : 100,
+    "min" : 1
+  }
+  cluster: string
+
+}

gms/api/src/main/pegasus/com/linkedin/datajob/DataJob.pdl

@@ -0,0 +1,33 @@
+namespace com.linkedin.datajob
+
+import com.linkedin.common.ChangeAuditStamps
+import com.linkedin.common.DataJobUrn
+import com.linkedin.common.Ownership
+
+
+
+/**
+ * Metadata bout DataJob
+ */
+record DataJob includes DataJobKey, ChangeAuditStamps {
+  /**
+   * DataJob urn
+   */
+  urn: DataJobUrn
+
+  /**
+   * Ownership Info
+   */
+  ownership: optional Ownership
+
+  /**
+   * Input and output datasets of job
+   */
+   inputOutput: optional DataJobInputOutput
+
+  /**
+   * Optional additional metadata about the job
+   */
+   info: optional DataJobInfo
+
+}