feat(ingest): SageMaker jobs and models

gms/api/src/main/snapshot/com.linkedin.datajob.dataJobs.snapshot.json

@@ -448,7 +448,7 @@
               "PIG" : "Pig type is for running Pig jobs.",
               "SQL" : "SQL is for running Presto, mysql queries etc"
             }
-          } ],
+          }, "string" ],
           "doc" : "Datajob type"
         }, {
           "name" : "flowUrn",
@@ -459,6 +459,25 @@
             "entityTypes" : [ "dataFlow" ],
             "name" : "IsPartOf"
           }
+        }, {
+          "name" : "status",
+          "type" : {
+            "type" : "enum",
+            "name" : "JobStatus",
+            "doc" : "Job statuses",
+            "symbols" : [ "STARTING", "IN_PROGRESS", "STOPPING", "STOPPED", "COMPLETED", "FAILED", "UNKNOWN" ],
+            "symbolDocs" : {
+              "COMPLETED" : "Jobs with successful completion.",
+              "FAILED" : "Jobs that have failed.",
+              "IN_PROGRESS" : "Jobs currently running.",
+              "STARTING" : "Jobs being initialized.",
+              "STOPPED" : "Jobs that have stopped.",
+              "STOPPING" : "Jobs being stopped.",
+              "UNKNOWN" : "Jobs with unknown status (either unmappable or unavailable)"
+            }
+          },
+          "doc" : "Status of the job",
+          "optional" : true
         } ],
         "Aspect" : {
           "name" : "dataJobInfo"
@@ -574,7 +593,7 @@
       "doc" : "Editable properties",
       "optional" : true
     } ]
-  }, "com.linkedin.datajob.DataJobInfo", "com.linkedin.datajob.DataJobInputOutput", "com.linkedin.datajob.DataJobKey", "com.linkedin.datajob.EditableDataJobProperties", "com.linkedin.datajob.azkaban.AzkabanJobType", {
+  }, "com.linkedin.datajob.DataJobInfo", "com.linkedin.datajob.DataJobInputOutput", "com.linkedin.datajob.DataJobKey", "com.linkedin.datajob.EditableDataJobProperties", "com.linkedin.datajob.JobStatus", "com.linkedin.datajob.azkaban.AzkabanJobType", {
     "type" : "typeref",
     "name" : "DataJobAspect",
     "namespace" : "com.linkedin.metadata.aspect",

gms/api/src/main/snapshot/com.linkedin.entity.aspects.snapshot.json

@@ -966,7 +966,7 @@
           "PIG" : "Pig type is for running Pig jobs.",
           "SQL" : "SQL is for running Presto, mysql queries etc"
         }
-      } ],
+      }, "string" ],
       "doc" : "Datajob type"
     }, {
       "name" : "flowUrn",
@@ -977,6 +977,25 @@
         "entityTypes" : [ "dataFlow" ],
         "name" : "IsPartOf"
       }
+    }, {
+      "name" : "status",
+      "type" : {
+        "type" : "enum",
+        "name" : "JobStatus",
+        "doc" : "Job statuses",
+        "symbols" : [ "STARTING", "IN_PROGRESS", "STOPPING", "STOPPED", "COMPLETED", "FAILED", "UNKNOWN" ],
+        "symbolDocs" : {
+          "COMPLETED" : "Jobs with successful completion.",
+          "FAILED" : "Jobs that have failed.",
+          "IN_PROGRESS" : "Jobs currently running.",
+          "STARTING" : "Jobs being initialized.",
+          "STOPPED" : "Jobs that have stopped.",
+          "STOPPING" : "Jobs being stopped.",
+          "UNKNOWN" : "Jobs with unknown status (either unmappable or unavailable)"
+        }
+      },
+      "doc" : "Status of the job",
+      "optional" : true
     } ],
     "Aspect" : {
       "name" : "dataJobInfo"
@@ -1046,7 +1065,7 @@
     "Aspect" : {
       "name" : "dataJobInputOutput"
     }
-  }, "com.linkedin.datajob.azkaban.AzkabanJobType", {
+  }, "com.linkedin.datajob.JobStatus", "com.linkedin.datajob.azkaban.AzkabanJobType", {
     "type" : "record",
     "name" : "DatasetDeprecation",
     "namespace" : "com.linkedin.dataset",
@@ -2375,6 +2394,7 @@
       "name" : "MLModelProperties",
       "namespace" : "com.linkedin.ml.metadata",
       "doc" : "Properties associated with a ML Model",
+      "include" : [ "com.linkedin.common.CustomProperties" ],
       "fields" : [ {
         "name" : "description",
         "type" : "string",

gms/api/src/main/snapshot/com.linkedin.entity.entities.snapshot.json

@@ -1241,7 +1241,7 @@
           "PIG" : "Pig type is for running Pig jobs.",
           "SQL" : "SQL is for running Presto, mysql queries etc"
         }
-      } ],
+      }, "string" ],
       "doc" : "Datajob type"
     }, {
       "name" : "flowUrn",
@@ -1252,6 +1252,25 @@
         "entityTypes" : [ "dataFlow" ],
         "name" : "IsPartOf"
       }
+    }, {
+      "name" : "status",
+      "type" : {
+        "type" : "enum",
+        "name" : "JobStatus",
+        "doc" : "Job statuses",
+        "symbols" : [ "STARTING", "IN_PROGRESS", "STOPPING", "STOPPED", "COMPLETED", "FAILED", "UNKNOWN" ],
+        "symbolDocs" : {
+          "COMPLETED" : "Jobs with successful completion.",
+          "FAILED" : "Jobs that have failed.",
+          "IN_PROGRESS" : "Jobs currently running.",
+          "STARTING" : "Jobs being initialized.",
+          "STOPPED" : "Jobs that have stopped.",
+          "STOPPING" : "Jobs being stopped.",
+          "UNKNOWN" : "Jobs with unknown status (either unmappable or unavailable)"
+        }
+      },
+      "doc" : "Status of the job",
+      "optional" : true
     } ],
     "Aspect" : {
       "name" : "dataJobInfo"
@@ -1359,7 +1378,7 @@
     "Aspect" : {
       "name" : "editableDataJobProperties"
     }
-  }, "com.linkedin.datajob.azkaban.AzkabanJobType", {
+  }, "com.linkedin.datajob.JobStatus", "com.linkedin.datajob.azkaban.AzkabanJobType", {
     "type" : "record",
     "name" : "DataPlatformInfo",
     "namespace" : "com.linkedin.dataplatform",
@@ -2775,6 +2794,7 @@
                   "name" : "MLModelProperties",
                   "namespace" : "com.linkedin.ml.metadata",
                   "doc" : "Properties associated with a ML Model",
+                  "include" : [ "com.linkedin.common.CustomProperties" ],
                   "fields" : [ {
                     "name" : "description",
                     "type" : "string",
@@ -3435,7 +3455,7 @@
                   "Aspect" : {
                     "name" : "mlFeatureTableProperties"
                   }
-                }, "com.linkedin.common.Ownership", "com.linkedin.common.InstitutionalMemory", "com.linkedin.common.Status", "com.linkedin.common.Deprecation" ]
+                }, "com.linkedin.common.Ownership", "com.linkedin.common.InstitutionalMemory", "com.linkedin.common.Status", "com.linkedin.common.Deprecation", "com.linkedin.common.BrowsePaths" ]
               }
             },
             "doc" : "The list of metadata aspects associated with the MLFeatureTable. Depending on the use case, this can either be all, or a selection, of supported aspects."

gms/api/src/main/snapshot/com.linkedin.ml.mlModels.snapshot.json

@@ -133,6 +133,20 @@
       "name" : "cost"
     }
   }, "com.linkedin.common.CostType", "com.linkedin.common.CostValue", {
+    "type" : "record",
+    "name" : "CustomProperties",
+    "namespace" : "com.linkedin.common",
+    "doc" : "Misc. properties about an entity.",
+    "fields" : [ {
+      "name" : "customProperties",
+      "type" : {
+        "type" : "map",
+        "values" : "string"
+      },
+      "doc" : "Custom property bag.",
+      "default" : { }
+    } ]
+  }, {
     "type" : "typeref",
     "name" : "DataPlatformUrn",
     "namespace" : "com.linkedin.common",
@@ -510,6 +524,7 @@
       "name" : "MLModelProperties",
       "namespace" : "com.linkedin.ml.metadata",
       "doc" : "Properties associated with a ML Model",
+      "include" : [ "com.linkedin.common.CustomProperties" ],
       "fields" : [ {
         "name" : "description",
         "type" : "string",

metadata-ingestion/README.md

@@ -387,6 +387,18 @@ source:
     aws_secret_access_key: # Optional.
     aws_session_token: # Optional.
     aws_role: # Optional (Role chaining supported by using a sorted list).
+
+    extract_feature_groups: True # if feature groups should be ingested, default True
+    extract_models: True # if models should be ingested, default True
+    extract_jobs: # if jobs should be ingested, default True for all
+      auto_ml: True
+      compilation: True
+      edge_packaging: True
+      hyper_parameter_tuning: True
+      labeling: True
+      processing: True
+      training: True
+      transform: True
 ```
 
 ### Snowflake `snowflake`