From 77118418d66150c075b1b168ec7ccb530d48df89 Mon Sep 17 00:00:00 2001 From: Hari Ramani Date: Tue, 25 Jun 2019 17:20:33 -0700 Subject: [PATCH 1/4] Add Chaining Trigger --- .../stable/2018-06-01/datafactory.json | 24 ++++++++++ .../2018-06-01/entityTypes/Pipeline.json | 8 ++++ .../2018-06-01/entityTypes/Trigger.json | 44 +++++++++++++++++++ 3 files changed, 76 insertions(+) diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/datafactory.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/datafactory.json index 9cfa78d3833d..1227d20d6dfc 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/datafactory.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/datafactory.json @@ -4335,6 +4335,14 @@ }, "readOnly": true }, + "runDimension": { + "description": "Run dimension emitted by Pipeline run.", + "type": "object", + "additionalProperties": { + "type": "string" + }, + "readOnly": true + }, "invokedBy": { "description": "Entity that started the pipeline run.", "$ref": "#/definitions/PipelineRunInvokedBy", @@ -4574,6 +4582,22 @@ "type": "string" }, "readOnly": true + }, + "runDimension": { + "description": "Run dimention for which trigger was fired.", + "type": "object", + "additionalProperties": { + "type": "string" + }, + "readOnly": true + }, + "DependencyStatus": { + "description": "Status of the upstream pipelines.", + "type": "object", + "additionalProperties": { + "type": "string" + }, + "readOnly": true } }, "additionalProperties": { diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json index 2ce9f0ee9dce..7a539a3133e3 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Pipeline.json @@ -41,6 +41,14 @@ "type": "object" } }, + "runDimensions": { + "description": "Dimensions emitted by Pipeline.", + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Type: string (or Expression with resultType string)." + } + }, "folder": { "description": "The folder that this Pipeline is in. If not specified, Pipeline will appear at the root level.", "type": "object", diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Trigger.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Trigger.json index d0671194aa21..2cf942b5f1cc 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Trigger.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Trigger.json @@ -585,6 +585,50 @@ "required": [ "typeProperties" ] + }, + "ChainingTrigger": { + "description": "Trigger that schedules pipeline runs based on dependent pipelines successful completion.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/Trigger" + } + ], + "properties": { + "pipeline": { + "$ref": "../datafactory.json#/definitions/TriggerPipelineReference", + "description": "Pipeline for which runs are created when all dependent pipelines complete successfully." + }, + "typeProperties": { + "description": "Chaining Trigger properties.", + "x-ms-client-flatten": true, + "properties": { + "dependsOn": { + "type": "array", + "items": { + "$ref": "../datafactory.json#/definitions/PipelineReference" + }, + "description": "Dependent Pipelines." + }, + "retryPolicy": { + "$ref": "#/definitions/RetryPolicy", + "description": "Retry policy that will be applied for failed pipeline runs." + }, + "runDimension": { + "description": "Run Dimenstion property that needs to be emitted by dependent pipelines.", + "type": "string" + } + }, + "required": [ + "runDimension", + "dependsOn" + ] + } + }, + "required": [ + "pipeline", + "typeProperties" + ] } } } From 7762accf7fa0b6a8a3cb7f7b7544f60c82fb7b48 Mon Sep 17 00:00:00 2001 From: Hari Ramani Date: Mon, 19 Aug 2019 23:19:27 -0700 Subject: [PATCH 2/4] Add Chaining Trigger --- .../stable/2018-06-01/datafactory.json | 4 ++-- .../stable/2018-06-01/entityTypes/Trigger.json | 6 +----- .../examples/PipelineRuns_QueryByFactory.json | 10 ++++++++-- .../2018-06-01/examples/Pipelines_Create.json | 18 ++++++++++++++++++ 4 files changed, 29 insertions(+), 9 deletions(-) diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/datafactory.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/datafactory.json index 1227d20d6dfc..b73b45f6c79c 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/datafactory.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/datafactory.json @@ -4584,7 +4584,7 @@ "readOnly": true }, "runDimension": { - "description": "Run dimention for which trigger was fired.", + "description": "Run dimension for which trigger was fired.", "type": "object", "additionalProperties": { "type": "string" @@ -4595,7 +4595,7 @@ "description": "Status of the upstream pipelines.", "type": "object", "additionalProperties": { - "type": "string" + "type": "object" }, "readOnly": true } diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Trigger.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Trigger.json index 2cf942b5f1cc..666c74ecd57d 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Trigger.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Trigger.json @@ -610,12 +610,8 @@ }, "description": "Dependent Pipelines." }, - "retryPolicy": { - "$ref": "#/definitions/RetryPolicy", - "description": "Retry policy that will be applied for failed pipeline runs." - }, "runDimension": { - "description": "Run Dimenstion property that needs to be emitted by dependent pipelines.", + "description": "Run Dimension property that needs to be emitted by dependent pipelines.", "type": "string" } }, diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/examples/PipelineRuns_QueryByFactory.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/examples/PipelineRuns_QueryByFactory.json index b110f2399a3f..8a2579b02242 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/examples/PipelineRuns_QueryByFactory.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/examples/PipelineRuns_QueryByFactory.json @@ -45,7 +45,10 @@ "status": "Succeeded", "message": "", "lastUpdated": "2018-06-16T00:38:12.7314495Z", - "annotations": [] + "annotations": [], + "runDimension": { + "JobId": "79c1cc52-265f-41a5-9553-be65e736fbd3" + } }, { "runId": "16ac5348-ff82-4f95-a80d-638c1d47b721", @@ -63,7 +66,10 @@ "status": "Cancelled", "message": "", "lastUpdated": "2018-06-16T00:39:51.216097Z", - "annotations": [] + "annotations": [], + "runDimension": { + "JobId": "84a3c493-0628-4b44-852f-ef5b3a11bdab" + } } ] } diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/examples/Pipelines_Create.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/examples/Pipelines_Create.json index 9d0a1ba448e3..244732eeda4b 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/examples/Pipelines_Create.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/examples/Pipelines_Create.json @@ -61,12 +61,21 @@ "parameters": { "OutputBlobNameList": { "type": "Array" + }, + "JobId": { + "type": "String" } }, "variables": { "TestVariableArray": { "type": "Array" } + }, + "runDimensions": { + "JobId": { + "value": "@pipeline().parameters.JobId", + "type": "Expression" + } } } }, @@ -140,12 +149,21 @@ "parameters": { "OutputBlobNameList": { "type": "Array" + }, + "JobId": { + "type": "String" } }, "variables": { "TestVariableArray": { "type": "Array" } + }, + "runDimensions": { + "JobId": { + "value": "@pipeline().parameters.JobId", + "type": "Expression" + } } }, "etag": "0a0069d4-0000-0000-0000-5b245bd50000" From c1c2169a59479b6f08e64d9ecacc34da87fe75f5 Mon Sep 17 00:00:00 2001 From: Hari Ramani Date: Tue, 20 Aug 2019 11:19:29 -0700 Subject: [PATCH 3/4] Fix Issues --- .../Microsoft.DataFactory/stable/2018-06-01/datafactory.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/datafactory.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/datafactory.json index b73b45f6c79c..5185509ffee9 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/datafactory.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/datafactory.json @@ -4591,7 +4591,7 @@ }, "readOnly": true }, - "DependencyStatus": { + "dependencyStatus": { "description": "Status of the upstream pipelines.", "type": "object", "additionalProperties": { From 5e1bb35d5c3314d8f4fead76c3d69a2522be026b Mon Sep 17 00:00:00 2001 From: Hari Ramani Date: Wed, 28 Aug 2019 18:16:23 -0700 Subject: [PATCH 4/4] Update review comments --- .../stable/2018-06-01/datafactory.json | 4 ++-- .../stable/2018-06-01/entityTypes/Trigger.json | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/datafactory.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/datafactory.json index 5185509ffee9..32786cab9f67 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/datafactory.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/datafactory.json @@ -4335,8 +4335,8 @@ }, "readOnly": true }, - "runDimension": { - "description": "Run dimension emitted by Pipeline run.", + "runDimensions": { + "description": "Run dimensions emitted by Pipeline run.", "type": "object", "additionalProperties": { "type": "string" diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Trigger.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Trigger.json index 666c74ecd57d..6c66d196fad5 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Trigger.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/stable/2018-06-01/entityTypes/Trigger.json @@ -587,7 +587,7 @@ ] }, "ChainingTrigger": { - "description": "Trigger that schedules pipeline runs based on dependent pipelines successful completion.", + "description": "Trigger that allows the referenced pipeline to depend on other pipeline runs based on runDimension Name/Value pairs. Upstream pipelines should declare the same runDimension Name and their runs should have the values for those runDimensions. The referenced pipeline run would be triggered if the values for the runDimension match for all upstream pipeline runs.", "type": "object", "allOf": [ { @@ -597,7 +597,7 @@ "properties": { "pipeline": { "$ref": "../datafactory.json#/definitions/TriggerPipelineReference", - "description": "Pipeline for which runs are created when all dependent pipelines complete successfully." + "description": "Pipeline for which runs are created when all upstream pipelines complete successfully." }, "typeProperties": { "description": "Chaining Trigger properties.", @@ -608,10 +608,10 @@ "items": { "$ref": "../datafactory.json#/definitions/PipelineReference" }, - "description": "Dependent Pipelines." + "description": "Upstream Pipelines." }, "runDimension": { - "description": "Run Dimension property that needs to be emitted by dependent pipelines.", + "description": "Run Dimension property that needs to be emitted by upstream pipelines.", "type": "string" } },