From 1fed5b4a879ea21794df180f200d868617820335 Mon Sep 17 00:00:00 2001 From: najian Date: Sat, 3 Mar 2018 03:55:48 +0800 Subject: [PATCH] add databricks activity (#2568) --- .../entityTypes/LinkedService.json | 77 +++++++++++++++++++ .../entityTypes/Pipeline.json | 39 ++++++++++ 2 files changed, 116 insertions(+) diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/preview/2017-09-01-preview/entityTypes/LinkedService.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/preview/2017-09-01-preview/entityTypes/LinkedService.json index 777fd49f4776..b5fde7e4c344 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/preview/2017-09-01-preview/entityTypes/LinkedService.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/preview/2017-09-01-preview/entityTypes/LinkedService.json @@ -3847,6 +3847,18 @@ "encryptedCredential": { "type": "object", "description": "The encrypted credential used for authentication. Credentials are encrypted using the integration runtime credential manager. Type: string (or Expression with resultType string)." + }, + "headNodeSize": { + "type": "object", + "description": "Specifies the size of the head node for the HDInsight cluster." + }, + "dataNodeSize": { + "type": "object", + "description": "Specifies the size of the data node for the HDInsight cluster." + }, + "zookeeperNodeSize": { + "type": "object", + "description": "Specifies the size of the Zoo Keeper node for the HDInsight cluster." } }, "required": [ @@ -3919,6 +3931,71 @@ "accountName", "tenant" ] + }, + "AzureDatabricksLinkedService": { + "x-ms-discriminator-value": "AzureDatabricks", + "description": "Azure Databricks linked service.", + "type": "object", + "allOf": [ + { + "$ref": "#/definitions/LinkedService" + } + ], + "properties": { + "typeProperties": { + "description": "Azure Databricks linked service properties.", + "x-ms-client-flatten": true, + "$ref": "#/definitions/AzureDatabricksLinkedServiceTypeProperties" + } + }, + "required": [ + "typeProperties" + ] + }, + "AzureDatabricksLinkedServiceTypeProperties": { + "description": "Azure Databricks linked service properties.", + "properties": { + "domain": { + "type": "object", + "description": ".azuredatabricks.net, domain name of your Databricks deployment. Type: string (or Expression with resultType string)." + }, + "accessToken": { + "description": "Access token for databricks REST API. Refer to https://docs.azuredatabricks.net/api/latest/authentication.html. Type: string (or Expression with resultType string).", + "$ref": "../datafactory.json#/definitions/SecretBase" + }, + "existingClusterId": { + "type": "object", + "description": "The id of an existing cluster that will be used for all runs of this job. Type: string (or Expression with resultType string)." + }, + "newClusterVersion": { + "type": "object", + "description": "The Spark version of new cluster. Type: string (or Expression with resultType string)." + }, + "newClusterNumOfWorker": { + "type": "object", + "description": "Number of worker nodes that new cluster should have. A string formatted Int32, like '1' means numOfWorker is 1 or '1:10' means auto-scale from 1 as min and 10 as max. Type: string (or Expression with resultType string)." + }, + "newClusterNodeType": { + "type": "object", + "description": "The node types of new cluster. Type: string (or Expression with resultType string)." + }, + "newClusterSparkConf": { + "description": "a set of optional, user-specified Spark configuration key-value pairs.", + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Type: string (or Expression with resultType string)." + } + }, + "encryptedCredential": { + "type": "object", + "description": "The encrypted credential used for authentication. Credentials are encrypted using the integration runtime credential manager. Type: string (or Expression with resultType string)." + } + }, + "required": [ + "domain", + "accessToken" + ] } } } \ No newline at end of file diff --git a/specification/datafactory/resource-manager/Microsoft.DataFactory/preview/2017-09-01-preview/entityTypes/Pipeline.json b/specification/datafactory/resource-manager/Microsoft.DataFactory/preview/2017-09-01-preview/entityTypes/Pipeline.json index fcc9958ab94f..cb7b6713b8bc 100644 --- a/specification/datafactory/resource-manager/Microsoft.DataFactory/preview/2017-09-01-preview/entityTypes/Pipeline.json +++ b/specification/datafactory/resource-manager/Microsoft.DataFactory/preview/2017-09-01-preview/entityTypes/Pipeline.json @@ -2728,6 +2728,45 @@ "condition", "items" ] + }, + "DatabricksNotebookActivity": { + "description": "DatabricksNotebook activity.", + "x-ms-discriminator-value": "DatabricksNotebook", + "allOf": [ + { + "$ref": "#/definitions/ExecutionActivity" + } + ], + "properties": { + "typeProperties": { + "x-ms-client-flatten": true, + "description": "Databricks Notebook activity properties.", + "$ref": "#/definitions/DatabricksNotebookActivityTypeProperties" + } + }, + "required": [ + "typeProperties" + ] + }, + "DatabricksNotebookActivityTypeProperties": { + "description": "Databricks Notebook activity properties.", + "properties": { + "notebookPath": { + "type": "object", + "description": "The absolute path of the notebook to be run in the Databricks Workspace. This path must begin with a slash. Type: string (or Expression with resultType string)." + }, + "baseParameters": { + "description": "Base parameters to be used for each run of this job.If the notebook takes a parameter that is not specified, the default value from the notebook will be used.", + "type": "object", + "additionalProperties": { + "type": "object", + "description": "Type: string (or Expression with resultType string)." + } + } + }, + "required": [ + "notebookPath" + ] } } }