-
Notifications
You must be signed in to change notification settings - Fork 1
/
Copy pathsynapse.json
234 lines (229 loc) · 10.4 KB
/
synapse.json
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
{
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {
"deploymentId": {
"type": "String",
"metadata": {
"description": "Suffix added to all resource names to make them unique. Use a maximum of 3 random characters. Four or more characters will be ignored."
}
}
},
"variables": {
"sqlAdministratorLogin": "asa.sql.admin",
"workspaceName": "[concat('asaworkspace', parameters('deploymentId'))]",
"adlsStorageAccountName": "[concat('asadatalake', parameters('deploymentId'))]",
"defaultDataLakeStorageFilesystemName": "defaultfs",
"sparkComputeName": "SparkPool01",
"primary_region": "East US",
"computeSubnetId": "",
"storageBlobDataContributorRoleID": "ba92f5b4-2d11-453d-a403-e96b0029c9fe",
"defaultDataLakeStorageAccountUrl": "[concat('https://', variables('adlsStorageAccountName'), '.dfs.core.windows.net')]",
"sparkAutoScaleEnabled": "true",
"sparkMinNodeCount": "3",
"sparkMaxNodeCount": "4",
"sparkNodeCount": "0",
"sparkNodeSizeFamily": "MemoryOptimized",
"sparkNodeSize": "Small",
"sparkAutoPauseEnabled": "true",
"sparkAutoPauseDelayInMinutes": "15",
"sparkVersion": "2.4"
},
"resources": [
{
"type": "Microsoft.Storage/storageAccounts",
"apiVersion": "2019-06-01",
"name": "[variables('adlsStorageAccountName')]",
"location": "[variables('primary_region')]",
"dependsOn": [],
"tags": {},
"sku": {
"name": "Standard_LRS"
},
"kind": "StorageV2",
"properties": {
"accessTier": "Hot",
"supportsHttpsTrafficOnly": "true",
"isHnsEnabled": "true",
"largeFileSharesState": "Disabled"
},
"resources": [
{
"type": "blobServices/containers",
"apiVersion": "2018-07-01",
"name": "[concat('default/', variables('defaultDataLakeStorageFilesystemName'))]",
"dependsOn": [
"[concat('Microsoft.Storage/storageAccounts/', variables('adlsStorageAccountName'))]"
]
}
]
},
{
"type": "Microsoft.Storage/storageAccounts/blobServices",
"apiVersion": "2019-06-01",
"name": "[concat(variables('adlsStorageAccountName'), '/default')]",
"dependsOn": [
"[resourceId('Microsoft.Storage/storageAccounts', variables('adlsStorageAccountName'))]"
],
"sku": {
"name": "Standard_LRS",
"tier": "Standard"
},
"properties": {
"cors": {
"corsRules": []
},
"deleteRetentionPolicy": {
"enabled": false
}
}
},
{
"type": "Microsoft.Storage/storageAccounts/fileServices",
"apiVersion": "2019-06-01",
"name": "[concat(variables('adlsStorageAccountName'), '/default')]",
"dependsOn": [
"[resourceId('Microsoft.Storage/storageAccounts', variables('adlsStorageAccountName'))]"
],
"sku": {
"name": "Standard_LRS",
"tier": "Standard"
},
"properties": {
"cors": {
"corsRules": []
}
}
},
{
"type": "Microsoft.Synapse/workspaces",
"apiVersion": "2019-06-01-preview",
"name": "[variables('workspaceName')]",
"location": "[variables('primary_region')]",
"dependsOn": [
"[concat('Microsoft.Storage/storageAccounts/', variables('adlsStorageAccountName'), '/blobServices/default/containers/', variables('defaultDataLakeStorageFilesystemName'))]"
],
"identity": {
"type": "SystemAssigned"
},
"properties": {
"defaultDataLakeStorage": {
"accountUrl": "[variables('defaultDataLakeStorageAccountUrl')]",
"filesystem": "[variables('defaultDataLakeStorageFilesystemName')]"
},
"virtualNetworkProfile": {
"computeSubnetId": "[variables('computeSubnetId')]"
},
"sqlAdministratorLogin": "[variables('sqlAdministratorLogin')]"
},
"resources": [
{
"type": "firewallrules",
"apiVersion": "2019-06-01-preview",
"name": "allowAll",
"location": "[variables('primary_region')]",
"dependsOn": [
"[concat('Microsoft.Synapse/workspaces/', variables('workspaceName'))]"
],
"properties": {
"startIpAddress": "0.0.0.0",
"endIpAddress": "255.255.255.255"
}
}
]
},
{
"type": "Microsoft.Storage/storageAccounts/blobServices/containers/providers/roleAssignments",
"apiVersion": "2018-09-01-preview",
"name": "[concat(variables('adlsStorageAccountName'), '/default/', variables('defaultDataLakeStorageFilesystemName'), '/Microsoft.Authorization/', guid(concat(resourceGroup().id, '/', variables('storageBlobDataContributorRoleID'), '/', variables('workspaceName'))))]",
"location": "[variables('primary_region')]",
"dependsOn": [
"[concat('Microsoft.Synapse/workspaces/', variables('workspaceName'))]"
],
"properties": {
"roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', variables('storageBlobDataContributorRoleID'))]",
"principalId": "[reference(concat('Microsoft.Synapse/workspaces/', variables('workspaceName')), '2019-06-01-preview', 'Full').identity.principalId]",
"principalType": "ServicePrincipal"
}
},
{
"type": "Microsoft.Authorization/roleAssignments",
"apiVersion": "2018-09-01-preview",
"name": "[guid(concat(resourceGroup().id, '/', variables('storageBlobDataContributorRoleID'), '/', variables('workspaceName'), '2'))]",
"location": "[variables('primary_region')]",
"dependsOn": [
"[concat('Microsoft.Synapse/workspaces/', variables('workspaceName'))]"
],
"properties": {
"roleDefinitionId": "[resourceId('Microsoft.Authorization/roleDefinitions', variables('storageBlobDataContributorRoleID'))]",
"principalId": "[reference(concat('Microsoft.Synapse/workspaces/', variables('workspaceName')), '2019-06-01-preview', 'Full').identity.principalId]",
"principalType": "ServicePrincipal"
}
},
{
"type": "Microsoft.Synapse/workspaces/bigDataPools",
"apiVersion": "2019-06-01-preview",
"name": "[concat(variables('workspaceName'), '/', variables('sparkComputeName'))]",
"location": "[variables('primary_region')]",
"dependsOn": [
"[concat('Microsoft.Synapse/workspaces/', variables('workspaceName'))]"
],
"properties": {
"nodeCount": "[variables('sparkNodeCount')]",
"nodeSizeFamily": "[variables('sparkNodeSizeFamily')]",
"nodeSize": "[variables('sparkNodeSize')]",
"autoScale": {
"enabled": "[variables('sparkAutoScaleEnabled')]",
"minNodeCount": "[variables('sparkMinNodeCount')]",
"maxNodeCount": "[variables('sparkMaxNodeCount')]"
},
"autoPause": {
"enabled": "[variables('sparkAutoPauseEnabled')]",
"delayInMinutes": "[variables('sparkAutoPauseDelayInMinutes')]"
},
"sparkVersion": "[variables('sparkVersion')]"
}
},
{
"apiVersion": "2015-01-01",
"type": "Microsoft.Resources/deployments",
"name": "UpdateSparkPool01",
"dependsOn": [
"[resourceId('Microsoft.Synapse/workspaces/bigDataPools', variables('workspaceName'), variables('sparkComputeName'))]"
],
"properties":{
"mode": "Incremental",
"parameters": {},
"template": {
"$schema": "https://schema.management.azure.com/schemas/2015-01-01/deploymentTemplate.json#",
"contentVersion": "1.0.0.0",
"parameters": {},
"variables": {},
"resources": [
{
"type": "Microsoft.Synapse/workspaces/bigDataPools",
"apiVersion": "2019-06-01-preview",
"name": "[concat(variables('workspaceName'), '/', variables('sparkComputeName'))]",
"location": "[variables('primary_region')]",
"properties": {
"nodeCount": "[variables('sparkNodeCount')]",
"nodeSizeFamily": "[variables('sparkNodeSizeFamily')]",
"nodeSize": "[variables('sparkNodeSize')]",
"autoScale": {
"enabled": "[variables('sparkAutoScaleEnabled')]",
"minNodeCount": "[variables('sparkMinNodeCount')]",
"maxNodeCount": "[variables('sparkMaxNodeCount')]"
},
"autoPause": {
"enabled": "[variables('sparkAutoPauseEnabled')]",
"delayInMinutes": "[variables('sparkAutoPauseDelayInMinutes')]"
},
"sparkVersion": "[variables('sparkVersion')]"
}
}
]
}
}
}
]
}