From 3cb04ccc61a99896dfe7c2bee65e134a495f3c46 Mon Sep 17 00:00:00 2001 From: ross-p-smith Date: Tue, 15 Dec 2020 13:49:00 +0000 Subject: [PATCH 1/6] Added DevContainer --- .devcontainer/Dockerfile | 70 +++++++++++++++++++++++++++++++++ .devcontainer/devcontainer.json | 43 ++++++++++++++++++++ 2 files changed, 113 insertions(+) create mode 100644 .devcontainer/Dockerfile create mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile new file mode 100644 index 000000000000..c0ddf0f6d709 --- /dev/null +++ b/.devcontainer/Dockerfile @@ -0,0 +1,70 @@ +#------------------------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information. +#------------------------------------------------------------------------------------------------------------- + +FROM golang:1.14-buster + +# Avoid warnings by switching to noninteractive +ENV DEBIAN_FRONTEND=noninteractive + +# This Dockerfile adds a non-root 'vscode' user with sudo access. However, for Linux, +# this user's GID/UID must match your local user UID/GID to avoid permission issues +# with bind mounts. Update USER_UID / USER_GID if yours is not 1000. See +# https://aka.ms/vscode-remote/containers/non-root-user for details. +ARG USERNAME=vscode +ARG USER_UID=1000 +ARG USER_GID=$USER_UID + +# Terraform and tflint versions +ARG TERRAFORM_VERSION=0.14.2 + +ENV GO111MODULE=on + +# Configure apt, install packages and tools +RUN apt-get update \ + && apt-get -y install --no-install-recommends curl unzip apt-utils dialog \ + # + # Verify git, process tools, lsb-release (common in install instructions for CLIs) installed + && apt-get -y install git iproute2 procps lsb-release \ + # + # Install Azure CLI + && curl -sL https://aka.ms/InstallAzureCLIDeb | bash \ + # + # Create a non-root user to use if preferred - see https://aka.ms/vscode-remote/containers/non-root-user. + && groupadd --gid $USER_GID $USERNAME \ + && useradd -s /bin/bash --uid $USER_UID --gid $USER_GID -m $USERNAME \ + && echo $USERNAME \ + # + # Clean up + && apt-get autoremove -y \ + && apt-get clean -y \ + && rm -rf /var/lib/apt/lists/* + +ENV GIT_PROMPT_START='\033[1;36maztf-devcon>\033[0m\033[0;33m\w\a\033[0m' + +# Save command line history +RUN echo "export HISTFILE=/root/commandhistory/.bash_history" >> "/root/.bashrc" \ + && echo "export PROMPT_COMMAND='history -a'" >> "/root/.bashrc" \ + && mkdir -p /root/commandhistory \ + && touch /root/commandhistory/.bash_history + +# Git command prompt +RUN git clone https://github.com/magicmonty/bash-git-prompt.git ~/.bash-git-prompt --depth=1 \ + && echo "if [ -f \"$HOME/.bash-git-prompt/gitprompt.sh\" ]; then GIT_PROMPT_ONLY_IN_REPO=1 && source $HOME/.bash-git-prompt/gitprompt.sh; fi" >> "/root/.bashrc" + +# Install Go tools +RUN \ + # --> Delve for debugging + go get github.com/go-delve/delve/cmd/dlv@v1.5.0 \ + # --> Go language server + && go get golang.org/x/tools/gopls@v0.5.5 \ + # --> Go symbols and outline for go to symbol support and test support + && go get github.com/acroca/go-symbols@v0.1.1 && go get github.com/ramya-rao-a/go-outline@7182a932836a71948db4a81991a494751eccfe77 + +RUN \ + # Install Terraform + mkdir -p /tmp/docker-downloads \ + && curl -sSL -o /tmp/docker-downloads/terraform.zip https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip \ + && unzip /tmp/docker-downloads/terraform.zip \ + && mv terraform /usr/local/bin \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json new file mode 100644 index 000000000000..0213bcb88421 --- /dev/null +++ b/.devcontainer/devcontainer.json @@ -0,0 +1,43 @@ +// For format details, see https://aka.ms/vscode-remote/devcontainer.json or this file's README at: +// https://github.com/microsoft/vscode-dev-containers/tree/v0.106.0/containers/go +{ + "name": "Go", + "dockerFile": "Dockerfile", + "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ], + + // Set *default* container specific settings.json values on container create. + "settings": { + "terminal.integrated.shell.linux": "/bin/bash", + "go.gopath": "/go", + "go.useLanguageServer": true, + "[go]": { + "editor.snippetSuggestions": "none", + "editor.formatOnSave": true, + "editor.codeActionsOnSave": { + "source.organizeImports": true, + } + }, + "gopls": { + "usePlaceholders": true, // add parameter placeholders when completing a function + // Experimental settings + "completeUnimported": true, // autocomplete unimported packages + "watchFileChanges": true, // watch file changes outside of the editor + "deepCompletion": true, // enable deep completion + }, + "files.eol": "\n", // formatting only supports LF line endings + }, + + // Add the IDs of extensions you want installed when the container is created. + "extensions": [ + "ms-vscode.go" + ], + + // Use 'forwardPorts' to make a list of ports inside the container available locally. + // "forwardPorts": [], + + // Use 'postCreateCommand' to run commands after the container is created. + "postCreateCommand": "make tools", + + // Uncomment to connect as a non-root user. See https://aka.ms/vscode-remote/containers/non-root. + // "remoteUser": "vscode" +} \ No newline at end of file From 139b8ebad631bca5cf62646f8790dc8ff1d026b2 Mon Sep 17 00:00:00 2001 From: ross-p-smith Date: Thu, 17 Dec 2020 15:42:41 +0000 Subject: [PATCH 2/6] Add ACL to Root File SYstem --- .devcontainer/devcontainer.json | 2 +- .../storage/parse/storage_filesystem_ace.go | 69 ++++++++++ ...rage_data_lake_gen2_filesystem_resource.go | 100 ++++++++++++++- .../storage_data_lake_gen2_path_resource.go | 85 +------------ .../storage_adls_access_control_permission.go | 20 +++ examples/storage/storage_adls_acls/main.tf | 119 ++++++++++++++++++ 6 files changed, 312 insertions(+), 83 deletions(-) create mode 100644 azurerm/internal/services/storage/parse/storage_filesystem_ace.go create mode 100644 azurerm/internal/services/storage/validate/storage_adls_access_control_permission.go create mode 100644 examples/storage/storage_adls_acls/main.tf diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json index 0213bcb88421..6c0838c9e3da 100644 --- a/.devcontainer/devcontainer.json +++ b/.devcontainer/devcontainer.json @@ -29,7 +29,7 @@ // Add the IDs of extensions you want installed when the container is created. "extensions": [ - "ms-vscode.go" + "golang.go" ], // Use 'forwardPorts' to make a list of ports inside the container available locally. diff --git a/azurerm/internal/services/storage/parse/storage_filesystem_ace.go b/azurerm/internal/services/storage/parse/storage_filesystem_ace.go new file mode 100644 index 000000000000..0d3396f17777 --- /dev/null +++ b/azurerm/internal/services/storage/parse/storage_filesystem_ace.go @@ -0,0 +1,69 @@ +package parse + +import ( + "github.com/google/uuid" + "github.com/tombuildsstuff/giovanni/storage/accesscontrol" +) + +func ExpandArmDataLakeGen2AceList(input []interface{}) (*accesscontrol.ACL, error) { + if len(input) == 0 { + return nil, nil + } + aceList := make([]accesscontrol.ACE, len(input)) + + for i := 0; i < len(input); i++ { + v := input[i].(map[string]interface{}) + + isDefault := false + if scopeRaw, ok := v["scope"]; ok { + if scopeRaw.(string) == "default" { + isDefault = true + } + } + + tagType := accesscontrol.TagType(v["type"].(string)) + + var id *uuid.UUID + if raw, ok := v["id"]; ok && raw != "" { + idTemp, err := uuid.Parse(raw.(string)) + if err != nil { + return nil, err + } + id = &idTemp + } + + permissions := v["permissions"].(string) + + ace := accesscontrol.ACE{ + IsDefault: isDefault, + TagType: tagType, + TagQualifier: id, + Permissions: permissions, + } + aceList[i] = ace + } + + return &accesscontrol.ACL{Entries: aceList}, nil +} + +func FlattenArmDataLakeGen2AceList(acl accesscontrol.ACL) []interface{} { + output := make([]interface{}, len(acl.Entries)) + + for i, v := range acl.Entries { + ace := make(map[string]interface{}) + + scope := "access" + if v.IsDefault { + scope = "default" + } + ace["scope"] = scope + ace["type"] = string(v.TagType) + if v.TagQualifier != nil { + ace["id"] = v.TagQualifier.String() + } + ace["permissions"] = v.Permissions + + output[i] = ace + } + return output +} diff --git a/azurerm/internal/services/storage/storage_data_lake_gen2_filesystem_resource.go b/azurerm/internal/services/storage/storage_data_lake_gen2_filesystem_resource.go index faa8e66dc2ea..e9b15dce2c94 100644 --- a/azurerm/internal/services/storage/storage_data_lake_gen2_filesystem_resource.go +++ b/azurerm/internal/services/storage/storage_data_lake_gen2_filesystem_resource.go @@ -8,6 +8,7 @@ import ( "time" "github.com/hashicorp/terraform-plugin-sdk/helper/schema" + "github.com/hashicorp/terraform-plugin-sdk/helper/validation" "github.com/terraform-providers/terraform-provider-azurerm/azurerm/helpers/tf" "github.com/terraform-providers/terraform-provider-azurerm/azurerm/internal/clients" "github.com/terraform-providers/terraform-provider-azurerm/azurerm/internal/services/storage/parse" @@ -15,6 +16,8 @@ import ( "github.com/terraform-providers/terraform-provider-azurerm/azurerm/internal/timeouts" "github.com/terraform-providers/terraform-provider-azurerm/azurerm/utils" "github.com/tombuildsstuff/giovanni/storage/2019-12-12/datalakestore/filesystems" + "github.com/tombuildsstuff/giovanni/storage/2019-12-12/datalakestore/paths" + "github.com/tombuildsstuff/giovanni/storage/accesscontrol" ) func resourceStorageDataLakeGen2FileSystem() *schema.Resource { @@ -73,6 +76,37 @@ func resourceStorageDataLakeGen2FileSystem() *schema.Resource { }, "properties": MetaDataSchema(), + + "ace": { + Type: schema.TypeList, + Optional: true, + Computed: true, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "scope": { + Type: schema.TypeString, + Optional: true, + ValidateFunc: validation.StringInSlice([]string{"default", "access"}, false), + Default: "access", + }, + "type": { + Type: schema.TypeString, + Required: true, + ValidateFunc: validation.StringInSlice([]string{"user", "group", "mask", "other"}, false), + }, + "id": { + Type: schema.TypeString, + Optional: true, + ValidateFunc: validation.IsUUID, + }, + "permissions": { + Type: schema.TypeString, + Required: true, + ValidateFunc: validate.ADLSAccessControlPermissions, + }, + }, + }, + }, }, } } @@ -80,6 +114,7 @@ func resourceStorageDataLakeGen2FileSystem() *schema.Resource { func resourceStorageDataLakeGen2FileSystemCreate(d *schema.ResourceData, meta interface{}) error { accountsClient := meta.(*clients.Client).Storage.AccountsClient client := meta.(*clients.Client).Storage.FileSystemsClient + pathClient := meta.(*clients.Client).Storage.ADLSGen2PathsClient ctx, cancel := timeouts.ForCreate(meta.(*clients.Client).StopContext, d) defer cancel() @@ -88,6 +123,12 @@ func resourceStorageDataLakeGen2FileSystemCreate(d *schema.ResourceData, meta in return err } + aceRaw := d.Get("ace").([]interface{}) + acl, err := parse.ExpandArmDataLakeGen2AceList(aceRaw) + if err != nil { + return fmt.Errorf("Error parsing ace list: %s", err) + } + // confirm the storage account exists, otherwise Data Plane API requests will fail storageAccount, err := accountsClient.GetProperties(ctx, storageID.ResourceGroup, storageID.Name, "") if err != nil { @@ -123,6 +164,21 @@ func resourceStorageDataLakeGen2FileSystemCreate(d *schema.ResourceData, meta in return fmt.Errorf("Error creating File System %q in Storage Account %q: %s", fileSystemName, storageID.Name, err) } + if acl != nil { + log.Printf("[INFO] Creating acl %q in File System %q in Storage Account %q.", acl, fileSystemName, storageID.Name) + var aclString *string + v := acl.String() + aclString = &v + accessControlInput := paths.SetAccessControlInput{ + ACL: aclString, + Owner: nil, + Group: nil, + } + if _, err := pathClient.SetAccessControl(ctx, storageID.Name, fileSystemName, "/", accessControlInput); err != nil { + return fmt.Errorf("Error setting access control for root path in File System %q in Storage Account %q: %s", fileSystemName, storageID.Name, err) + } + } + d.SetId(id) return resourceStorageDataLakeGen2FileSystemRead(d, meta) } @@ -130,6 +186,7 @@ func resourceStorageDataLakeGen2FileSystemCreate(d *schema.ResourceData, meta in func resourceStorageDataLakeGen2FileSystemUpdate(d *schema.ResourceData, meta interface{}) error { accountsClient := meta.(*clients.Client).Storage.AccountsClient client := meta.(*clients.Client).Storage.FileSystemsClient + pathClient := meta.(*clients.Client).Storage.ADLSGen2PathsClient ctx, cancel := timeouts.ForUpdate(meta.(*clients.Client).StopContext, d) defer cancel() @@ -143,6 +200,12 @@ func resourceStorageDataLakeGen2FileSystemUpdate(d *schema.ResourceData, meta in return err } + aceRaw := d.Get("ace").([]interface{}) + acl, err := parse.ExpandArmDataLakeGen2AceList(aceRaw) + if err != nil { + return fmt.Errorf("Error parsing ace list: %s", err) + } + // confirm the storage account exists, otherwise Data Plane API requests will fail storageAccount, err := accountsClient.GetProperties(ctx, storageID.ResourceGroup, storageID.Name, "") if err != nil { @@ -164,12 +227,28 @@ func resourceStorageDataLakeGen2FileSystemUpdate(d *schema.ResourceData, meta in return fmt.Errorf("Error updating Properties for File System %q in Storage Account %q: %s", id.DirectoryName, id.AccountName, err) } - return resourceStorageDataLakeGen2FileSystemRead(d, meta) + if acl != nil { + log.Printf("[INFO] Creating acl %q in File System %q in Storage Account %q.", acl, id.DirectoryName, id.AccountName) + var aclString *string + v := acl.String() + aclString = &v + accessControlInput := paths.SetAccessControlInput{ + ACL: aclString, + Owner: nil, + Group: nil, + } + if _, err := pathClient.SetAccessControl(ctx, id.AccountName, id.DirectoryName, "/", accessControlInput); err != nil { + return fmt.Errorf("Error setting access control for root path in File System %q in Storage Account %q: %s", id.DirectoryName, id.AccountName, err) + } + } + + return resourceArmStorageDataLakeGen2FileSystemRead(d, meta) } func resourceStorageDataLakeGen2FileSystemRead(d *schema.ResourceData, meta interface{}) error { accountsClient := meta.(*clients.Client).Storage.AccountsClient client := meta.(*clients.Client).Storage.FileSystemsClient + pathClient := meta.(*clients.Client).Storage.ADLSGen2PathsClient ctx, cancel := timeouts.ForRead(meta.(*clients.Client).StopContext, d) defer cancel() @@ -213,6 +292,25 @@ func resourceStorageDataLakeGen2FileSystemRead(d *schema.ResourceData, meta inte return fmt.Errorf("Error setting `properties`: %+v", err) } + // The above `getStatus` API request doesn't return the ACLs + // Have to make a `getAccessControl` request, but that doesn't return all fields either! + pathResponse, err := pathClient.GetProperties(ctx, id.AccountName, id.DirectoryName, "/", paths.GetPropertiesActionGetAccessControl) + if err != nil { + if utils.ResponseWasNotFound(pathResponse.Response) { + log.Printf("[INFO] Root path does not exist in File System %q in Storage Account %q - removing from state...", id.DirectoryName, id.AccountName) + d.SetId("") + return nil + } + + return fmt.Errorf("Error retrieving ACLs for Root path in File System %q in Storage Account %q: %+v", id.DirectoryName, id.AccountName, err) + } + + acl, err := accesscontrol.ParseACL(pathResponse.ACL) + if err != nil { + return fmt.Errorf("Error parsing response ACL %q: %s", pathResponse.ACL, err) + } + d.Set("ace", parse.FlattenArmDataLakeGen2AceList(acl)) + return nil } diff --git a/azurerm/internal/services/storage/storage_data_lake_gen2_path_resource.go b/azurerm/internal/services/storage/storage_data_lake_gen2_path_resource.go index 4939ace72d36..2ea3de37b174 100644 --- a/azurerm/internal/services/storage/storage_data_lake_gen2_path_resource.go +++ b/azurerm/internal/services/storage/storage_data_lake_gen2_path_resource.go @@ -6,7 +6,6 @@ import ( "log" "time" - "github.com/google/uuid" "github.com/hashicorp/terraform-plugin-sdk/helper/schema" "github.com/hashicorp/terraform-plugin-sdk/helper/validation" "github.com/terraform-providers/terraform-provider-azurerm/azurerm/helpers/tf" @@ -132,7 +131,7 @@ func resourceStorageDataLakeGen2Path() *schema.Resource { "permissions": { Type: schema.TypeString, Required: true, - ValidateFunc: validateADLSAccessControlPermissions, + ValidateFunc: validate.ADLSAccessControlPermissions, }, }, }, @@ -185,7 +184,7 @@ func resourceStorageDataLakeGen2PathCreate(d *schema.ResourceData, meta interfac return fmt.Errorf("Unhandled resource type %q", resourceString) } aceRaw := d.Get("ace").([]interface{}) - acl, err := expandDataLakeGen2PathAceList(aceRaw) + acl, err := parse.ExpandArmDataLakeGen2AceList(aceRaw) if err != nil { return fmt.Errorf("Error parsing ace list: %s", err) } @@ -249,7 +248,7 @@ func resourceStorageDataLakeGen2PathUpdate(d *schema.ResourceData, meta interfac path := d.Get("path").(string) aceRaw := d.Get("ace").([]interface{}) - acl, err := expandDataLakeGen2PathAceList(aceRaw) + acl, err := parse.ExpandArmDataLakeGen2AceList(aceRaw) if err != nil { return fmt.Errorf("Error parsing ace list: %s", err) } @@ -337,7 +336,7 @@ func resourceStorageDataLakeGen2PathRead(d *schema.ResourceData, meta interface{ if err != nil { return fmt.Errorf("Error parsing response ACL %q: %s", resp.ACL, err) } - d.Set("ace", flattenDataLakeGen2PathAceList(acl)) + d.Set("ace", parse.FlattenArmDataLakeGen2AceList(acl)) return nil } @@ -361,79 +360,3 @@ func resourceStorageDataLakeGen2PathDelete(d *schema.ResourceData, meta interfac return nil } - -func expandDataLakeGen2PathAceList(input []interface{}) (*accesscontrol.ACL, error) { - if len(input) == 0 { - return nil, nil - } - aceList := make([]accesscontrol.ACE, len(input)) - - for i := 0; i < len(input); i++ { - v := input[i].(map[string]interface{}) - - isDefault := false - if scopeRaw, ok := v["scope"]; ok { - if scopeRaw.(string) == "default" { - isDefault = true - } - } - - tagType := accesscontrol.TagType(v["type"].(string)) - - var id *uuid.UUID - if raw, ok := v["id"]; ok && raw != "" { - idTemp, err := uuid.Parse(raw.(string)) - if err != nil { - return nil, err - } - id = &idTemp - } - - permissions := v["permissions"].(string) - - ace := accesscontrol.ACE{ - IsDefault: isDefault, - TagType: tagType, - TagQualifier: id, - Permissions: permissions, - } - aceList[i] = ace - } - - return &accesscontrol.ACL{Entries: aceList}, nil -} - -func flattenDataLakeGen2PathAceList(acl accesscontrol.ACL) []interface{} { - output := make([]interface{}, len(acl.Entries)) - - for i, v := range acl.Entries { - ace := make(map[string]interface{}) - - scope := "access" - if v.IsDefault { - scope = "default" - } - ace["scope"] = scope - ace["type"] = string(v.TagType) - if v.TagQualifier != nil { - ace["id"] = v.TagQualifier.String() - } - ace["permissions"] = v.Permissions - - output[i] = ace - } - return output -} - -func validateADLSAccessControlPermissions(i interface{}, k string) (warnings []string, errors []error) { - v, ok := i.(string) - if !ok { - errors = append(errors, fmt.Errorf("expected type of %s to be string", k)) - return warnings, errors - } - if err := accesscontrol.ValidateACEPermissions(v); err != nil { - errors = append(errors, fmt.Errorf("value of %s not valid: %s", k, err)) - return warnings, errors - } - return warnings, errors -} diff --git a/azurerm/internal/services/storage/validate/storage_adls_access_control_permission.go b/azurerm/internal/services/storage/validate/storage_adls_access_control_permission.go new file mode 100644 index 000000000000..023c1c71154b --- /dev/null +++ b/azurerm/internal/services/storage/validate/storage_adls_access_control_permission.go @@ -0,0 +1,20 @@ +package validate + +import ( + "fmt" + + "github.com/tombuildsstuff/giovanni/storage/accesscontrol" +) + +func ADLSAccessControlPermissions(i interface{}, k string) (warnings []string, errors []error) { + v, ok := i.(string) + if !ok { + errors = append(errors, fmt.Errorf("expected type of %s to be string", k)) + return warnings, errors + } + if err := accesscontrol.ValidateACEPermissions(v); err != nil { + errors = append(errors, fmt.Errorf("value of %s not valid: %s", k, err)) + return warnings, errors + } + return warnings, errors +} diff --git a/examples/storage/storage_adls_acls/main.tf b/examples/storage/storage_adls_acls/main.tf new file mode 100644 index 000000000000..753e4ab64c36 --- /dev/null +++ b/examples/storage/storage_adls_acls/main.tf @@ -0,0 +1,119 @@ +provider "azurerm" { + features {} +} + +resource "azurerm_resource_group" "example" { + name = "${var.prefix}-resources" + location = var.location +} + +resource "azuread_application" "example" { + name = "${var.prefix}_aad_app" +} + +resource "azuread_service_principal" "example" { + application_id = azuread_application.example.application_id +} + +resource "azurerm_storage_account" "example" { + name = "${var.prefix}storageacct" + resource_group_name = azurerm_resource_group.example.name + location = azurerm_resource_group.example.location + + account_tier = "Standard" + account_kind = "StorageV2" + account_replication_type = "LRS" + is_hns_enabled = true +} + +data "azurerm_client_config" "current" { +} + +resource "azurerm_role_assignment" "storageAccountRoleAssignment" { + scope = azurerm_storage_account.example.id + role_definition_name = "Storage Blob Data Contributor" + principal_id = data.azurerm_client_config.current.object_id +} + +resource "azurerm_storage_data_lake_gen2_filesystem" "example" { + name = "example" + storage_account_id = azurerm_storage_account.example.id + ace { + type = "user" + permissions = "rwx" + } + ace { + type = "user" + id = azuread_service_principal.example.object_id + permissions = "--x" + } + ace { + type = "group" + permissions = "r-x" + } + ace { + type = "mask" + permissions = "r-x" + } + ace { + type = "other" + permissions = "---" + } + depends_on = [ + azurerm_role_assignment.storageAccountRoleAssignment + ] +} + +resource "azurerm_storage_data_lake_gen2_path" "example" { + storage_account_id = azurerm_storage_account.example.id + filesystem_name = azurerm_storage_data_lake_gen2_filesystem.example.name + path = "testpath" + resource = "directory" + ace { + type = "user" + permissions = "r-x" + } + ace { + type = "user" + id = azuread_service_principal.example.object_id + permissions = "r-x" + } + ace { + type = "group" + permissions = "-wx" + } + ace { + type = "mask" + permissions = "--x" + } + ace { + type = "other" + permissions = "--x" + } + ace { + scope = "default" + type = "user" + permissions = "r-x" + } + ace { + scope = "default" + type = "user" + id = azuread_service_principal.example.object_id + permissions = "r-x" + } + ace { + scope = "default" + type = "group" + permissions = "-wx" + } + ace { + scope = "default" + type = "mask" + permissions = "--x" + } + ace { + scope = "default" + type = "other" + permissions = "--x" + } +} From f15381101fbc099c869f850873579c112d431524 Mon Sep 17 00:00:00 2001 From: ross-p-smith Date: Thu, 17 Dec 2020 16:42:13 +0000 Subject: [PATCH 3/6] Added documentation and fixed whitespace --- ...ge_data_lake_gen2_filesystem.html.markdown | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/website/docs/r/storage_data_lake_gen2_filesystem.html.markdown b/website/docs/r/storage_data_lake_gen2_filesystem.html.markdown index c4fbd1a07571..4873ece5f8b5 100644 --- a/website/docs/r/storage_data_lake_gen2_filesystem.html.markdown +++ b/website/docs/r/storage_data_lake_gen2_filesystem.html.markdown @@ -48,9 +48,26 @@ The following arguments are supported: * `storage_account_id` - (Required) Specifies the ID of the Storage Account in which the Data Lake Gen2 File System should exist. Changing this forces a new resource to be created. +* `properties` - (Optional) A mapping of Key to Base64-Encoded Values which should be assigned to this Data Lake Gen2 File System. + +* `ace` - (Optional) One or more `ace` blocks as defined below to specify the entries for the ACL for the path. + ~> **NOTE:** The Storage Account requires `account_kind` to be either `StorageV2` or `BlobStorage`. In addition, `is_hns_enabled` has to be set to `true`. -* `properties` - (Optional) A mapping of Key to Base64-Encoded Values which should be assigned to this Data Lake Gen2 File System. +--- + +An `ace` block supports the following: + +* `scope` - (Optional) Specifies whether the ACE represents an `access` entry or a `default` entry. Default value is `access`. + +* `type` - (Required) Specifies the type of entry. Can be `user`, `group`, `mask` or `other`. + +* `id` - (Optional) Specifies the Object ID of the Azure Active Directory User or Group that the entry relates to. Only valid for `user` or `group` entries. + +* `permissions` - (Required) Specifies the permissions for the entry in `rwx` form. For example, `rwx` gives full permissions but `r--` only gives read permissions. + +More details on ACLs can be found here: https://docs.microsoft.com/en-us/azure/storage/blobs/data-lake-storage-access-control#access-control-lists-on-files-and-directories + ## Attributes Reference From 56c016ffb1c6d111ab50f1ab8c0a2b2e9c1f1c5f Mon Sep 17 00:00:00 2001 From: ross-p-smith Date: Thu, 17 Dec 2020 15:42:41 +0000 Subject: [PATCH 4/6] Add ACL to Root File SYstem --- .../storage/parse/storage_filesystem_ace.go | 4 +- ...rage_data_lake_gen2_filesystem_resource.go | 8 +- ...data_lake_gen2_filesystem_resource_test.go | 127 ++++++++++++++++++ .../storage_data_lake_gen2_path_resource.go | 6 +- ...orage_data_lake_gen2_path_resource_test.go | 4 +- 5 files changed, 138 insertions(+), 11 deletions(-) diff --git a/azurerm/internal/services/storage/parse/storage_filesystem_ace.go b/azurerm/internal/services/storage/parse/storage_filesystem_ace.go index 0d3396f17777..57797f08106e 100644 --- a/azurerm/internal/services/storage/parse/storage_filesystem_ace.go +++ b/azurerm/internal/services/storage/parse/storage_filesystem_ace.go @@ -5,7 +5,7 @@ import ( "github.com/tombuildsstuff/giovanni/storage/accesscontrol" ) -func ExpandArmDataLakeGen2AceList(input []interface{}) (*accesscontrol.ACL, error) { +func ExpandDataLakeGen2AceList(input []interface{}) (*accesscontrol.ACL, error) { if len(input) == 0 { return nil, nil } @@ -46,7 +46,7 @@ func ExpandArmDataLakeGen2AceList(input []interface{}) (*accesscontrol.ACL, erro return &accesscontrol.ACL{Entries: aceList}, nil } -func FlattenArmDataLakeGen2AceList(acl accesscontrol.ACL) []interface{} { +func FlattenDataLakeGen2AceList(acl accesscontrol.ACL) []interface{} { output := make([]interface{}, len(acl.Entries)) for i, v := range acl.Entries { diff --git a/azurerm/internal/services/storage/storage_data_lake_gen2_filesystem_resource.go b/azurerm/internal/services/storage/storage_data_lake_gen2_filesystem_resource.go index e9b15dce2c94..b58f79d1ad21 100644 --- a/azurerm/internal/services/storage/storage_data_lake_gen2_filesystem_resource.go +++ b/azurerm/internal/services/storage/storage_data_lake_gen2_filesystem_resource.go @@ -124,7 +124,7 @@ func resourceStorageDataLakeGen2FileSystemCreate(d *schema.ResourceData, meta in } aceRaw := d.Get("ace").([]interface{}) - acl, err := parse.ExpandArmDataLakeGen2AceList(aceRaw) + acl, err := parse.ExpandDataLakeGen2AceList(aceRaw) if err != nil { return fmt.Errorf("Error parsing ace list: %s", err) } @@ -201,7 +201,7 @@ func resourceStorageDataLakeGen2FileSystemUpdate(d *schema.ResourceData, meta in } aceRaw := d.Get("ace").([]interface{}) - acl, err := parse.ExpandArmDataLakeGen2AceList(aceRaw) + acl, err := parse.ExpandDataLakeGen2AceList(aceRaw) if err != nil { return fmt.Errorf("Error parsing ace list: %s", err) } @@ -242,7 +242,7 @@ func resourceStorageDataLakeGen2FileSystemUpdate(d *schema.ResourceData, meta in } } - return resourceArmStorageDataLakeGen2FileSystemRead(d, meta) + return resourceStorageDataLakeGen2FileSystemRead(d, meta) } func resourceStorageDataLakeGen2FileSystemRead(d *schema.ResourceData, meta interface{}) error { @@ -309,7 +309,7 @@ func resourceStorageDataLakeGen2FileSystemRead(d *schema.ResourceData, meta inte if err != nil { return fmt.Errorf("Error parsing response ACL %q: %s", pathResponse.ACL, err) } - d.Set("ace", parse.FlattenArmDataLakeGen2AceList(acl)) + d.Set("ace", parse.FlattenDataLakeGen2AceList(acl)) return nil } diff --git a/azurerm/internal/services/storage/storage_data_lake_gen2_filesystem_resource_test.go b/azurerm/internal/services/storage/storage_data_lake_gen2_filesystem_resource_test.go index cd9eb7d8a379..f8593cabd858 100644 --- a/azurerm/internal/services/storage/storage_data_lake_gen2_filesystem_resource_test.go +++ b/azurerm/internal/services/storage/storage_data_lake_gen2_filesystem_resource_test.go @@ -46,6 +46,43 @@ func TestAccStorageDataLakeGen2FileSystem_requiresImport(t *testing.T) { }) } +func TestAccStorageDataLakeGen2FileSystem_withDefaultACL(t *testing.T) { + data := acceptance.BuildTestData(t, "azurerm_storage_data_lake_gen2_filesystem", "test") + r := StorageDataLakeGen2FileSystemResource{} + + data.ResourceTest(t, r, []resource.TestStep{ + { + Config: r.withDefaultACL(data), + Check: resource.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.RequiresImportErrorStep(r.requiresImport), + }) +} + +func TestAccStorageDataLakeGen2FileSystem_UpdateDefaultACL(t *testing.T) { + data := acceptance.BuildTestData(t, "azurerm_storage_data_lake_gen2_filesystem", "test") + r := StorageDataLakeGen2FileSystemResource{} + + data.ResourceTest(t, r, []resource.TestStep{ + { + Config: r.withDefaultACL(data), + Check: resource.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep(), + { + Config: r.withExecuteACLForSPN(data), + Check: resource.ComposeTestCheckFunc( + check.That(data.ResourceName).ExistsInAzure(r), + ), + }, + data.ImportStep(), + }) +} + func TestAccStorageDataLakeGen2FileSystem_properties(t *testing.T) { data := acceptance.BuildTestData(t, "azurerm_storage_data_lake_gen2_filesystem", "test") r := StorageDataLakeGen2FileSystemResource{} @@ -168,3 +205,93 @@ resource "azurerm_storage_account" "test" { } `, data.RandomInteger, data.Locations.Primary, data.RandomString) } + +func (r StorageDataLakeGen2FileSystemResource) withDefaultACL(data acceptance.TestData) string { + template := r.template(data) + return fmt.Sprintf(` +%s + +data "azurerm_client_config" "current" { +} + +resource "azurerm_role_assignment" "storageAccountRoleAssignment" { + scope = azurerm_storage_account.test.id + role_definition_name = "Storage Blob Data Contributor" + principal_id = data.azurerm_client_config.current.object_id +} + +resource "azurerm_storage_data_lake_gen2_filesystem" "test" { + name = "acctest-%[2]d" + storage_account_id = azurerm_storage_account.test.id + ace { + type = "user" + permissions = "rwx" + } + ace { + type = "group" + permissions = "r-x" + } + ace { + type = "other" + permissions = "---" + } + depends_on = [ + azurerm_role_assignment.storageAccountRoleAssignment + ] +} +`, template, data.RandomInteger) +} + +func (r StorageDataLakeGen2FileSystemResource) withExecuteACLForSPN(data acceptance.TestData) string { + template := r.template(data) + return fmt.Sprintf(` +%s + +data "azurerm_client_config" "current" { +} + +resource "azurerm_role_assignment" "storageAccountRoleAssignment" { + scope = azurerm_storage_account.test.id + role_definition_name = "Storage Blob Data Contributor" + principal_id = data.azurerm_client_config.current.object_id +} + +resource "azuread_application" "test" { + name = "acctestspa%[2]d" +} + +resource "azuread_service_principal" "test" { + application_id = azuread_application.test.application_id +} + +resource "azurerm_storage_data_lake_gen2_filesystem" "test" { + name = "acctest-%[2]d" + storage_account_id = azurerm_storage_account.test.id + ace { + type = "user" + permissions = "rwx" + } + ace { + type = "user" + id = azuread_service_principal.test.object_id + permissions = "--x" + } + ace { + type = "group" + permissions = "r-x" + } + ace { + type = "mask" + permissions = "r-x" + } + ace { + type = "other" + permissions = "---" + } + depends_on = [ + azurerm_role_assignment.storageAccountRoleAssignment, + azuread_service_principal.test + ] +} +`, template, data.RandomInteger) +} diff --git a/azurerm/internal/services/storage/storage_data_lake_gen2_path_resource.go b/azurerm/internal/services/storage/storage_data_lake_gen2_path_resource.go index 2ea3de37b174..869c77c52bc5 100644 --- a/azurerm/internal/services/storage/storage_data_lake_gen2_path_resource.go +++ b/azurerm/internal/services/storage/storage_data_lake_gen2_path_resource.go @@ -184,7 +184,7 @@ func resourceStorageDataLakeGen2PathCreate(d *schema.ResourceData, meta interfac return fmt.Errorf("Unhandled resource type %q", resourceString) } aceRaw := d.Get("ace").([]interface{}) - acl, err := parse.ExpandArmDataLakeGen2AceList(aceRaw) + acl, err := parse.ExpandDataLakeGen2AceList(aceRaw) if err != nil { return fmt.Errorf("Error parsing ace list: %s", err) } @@ -248,7 +248,7 @@ func resourceStorageDataLakeGen2PathUpdate(d *schema.ResourceData, meta interfac path := d.Get("path").(string) aceRaw := d.Get("ace").([]interface{}) - acl, err := parse.ExpandArmDataLakeGen2AceList(aceRaw) + acl, err := parse.ExpandDataLakeGen2AceList(aceRaw) if err != nil { return fmt.Errorf("Error parsing ace list: %s", err) } @@ -336,7 +336,7 @@ func resourceStorageDataLakeGen2PathRead(d *schema.ResourceData, meta interface{ if err != nil { return fmt.Errorf("Error parsing response ACL %q: %s", resp.ACL, err) } - d.Set("ace", parse.FlattenArmDataLakeGen2AceList(acl)) + d.Set("ace", parse.FlattenDataLakeGen2AceList(acl)) return nil } diff --git a/azurerm/internal/services/storage/storage_data_lake_gen2_path_resource_test.go b/azurerm/internal/services/storage/storage_data_lake_gen2_path_resource_test.go index 8100068f312d..4a74c14c2a90 100644 --- a/azurerm/internal/services/storage/storage_data_lake_gen2_path_resource_test.go +++ b/azurerm/internal/services/storage/storage_data_lake_gen2_path_resource_test.go @@ -46,7 +46,7 @@ func TestAccStorageDataLakeGen2Path_requiresImport(t *testing.T) { }) } -func TestAccStorageDataLakeGen2Path_withSimpleACL(t *testing.T) { +func TestAccStorageDataLakeGen2Path_withSimpleACLAndUpdate(t *testing.T) { data := acceptance.BuildTestData(t, "azurerm_storage_data_lake_gen2_path", "test") r := StorageDataLakeGen2PathResource{} @@ -68,7 +68,7 @@ func TestAccStorageDataLakeGen2Path_withSimpleACL(t *testing.T) { }) } -func TestAccStorageDataLakeGen2Path_withSimpleACLAndUpdate(t *testing.T) { +func TestAccStorageDataLakeGen2Path_withSimpleACL(t *testing.T) { data := acceptance.BuildTestData(t, "azurerm_storage_data_lake_gen2_path", "test") r := StorageDataLakeGen2PathResource{} From 649298eaf34c283223eb0d6a754fe6e180cd1037 Mon Sep 17 00:00:00 2001 From: ross-p-smith Date: Sat, 2 Jan 2021 15:55:59 +0000 Subject: [PATCH 5/6] Added terrafmt spacing fix back in --- .../storage_data_lake_gen2_filesystem_resource_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azurerm/internal/services/storage/storage_data_lake_gen2_filesystem_resource_test.go b/azurerm/internal/services/storage/storage_data_lake_gen2_filesystem_resource_test.go index f8593cabd858..1e224dd445d9 100644 --- a/azurerm/internal/services/storage/storage_data_lake_gen2_filesystem_resource_test.go +++ b/azurerm/internal/services/storage/storage_data_lake_gen2_filesystem_resource_test.go @@ -289,8 +289,8 @@ resource "azurerm_storage_data_lake_gen2_filesystem" "test" { permissions = "---" } depends_on = [ - azurerm_role_assignment.storageAccountRoleAssignment, - azuread_service_principal.test + azurerm_role_assignment.storageAccountRoleAssignment, + azuread_service_principal.test ] } `, template, data.RandomInteger) From e6df43974c5b3d5669366cd093260666c28f5e79 Mon Sep 17 00:00:00 2001 From: ross-p-smith Date: Fri, 15 Jan 2021 15:59:34 +0000 Subject: [PATCH 6/6] Removed Devcontainer --- .devcontainer/Dockerfile | 70 --------------------------------- .devcontainer/devcontainer.json | 43 -------------------- 2 files changed, 113 deletions(-) delete mode 100644 .devcontainer/Dockerfile delete mode 100644 .devcontainer/devcontainer.json diff --git a/.devcontainer/Dockerfile b/.devcontainer/Dockerfile deleted file mode 100644 index c0ddf0f6d709..000000000000 --- a/.devcontainer/Dockerfile +++ /dev/null @@ -1,70 +0,0 @@ -#------------------------------------------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See https://go.microsoft.com/fwlink/?linkid=2090316 for license information. -#------------------------------------------------------------------------------------------------------------- - -FROM golang:1.14-buster - -# Avoid warnings by switching to noninteractive -ENV DEBIAN_FRONTEND=noninteractive - -# This Dockerfile adds a non-root 'vscode' user with sudo access. However, for Linux, -# this user's GID/UID must match your local user UID/GID to avoid permission issues -# with bind mounts. Update USER_UID / USER_GID if yours is not 1000. See -# https://aka.ms/vscode-remote/containers/non-root-user for details. -ARG USERNAME=vscode -ARG USER_UID=1000 -ARG USER_GID=$USER_UID - -# Terraform and tflint versions -ARG TERRAFORM_VERSION=0.14.2 - -ENV GO111MODULE=on - -# Configure apt, install packages and tools -RUN apt-get update \ - && apt-get -y install --no-install-recommends curl unzip apt-utils dialog \ - # - # Verify git, process tools, lsb-release (common in install instructions for CLIs) installed - && apt-get -y install git iproute2 procps lsb-release \ - # - # Install Azure CLI - && curl -sL https://aka.ms/InstallAzureCLIDeb | bash \ - # - # Create a non-root user to use if preferred - see https://aka.ms/vscode-remote/containers/non-root-user. - && groupadd --gid $USER_GID $USERNAME \ - && useradd -s /bin/bash --uid $USER_UID --gid $USER_GID -m $USERNAME \ - && echo $USERNAME \ - # - # Clean up - && apt-get autoremove -y \ - && apt-get clean -y \ - && rm -rf /var/lib/apt/lists/* - -ENV GIT_PROMPT_START='\033[1;36maztf-devcon>\033[0m\033[0;33m\w\a\033[0m' - -# Save command line history -RUN echo "export HISTFILE=/root/commandhistory/.bash_history" >> "/root/.bashrc" \ - && echo "export PROMPT_COMMAND='history -a'" >> "/root/.bashrc" \ - && mkdir -p /root/commandhistory \ - && touch /root/commandhistory/.bash_history - -# Git command prompt -RUN git clone https://github.com/magicmonty/bash-git-prompt.git ~/.bash-git-prompt --depth=1 \ - && echo "if [ -f \"$HOME/.bash-git-prompt/gitprompt.sh\" ]; then GIT_PROMPT_ONLY_IN_REPO=1 && source $HOME/.bash-git-prompt/gitprompt.sh; fi" >> "/root/.bashrc" - -# Install Go tools -RUN \ - # --> Delve for debugging - go get github.com/go-delve/delve/cmd/dlv@v1.5.0 \ - # --> Go language server - && go get golang.org/x/tools/gopls@v0.5.5 \ - # --> Go symbols and outline for go to symbol support and test support - && go get github.com/acroca/go-symbols@v0.1.1 && go get github.com/ramya-rao-a/go-outline@7182a932836a71948db4a81991a494751eccfe77 - -RUN \ - # Install Terraform - mkdir -p /tmp/docker-downloads \ - && curl -sSL -o /tmp/docker-downloads/terraform.zip https://releases.hashicorp.com/terraform/${TERRAFORM_VERSION}/terraform_${TERRAFORM_VERSION}_linux_amd64.zip \ - && unzip /tmp/docker-downloads/terraform.zip \ - && mv terraform /usr/local/bin \ No newline at end of file diff --git a/.devcontainer/devcontainer.json b/.devcontainer/devcontainer.json deleted file mode 100644 index 6c0838c9e3da..000000000000 --- a/.devcontainer/devcontainer.json +++ /dev/null @@ -1,43 +0,0 @@ -// For format details, see https://aka.ms/vscode-remote/devcontainer.json or this file's README at: -// https://github.com/microsoft/vscode-dev-containers/tree/v0.106.0/containers/go -{ - "name": "Go", - "dockerFile": "Dockerfile", - "runArgs": [ "--cap-add=SYS_PTRACE", "--security-opt", "seccomp=unconfined" ], - - // Set *default* container specific settings.json values on container create. - "settings": { - "terminal.integrated.shell.linux": "/bin/bash", - "go.gopath": "/go", - "go.useLanguageServer": true, - "[go]": { - "editor.snippetSuggestions": "none", - "editor.formatOnSave": true, - "editor.codeActionsOnSave": { - "source.organizeImports": true, - } - }, - "gopls": { - "usePlaceholders": true, // add parameter placeholders when completing a function - // Experimental settings - "completeUnimported": true, // autocomplete unimported packages - "watchFileChanges": true, // watch file changes outside of the editor - "deepCompletion": true, // enable deep completion - }, - "files.eol": "\n", // formatting only supports LF line endings - }, - - // Add the IDs of extensions you want installed when the container is created. - "extensions": [ - "golang.go" - ], - - // Use 'forwardPorts' to make a list of ports inside the container available locally. - // "forwardPorts": [], - - // Use 'postCreateCommand' to run commands after the container is created. - "postCreateCommand": "make tools", - - // Uncomment to connect as a non-root user. See https://aka.ms/vscode-remote/containers/non-root. - // "remoteUser": "vscode" -} \ No newline at end of file