Skip to content

Commit

Permalink
Add support for cli login for storage plugin (getporter#30)
Browse files Browse the repository at this point in the history
* Add support for cli login for storage plugin

Signed-off-by: Simon Davies <[email protected]>

* updates from review

Signed-off-by: Simon Davies <[email protected]>

* missed review comments

Signed-off-by: Simon Davies <[email protected]>

* fix error message

Signed-off-by: Simon Davies <[email protected]>

* updated BOM handling

Signed-off-by: Simon Davies <[email protected]>
  • Loading branch information
simongdavies authored Nov 17, 2020
1 parent 43ca733 commit 0e8b7e7
Show file tree
Hide file tree
Showing 8 changed files with 374 additions and 6 deletions.
2 changes: 1 addition & 1 deletion .vscode/tasks.json
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@
"command": "./.vscode/scripts/runporterinbackground.sh",
"options": {
"env": {
"PORTER_RUN_PLUGIN_IN_DEBUGGER": "secrets.azure.keyvault",
"PORTER_RUN_PLUGIN_IN_DEBUGGER": "storage.azure.blob",
"PORTER_PLUGIN_WORKING_DIRECTORY": "${workspaceRoot}/cmd/azure",
"PORTER_DEBUGGER_PORT": "2345",
"PORTER_HOME": "/home/${env:USER}/.porter"
Expand Down
33 changes: 29 additions & 4 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -23,20 +23,45 @@ Storage plugins allow Porter to store data, such as claims, parameters and crede

### Blob

The `azure.blob` plugin stores data in Azure Blob Storage.
The `azure.blob` plugin stores data in Azure Blob Storage. The plugin requires a storage account name and storage account key. This can be provided as a connection string in an environment variable or can be looked up at run time if the user is logged in with the Azure CLI.

1. [Create a storage account][account]
1. [Create a container][container] named `porter`.
1. Open, or create, `~/.porter/config.toml`.

#### Use a connection string

1. Add the following line to activate the Azure blob storage plugin:

```toml
default-storage-plugin = "azure.blob"
```

1. [Create a storage account][account]
1. [Create a container][container] named `porter`.
1. [Copy the connection string][connstring] for the storage account. Then set it as an environment variable named
`AZURE_STORAGE_CONNECTION_STRING`.

#### Use the Azure CLI

1. Add the following lines to activate the Azure blob storage plugin and configure storage account details:

```toml
default-storage = "azureblob"

[[storage]]
name = "azureblob"
plugin = "azure.blob"

[storage.config]
account="storage account name"
resource-group="storage account resource group"

```

If the machine you are using is already logged in with the Azure CLI, then the same security context will be used to lookup the keys for the storage account. By default it will use the current subscription (the one returned by the command `az account show`). To set the subscription explicitly add the following line to the `[storage.config]`.

```toml
subscription-id="storage account subscription id"
```

## Secrets

Secrets plugins allow Porter to inject secrets into credential or parameter sets.
Expand Down
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ require (
github.com/Azure/go-autorest/autorest v0.11.2
github.com/Azure/go-autorest/autorest/adal v0.9.0
github.com/Azure/go-autorest/autorest/azure/auth v0.5.0
github.com/Azure/go-autorest/autorest/azure/cli v0.4.0
github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
github.com/Azure/go-autorest/autorest/validation v0.3.0 // indirect
github.com/cnabio/cnab-go v0.13.4-0.20200817181428-9005c1da4354
Expand Down
9 changes: 9 additions & 0 deletions pkg/azure/azureconfig/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,15 @@ type Config struct {
// string should be loaded.
EnvConnectionString string `json:"env"`

// StorageAccount contains the name of the storage account to be used by the Azure storage plugin, if the azure connection environment variable is not set and this proeprty and StorageAccountResourceGroup are populated and the user is logged in with the Azure CLI
// the Storage Account Key will be looked up at runtime using the logged in users credentials
StorageAccount string `json:"account"`
// StorageAccountResourceGroup contains the name of the resource group containing the storage account to be used by the Azure storage plugin, if the azure connection environment variable is not set and this property and StorageAccount are populated and the user is logged in with the Azure CLI
// the Storage Account Key will be looked up at runtime using the logged in users credentials
StorageAccountResourceGroup string `json:"resource-group"`
// StorageAccountSubscriptionId contains the subscription id of the subscription to be used when looking up the Storage Account Key, if this is not set then the current CLI subscription will be used
StorageAccountSubscriptionId string `json:"subscription-id"`

// EnvAzurePrefix is the prefix applied to every azure
// environment variable For example, for a prefix of "DEV_AZURE_", the
// variables would be "DEV_AZURE_TENANT_ID", "DEV_AZURE_CLIENT_ID",
Expand Down
138 changes: 137 additions & 1 deletion pkg/azure/blob/credentials.go
Original file line number Diff line number Diff line change
@@ -1,12 +1,22 @@
package blob

import (
"bufio"
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"os"
"path"
"regexp"

"get.porter.sh/plugin/azure/pkg/azure/azureconfig"

"github.com/Azure/azure-pipeline-go/pipeline"
"github.com/Azure/azure-sdk-for-go/profiles/latest/storage/mgmt/storage"
"github.com/Azure/azure-storage-blob-go/azblob"
"github.com/Azure/go-autorest/autorest/azure/auth"
"github.com/hashicorp/go-hclog"
"github.com/pkg/errors"
)
Expand All @@ -17,6 +27,22 @@ type CredentialSet struct {
}

const ConnectionEnvironmentVariable = "AZURE_STORAGE_CONNECTION_STRING"
const PublicCloud = "AzureCloud"
const AzureDirectory = ".azure"
const AzureProile = "azureProfile.json"
const UserAgent = "porter.azure.storage.plugin"
const BOM = '\uFEFF'

type AvailableSubscription struct {
SubscriptionId string `json:"id"`
State string `json:"state"`
IsDefault bool `json:"isDefault"`
EnvironmentName string `json:"environmentName"`
}

type AvailableSubscriptions struct {
Subscriptions []AvailableSubscription `json:"subscriptions"`
}

func GetCredentials(cfg azureconfig.Config, l hclog.Logger) (CredentialSet, error) {
var credsEnv = cfg.EnvConnectionString
Expand All @@ -26,7 +52,14 @@ func GetCredentials(cfg azureconfig.Config, l hclog.Logger) (CredentialSet, erro

connString := os.Getenv(credsEnv)
if connString == "" {
return CredentialSet{}, errors.Errorf("environment variable %s containing the azure storage connection string was not set\n%#v", credsEnv, cfg)
cred, useCli, err := GetCredentialsFromCli(cfg, l)
if !useCli {
return CredentialSet{}, errors.Errorf("environment variable %s containing the azure storage connection string was not set:\n%#v", credsEnv, cfg)
}
if err != nil {
return CredentialSet{}, errors.Errorf("%v\n%#v", err, cfg)
}
return cred, nil
}

accountName, accountKey, err := parseConnectionString(connString)
Expand All @@ -43,6 +76,109 @@ func GetCredentials(cfg azureconfig.Config, l hclog.Logger) (CredentialSet, erro
return CredentialSet{Credential: *cred, Pipeline: pipe}, nil
}

func GetCredentialsFromCli(cfg azureconfig.Config, l hclog.Logger) (CredentialSet, bool, error) {

if cfg.StorageAccount == "" && cfg.StorageAccountResourceGroup == "" {
return CredentialSet{}, false, nil
}

if cfg.StorageAccount == "" {
return CredentialSet{}, true, errors.New("account is not set - cannot login with Azure CLI")
}

if cfg.StorageAccountResourceGroup == "" {
return CredentialSet{}, true, errors.New("resource-group is not set - cannot login with Azure CLI")
}

authorizer, err := auth.NewAuthorizerFromCLI()
if err != nil {
return CredentialSet{}, true, errors.Wrap(err, "Failed to login with Azure CLI")
}
subscriptionId := cfg.StorageAccountSubscriptionId
if subscriptionId == "" {
subscriptionId, err = getCurrentAzureSubscriptionFromCli()
if err != nil {
return CredentialSet{}, true, err
}
}
accountsClient := storage.NewAccountsClient(subscriptionId)
accountsClient.Authorizer = authorizer
err = accountsClient.AddToUserAgent(UserAgent)
if err != nil {
l.Debug(fmt.Sprintf("Error updating User Agent string for Azure: %v", err))
}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
result, err := accountsClient.ListKeys(ctx, cfg.StorageAccountResourceGroup, cfg.StorageAccount, "")
if err != nil {
return CredentialSet{}, true, errors.Wrap(err, "Failed to get storage account keys")
}
storageAccountKey := (*result.Keys)[0]
cred, err := azblob.NewSharedKeyCredential(cfg.StorageAccount, *storageAccountKey.Value)
if err != nil {
return CredentialSet{}, true, errors.Wrap(err, "Failed to create storage account credential")
}
pipe := azblob.NewPipeline(cred, azblob.PipelineOptions{})
return CredentialSet{Credential: *cred, Pipeline: pipe}, true, nil

}

func getCurrentAzureSubscriptionFromCli() (string, error) {

home, err := os.UserHomeDir()
if err != nil {
return "", errors.Wrap(err, "Error getting home directory")
}

return getCurrentAzureSubscriptionFromProfile(path.Join(home, AzureDirectory, AzureProile))
}

func getCurrentAzureSubscriptionFromProfile(filename string) (string, error) {

file, err := os.Open(filename)
if err != nil {
return "", errors.Wrap(err, "Error getting azure profile")
}
defer file.Close()

// azureProfile can have BOM so check for BOM before decoding

reader := bufio.NewReader(file)
if err := removeBOM(reader); err != nil {
return "", err
}

data, err := ioutil.ReadAll(reader)
if err != nil {
return "", errors.Wrap(err, "Error reading Azure profile")
}

var subscriptions AvailableSubscriptions
if err := json.Unmarshal(data, &subscriptions); err != nil {
return "", errors.Wrap(err, "Failed to decode Azure Profile")
}

for _, availableSubscription := range subscriptions.Subscriptions {
if availableSubscription.EnvironmentName == PublicCloud && availableSubscription.IsDefault {
return availableSubscription.SubscriptionId, nil
}
}

return "", errors.New("Failed to get current subscription from cli config")
}

func removeBOM(reader *bufio.Reader) error {
rune, _, err := reader.ReadRune()
if err != nil && err != io.EOF {
return errors.Wrap(err, "Error testing azure profile for BOM")
}
if rune != BOM && err != io.EOF {
if err := reader.UnreadRune(); err != nil {
return errors.Wrap(err, "Failed to unread rune")
}
}
return nil
}
func parseConnectionString(connString string) (name string, key string, err error) {
keyRegex := regexp.MustCompile("AccountKey=([^;]+)")
keyMatch := keyRegex.FindAllStringSubmatch(connString, -1)
Expand Down
133 changes: 133 additions & 0 deletions pkg/azure/blob/credentials_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,133 @@
package blob

import (
"fmt"
"os"
"path"
"strings"
"testing"

"get.porter.sh/plugin/azure/pkg/azure/azureconfig"
"github.com/Azure/go-autorest/autorest/azure/cli"
"github.com/hashicorp/go-hclog"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)

func Test_GetCredentials(t *testing.T) {
testcases := []struct {
name string
envVarsToSet map[string]string
config *azureconfig.Config
wantError string
}{
{
"Missing Environment Variables",
map[string]string{},
&azureconfig.Config{},
"environment variable AZURE_STORAGE_CONNECTION_STRING containing the azure storage connection string was not set:\nazureconfig.Config{EnvConnectionString:\"\", StorageAccount:\"\", StorageAccountResourceGroup:\"\", StorageAccountSubscriptionId:\"\", EnvAzurePrefix:\"\", Vault:\"\"}",
},
{
"Invalid Connection string",
map[string]string{
"AZURE_STORAGE_CONNECTION_STRING": "Invalid",
},
&azureconfig.Config{},
"unexpected format for AZURE_STORAGE_CONNECTION_STRING, could not find AccountName=NAME and AccountKey=KEY in it",
},
{
"Valid Connection string",
map[string]string{
"AZURE_STORAGE_CONNECTION_STRING": "DefaultEndpointsProtocol=https;AccountName=bmFtZQo=;AccountKey=a2V5Cg==;EndpointSuffix=core.windows.net",
},
&azureconfig.Config{},
"",
},
{
"Missing Storage Acccount Resource Group",
map[string]string{},
&azureconfig.Config{
StorageAccount: "account",
},
"resource-group is not set - cannot login with Azure CLI\nazureconfig.Config{EnvConnectionString:\"\", StorageAccount:\"account\", StorageAccountResourceGroup:\"\", StorageAccountSubscriptionId:\"\", EnvAzurePrefix:\"\", Vault:\"\"}",
},
{
"Missing Storage Acccount Name",
map[string]string{},
&azureconfig.Config{
StorageAccountResourceGroup: "group",
},
"account is not set - cannot login with Azure CLI\nazureconfig.Config{EnvConnectionString:\"\", StorageAccount:\"\", StorageAccountResourceGroup:\"group\", StorageAccountSubscriptionId:\"\", EnvAzurePrefix:\"\", Vault:\"\"}",
},
}
for _, tc := range testcases {

t.Run(tc.name, func(t *testing.T) {

logger := hclog.New(&hclog.LoggerOptions{
Name: strings.ReplaceAll(tc.name, " ", "_"),
Output: os.Stderr,
Level: hclog.Error,
})

for k, v := range tc.envVarsToSet {
os.Setenv(k, v)
}

defer func() {
for k := range tc.envVarsToSet {
os.Unsetenv(k)
}
}()

cred, err := GetCredentials(*tc.config, logger)
if tc.wantError == "" {
require.NoError(t, err, "GetCredentials should have not returned an error")
assert.NotNil(t, cred)
} else {
require.Error(t, err, "GetCredentials should have returned an error")
assert.EqualError(t, err, tc.wantError)
}
})
}
}

func Test_LoginwithCLI(t *testing.T) {

logger := hclog.New(&hclog.LoggerOptions{
Name: t.Name(),
Output: os.Stderr,
Level: hclog.Error,
})

config := &azureconfig.Config{
StorageAccount: "account",
StorageAccountResourceGroup: "group",
}

_, err := GetCredentials(*config, logger)
require.Error(t, err, "GetCredentials should have returned an error")
if isLoggedInWithAzureCLI() {
assert.Contains(t, err.Error(), "Failed to get storage account keys:")
} else {
assert.Contains(t, err.Error(), "Failed to login with Azure CLI:")
}
}
func Test_ParseAzureProfile(t *testing.T) {

files := []string{"profile_with_bom.json", "profile_without_bom.json"}
for _, filename := range files {
testName := fmt.Sprintf("parsing %s", filename)
t.Run(testName, func(t *testing.T) {
testdata := path.Join("testdata", filename)
subscriptionId, err := getCurrentAzureSubscriptionFromProfile(testdata)
assert.NoError(t, err, "Expected no error parsing Azure Profile")
assert.Equal(t, "8b5ab980-0253-40d6-b22a-61b3f9d94491", subscriptionId, "Expected Subscription not found parsing Azure Profile")
})
}
}

func isLoggedInWithAzureCLI() bool {
_, err := cli.GetTokenFromCLI("https://management.azure.com/")
return err == nil
}
Loading

0 comments on commit 0e8b7e7

Please sign in to comment.