api: Add NewWithCredentials() (#646)

This PR adds a new API

  - NewWithCredentials()

Internally NewWithCredentials is now used with
all APIs such as New(), NewV4(), NewV2() and NewWithRegion.

Also brings a new package called `credentials` to manage
various credentials type, currently the credentials
package supports

  - Reading file from `.aws/credentials`, `.mc/config.json`
  - Reading env variables for AWS*, MINIO*
  - Fetching from IAM roles assigned to an EC2 instance.
  - Static credentials which is the current default behavior.

Example code using IAM.

```go
        iam := credentials.NewIAM("")
        s3Client, err := minio.NewWithCredentials("s3.amazonaws.com", iam, true, "")
        if err != nil {
                log.Fatalln(err)
        }

        buckets, err := s3Client.ListBuckets()
        if err != nil {
                log.Fatalln(err)
        }
        for _, bucket := range buckets {
                log.Println(bucket)
        }
```

Fixes #643

api-presigned.go

@@ -122,21 +122,38 @@ func (c Client) PresignedPostPolicy(p *PostPolicy) (u *url.URL, formData map[str
 		return nil, nil, err
 	}
 
+	// Get credentials from the configured credentials provider.
+	credValues, err := c.credsProvider.Get()
+	if err != nil {
+		return nil, nil, err
+	}
+
+	var (
+		signerType      = credValues.SignerType
+		sessionToken    = credValues.SessionToken
+		accessKeyID     = credValues.AccessKeyID
+		secretAccessKey = credValues.SecretAccessKey
+	)
+
+	if signerType.IsAnonymous() {
+		return nil, nil, ErrInvalidArgument("Presigned operations are not supported for anonymous credentials")
+	}
+
 	// Keep time.
 	t := time.Now().UTC()
 	// For signature version '2' handle here.
-	if c.signature.isV2() {
+	if signerType.IsV2() {
 		policyBase64 := p.base64()
 		p.formData["policy"] = policyBase64
 		// For Google endpoint set this value to be 'GoogleAccessId'.
 		if s3utils.IsGoogleEndpoint(c.endpointURL) {
-			p.formData["GoogleAccessId"] = c.accessKeyID
+			p.formData["GoogleAccessId"] = accessKeyID
 		} else {
 			// For all other endpoints set this value to be 'AWSAccessKeyId'.
-			p.formData["AWSAccessKeyId"] = c.accessKeyID
+			p.formData["AWSAccessKeyId"] = accessKeyID
 		}
 		// Sign the policy.
-		p.formData["signature"] = s3signer.PostPresignSignatureV2(policyBase64, c.secretAccessKey)
+		p.formData["signature"] = s3signer.PostPresignSignatureV2(policyBase64, secretAccessKey)
 		return u, p.formData, nil
 	}
 
@@ -159,7 +176,7 @@ func (c Client) PresignedPostPolicy(p *PostPolicy) (u *url.URL, formData map[str
 	}
 
 	// Add a credential policy.
-	credential := s3signer.GetCredential(c.accessKeyID, location, t)
+	credential := s3signer.GetCredential(accessKeyID, location, t)
 	if err = p.addNewPolicy(policyCondition{
 		matchType: "eq",
 		condition: "$x-amz-credential",
@@ -168,13 +185,27 @@ func (c Client) PresignedPostPolicy(p *PostPolicy) (u *url.URL, formData map[str
 		return nil, nil, err
 	}
 
+	if sessionToken != "" {
+		if err = p.addNewPolicy(policyCondition{
+			matchType: "eq",
+			condition: "$x-amz-security-token",
+			value:     sessionToken,
+		}); err != nil {
+			return nil, nil, err
+		}
+	}
+
 	// Get base64 encoded policy.
 	policyBase64 := p.base64()
+
 	// Fill in the form data.
 	p.formData["policy"] = policyBase64
 	p.formData["x-amz-algorithm"] = signV4Algorithm
 	p.formData["x-amz-credential"] = credential
 	p.formData["x-amz-date"] = t.Format(iso8601DateFormat)
-	p.formData["x-amz-signature"] = s3signer.PostPresignSignatureV4(policyBase64, t, c.secretAccessKey, location)
+	if sessionToken != "" {
+		p.formData["x-amz-security-token"] = sessionToken
+	}
+	p.formData["x-amz-signature"] = s3signer.PostPresignSignatureV4(policyBase64, t, secretAccessKey, location)
 	return u, p.formData, nil
 }

api-put-bucket.go

@@ -1,5 +1,6 @@
 /*
- * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc.
+ * Minio Go Library for Amazon S3 Compatible Cloud Storage
+ * (C) 2015, 2016, 2017 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -28,6 +29,7 @@ import (
 	"net/url"
 	"path"
 
+	"github.com/minio/minio-go/pkg/credentials"
 	"github.com/minio/minio-go/pkg/policy"
 	"github.com/minio/minio-go/pkg/s3signer"
 )
@@ -135,9 +137,32 @@ func (c Client) makeBucketRequest(bucketName string, location string) (*http.Req
 	// set UserAgent for the request.
 	c.setUserAgent(req)
 
-	// set sha256 sum for signature calculation only with
-	// signature version '4'.
-	if c.signature.isV4() {
+	// Get credentials from the configured credentials provider.
+	value, err := c.credsProvider.Get()
+	if err != nil {
+		return nil, err
+	}
+
+	var (
+		signerType      = value.SignerType
+		accessKeyID     = value.AccessKeyID
+		secretAccessKey = value.SecretAccessKey
+		sessionToken    = value.SessionToken
+	)
+
+	// Custom signer set then override the behavior.
+	if c.overrideSignerType != credentials.SignatureDefault {
+		signerType = c.overrideSignerType
+	}
+
+	// If signerType returned by credentials helper is anonymous,
+	// then do not sign regardless of signerType override.
+	if value.SignerType == credentials.SignatureAnonymous {
+		signerType = credentials.SignatureAnonymous
+	}
+
+	// set sha256 sum for signature calculation only with signature version '4'.
+	if signerType.IsV4() {
 		req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum256([]byte{})))
 	}
 
@@ -155,19 +180,19 @@ func (c Client) makeBucketRequest(bucketName string, location string) (*http.Req
 		req.ContentLength = int64(len(createBucketConfigBytes))
 		// Set content-md5.
 		req.Header.Set("Content-Md5", base64.StdEncoding.EncodeToString(sumMD5(createBucketConfigBytes)))
-		if c.signature.isV4() {
+		if signerType.IsV4() {
 			// Set sha256.
 			req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum256(createBucketConfigBytes)))
 		}
 	}
 
 	// Sign the request.
-	if c.signature.isV4() {
+	if signerType.IsV4() {
 		// Signature calculated for MakeBucket request should be for 'us-east-1',
 		// regardless of the bucket's location constraint.
-		req = s3signer.SignV4(*req, c.accessKeyID, c.secretAccessKey, "us-east-1")
-	} else if c.signature.isV2() {
-		req = s3signer.SignV2(*req, c.accessKeyID, c.secretAccessKey)
+		req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1")
+	} else if signerType.IsV2() {
+		req = s3signer.SignV2(*req, accessKeyID, secretAccessKey)
 	}
 
 	// Return signed request.

api-put-bucket_test.go

@@ -1,5 +1,6 @@
 /*
- * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc.
+ * Minio Go Library for Amazon S3 Compatible Cloud Storage
+ * (C) 2015, 2016, 2017 Minio, Inc.
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -27,6 +28,7 @@ import (
 	"path"
 	"testing"
 
+	"github.com/minio/minio-go/pkg/credentials"
 	"github.com/minio/minio-go/pkg/s3signer"
 )
 
@@ -48,8 +50,32 @@ func TestMakeBucketRequest(t *testing.T) {
 		// set UserAgent for the request.
 		c.setUserAgent(req)
 
+		// Get credentials from the configured credentials provider.
+		value, err := c.credsProvider.Get()
+		if err != nil {
+			return nil, err
+		}
+
+		var (
+			signerType      = value.SignerType
+			accessKeyID     = value.AccessKeyID
+			secretAccessKey = value.SecretAccessKey
+			sessionToken    = value.SessionToken
+		)
+
+		// Custom signer set then override the behavior.
+		if c.overrideSignerType != credentials.SignatureDefault {
+			signerType = c.overrideSignerType
+		}
+
+		// If signerType returned by credentials helper is anonymous,
+		// then do not sign regardless of signerType override.
+		if value.SignerType == credentials.SignatureAnonymous {
+			signerType = credentials.SignatureAnonymous
+		}
+
 		// set sha256 sum for signature calculation only with signature version '4'.
-		if c.signature.isV4() {
+		if signerType.IsV4() {
 			req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum256([]byte{})))
 		}
 
@@ -67,19 +93,19 @@ func TestMakeBucketRequest(t *testing.T) {
 			req.ContentLength = int64(len(createBucketConfigBytes))
 			// Set content-md5.
 			req.Header.Set("Content-Md5", base64.StdEncoding.EncodeToString(sumMD5(createBucketConfigBytes)))
-			if c.signature.isV4() {
+			if signerType.IsV4() {
 				// Set sha256.
 				req.Header.Set("X-Amz-Content-Sha256", hex.EncodeToString(sum256(createBucketConfigBytes)))
 			}
 		}
 
 		// Sign the request.
-		if c.signature.isV4() {
+		if signerType.IsV4() {
 			// Signature calculated for MakeBucket request should be for 'us-east-1',
 			// regardless of the bucket's location constraint.
-			req = s3signer.SignV4(*req, c.accessKeyID, c.secretAccessKey, "us-east-1")
-		} else if c.signature.isV2() {
-			req = s3signer.SignV2(*req, c.accessKeyID, c.secretAccessKey)
+			req = s3signer.SignV4(*req, accessKeyID, secretAccessKey, sessionToken, "us-east-1")
+		} else if signerType.IsV2() {
+			req = s3signer.SignV2(*req, accessKeyID, secretAccessKey)
 		}
 
 		// Return signed request.
@@ -246,7 +272,7 @@ func TestMakeBucketRequest(t *testing.T) {
 			}
 
 			if expectedReq.Header.Get("X-Amz-Content-Sha256") != actualReq.Header.Get("X-Amz-Content-Sha256") {
-				t.Errorf("Test %d: 'X-Amz-Content-Sha256' header of the expected request doesn't match with that of the actual request", i+1)
+				t.Errorf("Test %d: 'X-Amz-Content-Sha256' header of the expected request %s doesn't match with that of the actual request %s", i+1, expectedReq.Header.Get("X-Amz-Content-Sha256"), actualReq.Header.Get("X-Amz-Content-Sha256"))
 			}
 			if expectedReq.Header.Get("User-Agent") != actualReq.Header.Get("User-Agent") {
 				t.Errorf("Test %d: Expected 'User-Agent' header to be \"%s\",but found \"%s\" instead", i+1, expectedReq.Header.Get("User-Agent"), actualReq.Header.Get("User-Agent"))

api-put-object-file.go

@@ -182,7 +182,7 @@ func (c Client) putObjectMultipartFromFile(bucketName, objectName string, fileRe
 				hashAlgos := make(map[string]hash.Hash)
 				hashSums := make(map[string][]byte)
 				hashAlgos["md5"] = md5.New()
-				if c.signature.isV4() && !c.secure {
+				if c.overrideSignerType.IsV4() && !c.secure {
 					hashAlgos["sha256"] = sha256.New()
 				}
 

api-put-object-multipart.go

@@ -213,7 +213,7 @@ func (c Client) putObjectMultipartStream(bucketName, objectName string, reader i
 		hashSums := make(map[string][]byte)
 		hashAlgos := make(map[string]hash.Hash)
 		hashAlgos["md5"] = md5.New()
-		if c.signature.isV4() && !c.secure {
+		if c.overrideSignerType.IsV4() && !c.secure {
 			hashAlgos["sha256"] = sha256.New()
 		}
 

api-put-object-progress.go

@@ -20,6 +20,7 @@ import (
 	"io"
 	"strings"
 
+	"github.com/minio/minio-go/pkg/credentials"
 	"github.com/minio/minio-go/pkg/encrypt"
 	"github.com/minio/minio-go/pkg/s3utils"
 )
@@ -103,6 +104,7 @@ func (c Client) PutObjectWithMetadata(bucketName, objectName string, reader io.R
 	if size < minPartSize && size >= 0 {
 		return c.putObjectSingle(bucketName, objectName, reader, size, metaData, progress)
 	}
+
 	// For all sizes greater than 5MiB do multipart.
 	n, err = c.putObjectMultipart(bucketName, objectName, reader, size, metaData, progress)
 	if err != nil {
@@ -143,8 +145,8 @@ func (c Client) PutObjectStreamingWithProgress(bucketName, objectName string, re
 			BucketName: bucketName,
 		}
 	}
-	// This method should return error with signature v2 minioClient.
-	if c.signature.isV2() {
+
+	if c.overrideSignerType.IsV2() {
 		return 0, ErrorResponse{
 			Code:       "NotImplemented",
 			Message:    "AWS streaming signature v4 is not supported with minio client initialized for AWS signature v2",
@@ -173,8 +175,8 @@ func (c Client) PutObjectStreamingWithProgress(bucketName, objectName string, re
 		return c.putObjectMultipartStream(bucketName, objectName, reader, size, metadata, progress)
 	}
 
-	// Set signature type to streaming signature v4.
-	c.signature = SignatureV4Streaming
+	// Set streaming signature.
+	c.overrideSignerType = credentials.SignatureV4Streaming
 
 	if size < minPartSize && size >= 0 {
 		return c.putObjectNoChecksum(bucketName, objectName, reader, size, metadata, progress)

api-put-object-readat.go

@@ -146,7 +146,7 @@ func (c Client) putObjectMultipartFromReadAt(bucketName, objectName string, read
 				hashSums := make(map[string][]byte)
 				hashAlgos := make(map[string]hash.Hash)
 				hashAlgos["md5"] = md5.New()
-				if c.signature.isV4() && !c.secure {
+				if c.overrideSignerType.IsV4() && !c.secure {
 					hashAlgos["sha256"] = sha256.New()
 				}
 

api-put-object.go

@@ -200,7 +200,7 @@ func (c Client) putObjectSingle(bucketName, objectName string, reader io.Reader,
 	hashAlgos := make(map[string]hash.Hash)
 	hashSums := make(map[string][]byte)
 	hashAlgos["md5"] = md5.New()
-	if c.signature.isV4() && !c.secure {
+	if c.overrideSignerType.IsV4() && !c.secure {
 		hashAlgos["sha256"] = sha256.New()
 	}