Introduce client level custom settings with SetS3TransferAccelerate()…

… API. (#586)

Fixes #585

api-error-response.go

@@ -149,6 +149,16 @@ func httpRespToErrorResponse(resp *http.Response, bucketName, objectName string)
 	return errResp
 }
 
+// ErrTransferAccelerationBucket - bucket name is invalid to be used with transfer acceleration.
+func ErrTransferAccelerationBucket(bucketName string) error {
+	msg := fmt.Sprintf("The name of the bucket used for Transfer Acceleration must be DNS-compliant and must not contain periods (\".\").")
+	return ErrorResponse{
+		Code:       "InvalidArgument",
+		Message:    msg,
+		BucketName: bucketName,
+	}
+}
+
 // ErrEntityTooLarge - Input size is larger than supported maximum.
 func ErrEntityTooLarge(totalSize, maxObjectSize int64, bucketName, objectName string) error {
 	msg := fmt.Sprintf("Your proposed upload size ‘%d’ exceeds the maximum allowed object size ‘%d’ for single PUT operation.", totalSize, maxObjectSize)

api.go

@@ -71,6 +71,9 @@ type Client struct {
 	isTraceEnabled bool
 	traceOutput    io.Writer
 
+	// S3 specific accelerated endpoint.
+	s3AccelerateEndpoint string
+
 	// Random seed.
 	random *rand.Rand
 }
@@ -206,8 +209,7 @@ func privateNew(endpoint, accessKeyID, secretAccessKey string, secure bool) (*Cl
 
 // SetAppInfo - add application details to user agent.
 func (c *Client) SetAppInfo(appName string, appVersion string) {
-	// if app name and version is not set, we do not a new user
-	// agent.
+	// if app name and version not set, we do not set a new user agent.
 	if appName != "" && appVersion != "" {
 		c.appInfo = struct {
 			appName    string
@@ -258,8 +260,18 @@ func (c *Client) TraceOff() {
 	c.isTraceEnabled = false
 }
 
-// requestMetadata - is container for all the values to make a
-// request.
+// SetS3TransferAccelerate - turns s3 accelerated endpoint on or off for all your
+// requests. This feature is only specific to S3 for all other endpoints this
+// function does nothing. To read further details on s3 transfer acceleration
+// please vist -
+// http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html
+func (c *Client) SetS3TransferAccelerate(accelerateEndpoint string) {
+	if s3utils.IsAmazonEndpoint(c.endpointURL) {
+		c.s3AccelerateEndpoint = accelerateEndpoint
+	}
+}
+
+// requestMetadata - is container for all the values to make a request.
 type requestMetadata struct {
 	// If set newRequest presigns the URL.
 	presignURL bool
@@ -601,7 +613,7 @@ func (c Client) newRequest(method string, metadata requestMetadata) (req *http.R
 		req.Body = ioutil.NopCloser(metadata.contentBody)
 	}
 
-	// FIXEM: Enable this when Google Cloud Storage properly supports 100-continue.
+	// FIXME: Enable this when Google Cloud Storage properly supports 100-continue.
 	// Skip setting 'expect' header for Google Cloud Storage, there
 	// are some known issues - https://github.com/restic/restic/issues/520
 	if !s3utils.IsGoogleEndpoint(c.endpointURL) {
@@ -669,22 +681,35 @@ func (c Client) setUserAgent(req *http.Request) {
 
 // makeTargetURL make a new target url.
 func (c Client) makeTargetURL(bucketName, objectName, bucketLocation string, queryValues url.Values) (*url.URL, error) {
+	// Save if target url will have buckets which suppport virtual host.
+	isVirtualHostStyle := s3utils.IsVirtualHostSupported(c.endpointURL, bucketName)
+
 	host := c.endpointURL.Host
 	// For Amazon S3 endpoint, try to fetch location based endpoint.
 	if s3utils.IsAmazonEndpoint(c.endpointURL) {
-		// Fetch new host based on the bucket location.
-		host = getS3Endpoint(bucketLocation)
+		if c.s3AccelerateEndpoint != "" {
+			// http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html
+			// Disable transfer acceleration for non-compliant bucket names.
+			if strings.Contains(bucketName, ".") {
+				return nil, ErrTransferAccelerationBucket(bucketName)
+			}
+			// If transfer acceleration is requested set new host.
+			// For more details about enabling transfer acceleration read here.
+			// http://docs.aws.amazon.com/AmazonS3/latest/dev/transfer-acceleration.html
+			host = c.s3AccelerateEndpoint
+		} else {
+			// Fetch new host based on the bucket location.
+			host = getS3Endpoint(bucketLocation)
+		}
 	}
+
 	// Save scheme.
 	scheme := c.endpointURL.Scheme
 
 	urlStr := scheme + "://" + host + "/"
 	// Make URL only if bucketName is available, otherwise use the
 	// endpoint URL.
 	if bucketName != "" {
-		// Save if target url will have buckets which suppport virtual host.
-		isVirtualHostStyle := s3utils.IsVirtualHostSupported(c.endpointURL, bucketName)
-
 		// If endpoint supports virtual host style use that always.
 		// Currently only S3 and Google Cloud Storage would support
 		// virtual host style.

bucket-cache.go

@@ -164,13 +164,6 @@ func (c Client) getBucketLocationRequest(bucketName string) (*http.Request, erro
 
 	// Set get bucket location always as path style.
 	targetURL := c.endpointURL
-
-	// Requesting a bucket location from an accelerate endpoint returns a 400,
-	// so default to us-east-1 for the lookup
-	if s3utils.IsAmazonS3AccelerateEndpoint(c.endpointURL) {
-		targetURL.Host = getS3Endpoint("us-east-1")
-	}
-
 	targetURL.Path = path.Join(bucketName, "") + "/"
 	targetURL.RawQuery = urlValues.Encode()
 

docs/API.md

@@ -60,7 +60,7 @@ func main() {
 |[`ListBuckets`](#ListBuckets)   |[`PutObject`](#PutObject)   |[`PresignedPutObject`](#PresignedPutObject)   | [`GetBucketPolicy`](#GetBucketPolicy)  | [`SetCustomTransport`](#SetCustomTransport) |
 |[`BucketExists`](#BucketExists)   |[`CopyObject`](#CopyObject)   |[`PresignedPostPolicy`](#PresignedPostPolicy)   |  [`ListBucketPolicies`](#ListBucketPolicies)  | [`TraceOn`](#TraceOn) |
 | [`RemoveBucket`](#RemoveBucket)  |[`StatObject`](#StatObject)   |   |  [`SetBucketNotification`](#SetBucketNotification)  | [`TraceOff`](#TraceOff) |
-|[`ListObjects`](#ListObjects)   |[`RemoveObject`](#RemoveObject)   |   |  [`GetBucketNotification`](#GetBucketNotification)   |
+|[`ListObjects`](#ListObjects)  |[`RemoveObject`](#RemoveObject)   |   |  [`GetBucketNotification`](#GetBucketNotification)  | [`SetS3TransferAccelerate`](#SetS3TransferAccelerate) |
 |[`ListObjectsV2`](#ListObjectsV2) | [`RemoveObjects`](#RemoveObjects) |   | [`RemoveAllBucketNotification`](#RemoveAllBucketNotification)  |
 |[`ListIncompleteUploads`](#ListIncompleteUploads) | [`RemoveIncompleteUpload`](#RemoveIncompleteUpload) |   |  [`ListenBucketNotification`](#ListenBucketNotification)  |
 |   | [`FPutObject`](#FPutObject)  |   |   |
@@ -1148,6 +1148,16 @@ __Parameters__
 ### TraceOff()
 Disables HTTP tracing.
 
+<a name="SetS3TransferAccelerate"></a>
+### SetS3TransferAccelerate(acceleratedEndpoint string)
+Set new AWS S3 transfer acceleration endpoint for your requests.
+NOTE: This API is a no-op for non-S3 endpoints.
+
+__Parameters__
+
+| Param  | Type  | Description  |
+|---|---|---|
+|`acceleratedEndpoint`  | _string_  | Set to new S3 transfer acceleration endpoint.|
 
 ## 7. Explore Further
 

examples/s3/putobject-s3-accelerate.go

@@ -0,0 +1,56 @@
+// +build ignore
+
+/*
+ * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2016 Minio, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package main
+
+import (
+	"log"
+	"os"
+
+	"github.com/minio/minio-go"
+)
+
+func main() {
+	// Note: YOUR-ACCESSKEYID, YOUR-SECRETACCESSKEY, my-testfile, my-bucketname and
+	// my-objectname are dummy values, please replace them with original values.
+
+	// Requests are always secure (HTTPS) by default. Set secure=false to enable insecure (HTTP) access.
+	// This boolean value is the last argument for New().
+
+	// New returns an Amazon S3 compatible client object. API compatibility (v2 or v4) is automatically
+	// determined based on the Endpoint value.
+	s3Client, err := minio.New("s3.amazonaws.com", "YOUR-ACCESSKEYID", "YOUR-SECRETACCESSKEY", true)
+	if err != nil {
+		log.Fatalln(err)
+	}
+
+	// Enable S3 transfer accelerate endpoint.
+	s3Client.S3TransferAccelerate("s3-accelerate.amazonaws.com")
+
+	object, err := os.Open("my-testfile")
+	if err != nil {
+		log.Fatalln(err)
+	}
+	defer object.Close()
+
+	n, err := s3Client.PutObject("my-bucketname", "my-objectname", object, "application/octet-stream")
+	if err != nil {
+		log.Fatalln(err)
+	}
+	log.Println("Uploaded", "my-objectname", " of size: ", n, "Successfully.")
+}

pkg/s3utils/utils.go

@@ -85,10 +85,6 @@ func IsAmazonEndpoint(endpointURL url.URL) bool {
 		return true
 	}
 
-	if IsAmazonS3AccelerateEndpoint(endpointURL) {
-		return true
-	}
-
 	return endpointURL.Host == "s3.amazonaws.com"
 }
 
@@ -105,14 +101,6 @@ func IsAmazonChinaEndpoint(endpointURL url.URL) bool {
 	return endpointURL.Host == "s3.cn-north-1.amazonaws.com.cn"
 }
 
-// IsAmazonS3AccelerateEndpoint - Match if it is an Amazon S3 Accelerate
-func IsAmazonS3AccelerateEndpoint(endpointURL url.URL) bool {
-	if endpointURL == sentinelURL {
-		return false
-	}
-	return endpointURL.Host == "s3-accelerate.amazonaws.com"
-}
-
 // IsGoogleEndpoint - Match if it is exactly Google cloud storage endpoint.
 func IsGoogleEndpoint(endpointURL url.URL) bool {
 	if endpointURL == sentinelURL {