Skip to content

Commit

Permalink
For GCS if size cannot be found use tempfile.
Browse files Browse the repository at this point in the history
This is added as a fallback option instead of returning
an error we can just add a limit reader upto 5GB and
attempt to upload the file instead.

Related to restic/restic#996
  • Loading branch information
harshavardhana authored and minio-trusted committed Jun 16, 2017
1 parent f6d5df6 commit 42f1a39
Show file tree
Hide file tree
Showing 2 changed files with 27 additions and 5 deletions.
8 changes: 3 additions & 5 deletions api-put-object-progress.go
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
package minio

import (
"fmt"
"io"
"strings"

Expand Down Expand Up @@ -85,17 +86,14 @@ func (c Client) PutObjectWithMetadata(bucketName, objectName string, reader io.R
// NOTE: Google Cloud Storage does not implement Amazon S3 Compatible multipart PUT.
// So we fall back to single PUT operation with the maximum limit of 5GiB.
if s3utils.IsGoogleEndpoint(c.endpointURL) {
if size <= -1 {
if size > maxSinglePutObjectSize {
return 0, ErrorResponse{
Code: "NotImplemented",
Message: "Content-Length cannot be negative for file uploads to Google Cloud Storage.",
Message: fmt.Sprintf("Invalid Content-Length %d for file uploads to Google Cloud Storage.", size),
Key: objectName,
BucketName: bucketName,
}
}
if size > maxSinglePutObjectSize {
return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName)
}
// Do not compute MD5 for Google Cloud Storage. Uploads up to 5GiB in size.
return c.putObjectNoChecksum(bucketName, objectName, reader, size, metaData, progress)
}
Expand Down
24 changes: 24 additions & 0 deletions api-put-object.go
Original file line number Diff line number Diff line change
Expand Up @@ -170,6 +170,30 @@ func (c Client) putObjectNoChecksum(bucketName, objectName string, reader io.Rea
return 0, ErrEntityTooLarge(size, maxSinglePutObjectSize, bucketName, objectName)
}

if size <= -1 {
// Initialize a new temporary file.
var tmpFile *tempFile
tmpFile, err = newTempFile("single$-putobject-single")
if err != nil {
return 0, err
}
defer tmpFile.Close()
size, err = io.Copy(tmpFile, io.LimitReader(reader, maxSinglePutObjectSize))
if err != nil {
return 0, err
}
// Seek back to beginning of the temporary file.
if _, err = tmpFile.Seek(0, 0); err != nil {
return 0, err
}
reader = tmpFile
} else {
readerAt, ok := reader.(io.ReaderAt)
if ok {
reader = io.NewSectionReader(readerAt, 0, size)
}
}

// Update progress reader appropriately to the latest offset as we
// read from the source.
readSeeker := newHook(reader, progress)
Expand Down

0 comments on commit 42f1a39

Please sign in to comment.