From 86bdf05b619bf409241dfd7d3a99d2a7a350b7c3 Mon Sep 17 00:00:00 2001 From: Harshavardhana Date: Wed, 26 Apr 2017 00:46:06 -0700 Subject: [PATCH] api, core: Add new Core API {Get,Stat}Object with pre-conditions. Implements a new API to provide a way to set headers for GetObject(), StatObject() request such as to - read partial data starting at offsets. - read only if etag matches. - read only if modtime matches. - read only if etag doesn't match. - read only if modtime doesn't match. Fixes #669 --- api-get-object-file.go | 7 +- api-get-object.go | 35 +++++----- api-stat.go | 25 +++++++- core.go | 13 ++++ core_test.go | 142 +++++++++++++++++++++++++++++++++++++++++ request-headers.go | 102 +++++++++++++++++++++++++++++ 6 files changed, 305 insertions(+), 19 deletions(-) create mode 100644 request-headers.go diff --git a/api-get-object-file.go b/api-get-object-file.go index a38fc852a7..9f3bdcc5a8 100644 --- a/api-get-object-file.go +++ b/api-get-object-file.go @@ -78,8 +78,13 @@ func (c Client) FGetObject(bucketName, objectName, filePath string) error { return err } + // Initialize get object conditions and set the appropriate + // range offsets to read from. + reqHeaders := NewGetReqHeaders() + reqHeaders.SetRange(st.Size(), 0) + // Seek to current position for incoming reader. - objectReader, objectStat, err := c.getObject(bucketName, objectName, st.Size(), 0) + objectReader, objectStat, err := c.getObject(bucketName, objectName, reqHeaders) if err != nil { return err } diff --git a/api-get-object.go b/api-get-object.go index 8066f70f23..bf97aa0ff7 100644 --- a/api-get-object.go +++ b/api-get-object.go @@ -1,5 +1,5 @@ /* - * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc. + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016, 2017 Minio, Inc. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -97,16 +97,19 @@ func (c Client) GetObject(bucketName, objectName string) (*Object, error) { if req.isFirstReq { // First request is a Read/ReadAt. if req.isReadOp { + reqHeaders := NewGetReqHeaders() // Differentiate between wanting the whole object and just a range. if req.isReadAt { // If this is a ReadAt request only get the specified range. // Range is set with respect to the offset and length of the buffer requested. // Do not set objectInfo from the first readAt request because it will not get // the whole object. - httpReader, _, err = c.getObject(bucketName, objectName, req.Offset, int64(len(req.Buffer))) + reqHeaders.SetRange(req.Offset, req.Offset+int64(len(req.Buffer))-1) + httpReader, _, err = c.getObject(bucketName, objectName, reqHeaders) } else { + reqHeaders.SetRange(req.Offset, 0) // First request is a Read request. - httpReader, objectInfo, err = c.getObject(bucketName, objectName, req.Offset, 0) + httpReader, objectInfo, err = c.getObject(bucketName, objectName, reqHeaders) } if err != nil { resCh <- getResponse{ @@ -166,6 +169,7 @@ func (c Client) GetObject(bucketName, objectName string) (*Object, error) { // new ones when they haven't been already. // All readAt requests are new requests. if req.DidOffsetChange || !req.beenRead { + reqHeaders := NewGetReqHeaders() if httpReader != nil { // Close previously opened http reader. httpReader.Close() @@ -173,9 +177,12 @@ func (c Client) GetObject(bucketName, objectName string) (*Object, error) { // If this request is a readAt only get the specified range. if req.isReadAt { // Range is set with respect to the offset and length of the buffer requested. - httpReader, _, err = c.getObject(bucketName, objectName, req.Offset, int64(len(req.Buffer))) + reqHeaders.SetRange(req.Offset, req.Offset+int64(len(req.Buffer))-1) + httpReader, _, err = c.getObject(bucketName, objectName, reqHeaders) } else { - httpReader, objectInfo, err = c.getObject(bucketName, objectName, req.Offset, 0) + // Range is set with respect to the offset. + reqHeaders.SetRange(req.Offset, 0) + httpReader, objectInfo, err = c.getObject(bucketName, objectName, reqHeaders) } if err != nil { resCh <- getResponse{ @@ -230,8 +237,8 @@ type getResponse struct { objectInfo ObjectInfo // Used for the first request. } -// Object represents an open object. It implements Read, ReadAt, -// Seeker, Close for a HTTP stream. +// Object represents an open object. It implements +// Reader, ReaderAt, Seeker, Closer for a HTTP stream. type Object struct { // Mutex. mutex *sync.Mutex @@ -594,7 +601,7 @@ func newObject(reqCh chan<- getRequest, resCh <-chan getResponse, doneCh chan<- // // For more information about the HTTP Range header. // go to http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35. -func (c Client) getObject(bucketName, objectName string, offset, length int64) (io.ReadCloser, ObjectInfo, error) { +func (c Client) getObject(bucketName, objectName string, reqHeaders RequestHeaders) (io.ReadCloser, ObjectInfo, error) { // Validate input arguments. if err := isValidBucketName(bucketName); err != nil { return nil, ObjectInfo{}, err @@ -603,15 +610,10 @@ func (c Client) getObject(bucketName, objectName string, offset, length int64) ( return nil, ObjectInfo{}, err } + // Set all the necessary reqHeaders. customHeader := make(http.Header) - // Set ranges if length and offset are valid. - // See https://tools.ietf.org/html/rfc7233#section-3.1 for reference. - if length > 0 && offset >= 0 { - customHeader.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1)) - } else if offset > 0 && length == 0 { - customHeader.Set("Range", fmt.Sprintf("bytes=%d-", offset)) - } else if length < 0 && offset == 0 { - customHeader.Set("Range", fmt.Sprintf("bytes=%d", length)) + for key, value := range reqHeaders.Header { + customHeader[key] = value } // Execute GET on objectName. @@ -645,6 +647,7 @@ func (c Client) getObject(bucketName, objectName string, offset, length int64) ( Region: resp.Header.Get("x-amz-bucket-region"), } } + // Get content-type. contentType := strings.TrimSpace(resp.Header.Get("Content-Type")) if contentType == "" { diff --git a/api-stat.go b/api-stat.go index e3bb115d4f..3b33ec4b80 100644 --- a/api-stat.go +++ b/api-stat.go @@ -85,11 +85,30 @@ func (c Client) StatObject(bucketName, objectName string) (ObjectInfo, error) { if err := isValidObjectName(objectName); err != nil { return ObjectInfo{}, err } + reqHeaders := NewHeadReqHeaders() + return c.statObject(bucketName, objectName, reqHeaders) +} + +// Lower level API for statObject supporting pre-conditions and range headers. +func (c Client) statObject(bucketName, objectName string, reqHeaders RequestHeaders) (ObjectInfo, error) { + // Input validation. + if err := isValidBucketName(bucketName); err != nil { + return ObjectInfo{}, err + } + if err := isValidObjectName(objectName); err != nil { + return ObjectInfo{}, err + } + + customHeader := make(http.Header) + for k, v := range reqHeaders.Header { + customHeader[k] = v + } // Execute HEAD on objectName. resp, err := c.executeMethod("HEAD", requestMetadata{ - bucketName: bucketName, - objectName: objectName, + bucketName: bucketName, + objectName: objectName, + customHeader: customHeader, }) defer closeResponse(resp) if err != nil { @@ -122,6 +141,7 @@ func (c Client) StatObject(bucketName, objectName string) (ObjectInfo, error) { } } } + // Parse Last-Modified has http time format. date, err := time.Parse(http.TimeFormat, resp.Header.Get("Last-Modified")) if err != nil { @@ -135,6 +155,7 @@ func (c Client) StatObject(bucketName, objectName string) (ObjectInfo, error) { Region: resp.Header.Get("x-amz-bucket-region"), } } + // Fetch content type if any present. contentType := strings.TrimSpace(resp.Header.Get("Content-Type")) if contentType == "" { diff --git a/core.go b/core.go index 90154d945b..be9388cecf 100644 --- a/core.go +++ b/core.go @@ -98,3 +98,16 @@ func (c Core) GetBucketPolicy(bucket string) (policy.BucketAccessPolicy, error) func (c Core) PutBucketPolicy(bucket string, bucketPolicy policy.BucketAccessPolicy) error { return c.putBucketPolicy(bucket, bucketPolicy) } + +// GetObject is a lower level API implemented to support reading +// partial objects and also downloading objects with special conditions +// matching etag, modtime etc. +func (c Core) GetObject(bucketName, objectName string, reqHeaders RequestHeaders) (io.ReadCloser, ObjectInfo, error) { + return c.getObject(bucketName, objectName, reqHeaders) +} + +// StatObject is a lower level API implemented to support special +// conditions matching etag, modtime on a request. +func (c Core) StatObject(bucketName, objectName string, reqHeaders RequestHeaders) (ObjectInfo, error) { + return c.statObject(bucketName, objectName, reqHeaders) +} diff --git a/core_test.go b/core_test.go index 52716ce623..f27d4a50d4 100644 --- a/core_test.go +++ b/core_test.go @@ -17,6 +17,8 @@ package minio import ( + "bytes" + "io" "math/rand" "os" "reflect" @@ -24,6 +26,146 @@ import ( "time" ) +// Tests for Core GetObject() function. +func TestGetObjectCore(t *testing.T) { + if testing.Short() { + t.Skip("skipping functional tests for the short runs") + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio core client object. + c, err := NewCore( + os.Getenv("S3_ADDRESS"), + os.Getenv("ACCESS_KEY"), + os.Getenv("SECRET_KEY"), + mustParseBool(os.Getenv("S3_SECURE")), + ) + if err != nil { + t.Fatal("Error:", err) + } + + // Enable tracing, write to stderr. + c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("Minio-go-FunctionalTest", "0.1.0") + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test") + + // Make a new bucket. + err = c.MakeBucket(bucketName, "us-east-1") + if err != nil { + t.Fatal("Error:", err, bucketName) + } + + // Generate data more than 32K + buf := bytes.Repeat([]byte("3"), rand.Intn(1<<20)+32*1024) + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + n, err := c.Client.PutObject(bucketName, objectName, bytes.NewReader(buf), "binary/octet-stream") + if err != nil { + t.Fatal("Error:", err, bucketName, objectName) + } + + if n != int64(len(buf)) { + t.Fatalf("Error: number of bytes does not match, want %v, got %v\n", len(buf), n) + } + + reqHeaders := NewGetReqHeaders() + + offset := int64(2048) + + // read directly + buf1 := make([]byte, 512) + buf2 := make([]byte, 512) + buf3 := make([]byte, n) + + reqHeaders.SetRange(offset, offset+int64(len(buf1))-1) + reader, objectInfo, err := c.GetObject(bucketName, objectName, reqHeaders) + if err != nil { + t.Fatal(err) + } + io.ReadFull(reader, buf1) + reader.Close() + + if objectInfo.Size != int64(len(buf1)) { + t.Fatalf("Error: GetObject read shorter bytes before reaching EOF, want %v, got %v\n", objectInfo.Size, len(buf1)) + } + if !bytes.Equal(buf1, buf[offset:offset+512]) { + t.Fatal("Error: Incorrect read between two GetObject from same offset.") + } + offset += 512 + + reqHeaders.SetRange(offset, offset+int64(len(buf2))-1) + reader, objectInfo, err = c.GetObject(bucketName, objectName, reqHeaders) + if err != nil { + t.Fatal(err) + } + io.ReadFull(reader, buf2) + reader.Close() + + if objectInfo.Size != int64(len(buf2)) { + t.Fatalf("Error: GetObject read shorter bytes before reaching EOF, want %v, got %v\n", objectInfo.Size, len(buf2)) + } + if !bytes.Equal(buf2, buf[offset:offset+512]) { + t.Fatal("Error: Incorrect read between two GetObject from same offset.") + } + + reqHeaders.SetRange(0, int64(len(buf3))) + reader, objectInfo, err = c.GetObject(bucketName, objectName, reqHeaders) + if err != nil { + t.Fatal(err) + } + io.ReadFull(reader, buf3) + reader.Close() + + if objectInfo.Size != int64(len(buf3)) { + t.Fatalf("Error: GetObject read shorter bytes before reaching EOF, want %v, got %v\n", objectInfo.Size, len(buf3)) + } + if !bytes.Equal(buf3, buf) { + t.Fatal("Error: Incorrect data read in GetObject, than what was previously upoaded.") + } + + reqHeaders = NewGetReqHeaders() + reqHeaders.SetMatchETag("etag") + _, _, err = c.GetObject(bucketName, objectName, reqHeaders) + if err == nil { + t.Fatal("Unexpected GetObject should fail with mismatching etags") + } + if errResp := ToErrorResponse(err); errResp.Code != "PreconditionFailed" { + t.Fatalf("Expected \"PreconditionFailed\" as code, got %s instead", errResp.Code) + } + + reqHeaders = NewGetReqHeaders() + reqHeaders.SetMatchETagExcept("etag") + reader, objectInfo, err = c.GetObject(bucketName, objectName, reqHeaders) + if err != nil { + t.Fatal(err) + } + io.ReadFull(reader, buf3) + reader.Close() + + if objectInfo.Size != int64(len(buf3)) { + t.Fatalf("Error: GetObject read shorter bytes before reaching EOF, want %v, got %v\n", objectInfo.Size, len(buf3)) + } + if !bytes.Equal(buf3, buf) { + t.Fatal("Error: Incorrect data read in GetObject, than what was previously upoaded.") + } + + err = c.RemoveObject(bucketName, objectName) + if err != nil { + t.Fatal("Error: ", err) + } + err = c.RemoveBucket(bucketName) + if err != nil { + t.Fatal("Error:", err) + } +} + // Tests get bucket policy core API. func TestGetBucketPolicy(t *testing.T) { if testing.Short() { diff --git a/request-headers.go b/request-headers.go new file mode 100644 index 0000000000..c463b1cede --- /dev/null +++ b/request-headers.go @@ -0,0 +1,102 @@ +/* + * Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2016 Minio, Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package minio + +import ( + "fmt" + "net/http" + "time" +) + +// RequestHeaders - implement methods for setting special +// request headers for GET, HEAD object operations. +// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html +type RequestHeaders struct { + http.Header +} + +// NewGetReqHeaders - initializes a new request headers for GET request. +func NewGetReqHeaders() RequestHeaders { + return RequestHeaders{ + Header: make(http.Header), + } +} + +// NewHeadReqHeaders - initializes a new request headers for HEAD request. +func NewHeadReqHeaders() RequestHeaders { + return RequestHeaders{ + Header: make(http.Header), + } +} + +// SetMatchETag - set match etag. +func (c RequestHeaders) SetMatchETag(etag string) error { + if etag == "" { + return ErrInvalidArgument("ETag cannot be empty.") + } + c.Set("If-Match", etag) + return nil +} + +// SetMatchETagExcept - set match etag except. +func (c RequestHeaders) SetMatchETagExcept(etag string) error { + if etag == "" { + return ErrInvalidArgument("ETag cannot be empty.") + } + c.Set("If-None-Match", etag) + return nil +} + +// SetUnmodified - set unmodified time since. +func (c RequestHeaders) SetUnmodified(modTime time.Time) error { + if modTime.IsZero() { + return ErrInvalidArgument("Modified since cannot be empty.") + } + c.Set("If-Unmodified-Since", modTime.Format(http.TimeFormat)) + return nil +} + +// SetModified - set modified time since. +func (c RequestHeaders) SetModified(modTime time.Time) error { + if modTime.IsZero() { + return ErrInvalidArgument("Modified since cannot be empty.") + } + c.Set("If-Modified-Since", modTime.Format(http.TimeFormat)) + return nil +} + +// SetRange - set the start and end offset of the object to be read. +// See https://tools.ietf.org/html/rfc7233#section-3.1 for reference. +func (c RequestHeaders) SetRange(start, end int64) error { + switch { + case start <= 0 && end < 0: + // Read everything until the 'end'. `bytes=-N` + c.Set("Range", fmt.Sprintf("bytes=%d", end)) + case start > 0 && end == 0: + // Read everything starting from offset 'start'. `bytes=N-` + c.Set("Range", fmt.Sprintf("bytes=%d-", start)) + case start >= 0 && end >= 0 && end >= start: + // Read everything starting at 'start' till the 'end'. `bytes=N-M` + c.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end)) + } + // All other cases such as + // bytes=-N- + // bytes=N-M where M < N + // These return error and are not supported. + return ErrInvalidArgument(fmt.Sprintf("Invalid range start and end specified bytes=%d-%d", + start, end)) +}