Skip to content

Commit

Permalink
api, core: Introduce a new Core API with GetObject pre-conditions.
Browse files Browse the repository at this point in the history
Implements a new API to provide a way to set pre-conditions for
GetObject() request such as to

 - read partial data starting at offsets.
 - read only if etag matches.
 - read only if modtime matches.
 - read only if etag doesn't match.
 - read only if modtime doesn't match.

Fixes #669
  • Loading branch information
harshavardhana authored and minio-trusted committed Apr 26, 2017
1 parent 550c8c2 commit b82a6f5
Show file tree
Hide file tree
Showing 5 changed files with 130 additions and 19 deletions.
82 changes: 82 additions & 0 deletions api-get-conditions.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
/*
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2016 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package minio

import (
"fmt"
"net/http"
"time"
)

// GetConditions - get conditions implement methods for setting
// conditions for a GetObject request.
// http://docs.aws.amazon.com/AmazonS3/latest/API/RESTObjectGET.html
type GetConditions struct {
http.Header
}

// SetMatchETag - set match etag.
func (c GetConditions) SetMatchETag(etag string) error {
if etag == "" {
return ErrInvalidArgument("ETag cannot be empty.")
}
c.Set("If-Match", etag)
return nil
}

// SetMatchETagExcept - set match etag except.
func (c GetConditions) SetMatchETagExcept(etag string) error {
if etag == "" {
return ErrInvalidArgument("ETag cannot be empty.")
}
c.Set("If-None-Match", etag)
return nil
}

// SetUnmodified - set unmodified time since.
func (c GetConditions) SetUnmodified(modTime time.Time) error {
if modTime.IsZero() {
return ErrInvalidArgument("Modified since cannot be empty.")
}
c.Set("If-Unmodified-Since", modTime.Format(http.TimeFormat))
return nil
}

// SetModified - set modified time since.
func (c GetConditions) SetModified(modTime time.Time) error {
if modTime.IsZero() {
return ErrInvalidArgument("Modified since cannot be empty.")
}
c.Set("If-Modified-Since", modTime.Format(http.TimeFormat))
return nil
}

// SetRange - set the start and end offset of the object to be read.
// See https://tools.ietf.org/html/rfc7233#section-3.1 for reference.
func (c GetConditions) SetRange(start, end int64) error {
if start < 0 || end > 0 && end < start {
return ErrInvalidArgument("Range start less than 0 or range end less than range start.")
}
if end == 0 {
// Read everything starting from offset 'start'.
c.Set("Range", fmt.Sprintf("bytes=%d-", start))
} else {
// Read everything starting at 'start' till the 'end'.
c.Set("Range", fmt.Sprintf("bytes=%d-%d", start, end))
}
return nil
}
7 changes: 6 additions & 1 deletion api-get-object-file.go
Original file line number Diff line number Diff line change
Expand Up @@ -78,8 +78,13 @@ func (c Client) FGetObject(bucketName, objectName, filePath string) error {
return err
}

// Initialize get object conditions and set the appropriate
// range offsets to read from.
conds := GetConditions{}
conds.SetRange(st.Size(), 0)

// Seek to current position for incoming reader.
objectReader, objectStat, err := c.getObject(bucketName, objectName, st.Size(), 0)
objectReader, objectStat, err := c.getObject(bucketName, objectName, conds)
if err != nil {
return err
}
Expand Down
17 changes: 17 additions & 0 deletions api-get-object-partial.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,17 @@
package minio

/*
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2017 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
36 changes: 18 additions & 18 deletions api-get-object.go
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016 Minio, Inc.
* Minio Go Library for Amazon S3 Compatible Cloud Storage (C) 2015, 2016, 2017 Minio, Inc.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -97,16 +97,19 @@ func (c Client) GetObject(bucketName, objectName string) (*Object, error) {
if req.isFirstReq {
// First request is a Read/ReadAt.
if req.isReadOp {
conds := GetConditions{}
// Differentiate between wanting the whole object and just a range.
if req.isReadAt {
// If this is a ReadAt request only get the specified range.
// Range is set with respect to the offset and length of the buffer requested.
// Do not set objectInfo from the first readAt request because it will not get
// the whole object.
httpReader, _, err = c.getObject(bucketName, objectName, req.Offset, int64(len(req.Buffer)))
conds.SetRange(req.Offset, req.Offset+int64(len(req.Buffer))-1)
httpReader, _, err = c.getObject(bucketName, objectName, conds)
} else {
conds.SetRange(req.Offset, 0)
// First request is a Read request.
httpReader, objectInfo, err = c.getObject(bucketName, objectName, req.Offset, 0)
httpReader, objectInfo, err = c.getObject(bucketName, objectName, conds)
}
if err != nil {
resCh <- getResponse{
Expand Down Expand Up @@ -166,16 +169,19 @@ func (c Client) GetObject(bucketName, objectName string) (*Object, error) {
// new ones when they haven't been already.
// All readAt requests are new requests.
if req.DidOffsetChange || !req.beenRead {
conds := GetConditions{}
if httpReader != nil {
// Close previously opened http reader.
httpReader.Close()
}
// If this request is a readAt only get the specified range.
if req.isReadAt {
// Range is set with respect to the offset and length of the buffer requested.
httpReader, _, err = c.getObject(bucketName, objectName, req.Offset, int64(len(req.Buffer)))
conds.SetRange(req.Offset, req.Offset+int64(len(req.Buffer))-1)
httpReader, _, err = c.getObject(bucketName, objectName, conds)
} else {
httpReader, objectInfo, err = c.getObject(bucketName, objectName, req.Offset, 0)
conds.SetRange(req.Offset, 0)
httpReader, objectInfo, err = c.getObject(bucketName, objectName, conds)
}
if err != nil {
resCh <- getResponse{
Expand Down Expand Up @@ -230,8 +236,8 @@ type getResponse struct {
objectInfo ObjectInfo // Used for the first request.
}

// Object represents an open object. It implements Read, ReadAt,
// Seeker, Close for a HTTP stream.
// Object represents an open object. It implements
// Reader, ReaderAt, Seeker, Closer for a HTTP stream.
type Object struct {
// Mutex.
mutex *sync.Mutex
Expand Down Expand Up @@ -594,7 +600,7 @@ func newObject(reqCh chan<- getRequest, resCh <-chan getResponse, doneCh chan<-
//
// For more information about the HTTP Range header.
// go to http://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.35.
func (c Client) getObject(bucketName, objectName string, offset, length int64) (io.ReadCloser, ObjectInfo, error) {
func (c Client) getObject(bucketName, objectName string, conditions GetConditions) (io.ReadCloser, ObjectInfo, error) {
// Validate input arguments.
if err := isValidBucketName(bucketName); err != nil {
return nil, ObjectInfo{}, err
Expand All @@ -603,15 +609,10 @@ func (c Client) getObject(bucketName, objectName string, offset, length int64) (
return nil, ObjectInfo{}, err
}

// Set all the necessary conditions.
customHeader := make(http.Header)
// Set ranges if length and offset are valid.
// See https://tools.ietf.org/html/rfc7233#section-3.1 for reference.
if length > 0 && offset >= 0 {
customHeader.Set("Range", fmt.Sprintf("bytes=%d-%d", offset, offset+length-1))
} else if offset > 0 && length == 0 {
customHeader.Set("Range", fmt.Sprintf("bytes=%d-", offset))
} else if length < 0 && offset == 0 {
customHeader.Set("Range", fmt.Sprintf("bytes=%d", length))
for key, value := range conditions.Header {
customHeader[key] = value
}

// Execute GET on objectName.
Expand Down Expand Up @@ -645,6 +646,7 @@ func (c Client) getObject(bucketName, objectName string, offset, length int64) (
Region: resp.Header.Get("x-amz-bucket-region"),
}
}

// Get content-type.
contentType := strings.TrimSpace(resp.Header.Get("Content-Type"))
if contentType == "" {
Expand All @@ -656,7 +658,5 @@ func (c Client) getObject(bucketName, objectName string, offset, length int64) (
objectStat.Size = resp.ContentLength
objectStat.LastModified = date
objectStat.ContentType = contentType

// do not close body here, caller will close
return resp.Body, objectStat, nil
}
7 changes: 7 additions & 0 deletions core.go
Original file line number Diff line number Diff line change
Expand Up @@ -98,3 +98,10 @@ func (c Core) GetBucketPolicy(bucket string) (policy.BucketAccessPolicy, error)
func (c Core) PutBucketPolicy(bucket string, bucketPolicy policy.BucketAccessPolicy) error {
return c.putBucketPolicy(bucket, bucketPolicy)
}

// GetObject is a lower level API implemented to support reading
// partial objects and also downloading objects with special conditions
// matching etag, modtime etc.
func (c Core) GetObject(bucketName, objectName string, conditions GetConditions) (io.ReadCloser, ObjectInfo, error) {
return c.getObject(bucketName, objectName, conditions)
}

0 comments on commit b82a6f5

Please sign in to comment.