Skip to content

Commit

Permalink
RemoveObjects() to return delete-marker & delete-marker-version-id
Browse files Browse the repository at this point in the history 
This is a breaking change, major version needs to be updated.
  • Loading branch information
Anis Elleuch committed Feb 20, 2021
1 parent 6698ff4 commit f44ea79
Show file tree
Hide file tree
Showing 3 changed files with 72 additions and 49 deletions.
107 changes: 64 additions & 43 deletions api-remove.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,10 +83,11 @@ func (c Client) RemoveObject(ctx context.Context, bucketName, objectName string,
return err
}

return c.removeObject(ctx, bucketName, objectName, opts)
res := c.removeObject(ctx, bucketName, objectName, opts)
return res.Err
}

func (c Client) removeObject(ctx context.Context, bucketName, objectName string, opts RemoveObjectOptions) error {
func (c Client) removeObject(ctx context.Context, bucketName, objectName string, opts RemoveObjectOptions) RemoveObjectResult {

// Get resources properly escaped and lined up before
// using them in http request.
Expand Down Expand Up @@ -122,26 +123,36 @@ func (c Client) removeObject(ctx context.Context, bucketName, objectName string,
})
defer closeResponse(resp)
if err != nil {
return err
return RemoveObjectResult{Err: err}
}
if resp != nil {
// if some unexpected error happened and max retry is reached, we want to let client know
if resp.StatusCode != http.StatusNoContent {
return httpRespToErrorResponse(resp, bucketName, objectName)
err := httpRespToErrorResponse(resp, bucketName, objectName)
return RemoveObjectResult{Err: err}
}
}

// DeleteObject always responds with http '204' even for
// objects which do not exist. So no need to handle them
// specifically.
return nil
return RemoveObjectResult{
ObjectName: objectName,
ObjectVersionID: opts.VersionID,
DeleteMarker: resp.Header.Get("x-amz-delete-marker") == "true",
DeleteMarkerVersionID: resp.Header.Get("x-amz-version-id"),
}
}

// RemoveObjectError - container of Multi Delete S3 API error
type RemoveObjectError struct {
ObjectName string
VersionID string
Err error
// RemoveObjectResult - container of Multi Delete S3 API result
type RemoveObjectResult struct {
ObjectName string
ObjectVersionID string

DeleteMarker bool
DeleteMarkerVersionID string

Err error
}

// generateRemoveMultiObjects - generate the XML request for remove multi objects request
Expand All @@ -153,31 +164,42 @@ func generateRemoveMultiObjectsRequest(objects []ObjectInfo) []byte {
VersionID: obj.VersionID,
})
}
xmlBytes, _ := xml.Marshal(deleteMultiObjects{Objects: delObjects, Quiet: true})
xmlBytes, _ := xml.Marshal(deleteMultiObjects{Objects: delObjects, Quiet: false})
return xmlBytes
}

// processRemoveMultiObjectsResponse - parse the remove multi objects web service
// and return the success/failure result status for each object
func processRemoveMultiObjectsResponse(body io.Reader, objects []ObjectInfo, errorCh chan<- RemoveObjectError) {
func processRemoveMultiObjectsResponse(body io.Reader, objects []ObjectInfo, resultCh chan<- RemoveObjectResult) {
// Parse multi delete XML response
rmResult := &deleteMultiObjectsResult{}
err := xmlDecoder(body, rmResult)
if err != nil {
errorCh <- RemoveObjectError{ObjectName: "", Err: err}
resultCh <- RemoveObjectResult{ObjectName: "", Err: err}
return
}

// Fill deletion that returned success
for _, obj := range rmResult.DeletedObjects {
resultCh <- RemoveObjectResult{
ObjectName: obj.Key,
ObjectVersionID: obj.VersionID,
// Only filled with versioned buckets
DeleteMarker: obj.DeleteMarker,
DeleteMarkerVersionID: obj.DeleteMarkerVersionID,
}
}

// Fill deletion that returned an error.
for _, obj := range rmResult.UnDeletedObjects {
// Version does not exist is not an error ignore and continue.
switch obj.Code {
case "InvalidArgument", "NoSuchVersion":
continue
}
errorCh <- RemoveObjectError{
ObjectName: obj.Key,
VersionID: obj.VersionID,
resultCh <- RemoveObjectResult{
ObjectName: obj.Key,
ObjectVersionID: obj.VersionID,
Err: ErrorResponse{
Code: obj.Code,
Message: obj.Message,
Expand All @@ -193,29 +215,30 @@ type RemoveObjectsOptions struct {

// RemoveObjects removes multiple objects from a bucket while
// it is possible to specify objects versions which are received from
// objectsCh. Remove failures are sent back via error channel.
func (c Client) RemoveObjects(ctx context.Context, bucketName string, objectsCh <-chan ObjectInfo, opts RemoveObjectsOptions) <-chan RemoveObjectError {
errorCh := make(chan RemoveObjectError, 1)
// objectsCh. Remove results, successes and failures are sent back via
// RemoveObjectResult channel
func (c Client) RemoveObjects(ctx context.Context, bucketName string, objectsCh <-chan ObjectInfo, opts RemoveObjectsOptions) <-chan RemoveObjectResult {
resultCh := make(chan RemoveObjectResult, 1)

// Validate if bucket name is valid.
if err := s3utils.CheckValidBucketName(bucketName); err != nil {
defer close(errorCh)
errorCh <- RemoveObjectError{
defer close(resultCh)
resultCh <- RemoveObjectResult{
Err: err,
}
return errorCh
return resultCh
}
// Validate objects channel to be properly allocated.
if objectsCh == nil {
defer close(errorCh)
errorCh <- RemoveObjectError{
defer close(resultCh)
resultCh <- RemoveObjectResult{
Err: errInvalidArgument("Objects channel cannot be nil"),
}
return errorCh
return resultCh
}

go c.removeObjects(ctx, bucketName, objectsCh, errorCh, opts)
return errorCh
go c.removeObjects(ctx, bucketName, objectsCh, resultCh, opts)
return resultCh
}

// Return true if the character is within the allowed characters in an XML 1.0 document
Expand All @@ -239,14 +262,14 @@ func hasInvalidXMLChar(str string) bool {
}

// Generate and call MultiDelete S3 requests based on entries received from objectsCh
func (c Client) removeObjects(ctx context.Context, bucketName string, objectsCh <-chan ObjectInfo, errorCh chan<- RemoveObjectError, opts RemoveObjectsOptions) {
func (c Client) removeObjects(ctx context.Context, bucketName string, objectsCh <-chan ObjectInfo, resultCh chan<- RemoveObjectResult, opts RemoveObjectsOptions) {
maxEntries := 1000
finish := false
urlValues := make(url.Values)
urlValues.Set("delete", "")

// Close error channel when Multi delete finishes.
defer close(errorCh)
// Close result channel when Multi delete finishes.
defer close(resultCh)

// Loop over entries by 1000 and call MultiDelete requests
for {
Expand All @@ -260,22 +283,20 @@ func (c Client) removeObjects(ctx context.Context, bucketName string, objectsCh
for object := range objectsCh {
if hasInvalidXMLChar(object.Key) {
// Use single DELETE so the object name will be in the request URL instead of the multi-delete XML document.
err := c.removeObject(ctx, bucketName, object.Key, RemoveObjectOptions{
removeResult := c.removeObject(ctx, bucketName, object.Key, RemoveObjectOptions{
VersionID: object.VersionID,
GovernanceBypass: opts.GovernanceBypass,
})
if err != nil {
if err := removeResult.Err; err != nil {
// Version does not exist is not an error ignore and continue.
switch ToErrorResponse(err).Code {
case "InvalidArgument", "NoSuchVersion":
continue
}
errorCh <- RemoveObjectError{
ObjectName: object.Key,
VersionID: object.VersionID,
Err: err,
}
resultCh <- removeResult
}

resultCh <- removeResult
continue
}

Expand Down Expand Up @@ -315,22 +336,22 @@ func (c Client) removeObjects(ctx context.Context, bucketName string, objectsCh
if resp != nil {
if resp.StatusCode != http.StatusOK {
e := httpRespToErrorResponse(resp, bucketName, "")
errorCh <- RemoveObjectError{ObjectName: "", Err: e}
resultCh <- RemoveObjectResult{ObjectName: "", Err: e}
}
}
if err != nil {
for _, b := range batch {
errorCh <- RemoveObjectError{
ObjectName: b.Key,
VersionID: b.VersionID,
Err: err,
resultCh <- RemoveObjectResult{
ObjectName: b.Key,
ObjectVersionID: b.VersionID,
Err: err,
}
}
continue
}

// Process multiobjects remove xml response
processRemoveMultiObjectsResponse(resp.Body, batch, errorCh)
processRemoveMultiObjectsResponse(resp.Body, batch, resultCh)

closeResponse(resp)
}
Expand Down
2 changes: 1 addition & 1 deletion api-s3-datatypes.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -335,7 +335,7 @@ type deletedObject struct {
VersionID string `xml:"VersionId,omitempty"`
// These fields are ignored.
DeleteMarker bool
DeleteMarkerVersionID string
DeleteMarkerVersionID string `xml:"DeleteMarkerVersionId,omitempty"`
}

// nonDeletedObject container for Error element (failed deletion) in MultiObjects Delete XML response
Expand Down
12 changes: 7 additions & 5 deletions examples/s3/removeobjects.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,8 +23,8 @@ import (
"context"
"log"

"github.com/minio/minio-go/v7"
"github.com/minio/minio-go/v7/pkg/credentials"
"github.com/minio/minio-go/v8"
"github.com/minio/minio-go/v8/pkg/credentials"
)

func main() {
Expand Down Expand Up @@ -60,11 +60,13 @@ func main() {
}()

// Call RemoveObjects API
errorCh := s3Client.RemoveObjects(context.Background(), "my-bucketname", objectsCh, minio.RemoveObjectsOptions{})
resultch := s3Client.RemoveObjects(context.Background(), "my-bucketname", objectsCh, minio.RemoveObjectsOptions{})

// Print errors received from RemoveObjects API
for e := range errorCh {
log.Fatalln("Failed to remove " + e.ObjectName + ", error: " + e.Err.Error())
for result := range resultCh {
if resultCh.Err != nil {
log.Fatalln("Failed to remove " + result.ObjectName + ", error: " + result.Err.Error())
}
}

log.Println("Success")
Expand Down

0 comments on commit f44ea79

Please sign in to comment.