fix: use seeker to avoid memory without multipart upload

refer #15754

.github/workflows/go-windows.yml

@@ -1,20 +1,23 @@
-name: Go
+name: Build (Windows)
 
 on:
   pull_request:
     branches:
     - master
-  push:
-    branches:
-    - master
+
+# This ensures that previous jobs for the PR are canceled when the PR is
+# updated.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
 
 jobs:
   build:
     name: Test on Go ${{ matrix.go-version }} and ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.17.x]
+        go-version: [1.18.x, 1.19.x]
         os: [windows-latest]
     steps:
       - name: Set up Go ${{ matrix.go-version }} on ${{ matrix.os }}

.github/workflows/go.yml

@@ -1,20 +1,23 @@
-name: Go
+name: Build (Linux)
 
 on:
   pull_request:
     branches:
     - master
-  push:
-    branches:
-    - master
+
+# This ensures that previous jobs for the PR are canceled when the PR is
+# updated.
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref }}
+  cancel-in-progress: true
 
 jobs:
   build:
     name: Test on Go ${{ matrix.go-version }} and ${{ matrix.os }}
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        go-version: [1.17.x, 1.18.x]
+        go-version: [1.18.x, 1.19.x]
         os: [ubuntu-latest]
     steps:
       - name: Set up Go ${{ matrix.go-version }} on ${{ matrix.os }}

.gitignore

@@ -1,4 +1,5 @@
 *~
 *.test
 validator
-golangci-lint
+golangci-lint
+functional_tests

.golangci.yml

@@ -12,8 +12,7 @@ linters:
     - govet
     - ineffassign
     - gosimple
-    - deadcode
-    - structcheck
+    - unused
     - gocritic
 
 issues:
@@ -25,3 +24,4 @@ issues:
       - "captLocal:"
       - "ifElseChain:"
       - "elseif:"
+      - "should have a package comment"

Makefile

@@ -9,7 +9,7 @@ checks: lint vet test examples functional-test
 
 lint:
 	@mkdir -p ${GOPATH}/bin
-	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin v1.45.2
+	@echo "Installing golangci-lint" && curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(GOPATH)/bin
 	@echo "Running $@ check"
 	@GO111MODULE=on ${GOPATH}/bin/golangci-lint cache clean
 	@GO111MODULE=on ${GOPATH}/bin/golangci-lint run --timeout=5m --config ./.golangci.yml
@@ -27,7 +27,8 @@ examples:
 	@cd ./examples/minio && $(foreach v,$(wildcard examples/minio/*.go),go build -mod=mod -o ${TMPDIR}/$(basename $(v)) $(notdir $(v)) || exit 1;)
 
 functional-test:
-	@GO111MODULE=on SERVER_ENDPOINT=localhost:9000 ACCESS_KEY=minio SECRET_KEY=minio123 ENABLE_HTTPS=1 MINT_MODE=full go run functional_tests.go
+	@GO111MODULE=on go build -race functional_tests.go
+	@SERVER_ENDPOINT=localhost:9000 ACCESS_KEY=minio SECRET_KEY=minio123 ENABLE_HTTPS=1 MINT_MODE=full ./functional_tests
 
 clean:
 	@echo "Cleaning up all the generated files"

api-error-response.go

@@ -67,14 +67,14 @@ type ErrorResponse struct {
 //
 // For example:
 //
-//   import s3 "github.com/minio/minio-go/v7"
-//   ...
-//   ...
-//   reader, stat, err := s3.GetObject(...)
-//   if err != nil {
-//      resp := s3.ToErrorResponse(err)
-//   }
-//   ...
+//	import s3 "github.com/minio/minio-go/v7"
+//	...
+//	...
+//	reader, stat, err := s3.GetObject(...)
+//	if err != nil {
+//	   resp := s3.ToErrorResponse(err)
+//	}
+//	...
 func ToErrorResponse(err error) ErrorResponse {
 	switch err := err.(type) {
 	case ErrorResponse:

api-list.go

@@ -32,11 +32,10 @@ import (
 // This call requires explicit authentication, no anonymous requests are
 // allowed for listing buckets.
 //
-//   api := client.New(....)
-//   for message := range api.ListBuckets(context.Background()) {
-//       fmt.Println(message)
-//   }
-//
+//	api := client.New(....)
+//	for message := range api.ListBuckets(context.Background()) {
+//	    fmt.Println(message)
+//	}
 func (c *Client) ListBuckets(ctx context.Context) ([]BucketInfo, error) {
 	// Execute GET on service.
 	resp, err := c.executeMethod(ctx, http.MethodGet, requestMetadata{contentSHA256Hex: emptySHA256Hex})
@@ -659,11 +658,10 @@ func (o *ListObjectsOptions) Set(key, value string) {
 
 // ListObjects returns objects list after evaluating the passed options.
 //
-//   api := client.New(....)
-//   for object := range api.ListObjects(ctx, "mytestbucket", minio.ListObjectsOptions{Prefix: "starthere", Recursive:true}) {
-//       fmt.Println(object)
-//   }
-//
+//	api := client.New(....)
+//	for object := range api.ListObjects(ctx, "mytestbucket", minio.ListObjectsOptions{Prefix: "starthere", Recursive:true}) {
+//	    fmt.Println(object)
+//	}
 func (c *Client) ListObjects(ctx context.Context, bucketName string, opts ListObjectsOptions) <-chan ObjectInfo {
 	if opts.WithVersions {
 		return c.listObjectVersions(ctx, bucketName, opts)
@@ -694,12 +692,12 @@ func (c *Client) ListObjects(ctx context.Context, bucketName string, opts ListOb
 // If you enable recursive as 'true' this function will return back all
 // the multipart objects in a given bucket name.
 //
-//   api := client.New(....)
-//   // Recurively list all objects in 'mytestbucket'
-//   recursive := true
-//   for message := range api.ListIncompleteUploads(context.Background(), "mytestbucket", "starthere", recursive) {
-//       fmt.Println(message)
-//   }
+//	api := client.New(....)
+//	// Recurively list all objects in 'mytestbucket'
+//	recursive := true
+//	for message := range api.ListIncompleteUploads(context.Background(), "mytestbucket", "starthere", recursive) {
+//	    fmt.Println(message)
+//	}
 func (c *Client) ListIncompleteUploads(ctx context.Context, bucketName, objectPrefix string, recursive bool) <-chan ObjectMultipartInfo {
 	return c.listIncompleteUploads(ctx, bucketName, objectPrefix, recursive)
 }
@@ -916,7 +914,7 @@ func (c *Client) findUploadIDs(ctx context.Context, bucketName, objectName strin
 }
 
 // listObjectPartsQuery (List Parts query)
-//     - lists some or all (up to 1000) parts that have been uploaded
+//   - lists some or all (up to 1000) parts that have been uploaded
 //     for a specific multipart upload
 //
 // You can use the request parameters as selection criteria to return

api-put-object-common.go

@@ -65,10 +65,9 @@ func isReadAt(reader io.Reader) (ok bool) {
 // NOTE: Assumption here is that for any object to be uploaded to any S3 compatible
 // object storage it will have the following parameters as constants.
 //
-//  maxPartsCount - 10000
-//  minPartSize - 16MiB
-//  maxMultipartPutObjectSize - 5TiB
-//
+//	maxPartsCount - 10000
+//	minPartSize - 16MiB
+//	maxMultipartPutObjectSize - 5TiB
 func OptimalPartInfo(objectSize int64, configuredPartSize uint64) (totalPartsCount int, partSize int64, lastPartSize int64, err error) {
 	// object size is '-1' set it to 5TiB.
 	var unknownSize bool

api-put-object-streaming.go

@@ -434,19 +434,29 @@ func (c *Client) putObject(ctx context.Context, bucketName, objectName string, r
 
 	var md5Base64 string
 	if opts.SendContentMd5 {
-		// Create a buffer.
-		buf := make([]byte, size)
+		// Calculate md5sum.
+		hash := c.md5Hasher()
+
+		seeker, ok := reader.(io.Seeker)
+		if ok {
+			if _, err := io.Copy(hash, reader); err != nil {
+				return UploadInfo{}, err
+			}
+			seeker.Seek(0, io.SeekStart) // Seek back to beginning
+		} else {
+			// Create a buffer.
+			buf := make([]byte, size)
 
-		length, rErr := readFull(reader, buf)
-		if rErr != nil && rErr != io.ErrUnexpectedEOF && rErr != io.EOF {
-			return UploadInfo{}, rErr
+			length, err := readFull(reader, buf)
+			if err != nil && err != io.ErrUnexpectedEOF && err != io.EOF {
+				return UploadInfo{}, err
+			}
+
+			hash.Write(buf[:length])
+			reader = bytes.NewReader(buf[:length])
 		}
 
-		// Calculate md5sum.
-		hash := c.md5Hasher()
-		hash.Write(buf[:length])
 		md5Base64 = base64.StdEncoding.EncodeToString(hash.Sum(nil))
-		reader = bytes.NewReader(buf[:length])
 		hash.Close()
 	}
 

api-remove.go

@@ -82,8 +82,8 @@ func (c *Client) RemoveBucketWithOptions(ctx context.Context, bucketName string,
 
 // RemoveBucket deletes the bucket name.
 //
-//  All objects (including all object versions and delete markers).
-//  in the bucket must be deleted before successfully attempting this request.
+//	All objects (including all object versions and delete markers).
+//	in the bucket must be deleted before successfully attempting this request.
 func (c *Client) RemoveBucket(ctx context.Context, bucketName string) error {
 	// Input validation.
 	if err := s3utils.CheckValidBucketName(bucketName); err != nil {

api.go

@@ -117,7 +117,7 @@ const (
 // User Agent should always following the below style.
 // Please open an issue to discuss any new changes here.
 //
-//       MinIO (OS; ARCH) LIB/VER APP/VER
+//	MinIO (OS; ARCH) LIB/VER APP/VER
 const (
 	libraryUserAgentPrefix = "MinIO (" + runtime.GOOS + "; " + runtime.GOARCH + ") "
 	libraryUserAgent       = libraryUserAgentPrefix + libraryName + "/" + libraryVersion
@@ -312,9 +312,9 @@ func (c *Client) SetS3TransferAccelerate(accelerateEndpoint string) {
 // Hash materials provides relevant initialized hash algo writers
 // based on the expected signature type.
 //
-//  - For signature v4 request if the connection is insecure compute only sha256.
-//  - For signature v4 request if the connection is secure compute only md5.
-//  - For anonymous request compute md5.
+//   - For signature v4 request if the connection is insecure compute only sha256.
+//   - For signature v4 request if the connection is secure compute only md5.
+//   - For anonymous request compute md5.
 func (c *Client) hashMaterials(isMd5Requested, isSha256Requested bool) (hashAlgos map[string]md5simd.Hasher, hashSums map[string][]byte) {
 	hashSums = make(map[string][]byte)
 	hashAlgos = make(map[string]md5simd.Hasher)

pkg/credentials/chain.go

@@ -31,18 +31,17 @@ package credentials
 // will cache that Provider for all calls to IsExpired(), until Retrieve is
 // called again after IsExpired() is true.
 //
-//     creds := credentials.NewChainCredentials(
-//         []credentials.Provider{
-//             &credentials.EnvAWSS3{},
-//             &credentials.EnvMinio{},
-//         })
-//
-//     // Usage of ChainCredentials.
-//     mc, err := minio.NewWithCredentials(endpoint, creds, secure, "us-east-1")
-//     if err != nil {
-//          log.Fatalln(err)
-//     }
+//	creds := credentials.NewChainCredentials(
+//	    []credentials.Provider{
+//	        &credentials.EnvAWSS3{},
+//	        &credentials.EnvMinio{},
+//	    })
 //
+//	// Usage of ChainCredentials.
+//	mc, err := minio.NewWithCredentials(endpoint, creds, secure, "us-east-1")
+//	if err != nil {
+//	     log.Fatalln(err)
+//	}
 type Chain struct {
 	Providers []Provider
 	curr      Provider

pkg/credentials/credentials.go

@@ -65,10 +65,11 @@ type Provider interface {
 // provider's struct.
 //
 // Example:
-//     type IAMCredentialProvider struct {
-//         Expiry
-//         ...
-//     }
+//
+//	type IAMCredentialProvider struct {
+//	    Expiry
+//	    ...
+//	}
 type Expiry struct {
 	// The date/time when to expire on
 	expiration time.Time

pkg/credentials/doc.go

@@ -28,35 +28,33 @@
 //
 // Example of using the environment variable credentials.
 //
-//     creds := NewFromEnv()
-//     // Retrieve the credentials value
-//     credValue, err := creds.Get()
-//     if err != nil {
-//         // handle error
-//     }
+//	creds := NewFromEnv()
+//	// Retrieve the credentials value
+//	credValue, err := creds.Get()
+//	if err != nil {
+//	    // handle error
+//	}
 //
 // Example of forcing credentials to expire and be refreshed on the next Get().
 // This may be helpful to proactively expire credentials and refresh them sooner
 // than they would naturally expire on their own.
 //
-//     creds := NewFromIAM("")
-//     creds.Expire()
-//     credsValue, err := creds.Get()
-//     // New credentials will be retrieved instead of from cache.
+//	creds := NewFromIAM("")
+//	creds.Expire()
+//	credsValue, err := creds.Get()
+//	// New credentials will be retrieved instead of from cache.
 //
-//
-// Custom Provider
+// # Custom Provider
 //
 // Each Provider built into this package also provides a helper method to generate
 // a Credentials pointer setup with the provider. To use a custom Provider just
 // create a type which satisfies the Provider interface and pass it to the
 // NewCredentials method.
 //
-//     type MyProvider struct{}
-//     func (m *MyProvider) Retrieve() (Value, error) {...}
-//     func (m *MyProvider) IsExpired() bool {...}
-//
-//     creds := NewCredentials(&MyProvider{})
-//     credValue, err := creds.Get()
+//	type MyProvider struct{}
+//	func (m *MyProvider) Retrieve() (Value, error) {...}
+//	func (m *MyProvider) IsExpired() bool {...}
 //
+//	creds := NewCredentials(&MyProvider{})
+//	credValue, err := creds.Get()
 package credentials

pkg/encrypt/server-side.go

@@ -67,9 +67,9 @@ var DefaultPBKDF PBKDF = func(password, salt []byte) ServerSide {
 
 // Type is the server-side-encryption method. It represents one of
 // the following encryption methods:
-//  - SSE-C: server-side-encryption with customer provided keys
-//  - KMS:   server-side-encryption with managed keys
-//  - S3:    server-side-encryption using S3 storage encryption
+//   - SSE-C: server-side-encryption with customer provided keys
+//   - KMS:   server-side-encryption with managed keys
+//   - S3:    server-side-encryption using S3 storage encryption
 type Type string
 
 const (