From b37640650d4f94e749bd8542d490580f5251e2a0 Mon Sep 17 00:00:00 2001 From: Klaus Post Date: Fri, 13 Jan 2023 06:58:54 +0100 Subject: [PATCH] Use provided sha256 hasher (#1749) --- api.go | 2 +- core_test.go | 24 ++++++++++++ pkg/credentials/file_test.go | 4 ++ pkg/signer/request-signature-streaming.go | 37 ++++++++++++------ .../request-signature-streaming_test.go | 38 ++++++++++++++++--- 5 files changed, 87 insertions(+), 18 deletions(-) diff --git a/api.go b/api.go index ecb5ca47e4..99a03df958 100644 --- a/api.go +++ b/api.go @@ -845,7 +845,7 @@ func (c *Client) newRequest(ctx context.Context, method string, metadata request // Additionally, we also look if the initialized client is secure, // if yes then we don't need to perform streaming signature. req = signer.StreamingSignV4(req, accessKeyID, - secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC()) + secretAccessKey, sessionToken, location, metadata.contentLength, time.Now().UTC(), c.sha256Hasher()) default: // Set sha256 sum for signature calculation only with signature version '4'. shaHeader := unsignedPayload diff --git a/core_test.go b/core_test.go index e58ad3f8b8..23b5a28dbf 100644 --- a/core_test.go +++ b/core_test.go @@ -41,6 +41,9 @@ const ( // Tests for Core GetObject() function. func TestGetObjectCore(t *testing.T) { + if os.Getenv(serverEndpoint) == "" { + t.Skip("SERVER_ENDPOINT not set") + } if testing.Short() { t.Skip("skipping functional tests for the short runs") } @@ -238,6 +241,9 @@ func TestGetObjectCore(t *testing.T) { // Tests GetObject to return Content-Encoding properly set // and overrides any auto decoding. func TestGetObjectContentEncoding(t *testing.T) { + if os.Getenv(serverEndpoint) == "" { + t.Skip("SERVER_ENDPOINT not set") + } if testing.Short() { t.Skip("skipping functional tests for the short runs") } @@ -311,6 +317,9 @@ func TestGetObjectContentEncoding(t *testing.T) { // Tests get bucket policy core API. func TestGetBucketPolicy(t *testing.T) { + if os.Getenv(serverEndpoint) == "" { + t.Skip("SERVER_ENDPOINT not set") + } if testing.Short() { t.Skip("skipping functional tests for short runs") } @@ -374,6 +383,9 @@ func TestGetBucketPolicy(t *testing.T) { // Tests Core CopyObject API implementation. func TestCoreCopyObject(t *testing.T) { + if os.Getenv(serverEndpoint) == "" { + t.Skip("SERVER_ENDPOINT not set") + } if testing.Short() { t.Skip("skipping functional tests for short runs") } @@ -497,6 +509,9 @@ func TestCoreCopyObject(t *testing.T) { // Test Core CopyObjectPart implementation func TestCoreCopyObjectPart(t *testing.T) { + if os.Getenv(serverEndpoint) == "" { + t.Skip("SERVER_ENDPOINT not set") + } if testing.Short() { t.Skip("skipping functional tests for short runs") } @@ -650,6 +665,9 @@ func TestCoreCopyObjectPart(t *testing.T) { // Test Core PutObject. func TestCorePutObject(t *testing.T) { + if os.Getenv(serverEndpoint) == "" { + t.Skip("SERVER_ENDPOINT not set") + } if testing.Short() { t.Skip("skipping functional tests for short runs") } @@ -744,6 +762,9 @@ func TestCorePutObject(t *testing.T) { } func TestCoreGetObjectMetadata(t *testing.T) { + if os.Getenv(serverEndpoint) == "" { + t.Skip("SERVER_ENDPOINT not set") + } if testing.Short() { t.Skip("skipping functional tests for the short runs") } @@ -801,6 +822,9 @@ func TestCoreGetObjectMetadata(t *testing.T) { } func TestCoreMultipartUpload(t *testing.T) { + if os.Getenv(serverEndpoint) == "" { + t.Skip("SERVER_ENDPOINT not set") + } if testing.Short() { t.Skip("skipping functional tests for the short runs") } diff --git a/pkg/credentials/file_test.go b/pkg/credentials/file_test.go index d42fd3a51d..fab48dc441 100644 --- a/pkg/credentials/file_test.go +++ b/pkg/credentials/file_test.go @@ -20,10 +20,14 @@ package credentials import ( "os" "path/filepath" + "runtime" "testing" ) func TestFileAWS(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("\"/bin/cat\": file does not exist") + } os.Clearenv() creds := NewFileAWSCredentials("credentials.sample", "") diff --git a/pkg/signer/request-signature-streaming.go b/pkg/signer/request-signature-streaming.go index cf2356fd90..1c2f1dc9d1 100644 --- a/pkg/signer/request-signature-streaming.go +++ b/pkg/signer/request-signature-streaming.go @@ -26,6 +26,8 @@ import ( "strconv" "strings" "time" + + md5simd "github.com/minio/md5-simd" ) // Reference for constants used below - @@ -90,14 +92,14 @@ func getStreamLength(dataLen, chunkSize int64, trailers http.Header) int64 { // buildChunkStringToSign - returns the string to sign given chunk data // and previous signature. -func buildChunkStringToSign(t time.Time, region, previousSig string, chunkData []byte) string { +func buildChunkStringToSign(t time.Time, region, previousSig, chunkChecksum string) string { stringToSignParts := []string{ streamingPayloadHdr, t.Format(iso8601DateFormat), getScope(region, t, ServiceTypeS3), previousSig, emptySHA256, - hex.EncodeToString(sum256(chunkData)), + chunkChecksum, } return strings.Join(stringToSignParts, "\n") @@ -105,13 +107,13 @@ func buildChunkStringToSign(t time.Time, region, previousSig string, chunkData [ // buildTrailerChunkStringToSign - returns the string to sign given chunk data // and previous signature. -func buildTrailerChunkStringToSign(t time.Time, region, previousSig string, chunkData []byte) string { +func buildTrailerChunkStringToSign(t time.Time, region, previousSig, chunkChecksum string) string { stringToSignParts := []string{ streamingTrailerHdr, t.Format(iso8601DateFormat), getScope(region, t, ServiceTypeS3), previousSig, - hex.EncodeToString(sum256(chunkData)), + chunkChecksum, } return strings.Join(stringToSignParts, "\n") @@ -148,21 +150,21 @@ func buildChunkHeader(chunkLen int64, signature string) []byte { } // buildChunkSignature - returns chunk signature for a given chunk and previous signature. -func buildChunkSignature(chunkData []byte, reqTime time.Time, region, +func buildChunkSignature(chunkCheckSum string, reqTime time.Time, region, previousSignature, secretAccessKey string, ) string { chunkStringToSign := buildChunkStringToSign(reqTime, region, - previousSignature, chunkData) + previousSignature, chunkCheckSum) signingKey := getSigningKey(secretAccessKey, region, reqTime, ServiceTypeS3) return getSignature(signingKey, chunkStringToSign) } // buildChunkSignature - returns chunk signature for a given chunk and previous signature. -func buildTrailerChunkSignature(chunkData []byte, reqTime time.Time, region, +func buildTrailerChunkSignature(chunkChecksum string, reqTime time.Time, region, previousSignature, secretAccessKey string, ) string { chunkStringToSign := buildTrailerChunkStringToSign(reqTime, region, - previousSignature, chunkData) + previousSignature, chunkChecksum) signingKey := getSigningKey(secretAccessKey, region, reqTime, ServiceTypeS3) return getSignature(signingKey, chunkStringToSign) } @@ -202,12 +204,17 @@ type StreamingReader struct { totalChunks int lastChunkSize int trailer http.Header + sh256 md5simd.Hasher } // signChunk - signs a chunk read from s.baseReader of chunkLen size. func (s *StreamingReader) signChunk(chunkLen int, addCrLf bool) { // Compute chunk signature for next header - signature := buildChunkSignature(s.chunkBuf[:chunkLen], s.reqTime, + s.sh256.Reset() + s.sh256.Write(s.chunkBuf[:chunkLen]) + chunckChecksum := hex.EncodeToString(s.sh256.Sum(nil)) + + signature := buildChunkSignature(chunckChecksum, s.reqTime, s.region, s.prevSignature, s.secretAccessKey) // For next chunk signature computation @@ -239,8 +246,11 @@ func (s *StreamingReader) addSignedTrailer(h http.Header) { s.chunkBuf = append(s.chunkBuf, []byte(strings.ToLower(k)+trailerKVSeparator+v[0]+"\n")...) } + s.sh256.Reset() + s.sh256.Write(s.chunkBuf) + chunkChecksum := hex.EncodeToString(s.sh256.Sum(nil)) // Compute chunk signature - signature := buildTrailerChunkSignature(s.chunkBuf, s.reqTime, + signature := buildTrailerChunkSignature(chunkChecksum, s.reqTime, s.region, s.prevSignature, s.secretAccessKey) // For next chunk signature computation @@ -273,7 +283,7 @@ func (s *StreamingReader) setStreamingAuthHeader(req *http.Request) { // StreamingSignV4 - provides chunked upload signatureV4 support by // implementing io.Reader. func StreamingSignV4(req *http.Request, accessKeyID, secretAccessKey, sessionToken, - region string, dataLen int64, reqTime time.Time, + region string, dataLen int64, reqTime time.Time, sh256 md5simd.Hasher, ) *http.Request { // Set headers needed for streaming signature. prepareStreamingRequest(req, sessionToken, dataLen, reqTime) @@ -294,6 +304,7 @@ func StreamingSignV4(req *http.Request, accessKeyID, secretAccessKey, sessionTok chunkNum: 1, totalChunks: int((dataLen+payloadChunkSize-1)/payloadChunkSize) + 1, lastChunkSize: int(dataLen % payloadChunkSize), + sh256: sh256, } if len(req.Trailer) > 0 { stReader.trailer = req.Trailer @@ -384,5 +395,9 @@ func (s *StreamingReader) Read(buf []byte) (int, error) { // Close - this method makes underlying io.ReadCloser's Close method available. func (s *StreamingReader) Close() error { + if s.sh256 != nil { + s.sh256.Close() + s.sh256 = nil + } return s.baseReadCloser.Close() } diff --git a/pkg/signer/request-signature-streaming_test.go b/pkg/signer/request-signature-streaming_test.go index 06efa556d4..52733b2aa9 100644 --- a/pkg/signer/request-signature-streaming_test.go +++ b/pkg/signer/request-signature-streaming_test.go @@ -19,13 +19,37 @@ package signer import ( "bytes" + fipssha256 "crypto/sha256" "encoding/hex" + "hash" "io" "net/http" "testing" "time" + + md5simd "github.com/minio/md5-simd" + "github.com/minio/sha256-simd" ) +// hashWrapper implements the md5simd.Hasher interface. +type hashWrapper struct { + hash.Hash +} + +func newSHA256Hasher() md5simd.Hasher { + return &hashWrapper{Hash: fipssha256.New()} +} + +func (m *hashWrapper) Close() { + m.Hash = nil +} + +func sum256hex(data []byte) string { + hash := sha256.New() + hash.Write(data) + return hex.EncodeToString(hash.Sum(nil)) +} + func TestGetSeedSignature(t *testing.T) { accessKeyID := "AKIAIOSFODNN7EXAMPLE" secretAccessKeyID := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" @@ -42,7 +66,7 @@ func TestGetSeedSignature(t *testing.T) { t.Fatalf("Failed to parse time - %v", err) } - req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", "us-east-1", int64(dataLen), reqTime) + req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", "us-east-1", int64(dataLen), reqTime, newSHA256Hasher()) actualSeedSignature := req.Body.(*StreamingReader).seedSignature expectedSeedSignature := "38cab3af09aa15ddf29e26e36236f60fb6bfb6243a20797ae9a8183674526079" @@ -58,7 +82,8 @@ func TestChunkSignature(t *testing.T) { location := "us-east-1" secretAccessKeyID := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" expectedSignature := "ad80c730a21e5b8d04586a2213dd63b9a0e99e0e2307b0ade35a65485a288648" - actualSignature := buildChunkSignature(chunkData, reqTime, location, previousSignature, secretAccessKeyID) + chunkCheckSum := sum256hex(chunkData) + actualSignature := buildChunkSignature(chunkCheckSum, reqTime, location, previousSignature, secretAccessKeyID) if actualSignature != expectedSignature { t.Errorf("Expected %s but received %s", expectedSignature, actualSignature) } @@ -72,7 +97,8 @@ func TestTrailerChunkSignature(t *testing.T) { location := "us-east-1" secretAccessKeyID := "wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY" expectedSignature := "41e14ac611e27a8bb3d66c3bad6856f209297767d5dd4fc87d8fa9e422e03faf" - actualSignature := buildTrailerChunkSignature(chunkData, reqTime, location, previousSignature, secretAccessKeyID) + chunkCheckSum := sum256hex(chunkData) + actualSignature := buildTrailerChunkSignature(chunkCheckSum, reqTime, location, previousSignature, secretAccessKeyID) if actualSignature != expectedSignature { t.Errorf("Expected %s but received %s", expectedSignature, actualSignature) } @@ -90,7 +116,7 @@ func TestSetStreamingAuthorization(t *testing.T) { dataLen := int64(65 * 1024) reqTime, _ := time.Parse(iso8601DateFormat, "20130524T000000Z") - req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", location, dataLen, reqTime) + req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", location, dataLen, reqTime, newSHA256Hasher()) expectedAuthorization := "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class,Signature=38cab3af09aa15ddf29e26e36236f60fb6bfb6243a20797ae9a8183674526079" @@ -117,7 +143,7 @@ func TestSetStreamingAuthorizationTrailer(t *testing.T) { dataLen := int64(65 * 1024) reqTime, _ := time.Parse(iso8601DateFormat, "20130524T000000Z") - req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", location, dataLen, reqTime) + req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", location, dataLen, reqTime, newSHA256Hasher()) // (order of signed headers is different) expectedAuthorization := "AWS4-HMAC-SHA256 Credential=AKIAIOSFODNN7EXAMPLE/20130524/us-east-1/s3/aws4_request,SignedHeaders=content-encoding;host;x-amz-content-sha256;x-amz-date;x-amz-decoded-content-length;x-amz-storage-class;x-amz-trailer,Signature=106e2a8a18243abcf37539882f36619c00e2dfc72633413f02d3b74544bfeb8e" @@ -145,7 +171,7 @@ func TestStreamingReader(t *testing.T) { baseReader := io.NopCloser(bytes.NewReader(bytes.Repeat([]byte("a"), 65*1024))) req.Body = baseReader - req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", location, dataLen, reqTime) + req = StreamingSignV4(req, accessKeyID, secretAccessKeyID, "", location, dataLen, reqTime, newSHA256Hasher()) b, err := io.ReadAll(req.Body) if err != nil {