From 8d6ca65be706ddf54eeab9d272a61dd31a3f8e25 Mon Sep 17 00:00:00 2001 From: Sean McGrail <549813+skmcgrail@users.noreply.github.com> Date: Fri, 5 Aug 2022 17:20:24 +0000 Subject: [PATCH] feature/s3/manager: Suppress WriterAt which can bypass Windows optimizations. (#1791) Fixes issue with Go 1.19 discovered in the S3 transfer manager's windows upload and downloader buffer pools. It looks like the update to Go 1.19 changes the code path that is used by the pool because io.NopCloser was updated to forward WriterTo if the underlying Writer implemented it. Looks likes we'd made some assumptions about this behavior not being present. This PR suppresses the WriterTo of the buffer pool to prevent the unexpected usage. --- feature/s3/manager/download.go | 10 ++++++++-- feature/s3/manager/writer_read_from.go | 8 ++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/feature/s3/manager/download.go b/feature/s3/manager/download.go index c3fbe021982..2ebcea585e3 100644 --- a/feature/s3/manager/download.go +++ b/feature/s3/manager/download.go @@ -100,7 +100,8 @@ func WithDownloaderClientOptions(opts ...func(*s3.Options)) func(*Downloader) { // interface. // // Example: -// // Load AWS Config +// +// // Load AWS Config // cfg, err := config.LoadDefaultConfig(context.TODO()) // if err != nil { // panic(err) @@ -153,6 +154,7 @@ func NewDownloader(c DownloadAPIClient, options ...func(*Downloader)) *Downloade // and GC runs. // // Example: +// // // pre-allocate in memory buffer, where headObject type is *s3.HeadObjectOutput // buf := make([]byte, int(headObject.ContentLength)) // // wrap with aws.WriteAtBuffer @@ -397,7 +399,11 @@ func (d *downloader) tryDownloadChunk(params *s3.GetObjectInput, w io.Writer) (i } d.setTotalBytes(resp) // Set total if not yet set. - n, err := io.Copy(w, resp.Body) + var src io.Reader = resp.Body + if d.cfg.BufferProvider != nil { + src = &suppressWriterAt{suppressed: src} + } + n, err := io.Copy(w, src) resp.Body.Close() if err != nil { return n, &errReadingBody{err: err} diff --git a/feature/s3/manager/writer_read_from.go b/feature/s3/manager/writer_read_from.go index 3df983a652a..fb10ec309ac 100644 --- a/feature/s3/manager/writer_read_from.go +++ b/feature/s3/manager/writer_read_from.go @@ -73,3 +73,11 @@ func (p *PooledBufferedReadFromProvider) GetReadFrom(writer io.Writer) (r Writer } return r, cleanup } + +type suppressWriterAt struct { + suppressed io.Reader +} + +func (s *suppressWriterAt) Read(p []byte) (n int, err error) { + return s.suppressed.Read(p) +}