feat!: retryable http client (#398)

If implemented, this will provide a default http client with retry.

The retry function is an exponential back off 0.25s * 2^n ± 10% and max
5 attempts.

The client is the default client of `auth.Client`

BREAKING CHANGE: `auth.DefaultClient` uses `retry.DefaultClient` instead of `http.DefaultClient`
Fixes: #147 
Co-authored-by: Shiwei Zhang <[email protected]>
Signed-off-by: Soule BA <[email protected]>

example_test.go

@@ -182,7 +182,7 @@ func TestMain(m *testing.M) {
 		panic(err)
 	}
 	remoteHost = u.Host
-	http.DefaultClient = httpsServer.Client()
+	http.DefaultTransport = httpsServer.Client().Transport
 
 	os.Exit(m.Run())
 }

registry/example_test.go

@@ -65,7 +65,7 @@ func TestMain(m *testing.M) {
 		panic(err)
 	}
 	host = u.Host
-	http.DefaultClient = ts.Client()
+	http.DefaultTransport = ts.Client().Transport
 
 	os.Exit(m.Run())
 }

registry/remote/auth/client.go

@@ -28,10 +28,12 @@ import (
 	"strings"
 
 	"oras.land/oras-go/v2/registry/remote/internal/errutil"
+	"oras.land/oras-go/v2/registry/remote/retry"
 )
 
 // DefaultClient is the default auth-decorated client.
 var DefaultClient = &Client{
+	Client: retry.DefaultClient,
 	Header: http.Header{
 		"User-Agent": {"oras-go"},
 	},
@@ -68,6 +70,11 @@ type Client struct {
 	// Client is the underlying HTTP client used to access the remote
 	// server.
 	// If nil, http.DefaultClient is used.
+	// It is possible to use the default retry client from the package
+	// `oras.land/oras-go/v2/registry/remote/retry`. That client is already available
+	// in the DefaultClient.
+	// It is also possible to use a custom client. For example, github.com/hashicorp/go-retryablehttp
+	// is a popular HTTP client that supports retries.
 	Client *http.Client
 
 	// Header contains the custom headers to be added to each request.

registry/remote/example_test.go

@@ -214,7 +214,7 @@ func TestMain(m *testing.M) {
 		panic(err)
 	}
 	host = u.Host
-	http.DefaultClient = ts.Client()
+	http.DefaultTransport = ts.Client().Transport
 
 	os.Exit(m.Run())
 }

registry/remote/retry/client.go

@@ -0,0 +1,114 @@
+/*
+Copyright The ORAS Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package retry
+
+import (
+	"net/http"
+	"time"
+)
+
+// DefaultClient is a client with the default retry policy.
+var DefaultClient = NewClient()
+
+// NewClient creates an HTTP client with the default retry policy.
+func NewClient() *http.Client {
+	return &http.Client{
+		Transport: NewTransport(nil),
+	}
+}
+
+// Transport is an HTTP transport with retry policy.
+type Transport struct {
+	// Base is the underlying HTTP transport to use.
+	// If nil, http.DefaultTransport is used for round trips.
+	Base http.RoundTripper
+
+	// Policy returns a retry Policy to use for the request.
+	// If nil, DefaultPolicy is used to determine if the request should be retried.
+	Policy func() Policy
+}
+
+// NewTransport creates an HTTP Transport with the default retry policy.
+func NewTransport(base http.RoundTripper) *Transport {
+	return &Transport{
+		Base: base,
+	}
+}
+
+// RoundTrip executes a single HTTP transaction, returning a Response for the
+// provided Request.
+// It relies on the configured Policy to determine if the request should be
+// retried and to backoff.
+func (t *Transport) RoundTrip(req *http.Request) (*http.Response, error) {
+	ctx := req.Context()
+	policy := t.policy()
+	attempt := 0
+	for {
+		resp, respErr := t.roundTrip(req)
+		duration, err := policy.Retry(attempt, resp, respErr)
+		if err != nil {
+			if respErr == nil {
+				resp.Body.Close()
+			}
+			return nil, err
+		}
+		if duration < 0 {
+			return resp, respErr
+		}
+
+		// rewind the body if possible
+		if req.Body != nil {
+			if req.GetBody == nil {
+				// body can't be rewound, so we can't retry
+				return resp, respErr
+			}
+			body, err := req.GetBody()
+			if err != nil {
+				// failed to rewind the body, so we can't retry
+				return resp, respErr
+			}
+			req.Body = body
+		}
+
+		// close the response body if needed
+		if respErr == nil {
+			resp.Body.Close()
+		}
+
+		timer := time.NewTimer(duration)
+		select {
+		case <-ctx.Done():
+			timer.Stop()
+			return nil, ctx.Err()
+		case <-timer.C:
+		}
+		attempt++
+	}
+}
+
+func (t *Transport) roundTrip(req *http.Request) (*http.Response, error) {
+	if t.Base == nil {
+		return http.DefaultTransport.RoundTrip(req)
+	}
+	return t.Base.RoundTrip(req)
+}
+
+func (t *Transport) policy() Policy {
+	if t.Policy == nil {
+		return DefaultPolicy
+	}
+	return t.Policy()
+}

registry/remote/retry/client_test.go

@@ -0,0 +1,97 @@
+/*
+Copyright The ORAS Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package retry
+
+import (
+	"bytes"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func Test_Client(t *testing.T) {
+	testCases := []struct {
+		name        string
+		attempts    int
+		retryAfter  bool
+		StatusCode  int
+		expectedErr bool
+	}{
+		{
+			name:     "successful request with 0 retry",
+			attempts: 1, retryAfter: false, StatusCode: http.StatusOK, expectedErr: false,
+		},
+		{
+			name: "successful request with 1 retry caused by rate limit",
+			// 1 request + 1 retry = 2 attempts
+			attempts: 2, retryAfter: true, StatusCode: http.StatusTooManyRequests, expectedErr: false,
+		},
+		{
+			name: "successful request with 1 retry caused by 408",
+			// 1 request + 1 retry = 2 attempts
+			attempts: 2, retryAfter: false, StatusCode: http.StatusRequestTimeout, expectedErr: false,
+		},
+		{
+			name: "successful request with 2 retries caused by 429",
+			// 1 request + 2 retries = 3 attempts
+			attempts: 3, retryAfter: false, StatusCode: http.StatusTooManyRequests, expectedErr: false,
+		},
+		{
+			name: "unsuccessful request with 6 retries caused by too many retries",
+			// 1 request + 6 retries = 7 attempts
+			attempts: 7, retryAfter: false, StatusCode: http.StatusServiceUnavailable, expectedErr: true,
+		},
+	}
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			count := 0
+			ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+				count++
+				if count < tc.attempts {
+					if tc.retryAfter {
+						w.Header().Set("Retry-After", "1")
+					}
+					http.Error(w, "error", tc.StatusCode)
+					return
+				}
+				w.WriteHeader(http.StatusOK)
+			}))
+			defer ts.Close()
+
+			req, err := http.NewRequest(http.MethodPost, ts.URL, bytes.NewReader([]byte("test")))
+			if err != nil {
+				t.Fatalf("failed to create test request: %v", err)
+			}
+
+			resp, err := DefaultClient.Do(req)
+			if err != nil {
+				t.Fatalf("failed to do test request: %v", err)
+			}
+			if tc.expectedErr {
+				if count != (tc.attempts - 1) {
+					t.Errorf("expected attempts %d, got %d", tc.attempts, count)
+				}
+				if resp.StatusCode != http.StatusServiceUnavailable {
+					t.Errorf("expected status code %d, got %d", http.StatusServiceUnavailable, resp.StatusCode)
+				}
+				return
+			}
+			if tc.attempts != count {
+				t.Errorf("expected attempts %d, got %d", tc.attempts, count)
+			}
+		})
+	}
+}