roachtest: teach cluster-init to reproduce startup crash

Teach the cluster-init test to reproduce the crash during bootup
described in issue cockroachdb#25771. The trick is to send requests with a session
cookie, as the admin UI would, while the cluster is waiting for init.

Note that the crash was caused by the authenticationMux, which is used
to protect secure clusters, but this test uses an insecure cluster. This
is relying on a wart: an authenticationMux is installed for UI assets
whether or not the cluster is secure. This could change in the future. A
better test would use a secure cluster, however roachprod/roachtest do
not easily support that at this time, nor is it easy to write a unit
test that exercises the crash more robustly.

Release note: None

pkg/cmd/roachtest/cluster_init.go

@@ -23,6 +23,8 @@ import (
 	"strings"
 	"time"
 
+	"github.com/cockroachdb/cockroach/pkg/server"
+	"github.com/cockroachdb/cockroach/pkg/server/serverpb"
 	"github.com/cockroachdb/cockroach/pkg/util/retry"
 	"golang.org/x/sync/errgroup"
 )
@@ -143,16 +145,37 @@ func runClusterInit(ctx context.Context, t *test, c *cluster) {
 				{"/_status/nodes", http.StatusNotFound},
 			}
 			for _, tc := range httpTests {
-				resp, err := http.Get(urlMap[1] + tc.endpoint)
-				if err != nil {
-					t.Fatalf("unexpected error hitting %s endpoint: %v", tc.endpoint, err)
-				}
-				defer resp.Body.Close()
-				if resp.StatusCode != tc.expectedStatus {
-					bodyBytes, _ := ioutil.ReadAll(resp.Body)
-					t.Fatalf("unexpected response code %d (expected %d) hitting %s endpoint: %v",
-						resp.StatusCode, tc.expectedStatus, tc.endpoint, string(bodyBytes))
+				for _, withCookie := range []bool{false, true} {
+					req, err := http.NewRequest("GET", urlMap[1]+tc.endpoint, nil /* body */)
+					if err != nil {
+						t.Fatalf("unexpected error while constructing request for %s: %s", tc.endpoint, err)
+					}
+					if withCookie {
+						// Prevent regression of #25771 by also sending authenticated
+						// requests, like would be sent if an admin UI were open against
+						// this node while it booted.
+						cookie, err := server.EncodeSessionCookie(&serverpb.SessionCookie{
+							// The actual contents of the cookie don't matter; the presence of
+							// a valid encoded cookie is enough to trigger the authentication
+							// code paths.
+						})
+						if err != nil {
+							t.Fatal(err)
+						}
+						req.AddCookie(cookie)
+					}
+					resp, err := http.DefaultClient.Do(req)
+					if err != nil {
+						t.Fatalf("unexpected error hitting %s endpoint: %v", tc.endpoint, err)
+					}
+					defer resp.Body.Close()
+					if resp.StatusCode != tc.expectedStatus {
+						bodyBytes, _ := ioutil.ReadAll(resp.Body)
+						t.Fatalf("unexpected response code %d (expected %d) hitting %s endpoint: %v",
+							resp.StatusCode, tc.expectedStatus, tc.endpoint, string(bodyBytes))
+					}
 				}
+
 			}
 
 			c.Run(ctx, c.Node(initNode),

pkg/server/authentication.go

@@ -139,7 +139,7 @@ func (s *authenticationServer) UserLogin(
 		ID:     id,
 		Secret: secret,
 	}
-	cookie, err := encodeSessionCookie(cookieValue)
+	cookie, err := EncodeSessionCookie(cookieValue)
 	if err != nil {
 		return nil, apiInternalError(ctx, err)
 	}
@@ -369,7 +369,8 @@ func (am *authenticationMux) ServeHTTP(w http.ResponseWriter, req *http.Request)
 	am.inner.ServeHTTP(w, req)
 }
 
-func encodeSessionCookie(sessionCookie *serverpb.SessionCookie) (*http.Cookie, error) {
+// EncodeSessionCookie encodes a SessionCookie proto into an http.Cookie.
+func EncodeSessionCookie(sessionCookie *serverpb.SessionCookie) (*http.Cookie, error) {
 	cookieValueBytes, err := protoutil.Marshal(sessionCookie)
 	if err != nil {
 		return nil, errors.Wrap(err, "session cookie could not be encoded")

pkg/server/authentication_test.go

@@ -545,7 +545,7 @@ func TestLogout(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	encodedCookie, err := encodeSessionCookie(cookie)
+	encodedCookie, err := EncodeSessionCookie(cookie)
 	if err != nil {
 		t.Fatal(err)
 	}

pkg/server/testserver.go

@@ -569,7 +569,7 @@ func (ts *TestServer) getAuthenticatedHTTPClientAndCookie() (
 				Secret: secret,
 			}
 			// Encode a session cookie and store it in a cookie jar.
-			cookie, err := encodeSessionCookie(rawCookie)
+			cookie, err := EncodeSessionCookie(rawCookie)
 			if err != nil {
 				return err
 			}