Merge #138717 #139958 #139963

138717: settings: increase number of cores test can use r=cthumuluru-crdb a=cthumuluru-crdb

Clutser setting updates on one node occasionally take over 45 seconds to propagate to other nodes. This test has failed a few times recently. The issue is not reproducible, even under stress testing with 10,000 repetitions. Increasing the timeout to 60 seconds to determine if it resolves the problem.

Epic: None
Fixes: #133732
Release note: None

139958: cdctest: fix fingerprint validator primary key cols check r=asg0451 a=wenyihu6

Previously, fingerprint validator could fail with sql smith due to precision
loss from JSON encoding & decoding of large numbers. We previously fixed the
validator regarding its values check. This patch fixes the primary key columns
check accordingly.

Part of: #124146
Release note: none

139963: kvclient: add a btree to store buffered writes r=stevendanna a=arulajmani

This patch introduces a btree to buffer writes on the txnWriteBuffer interceptor.

References #139053

Release note: None

Co-authored-by: Chandra Thumuluru <[email protected]>
Co-authored-by: Wenyi Hu <[email protected]>
Co-authored-by: Arul Ajmani <[email protected]>

build/bazelutil/check.sh

@@ -24,6 +24,7 @@ pkg/roachprod/prometheus/prometheus.go://go:generate mockgen -package=prometheus
 pkg/cmd/roachtest/clusterstats/collector.go://go:generate mockgen -package=clusterstats -destination mocks_generated_test.go github.com/cockroachdb/cockroach/pkg/roachprod/prometheus Client
 pkg/cmd/roachtest/tests/drt.go://go:generate mockgen -package tests -destination drt_generated_test.go github.com/cockroachdb/cockroach/pkg/roachprod/prometheus Client
 pkg/kv/kvclient/kvcoord/transport.go://go:generate mockgen -package=kvcoord -destination=mocks_generated_test.go . Transport
+pkg/kv/kvclient/kvcoord/txn_interceptor_write_buffer.go://go:generate ../../../util/interval/generic/gen.sh *bufferedWrite kvcoord
 pkg/kv/kvclient/rangecache/range_cache.go://go:generate mockgen -package=rangecachemock -destination=rangecachemock/mocks_generated.go . RangeDescriptorDB
 pkg/kv/kvclient/rangefeed/rangefeed.go://go:generate mockgen -destination=mocks_generated_test.go --package=rangefeed . DB
 pkg/kv/kvserver/concurrency/lock_table.go://go:generate ../../../util/interval/generic/gen.sh *keyLocks concurrency

pkg/ccl/changefeedccl/cdctest/nemeses.go

@@ -240,10 +240,6 @@ type NemesesOption struct {
 var NemesesOptions = []NemesesOption{
 	{
 		EnableFpValidator: true,
-		EnableSQLSmith:    false,
-	},
-	{
-		EnableFpValidator: false,
 		EnableSQLSmith:    true,
 	},
 }

pkg/ccl/changefeedccl/cdctest/validator.go

@@ -804,32 +804,16 @@ func (v *FingerprintValidator) applyRowUpdate(row validatorRow) (_err error) {
 		stmtBuf.WriteString(`)`)
 
 		// Also verify that the key matches the value.
-		type wrapper struct {
-			After map[string]interface{} `json:"after"`
-		}
-		var value wrapper
-		if err := gojson.Unmarshal([]byte(row.value), &value); err != nil {
-			return err
-		}
-		primaryKeyDatums := make([]interface{}, len(v.primaryKeyCols))
 		for idx, primaryKeyCol := range v.primaryKeyCols {
-			primaryKeyDatums[idx] = value.After[primaryKeyCol]
-		}
-		primaryKeyJSON, err := gojson.Marshal(primaryKeyDatums)
-		if err != nil {
-			return err
-		}
-
-		rowKey := row.key
-		if len(primaryKeyDatums) > 1 {
-			// format the key using the Go marshaller; otherwise, differences
-			// in formatting could lead to the comparison below failing
-			rowKey = asGoJSON(row.key)
-		}
-		if string(primaryKeyJSON) != rowKey {
-			v.failures = append(v.failures,
-				fmt.Sprintf(`key %s did not match expected key %s for value %s`,
-					rowKey, primaryKeyJSON, row.value))
+			valueJSON, err := afterJSON.FetchValKey(primaryKeyCol)
+			if err != nil {
+				return err
+			}
+			if rowKeyValue := keyJSONAsArray[idx]; valueJSON != rowKeyValue {
+				v.failures = append(v.failures,
+					fmt.Sprintf(`key %s did not match expected key %s for value %s`,
+						primaryKeyCol, rowKeyValue, valueJSON))
+			}
 		}
 	} else {
 		// DELETE
@@ -1096,22 +1080,3 @@ func fetchPrimaryKeyCols(sqlDB *gosql.DB, tableStr string) ([]string, error) {
 	}
 	return primaryKeyCols, nil
 }
-
-// asGoJSON tries to unmarshal the given string as JSON; if
-// successful, the struct is marshalled back to JSON. This is to
-// enforce the default formatting of the standard library marshaller,
-// allowing comparisons of JSON strings when we don't control the
-// formatting of the strings.
-func asGoJSON(s string) string {
-	var obj interface{}
-	if err := gojson.Unmarshal([]byte(s), &obj); err != nil {
-		return s
-	}
-
-	blob, err := gojson.Marshal(obj)
-	if err != nil {
-		return s
-	}
-
-	return string(blob)
-}

pkg/gen/misc.bzl

@@ -4,6 +4,8 @@ MISC_SRCS = [
     "//pkg/backup:data_driven_generated_test.go",
     "//pkg/ccl/kvccl/kvtenantccl/upgradeinterlockccl:generated_test.go",
     "//pkg/internal/team:TEAMS.yaml",
+    "//pkg/kv/kvclient/kvcoord:bufferedwrite_interval_btree.go",
+    "//pkg/kv/kvclient/kvcoord:bufferedwrite_interval_btree_test.go",
     "//pkg/kv/kvpb:batch_generated.go",
     "//pkg/kv/kvserver/concurrency:keylocks_interval_btree.go",
     "//pkg/kv/kvserver/concurrency:keylocks_interval_btree_test.go",

pkg/kv/kvclient/kvcoord/BUILD.bazel

@@ -1,6 +1,7 @@
 load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test", "gomock")
 load("//build:STRINGER.bzl", "stringer")
 load("//pkg/testutils:buildutil/buildutil.bzl", "disallowed_imports_test")
+load("//pkg/util/interval/generic:gen.bzl", "gen_interval_btree")
 
 go_library(
     name = "kvcoord",
@@ -33,6 +34,7 @@ go_library(
         "txn_interceptor_write_buffer.go",
         "txn_lock_gatekeeper.go",
         "txn_metrics.go",
+        ":bufferedwrite_interval_btree.go",  # keep
         ":gen-txnstate-stringer",  # keep
     ],
     importpath = "github.com/cockroachdb/cockroach/pkg/kv/kvclient/kvcoord",
@@ -152,6 +154,7 @@ go_test(
         "txn_interceptor_seq_num_allocator_test.go",
         "txn_interceptor_span_refresher_test.go",
         "txn_test.go",
+        ":bufferedwrite_interval_btree_test.go",  # keep
         ":mock_kvcoord",  # keep
     ],
     data = glob(["testdata/**"]),
@@ -261,3 +264,9 @@ disallowed_imports_test(
         "//pkg/storage",
     ],
 )
+
+gen_interval_btree(
+    name = "buffered_write_interval_btree",
+    package = "kvcoord",
+    type = "*bufferedWrite",
+)