Merge branch 'master' into k8s-createtree

CHANGELOG.md

@@ -2,6 +2,10 @@
 
 ## HEAD
 
+* Deprecate `--tree_ids_with_no_ephemeral_nodes` flag by setting it to all trees
+  (`*`) by default.
+* Updated golangci-lint to v1.47.3 (developers should update to this version)
+
 ## v1.4.2
 
 * #2568: Allow disabling the writes of ephemeral nodes to storage via the

README.md

@@ -211,7 +211,7 @@ and tests over the codebase.
 
 #### Install [golangci-lint](https://github.com/golangci/golangci-lint#local-installation).
 ```bash
-go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.46.1
+go install github.com/golangci/golangci-lint/cmd/golangci-lint@v1.47.3
 ```
 
 #### Run code generation, build, test and linters

docs/merkletree/treetex/main.go

@@ -19,7 +19,6 @@
 // Usage: go run main.go | xelatex
 // This should generate a PDF file called treetek.pdf containing a drawing of
 // the tree.
-//
 package main
 
 import (

experimental/batchmap/cmd/verify/verify.go

@@ -101,9 +101,7 @@ func main() {
 		if leaf == nil {
 			glog.Fatalf("couldn't find expected leaf %x in tile %x", needLeafPath, tile.Path)
 		}
-		// TODO(pavelkalinnikov): Remove nolint after fixing
-		// https://github.com/dominikh/go-tools/issues/921.
-		if !bytes.Equal(leaf.Hash, needValue) { // nolint: staticcheck
+		if !bytes.Equal(leaf.Hash, needValue) {
 			glog.Fatalf("wrong leaf value in tile %x, leaf %x: got %x, want %x", tile.Path, leaf.Path, leaf.Hash, needValue)
 		}
 

go.sum

@@ -661,6 +661,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k=
+github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU=
 go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4=
@@ -1115,6 +1116,7 @@ google.golang.org/api v0.75.0/go.mod h1:pU9QmyHLnzlpar1Mjt4IbapUCy8J+6HD6GeELN69
 google.golang.org/api v0.78.0/go.mod h1:1Sg78yoMLOhlQTeF+ARBoytAcH1NNyyl390YMy6rKmw=
 google.golang.org/api v0.80.0/go.mod h1:xY3nI94gbvBrE0J6NHXhxOmW97HG7Khjkku6AFB3Hyg=
 google.golang.org/api v0.84.0/go.mod h1:NTsGnUFJMYROtiquksZHBWtHfeMC7iYthki7Eq3pa8o=
+google.golang.org/api v0.86.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw=
 google.golang.org/api v0.90.0 h1:WMnUWAvihIClUYFNeFA69VTuR3duKS3IalMGDQcLvq8=
 google.golang.org/api v0.90.0/go.mod h1:+Sem1dnrKlrXMR/X0bPnMWyluQe4RsNoYfmNLhOIkzw=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=

integration/cloudbuild/testbase/Dockerfile

@@ -22,7 +22,7 @@ RUN apt-get update && apt-get install -y \
   xxd
 
 # Install golangci-lint. See docs at: https://golangci-lint.run/usage/install/.
-RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.46.1
+RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.47.3
 
 RUN mkdir protoc && \
     (cd protoc && \

integration/format/testdata/dump_tree_output_1000

@@ -12,23 +12,11 @@ leaves: {
   key: "CAI="
   value: "\xbb\x91zJ\xbb\xbbR\xc7\xe0>\xde\x1e\xe5\x9fC\x11\xe5J\x85\x0b\x00\x99\xd1^\xa2\xd8RB?;S5"
 }
-leaves: {
-  key: "CAM="
-  value: "\xc3\xe2VW\xbb\xed\x8f\x1eۂz\x05\xf1\x86;\xc21\x05\xae\x05\x8e\x05-o9\xb5\x05*|\x9b\x1c@"
-}
-internal_nodes: {
-  key: "BgA="
-  value: "7ה\x03g\xb8+)\xe4\n\r\xb6\x9a\x83\xf8\x11}ui\xe7\xf7N\xc1:\x87S\x06%\\'\x7f\x1b"
-}
 internal_nodes: {
   key: "BwA="
   value: "\xc6\xee@\x91R\xb3TI\x03\xf2\x86B-}c:\x9aۢ\xef\x0f/\xdb\xf1V\x8e-\r\xcf\x17?\x92"
 }
-internal_nodes: {
-  key: "BwI="
-  value: "\xe3\xda%\xfem\x86\x03v#T\x8e0\xf8 \x1e\xdb\xfb\xad\xef\\\x18\x8asX\xec\x0fN\xbe\"\x901\x1e"
-}
-internal_node_count: 3
+internal_node_count: 1
 
 prefix: "\x00\x00\x00\x00\x00\x00\x00"
 depth: 8
@@ -7116,10 +7104,6 @@ internal_nodes: {
   key: "AQA="
   value: "\xb2\nE\x80\x9e\x88X\xb4\xe5\x14\xa7ic\x80\xd1b**.u\n\xfd \t\x90e\xae\xc4\xee\x89Q\xdc"
 }
-internal_nodes: {
-  key: "AYA="
-  value: "\x8e\xcc\xe2\x04\x1c}\x0b\x0f\x1f\xa0K\xc4y\xb3mӠP\x8d\xa68\x02\x178\xf2@\xac\xd3\xd1/&\x8f"
-}
 internal_nodes: {
   key: "AgA="
   value: "A:\x85Z\x8aıqBκ\t9\x02\x0fF\xe9\x91s\x84\x07\"\x08\x08Ů\xfe\xea-\xf6\xe8R"
@@ -7132,10 +7116,6 @@ internal_nodes: {
   key: "AoA="
   value: "\xb1P\x17A\x02\xe2\xe40\xff\x16q\x9e\x1d\x91\xf6\xf0\x0e\x81\xf6\x9a\x81\xd0\r/\xac\x0e4]?3\x12r"
 }
-internal_nodes: {
-  key: "AsA="
-  value: "\xf0ݟo9d\x0f\x8f𠅼\r\x85\x07k\rV2\x84\x99 \xeb\x0c4\x06\xdd\xc4\xf7K\xf2\x98"
-}
 internal_nodes: {
   key: "AwA="
   value: "\xf9ϧ\x1d\xf4?Zٶ\x82\xfc\\g\xc9\xf2eyĭ\x03\x0eI\rX\xc5\xdd\xd4\xf7\\K\xd0G"
@@ -8012,5 +7992,5 @@ internal_nodes: {
   key: "Bzw="
   value: "\xd5u\xac\xe5b\xa0&\xaa\x92\xd9\xf6~FO\x9c\xa2>c\x80\xe3>rI\x86\xe5\x91'l\xaf"
 }
-internal_node_count: 230
+internal_node_count: 228
 

integration/format/testdata/dump_tree_output_871

@@ -12,19 +12,11 @@ leaves: {
   key: "CAI="
   value: "\xbb\x91zJ\xbb\xbbR\xc7\xe0>\xde\x1e\xe5\x9fC\x11\xe5J\x85\x0b\x00\x99\xd1^\xa2\xd8RB?;S5"
 }
-internal_nodes: {
-  key: "BgA="
-  value: "\xf6\x87\xc1\xbdh\x94\xb4Q\xdc\xf7\x1fZ+\x03\x9d\xce`\xf2\x87Џ\x89\x02\xc1iw\xa3AC\xbe\xa7N"
-}
 internal_nodes: {
   key: "BwA="
   value: "\xc6\xee@\x91R\xb3TI\x03\xf2\x86B-}c:\x9aۢ\xef\x0f/\xdb\xf1V\x8e-\r\xcf\x17?\x92"
 }
-internal_nodes: {
-  key: "BwI="
-  value: "\x07U\xae&\xaf\xea\xbd1\x10܍\xe6\xf0\xac\xf1\xe5)\x02\xddk\x80Q`/P\\\xa9\xe7\xfa_\xc3/"
-}
-internal_node_count: 3
+internal_node_count: 1
 
 prefix: "\x00\x00\x00\x00\x00\x00\x00"
 depth: 8
@@ -6576,18 +6568,10 @@ internal_nodes: {
   key: "A0A="
   value: "K\x1a\xbc\x93\xb8[20.Җ`\x9d\x12\x1f\x90W\xa3Ȥ~\r>\t0\x02\xfd\x07\tL\x03y"
 }
-internal_nodes: {
-  key: "AQA="
-  value: "$F\xb2T\xeb\x8c\x03\xf2\xef`3ۄ\xbfik\xba\x97\xb5\x89\x1c^\x81\xfa4\x0e\xf8\xf5#\"ӑ"
-}
 internal_nodes: {
   key: "AgA="
   value: "A:\x85Z\x8aıqBκ\t9\x02\x0fF\xe9\x91s\x84\x07\"\x08\x08Ů\xfe\xea-\xf6\xe8R"
 }
-internal_nodes: {
-  key: "AkA="
-  value: "\xf5<\x8c8C\n\x1c\xa6\xdbG\x94\xbaL\x85\xac\xf0x\xeb\xb2\x03i\xaf}\xeb\x1b\xa8\x96\x99\x19z>\xab"
-}
 internal_nodes: {
   key: "AwA="
   value: "\xf9ϧ\x1d\xf4?Zٶ\x82\xfc\\g\xc9\xf2eyĭ\x03\x0eI\rX\xc5\xdd\xd4\xf7\\K\xd0G"
@@ -6744,10 +6728,6 @@ internal_nodes: {
   key: "BVg="
   value: "5W\x98P\xfe\x125'\xb9K0\xcf\x08\x17\x92\xdb\xfb\x04\xda^\xf6\x03K\x04B\xd9ifHZ\xb6\xfe"
 }
-internal_nodes: {
-  key: "BWA="
-  value: "\xd6N^\x93e\x13Qd\xe1\xea\xc0_a\x04\x8dL\x9fu\xad\x05\xe6\xd1y&\xa3\xa1\xc3Et\xfd\xa7\xde"
-}
 internal_nodes: {
   key: "BgA="
   value: "\xec\x86&v\xd5C*\x15\xb3\xec\t\x89_\xf1*\xc1\xf02Ұ\xbdk\x08\xb0x\xfc\xdfȥ\xeaܾ"
@@ -6848,10 +6828,6 @@ internal_nodes: {
   key: "BmA="
   value: "\x15\xd6l\x92\xd7$*q\x9f\xbf\xd6\x7f\x88\xb0\xc2`\x1d\xc2y\xc3\xfe\x82bԥ\xc4\xe9\r}i\x97!"
 }
-internal_nodes: {
-  key: "BmQ="
-  value: "d\xbct\x8f\xdema\x7f\xd3\xf1\\\x14/l\xb3{{\x02\x87\xc4>\xdd,+\xfe\x87\x7f\xc8PTJ\xa7"
-}
 internal_nodes: {
   key: "Bw4="
   value: "\x99\xbc\xce}\xcd\xf2\xfe\xb9]\xecY:P7\x05~l\xfc\xab\x066 G\xa1y\x9c\x9a\xe4\xe1\r\x89\xb9"
@@ -6980,5 +6956,5 @@ internal_nodes: {
   key: "Bzw="
   value: "\xd5u\xac\xe5b\xa0&\xaa\x92\xd9\xf6~FO\x9c\xa2>c\x80\xe3>rI\x86\xe5\x91'l\xaf"
 }
-internal_node_count: 102
+internal_node_count: 98
 

integration/format/testdata/dump_tree_output_96

@@ -388,10 +388,6 @@ internal_nodes: {
   key: "A0A="
   value: "\xa2\x9b|=$\xce/\x86+\x0b\"\x92n\xcc\xc3\\/b7Lm\xdb\xd5\x19{\xfeKo\xe8\xbdtF"
 }
-internal_nodes: {
-  key: "AQA="
-  value: "\xf0K\xff\xed19\x0e\tL\xea\xfc\x12U\xf1\xf2α1z\x8a\xba\xe0\xb0t\x9a\x88\x15]U\xee\xce\xd0"
-}
 internal_nodes: {
   key: "AgA="
   value: "\x8f\x05[\xe4\x9bG\x8fW\x12\xeb\xe2\xd8[\x89˔m\xccP\x87\x18_\x1d.\xb66S\xa5\x99\x81\x08\xce"
@@ -764,5 +760,5 @@ internal_nodes: {
   key: "Bzw="
   value: "\xd7\x1a\xff\x1a\xea\xde\xc2\xe8\xe8<p;9\x16\x1f\x10\xde\xd4?\x0e\xe6\xdcʗ+\x8b\x0cf >7d"
 }
-internal_node_count: 95
+internal_node_count: 94
 

log/operation_manager_test.go

@@ -88,8 +88,9 @@ func (l logOpInfoMatcher) String() string {
 
 // Set up some log IDs in mock storage.
 // The following IDs have special behaviour:
-//  logIDThatFailsGetTreeOp: fail the GetTree() operation
-//  logIDWithNoDisplayName: return a tree with no DisplayName
+//
+//	logIDThatFailsGetTreeOp: fail the GetTree() operation
+//	logIDWithNoDisplayName: return a tree with no DisplayName
 func setupLogIDs(ctrl *gomock.Controller, logNames map[int64]string) (*storage.MockLogStorage, *storage.MockAdminStorage) {
 	ids := make([]int64, 0, len(logNames))
 	for id := range logNames {

log/sequencer.go

@@ -20,7 +20,6 @@ import (
 	"flag"
 	"fmt"
 	"strconv"
-	"strings"
 	"sync"
 	"time"
 
@@ -66,38 +65,8 @@ var (
 	QuotaIncreaseFactor = 1.1
 )
 
-// stringSet is a set of strings that can be parsed by flag package.
-type stringSet map[string]bool
-
-func (s *stringSet) String() string {
-	keys := make([]string, 0, len(*s))
-	for k, v := range *s {
-		if v {
-			keys = append(keys, k)
-		}
-	}
-	return strings.Join(keys, ",")
-}
-
-func (s *stringSet) Set(value string) error {
-	*s = make(stringSet)
-	for _, id := range strings.Split(value, ",") {
-		(*s)[id] = true
-	}
-	return nil
-}
-
-// The tree IDs for which sequencer does not store ephemeral node hashes.
-// Trillian releases up to v1.4.1 store the ephemeral hashes, which corresponds
-// to this slice being empty. Release v1.4.2 allows disabling this behaviour
-// for individual, or all trees (denoted by the "*" wildcard). The release
-// after v1.4.2 will switch to "*" behaviour unconditionally.
-var idsWithNoEphemeralNodes stringSet
-
-// TODO(pavelkalinnikov): Remove this flag in the next release.
-func init() {
-	flag.Var(&idsWithNoEphemeralNodes, "tree_ids_with_no_ephemeral_nodes", "Comma-separated list of tree IDs for which storing the ephemeral nodes is disabled, or * to disable it for all trees")
-}
+// TODO(https://github.com/google/trillian/issues/2786): Remove this flag in the next release.
+var _ = flag.String("tree_ids_with_no_ephemeral_nodes", "*", "[Deprecated] Comma-separated list of tree IDs for which storing the ephemeral nodes is disabled, or * to disable it for all trees")
 
 func quotaIncreaseFactor() float64 {
 	if QuotaIncreaseFactor < 1 {
@@ -221,18 +190,8 @@ func updateCompactRange(cr *compact.Range, leaves []*trillian.LogLeaf, label str
 		}
 	}
 
-	// Store or not store ephemeral nodes depending on the flag. This is a
-	// temporary safety measure, to test this on individual trees before fully
-	// disabling ephemeral nodes.
-	storeEphemeral := store
-	if idsWithNoEphemeralNodes[label] || idsWithNoEphemeralNodes["*"] {
-		storeEphemeral = nil
-		glog.Infof("%s: Not storing ephemeral nodes", label)
-	}
-
-	// TODO(pavelkalinnikov): Do not store the ephemeral node hashes
-	// unconditionally, because they are not used since v1.4.0.
-	hash, err := cr.GetRootHash(storeEphemeral)
+	// Note: Ephemeral nodes are not stored.
+	hash, err := cr.GetRootHash(nil)
 	if err != nil {
 		return nil, nil, err
 	}

log/sequencer_test.go

@@ -154,11 +154,6 @@ var (
 	})
 )
 
-func init() {
-	// The tree ID used by TestIntegrateBatch.
-	idsWithNoEphemeralNodes = map[string]bool{"154035": true}
-}
-
 func makeSLR(root *types.LogRootV1) *trillian.SignedLogRoot {
 	logRoot, _ := root.MarshalBinary()
 	return &trillian.SignedLogRoot{LogRoot: logRoot}

merkle/compact/nodes.go

@@ -22,11 +22,11 @@ import "math/bits"
 // index is its horizontal position in this level ordered from left to right.
 // Consider an example below where nodes are labeled as [<level> <index>].
 //
-//           [2 0]
-//          /     \
-//       [1 0]     \
-//       /   \      \
-//   [0 0]  [0 1]  [0 2]
+//	        [2 0]
+//	       /     \
+//	    [1 0]     \
+//	    /   \      \
+//	[0 0]  [0 1]  [0 2]
 type NodeID struct {
 	Level uint
 	Index uint64

merkle/compact/range.go

@@ -239,9 +239,9 @@ func getMergePath(begin, mid, end uint64) (uint, uint) {
 // some integers m, k >= 0.
 //
 // The sequence of sizes is returned encoded as bitmasks left and right, where:
-//  - a 1 bit in a bitmask denotes a sub-range of the corresponding size 2^k
-//  - left mask bits in LSB-to-MSB order encode the left part of the sequence
-//  - right mask bits in MSB-to-LSB order encode the right part
+//   - a 1 bit in a bitmask denotes a sub-range of the corresponding size 2^k
+//   - left mask bits in LSB-to-MSB order encode the left part of the sequence
+//   - right mask bits in MSB-to-LSB order encode the right part
 //
 // The corresponding values of m are not returned (they can be calculated from
 // begin and the sub-range sizes).