Allow not storing ephemeral node hashes (#2568)

This change adds --tree_ids_with_no_ephemeral_nodes flag to the sequencer,
which allows disabling the writes of ephemeral nodes for a subset of Trillian
trees. The plan is to disable it unconditionally in the next release, but the
flag helps to test the new behaviour safely on a subset of trees.

The stored ephemeral nodes are not used because all the higher level logic uses
compact ranges, operating only with the "perfect" nodes. Since commit df47465 on
Jul 13, 2021 ephemeral nodes are never read from the tree storage.

Not storing ephemeral nodes also has a positive performance effect. Previously,
every sequencing transaction for a typical tree of hundreds of millions entries
would write at least 4 tiles at the tree edge, because it "touched" all the
ephemeral nodes. Now, it will only touch perfect nodes, which for a typical
transaction are in the bottom 1-2 tiles.

Relatedly, now that only non-ephemeral node updates trigger a tile write, every
tile can be written at most 256 times, whereas previously tiles could be written
millions of times (e.g. the root tile of the tree). It led to issues like #1188
which is the result of needing to scan / query millions of rows.

CHANGELOG.md

@@ -2,6 +2,9 @@
 
 ## HEAD
 
+* #2568: Allow disabling the writes of ephemeral nodes to storage via the
+  `--tree_ids_with_no_ephemeral_nodes` flag to the sequencer.
+
 ## v1.4.1
 
 * `countFromInformationSchema` function to add support for MySQL 8.

log/sequencer.go

@@ -17,8 +17,10 @@ package log
 import (
 	"bytes"
 	"context"
+	"flag"
 	"fmt"
 	"strconv"
+	"strings"
 	"sync"
 	"time"
 
@@ -64,6 +66,22 @@ var (
 	QuotaIncreaseFactor = 1.1
 )
 
+// The tree IDs for which sequencer does not store ephemeral node hashes.
+// Trillian releases up to v1.4.1 store the ephemeral hashes, which corresponds
+// to this slice being empty. Release v1.4.2 allows disabling this behaviour
+// for individual, or all trees (denoted by the "*" wildcard). The release
+// after v1.4.2 will switch to "*" behaviour unconditionally.
+var idsWithNoEphemeralNodes = make(map[string]bool)
+
+// TODO(pavelkalinnikov): Remove this flag in the next release.
+func init() {
+	var ids string
+	flag.StringVar(&ids, "tree_ids_with_no_ephemeral_nodes", "", "Comma-separated list of tree IDs for which storing the ephemeral nodes is disabled, or * to disable it for all trees")
+	for _, id := range strings.Split(ids, ",") {
+		idsWithNoEphemeralNodes[id] = true
+	}
+}
+
 func quotaIncreaseFactor() float64 {
 	if QuotaIncreaseFactor < 1 {
 		QuotaIncreaseFactor = 1
@@ -185,8 +203,18 @@ func updateCompactRange(cr *compact.Range, leaves []*trillian.LogLeaf, label str
 			return nil, nil, err
 		}
 	}
-	// Store ephemeral nodes on the right border of the tree as well.
-	hash, err := cr.GetRootHash(store)
+
+	// Store or not store ephemeral nodes depending on the flag. This is a
+	// temporary safety measure, to test this on individual trees before fully
+	// disabling ephemeral nodes.
+	storeEphemeral := store
+	if idsWithNoEphemeralNodes[label] || idsWithNoEphemeralNodes["*"] {
+		storeEphemeral = nil
+	}
+
+	// TODO(pavelkalinnikov): Do not store the ephemeral node hashes
+	// unconditionally, because they are not used since v1.4.0.
+	hash, err := cr.GetRootHash(storeEphemeral)
 	if err != nil {
 		return nil, nil, err
 	}

log/sequencer_test.go

@@ -99,10 +99,6 @@ var (
 			ID:   compact.NewNodeID(0, 16),
 			Hash: testonly.MustDecodeBase64("L5Iyd7aFOVewxiRm29xD+EU+jvEo4RfufBijKdflWMk="),
 		},
-		{
-			ID:   compact.NewNodeID(5, 0),
-			Hash: testonly.MustDecodeBase64("R57DrKTGuZdjCNXjv6InGrm4rABLOn9yWpdHmYOoLwU="),
-		},
 	}
 
 	testRoot = &types.LogRootV1{
@@ -128,14 +124,6 @@ var (
 	}
 	// Nodes that will be stored after updating the tree of size 21.
 	updatedNodes21 = []tree.Node{
-		{
-			ID:   compact.NewNodeID(5, 0),
-			Hash: testonly.MustDecodeBase64("1oUtLDlyOWXLHLAvL3NvWaO4D9kr0oQYScylDlgjey4="),
-		},
-		{
-			ID:   compact.NewNodeID(3, 2),
-			Hash: testonly.MustDecodeBase64("1yCvo/9xbNIileBAEjc+c00GxVQQV7h54Tdmkc48uRU="),
-		},
 		{
 			ID:   compact.NewNodeID(1, 10),
 			Hash: testonly.MustDecodeBase64("S55qEsQMx90/eq1fSb87pYCB9WIYL7hBgiTY+B9LmPw="),
@@ -166,6 +154,11 @@ var (
 	})
 )
 
+func init() {
+	// The tree ID used by TestIntegrateBatch.
+	idsWithNoEphemeralNodes["154035"] = true
+}
+
 func makeSLR(root *types.LogRootV1) *trillian.SignedLogRoot {
 	logRoot, _ := root.MarshalBinary()
 	return &trillian.SignedLogRoot{LogRoot: logRoot}