From f3fa5a83ba37d752c4cea1014c2ec637d0cf3e09 Mon Sep 17 00:00:00 2001 From: Pavel Kalinnikov Date: Mon, 16 May 2022 23:17:08 +0000 Subject: [PATCH] Hide storing ephemeral nodes behind a per-tree flag --- .../format/testdata/dump_tree_output_1000 | 24 ++++++++++++-- .../format/testdata/dump_tree_output_871 | 28 ++++++++++++++-- .../format/testdata/dump_tree_output_96 | 6 +++- log/sequencer.go | 32 +++++++++++++++++-- log/sequencer_test.go | 4 +++ 5 files changed, 87 insertions(+), 7 deletions(-) diff --git a/integration/format/testdata/dump_tree_output_1000 b/integration/format/testdata/dump_tree_output_1000 index 759300c6a6..86e7c0f0d2 100644 --- a/integration/format/testdata/dump_tree_output_1000 +++ b/integration/format/testdata/dump_tree_output_1000 @@ -12,11 +12,23 @@ leaves: { key: "CAI=" value: "\xbb\x91zJ\xbb\xbbR\xc7\xe0>\xde\x1e\xe5\x9fC\x11\xe5J\x85\x0b\x00\x99\xd1^\xa2\xd8RB?;S5" } +leaves: { + key: "CAM=" + value: "\xc3\xe2VW\xbb\xed\x8f\x1eۂz\x05\xf1\x86;\xc21\x05\xae\x05\x8e\x05-o9\xb5\x05*|\x9b\x1c@" +} +internal_nodes: { + key: "BgA=" + value: "7ה\x03g\xb8+)\xe4\n\r\xb6\x9a\x83\xf8\x11}ui\xe7\xf7N\xc1:\x87S\x06%\\'\x7f\x1b" +} internal_nodes: { key: "BwA=" value: "\xc6\xee@\x91R\xb3TI\x03\xf2\x86B-}c:\x9aۢ\xef\x0f/\xdb\xf1V\x8e-\r\xcf\x17?\x92" } -internal_node_count: 1 +internal_nodes: { + key: "BwI=" + value: "\xe3\xda%\xfem\x86\x03v#T\x8e0\xf8 \x1e\xdb\xfb\xad\xef\\\x18\x8asX\xec\x0fN\xbe\"\x901\x1e" +} +internal_node_count: 3 prefix: "\x00\x00\x00\x00\x00\x00\x00" depth: 8 @@ -7104,6 +7116,10 @@ internal_nodes: { key: "AQA=" value: "\xb2\nE\x80\x9e\x88X\xb4\xe5\x14\xa7ic\x80\xd1b**.u\n\xfd \t\x90e\xae\xc4\xee\x89Q\xdc" } +internal_nodes: { + key: "AYA=" + value: "\x8e\xcc\xe2\x04\x1c}\x0b\x0f\x1f\xa0K\xc4y\xb3mӠP\x8d\xa68\x02\x178\xf2@\xac\xd3\xd1/&\x8f" +} internal_nodes: { key: "AgA=" value: "A:\x85Z\x8aıqBκ\t9\x02\x0fF\xe9\x91s\x84\x07\"\x08\x08Ů\xfe\xea-\xf6\xe8R" @@ -7116,6 +7132,10 @@ internal_nodes: { key: "AoA=" value: "\xb1P\x17A\x02\xe2\xe40\xff\x16q\x9e\x1d\x91\xf6\xf0\x0e\x81\xf6\x9a\x81\xd0\r/\xac\x0e4]?3\x12r" } +internal_nodes: { + key: "AsA=" + value: "\xf0ݟo9d\x0f\x8f𠅼\r\x85\x07k\rV2\x84\x99 \xeb\x0c4\x06\xdd\xc4\xf7K\xf2\x98" +} internal_nodes: { key: "AwA=" value: "\xf9ϧ\x1d\xf4?Zٶ\x82\xfc\\g\xc9\xf2eyĭ\x03\x0eI\rX\xc5\xdd\xd4\xf7\\K\xd0G" @@ -7992,5 +8012,5 @@ internal_nodes: { key: "Bzw=" value: "\xd5u\xac\xe5b\xa0&\xaa\x92\xd9\xf6~FO\x9c\xa2>c\x80\xe3>rI\x86\xe5\x91'l\xaf" } -internal_node_count: 228 +internal_node_count: 230 diff --git a/integration/format/testdata/dump_tree_output_871 b/integration/format/testdata/dump_tree_output_871 index b1cdca0627..e26f9f0b94 100644 --- a/integration/format/testdata/dump_tree_output_871 +++ b/integration/format/testdata/dump_tree_output_871 @@ -12,11 +12,19 @@ leaves: { key: "CAI=" value: "\xbb\x91zJ\xbb\xbbR\xc7\xe0>\xde\x1e\xe5\x9fC\x11\xe5J\x85\x0b\x00\x99\xd1^\xa2\xd8RB?;S5" } +internal_nodes: { + key: "BgA=" + value: "\xf6\x87\xc1\xbdh\x94\xb4Q\xdc\xf7\x1fZ+\x03\x9d\xce`\xf2\x87Џ\x89\x02\xc1iw\xa3AC\xbe\xa7N" +} internal_nodes: { key: "BwA=" value: "\xc6\xee@\x91R\xb3TI\x03\xf2\x86B-}c:\x9aۢ\xef\x0f/\xdb\xf1V\x8e-\r\xcf\x17?\x92" } -internal_node_count: 1 +internal_nodes: { + key: "BwI=" + value: "\x07U\xae&\xaf\xea\xbd1\x10܍\xe6\xf0\xac\xf1\xe5)\x02\xddk\x80Q`/P\\\xa9\xe7\xfa_\xc3/" +} +internal_node_count: 3 prefix: "\x00\x00\x00\x00\x00\x00\x00" depth: 8 @@ -6568,10 +6576,18 @@ internal_nodes: { key: "A0A=" value: "K\x1a\xbc\x93\xb8[20.Җ`\x9d\x12\x1f\x90W\xa3Ȥ~\r>\t0\x02\xfd\x07\tL\x03y" } +internal_nodes: { + key: "AQA=" + value: "$F\xb2T\xeb\x8c\x03\xf2\xef`3ۄ\xbfik\xba\x97\xb5\x89\x1c^\x81\xfa4\x0e\xf8\xf5#\"ӑ" +} internal_nodes: { key: "AgA=" value: "A:\x85Z\x8aıqBκ\t9\x02\x0fF\xe9\x91s\x84\x07\"\x08\x08Ů\xfe\xea-\xf6\xe8R" } +internal_nodes: { + key: "AkA=" + value: "\xf5<\x8c8C\n\x1c\xa6\xdbG\x94\xbaL\x85\xac\xf0x\xeb\xb2\x03i\xaf}\xeb\x1b\xa8\x96\x99\x19z>\xab" +} internal_nodes: { key: "AwA=" value: "\xf9ϧ\x1d\xf4?Zٶ\x82\xfc\\g\xc9\xf2eyĭ\x03\x0eI\rX\xc5\xdd\xd4\xf7\\K\xd0G" @@ -6728,6 +6744,10 @@ internal_nodes: { key: "BVg=" value: "5W\x98P\xfe\x125'\xb9K0\xcf\x08\x17\x92\xdb\xfb\x04\xda^\xf6\x03K\x04B\xd9ifHZ\xb6\xfe" } +internal_nodes: { + key: "BWA=" + value: "\xd6N^\x93e\x13Qd\xe1\xea\xc0_a\x04\x8dL\x9fu\xad\x05\xe6\xd1y&\xa3\xa1\xc3Et\xfd\xa7\xde" +} internal_nodes: { key: "BgA=" value: "\xec\x86&v\xd5C*\x15\xb3\xec\t\x89_\xf1*\xc1\xf02Ұ\xbdk\x08\xb0x\xfc\xdfȥ\xeaܾ" @@ -6828,6 +6848,10 @@ internal_nodes: { key: "BmA=" value: "\x15\xd6l\x92\xd7$*q\x9f\xbf\xd6\x7f\x88\xb0\xc2`\x1d\xc2y\xc3\xfe\x82bԥ\xc4\xe9\r}i\x97!" } +internal_nodes: { + key: "BmQ=" + value: "d\xbct\x8f\xdema\x7f\xd3\xf1\\\x14/l\xb3{{\x02\x87\xc4>\xdd,+\xfe\x87\x7f\xc8PTJ\xa7" +} internal_nodes: { key: "Bw4=" value: "\x99\xbc\xce}\xcd\xf2\xfe\xb9]\xecY:P7\x05~l\xfc\xab\x066 G\xa1y\x9c\x9a\xe4\xe1\r\x89\xb9" @@ -6956,5 +6980,5 @@ internal_nodes: { key: "Bzw=" value: "\xd5u\xac\xe5b\xa0&\xaa\x92\xd9\xf6~FO\x9c\xa2>c\x80\xe3>rI\x86\xe5\x91'l\xaf" } -internal_node_count: 98 +internal_node_count: 102 diff --git a/integration/format/testdata/dump_tree_output_96 b/integration/format/testdata/dump_tree_output_96 index b9d86bd086..16ed0903cb 100644 --- a/integration/format/testdata/dump_tree_output_96 +++ b/integration/format/testdata/dump_tree_output_96 @@ -388,6 +388,10 @@ internal_nodes: { key: "A0A=" value: "\xa2\x9b|=$\xce/\x86+\x0b\"\x92n\xcc\xc3\\/b7Lm\xdb\xd5\x19{\xfeKo\xe8\xbdtF" } +internal_nodes: { + key: "AQA=" + value: "\xf0K\xff\xed19\x0e\tL\xea\xfc\x12U\xf1\xf2α1z\x8a\xba\xe0\xb0t\x9a\x88\x15]U\xee\xce\xd0" +} internal_nodes: { key: "AgA=" value: "\x8f\x05[\xe4\x9bG\x8fW\x12\xeb\xe2\xd8[\x89˔m\xccP\x87\x18_\x1d.\xb66S\xa5\x99\x81\x08\xce" @@ -760,5 +764,5 @@ internal_nodes: { key: "Bzw=" value: "\xd7\x1a\xff\x1a\xea\xde\xc2\xe8\xe87d" } -internal_node_count: 94 +internal_node_count: 95 diff --git a/log/sequencer.go b/log/sequencer.go index e3e99d3b7d..06953f53ad 100644 --- a/log/sequencer.go +++ b/log/sequencer.go @@ -17,8 +17,10 @@ package log import ( "bytes" "context" + "flag" "fmt" "strconv" + "strings" "sync" "time" @@ -64,6 +66,22 @@ var ( QuotaIncreaseFactor = 1.1 ) +// The tree IDs for which sequencer does not store ephemeral node hashes. +// Trillian releases up to v1.4.1 store the ephemeral hashes, which corresponds +// to this slice being empty. Release v1.4.2 allows disabling this behaviour +// for individual, or all trees (denoted by the "*" wildcard). The release +// after v1.4.2 will switch to "*" behaviour unconditionally. +var idsWithNoEphemeralNodes = make(map[string]bool) + +// TODO(pavelkalinnikov): Remove this flag in the next release. +func init() { + var ids string + flag.StringVar(&ids, "tree_ids_with_no_ephemeral_nodes", "", "Comma-separated list of tree IDs for which storing the ephemeral nodes is disabled, or * to disable it for all trees") + for _, id := range strings.Split(ids, ",") { + idsWithNoEphemeralNodes[id] = true + } +} + func quotaIncreaseFactor() float64 { if QuotaIncreaseFactor < 1 { QuotaIncreaseFactor = 1 @@ -185,8 +203,18 @@ func updateCompactRange(cr *compact.Range, leaves []*trillian.LogLeaf, label str return nil, nil, err } } - // Note: The ephemeral node hashes are not stored. - hash, err := cr.GetRootHash(nil) + + // Store or not store ephemeral nodes depending on the flag. This is a + // temporary safety measure, to test this on individual trees before fully + // disabling ephemeral nodes. + storeEphemeral := store + if idsWithNoEphemeralNodes[label] || idsWithNoEphemeralNodes["*"] { + storeEphemeral = nil + } + + // TODO(pavelkalinnikov): Do not store the ephemeral node hashes + // unconditionally, because they are not used since v1.4.0. + hash, err := cr.GetRootHash(storeEphemeral) if err != nil { return nil, nil, err } diff --git a/log/sequencer_test.go b/log/sequencer_test.go index ba9520afda..1c4e55ce67 100644 --- a/log/sequencer_test.go +++ b/log/sequencer_test.go @@ -154,6 +154,10 @@ var ( }) ) +func init() { + idsWithNoEphemeralNodes["*"] = true +} + func makeSLR(root *types.LogRootV1) *trillian.SignedLogRoot { logRoot, _ := root.MarshalBinary() return &trillian.SignedLogRoot{LogRoot: logRoot}