From 0609182fa9a0dbce37e7bb56917e6f2531a0fe11 Mon Sep 17 00:00:00 2001
From: Pavel Kalinnikov <pkalinnikov@google.com>
Date: Mon, 25 Apr 2022 12:37:39 +0100
Subject: [PATCH] Remake dumplib into integration test (#2710)

dumplib was a tool which created an in-memory log storage of a particular size,
and printed its contents with various flags. The useful bit of this tool was a "golden"
unit test which checks regressions on the SubtreeProto format in storage.

This change removes the tool, refactors the test, and places it in integration/format.
---
 CHANGELOG.md                                  |   3 +
 integration/format/format.go                  | 175 +++++++
 .../format}/nochange_test.go                  |  79 ++--
 .../format}/testdata/dump_tree_output_1000    |   0
 .../format}/testdata/dump_tree_output_1024    |   0
 .../format}/testdata/dump_tree_output_871     |   0
 .../format}/testdata/dump_tree_output_96      |   0
 storage/tools/dump_tree/dumplib.go            | 428 ------------------
 storage/tools/dump_tree/main.go               |  77 ----
 9 files changed, 205 insertions(+), 557 deletions(-)
 create mode 100644 integration/format/format.go
 rename {storage/tools/dump_tree => integration/format}/nochange_test.go (50%)
 rename {storage/tools/dump_tree => integration/format}/testdata/dump_tree_output_1000 (100%)
 rename {storage/tools/dump_tree => integration/format}/testdata/dump_tree_output_1024 (100%)
 rename {storage/tools/dump_tree => integration/format}/testdata/dump_tree_output_871 (100%)
 rename {storage/tools/dump_tree => integration/format}/testdata/dump_tree_output_96 (100%)
 delete mode 100644 storage/tools/dump_tree/dumplib.go
 delete mode 100644 storage/tools/dump_tree/main.go

diff --git a/CHANGELOG.md b/CHANGELOG.md
index 0dd5523d96..ea518ef255 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -5,6 +5,9 @@
 * `countFromInformationSchema` function to add support for MySQL 8.
 
 ### Removals
+
+ * #2710: Unused `storage/tools/dumplib` was removed. The useful storage format
+  regression test moved to `integration/format`.
  * #2711: Unused `storage/tools/hasher` removed.
  * #2715: Packages under `merkle` are deprecated and to be removed. Use
    https://github.com/transparency-dev/merkle instead.
diff --git a/integration/format/format.go b/integration/format/format.go
new file mode 100644
index 0000000000..9d70be68a3
--- /dev/null
+++ b/integration/format/format.go
@@ -0,0 +1,175 @@
+// Copyright 2017 Google LLC. All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package format
+
+import (
+	"bytes"
+	"context"
+	"crypto/sha256"
+	"fmt"
+	"sort"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/google/trillian"
+	"github.com/google/trillian/log"
+	"github.com/google/trillian/monitoring"
+	"github.com/google/trillian/quota"
+	"github.com/google/trillian/storage"
+	"github.com/google/trillian/storage/cache"
+	"github.com/google/trillian/storage/memory"
+	"github.com/google/trillian/storage/storagepb"
+	"github.com/google/trillian/types"
+	"github.com/google/trillian/util/clock"
+	"github.com/transparency-dev/merkle"
+	"github.com/transparency-dev/merkle/rfc6962"
+	"google.golang.org/protobuf/encoding/prototext"
+	"google.golang.org/protobuf/types/known/durationpb"
+)
+
+func run(treeSize, batchSize int, leafFormat string) (string, error) {
+	ctx := context.Background()
+
+	ts := memory.NewTreeStorage()
+	ls := memory.NewLogStorage(ts, monitoring.InertMetricFactory{})
+	as := memory.NewAdminStorage(ts)
+	tree, err := createTree(ctx, as, ls)
+	if err != nil {
+		return "", err
+	}
+	log.InitMetrics(nil)
+
+	leaves := generateLeaves(treeSize, leafFormat)
+	if err := sequenceLeaves(ctx, ls, tree, leaves, batchSize); err != nil {
+		return "", err
+	}
+
+	// Read the latest LogRoot back.
+	var root types.LogRootV1
+	if err := ls.ReadWriteTransaction(ctx, tree, func(ctx context.Context, tx storage.LogTreeTX) error {
+		latest, err := tx.LatestSignedLogRoot(ctx)
+		if err != nil {
+			return err
+		}
+		return root.UnmarshalBinary(latest.LogRoot)
+	}); err != nil {
+		return "", fmt.Errorf("ReadWriteTransaction: %v", err)
+	}
+
+	return latestRevisions(ls, tree.TreeId, rfc6962.DefaultHasher)
+}
+
+func createTree(ctx context.Context, as storage.AdminStorage, ls storage.LogStorage) (*trillian.Tree, error) {
+	tree, err := storage.CreateTree(ctx, as, &trillian.Tree{
+		TreeType:        trillian.TreeType_LOG,
+		TreeState:       trillian.TreeState_ACTIVE,
+		MaxRootDuration: durationpb.New(0 * time.Millisecond),
+	})
+	if err != nil {
+		return nil, fmt.Errorf("CreateTree: %v", err)
+	}
+
+	logRoot, err := (&types.LogRootV1{RootHash: rfc6962.DefaultHasher.EmptyRoot()}).MarshalBinary()
+	if err != nil {
+		return nil, fmt.Errorf("MarshalBinary: %v", err)
+	}
+
+	if err = ls.ReadWriteTransaction(ctx, tree, func(ctx context.Context, tx storage.LogTreeTX) error {
+		return tx.StoreSignedLogRoot(ctx, &trillian.SignedLogRoot{LogRoot: logRoot})
+	}); err != nil {
+		return nil, fmt.Errorf("ReadWriteTransaction: %v", err)
+	}
+
+	return tree, nil
+}
+
+func generateLeaves(count int, format string) []*trillian.LogLeaf {
+	leaves := make([]*trillian.LogLeaf, 0, count)
+	for i := 0; i < count; i++ {
+		data := []byte(fmt.Sprintf(format, i))
+		hash := sha256.Sum256(data)
+		leaves = append(leaves, &trillian.LogLeaf{
+			LeafValue: data, LeafIdentityHash: hash[:], MerkleLeafHash: hash[:],
+		})
+	}
+	return leaves
+}
+
+func sequenceLeaves(ctx context.Context, ls storage.LogStorage, tree *trillian.Tree, leaves []*trillian.LogLeaf, batchSize int) error {
+	for i, size := 0, len(leaves); i < size; i += batchSize {
+		if left := size - i; left < batchSize {
+			batchSize = left
+		}
+		if _, err := ls.QueueLeaves(ctx, tree, leaves[i:i+batchSize], time.Now()); err != nil {
+			return fmt.Errorf("QueueLeaves: %v", err)
+		}
+
+		sequenced, err := log.IntegrateBatch(ctx, tree, batchSize, 0, 24*time.Hour, clock.System, ls, quota.Noop())
+		if err != nil {
+			return fmt.Errorf("IntegrateBatch: %v", err)
+		}
+		if got, want := sequenced, batchSize; got != want {
+			return fmt.Errorf("IntegrateBatch: got %d, want %d", got, want)
+		}
+	}
+	return nil
+}
+
+type treeAndRev struct {
+	subtree  *storagepb.SubtreeProto
+	revision int
+}
+
+func latestRevisions(ls storage.LogStorage, treeID int64, hasher merkle.LogHasher) (string, error) {
+	// vMap maps subtree prefixes (as strings) to the corresponding subtree proto and its revision
+	vMap := make(map[string]treeAndRev)
+	memory.DumpSubtrees(ls, treeID, func(k string, v *storagepb.SubtreeProto) {
+		// Relies on the btree key space for subtrees being /tree_id/subtree/id/revision.
+		pieces := strings.Split(k, "/")
+		if got, want := len(pieces), 5; got != want {
+			panic(fmt.Sprintf("Btree subtree key segments: got %d, want %d", got, want))
+		}
+
+		subID := pieces[3]
+		rev, err := strconv.Atoi(pieces[4])
+		if err != nil {
+			panic(fmt.Sprintf("Bad subtree key: %v: %v", k, err))
+		}
+
+		if rev > vMap[subID].revision {
+			vMap[subID] = treeAndRev{
+				subtree:  v,
+				revision: rev,
+			}
+		}
+	})
+
+	// Store the keys in sorted order.
+	keys := make([]string, 0, len(vMap))
+	for k := range vMap {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+
+	// The map should now contain the latest revisions per subtree.
+	out := new(bytes.Buffer)
+	for _, k := range keys {
+		subtree := vMap[k].subtree
+		cache.PopulateLogTile(subtree, hasher)
+		fmt.Fprintf(out, "%s\n", prototext.Format(subtree))
+	}
+	return out.String(), nil
+}
diff --git a/storage/tools/dump_tree/nochange_test.go b/integration/format/nochange_test.go
similarity index 50%
rename from storage/tools/dump_tree/nochange_test.go
rename to integration/format/nochange_test.go
index b11e0d01a4..0a241b05c7 100644
--- a/storage/tools/dump_tree/nochange_test.go
+++ b/integration/format/nochange_test.go
@@ -12,7 +12,10 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-package main
+// Package format contains an integration test which builds a log using an
+// in-memory storage end-to-end, and makes sure the SubtreeProto storage format
+// has no regressions.
+package format
 
 import (
 	"io/ioutil"
@@ -25,61 +28,33 @@ import (
 	"google.golang.org/protobuf/testing/protocmp"
 )
 
-// TestDBFormatNoChange ensures that the prefix, suffix, and protos stored in the database do not change.
-// This test compares the output from dump_tree against a previously saved output.
+// TestDBFormatNoChange ensures that SubtreeProto tiles stored in the database
+// do not change. This test compares against a previously saved output.
 func TestDBFormatNoChange(t *testing.T) {
 	for _, tc := range []struct {
-		desc string
-		file string
-		opts Options
+		name string
+		size int
 	}{
-		{
-			desc: "tree_size: 96",
-			file: "testdata/dump_tree_output_96",
-			opts: Options{
-				96, 50,
-				"Leaf %d",
-				true, false, false, false, false, true, false, false,
-			},
-		},
-		{
-			desc: "tree_size: 871",
-			file: "testdata/dump_tree_output_871",
-			opts: Options{
-				871, 50,
-				"Leaf %d",
-				true, false, false, false, false, true, false, false,
-			},
-		},
-		{
-			desc: "tree_size: 1000",
-			file: "testdata/dump_tree_output_1000",
-			opts: Options{
-				1000, 50,
-				"Leaf %d",
-				true, false, false, false, false, true, false, false,
-			},
-		},
-		{
-			desc: "tree_size: 1024",
-			file: "testdata/dump_tree_output_1024",
-			opts: Options{
-				1024, 50,
-				"Leaf %d",
-				true, false, false, false, false, true, false, false,
-			},
-		},
+		{name: "dump_tree_output_96", size: 96},
+		{name: "dump_tree_output_871", size: 871},
+		{name: "dump_tree_output_1000", size: 1000},
+		{name: "dump_tree_output_1024", size: 1024},
 	} {
-		out := Main(tc.opts)
-		saved, err := ioutil.ReadFile(tc.file)
-		if err != nil {
-			t.Fatalf("ReadFile(%v): %v", tc.file, err)
-		}
-		got := parseTiles(t, out)
-		want := parseTiles(t, string(saved))
-		if d := cmp.Diff(want, got, protocmp.Transform()); d != "" {
-			t.Errorf("Diff(-want,+got):\n%s", d)
-		}
+		t.Run(tc.name, func(t *testing.T) {
+			out, err := run(tc.size, 50, "Leaf %d")
+			if err != nil {
+				t.Fatalf("run: %v", err)
+			}
+			saved, err := ioutil.ReadFile("testdata/" + tc.name)
+			if err != nil {
+				t.Fatalf("ReadFile(%v): %v", tc.name, err)
+			}
+			got := parseTiles(t, out)
+			want := parseTiles(t, string(saved))
+			if d := cmp.Diff(want, got, protocmp.Transform()); d != "" {
+				t.Errorf("Diff(-want,+got):\n%s", d)
+			}
+		})
 	}
 }
 
diff --git a/storage/tools/dump_tree/testdata/dump_tree_output_1000 b/integration/format/testdata/dump_tree_output_1000
similarity index 100%
rename from storage/tools/dump_tree/testdata/dump_tree_output_1000
rename to integration/format/testdata/dump_tree_output_1000
diff --git a/storage/tools/dump_tree/testdata/dump_tree_output_1024 b/integration/format/testdata/dump_tree_output_1024
similarity index 100%
rename from storage/tools/dump_tree/testdata/dump_tree_output_1024
rename to integration/format/testdata/dump_tree_output_1024
diff --git a/storage/tools/dump_tree/testdata/dump_tree_output_871 b/integration/format/testdata/dump_tree_output_871
similarity index 100%
rename from storage/tools/dump_tree/testdata/dump_tree_output_871
rename to integration/format/testdata/dump_tree_output_871
diff --git a/storage/tools/dump_tree/testdata/dump_tree_output_96 b/integration/format/testdata/dump_tree_output_96
similarity index 100%
rename from storage/tools/dump_tree/testdata/dump_tree_output_96
rename to integration/format/testdata/dump_tree_output_96
diff --git a/storage/tools/dump_tree/dumplib.go b/storage/tools/dump_tree/dumplib.go
deleted file mode 100644
index 03a0d91f21..0000000000
--- a/storage/tools/dump_tree/dumplib.go
+++ /dev/null
@@ -1,428 +0,0 @@
-// Copyright 2017 Google LLC. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-package main
-
-import (
-	"bytes"
-	"context"
-	"crypto/sha256"
-	"encoding/base64"
-	"encoding/binary"
-	"encoding/hex"
-	"fmt"
-	"sort"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/golang/glog"
-	"github.com/google/trillian"
-	"github.com/google/trillian/log"
-	"github.com/google/trillian/monitoring"
-	"github.com/google/trillian/quota"
-	"github.com/google/trillian/storage"
-	"github.com/google/trillian/storage/cache"
-	"github.com/google/trillian/storage/memory"
-	"github.com/google/trillian/storage/storagepb"
-	"github.com/google/trillian/types"
-	"github.com/google/trillian/util/clock"
-	"github.com/transparency-dev/merkle"
-	"github.com/transparency-dev/merkle/compact"
-	"github.com/transparency-dev/merkle/rfc6962"
-	"google.golang.org/protobuf/encoding/prototext"
-	"google.golang.org/protobuf/proto"
-	"google.golang.org/protobuf/types/known/durationpb"
-)
-
-// A 32 bit magic number that is written at the start of record io files to identify the format.
-const recordIOMagic int32 = 0x3ed7230a
-
-type treeAndRev struct {
-	fullKey  string
-	subtree  *storagepb.SubtreeProto
-	revision int
-}
-
-// summarizeProto is an output formatter function that produces a single line summary.
-func summarizeProto(leafHashesFlag bool) func(s *storagepb.SubtreeProto) string {
-	return func(s *storagepb.SubtreeProto) string {
-		summary := fmt.Sprintf("p: %-20s d: %d lc: %3d ic: %3d\n",
-			hex.EncodeToString(s.Prefix),
-			s.Depth,
-			len(s.Leaves),
-			s.InternalNodeCount)
-
-		if leafHashesFlag {
-			for prefix, hash := range s.Leaves {
-				dp, err := base64.StdEncoding.DecodeString(prefix)
-				if err != nil {
-					glog.Fatalf("Failed to decode leaf prefix: %v", err)
-				}
-				summary += fmt.Sprintf("%s -> %s\n", hex.EncodeToString(dp), hex.EncodeToString(hash))
-			}
-		}
-
-		return summary
-	}
-}
-
-// fullProto is an output formatter function that produces a single line in proto text format.
-func fullProto(s *storagepb.SubtreeProto) string {
-	return fmt.Sprintf("%s\n", prototext.Format(s))
-}
-
-// recordIOProto is an output formatter that produces binary recordio format
-func recordIOProto(s *storagepb.SubtreeProto) string {
-	buf := new(bytes.Buffer)
-	data, err := proto.Marshal(s)
-	if err != nil {
-		glog.Fatalf("Failed to marshal subtree proto: %v", err)
-	}
-	dataLen := int64(len(data))
-	if err = binary.Write(buf, binary.BigEndian, dataLen); err != nil {
-		glog.Fatalf("binary.Write failed: %v", err)
-	}
-	var compLen int64
-	if err = binary.Write(buf, binary.BigEndian, compLen); err != nil {
-		glog.Fatalf("binary.Write failed: %v", err)
-	}
-	// buffer.Write() always returns a nil error
-	buf.Write(data)
-
-	return buf.String()
-}
-
-func sequence(tree *trillian.Tree, logStorage storage.LogStorage, count, batchSize int) {
-	glog.Infof("Sequencing batch of size %d", count)
-	sequenced, err := log.IntegrateBatch(context.TODO(), tree, batchSize, 0, 24*time.Hour, clock.System, logStorage, quota.Noop())
-	if err != nil {
-		glog.Fatalf("IntegrateBatch got: %v, want: no err", err)
-	}
-
-	if got, want := sequenced, count; got != want {
-		glog.Fatalf("IntegrateBatch got: %d sequenced, want: %d", got, want)
-	}
-}
-
-func createTree(as storage.AdminStorage, ls storage.LogStorage) *trillian.Tree {
-	ctx := context.TODO()
-	tree := &trillian.Tree{
-		TreeType:        trillian.TreeType_LOG,
-		TreeState:       trillian.TreeState_ACTIVE,
-		MaxRootDuration: durationpb.New(0 * time.Millisecond),
-	}
-	createdTree, err := storage.CreateTree(ctx, as, tree)
-	if err != nil {
-		glog.Fatalf("Create tree: %v", err)
-	}
-
-	logRoot, err := (&types.LogRootV1{RootHash: rfc6962.DefaultHasher.EmptyRoot()}).MarshalBinary()
-	if err != nil {
-		glog.Fatalf("MarshalBinary: %v", err)
-	}
-	sthZero := &trillian.SignedLogRoot{LogRoot: logRoot}
-
-	err = ls.ReadWriteTransaction(ctx, createdTree, func(ctx context.Context, tx storage.LogTreeTX) error {
-		if err := tx.StoreSignedLogRoot(ctx, sthZero); err != nil {
-			glog.Fatalf("StoreSignedLogRoot: %v", err)
-		}
-		return nil
-	})
-	if err != nil {
-		glog.Fatalf("ReadWriteTransaction: %v", err)
-	}
-
-	return createdTree
-}
-
-// Options are the commandline arguments one can pass to Main
-type Options struct {
-	TreeSize, BatchSize                          int
-	LeafFormat                                   string
-	LatestRevision, Summary, HexKeys, LeafHashes bool
-	RecordIO, Rebuild, Traverse, DumpLeaves      bool
-}
-
-// Main runs the dump_tree tool
-func Main(args Options) string {
-	ctx := context.Background()
-	validateFlagsOrDie(args.Summary, args.RecordIO)
-
-	glog.Info("Initializing memory log storage")
-	ts := memory.NewTreeStorage()
-	ls := memory.NewLogStorage(ts, monitoring.InertMetricFactory{})
-	as := memory.NewAdminStorage(ts)
-	tree := createTree(as, ls)
-
-	log.InitMetrics(nil)
-
-	// Create the initial tree head at size 0, which is required. And then sequence the leaves.
-	sequence(tree, ls, 0, args.BatchSize)
-	sequenceLeaves(ls, tree, args.TreeSize, args.BatchSize, args.LeafFormat)
-
-	// Read the latest STH back
-	var root types.LogRootV1
-	err := ls.ReadWriteTransaction(ctx, tree, func(ctx context.Context, tx storage.LogTreeTX) error {
-		var err error
-		sth, err := tx.LatestSignedLogRoot(ctx)
-		if err != nil {
-			glog.Fatalf("LatestSignedLogRoot: %v", err)
-		}
-		if err := root.UnmarshalBinary(sth.LogRoot); err != nil {
-			return fmt.Errorf("could not parse current log root: %v", err)
-		}
-
-		glog.Infof("STH at size %d has hash %s",
-			root.TreeSize,
-			hex.EncodeToString(root.RootHash))
-		return nil
-	})
-	if err != nil {
-		glog.Fatalf("ReadWriteTransaction: %v", err)
-	}
-
-	// All leaves are now sequenced into the tree. The current state is what we need.
-	glog.Info("Producing output")
-
-	if args.Traverse {
-		return traverseTreeStorage(ctx, ls, tree, args.TreeSize)
-	}
-
-	if args.DumpLeaves {
-		return dumpLeaves(ctx, ls, tree, args.TreeSize)
-	}
-
-	var formatter func(*storagepb.SubtreeProto) string
-	switch {
-	case args.Summary:
-		formatter = summarizeProto(args.LeafHashes)
-	case args.RecordIO:
-		formatter = recordIOProto
-		recordIOHdr()
-	default:
-		formatter = fullProto
-	}
-
-	if args.LatestRevision {
-		return latestRevisions(ls, tree.TreeId, rfc6962.DefaultHasher, formatter, args.Rebuild, args.HexKeys)
-	}
-	return allRevisions(ls, tree.TreeId, rfc6962.DefaultHasher, formatter, args.Rebuild, args.HexKeys)
-}
-
-func allRevisions(ls storage.LogStorage, treeID int64, hasher merkle.LogHasher, of func(*storagepb.SubtreeProto) string, rebuildInternal, hexKeysFlag bool) string {
-	out := new(bytes.Buffer)
-	memory.DumpSubtrees(ls, treeID, func(k string, v *storagepb.SubtreeProto) {
-		if rebuildInternal {
-			cache.PopulateLogTile(v, hasher)
-		}
-		if hexKeysFlag {
-			hexKeys(v)
-		}
-		fmt.Fprint(out, of(v))
-	})
-	return out.String()
-}
-
-func latestRevisions(ls storage.LogStorage, treeID int64, hasher merkle.LogHasher, of func(*storagepb.SubtreeProto) string, rebuildInternal, hexKeysFlag bool) string {
-	out := new(bytes.Buffer)
-	// vMap maps subtree prefixes (as strings) to the corresponding subtree proto and its revision
-	vMap := make(map[string]treeAndRev)
-	memory.DumpSubtrees(ls, treeID, func(k string, v *storagepb.SubtreeProto) {
-		// Relies on the btree key space for subtrees being /tree_id/subtree/<id>/<revision>
-		pieces := strings.Split(k, "/")
-		if got, want := len(pieces), 5; got != want {
-			glog.Fatalf("Wrong no of Btree subtree key segments. Got: %d, want: %d", got, want)
-		}
-
-		subID := pieces[3]
-		subtree := vMap[subID]
-		rev, err := strconv.Atoi(pieces[4])
-		if err != nil {
-			glog.Fatalf("Bad subtree key: %v", k)
-		}
-
-		if rev > subtree.revision {
-			vMap[subID] = treeAndRev{
-				fullKey:  k,
-				subtree:  v,
-				revision: rev,
-			}
-		}
-	})
-
-	// Store the keys in sorted order
-	var sKeys []string
-	for k := range vMap {
-		sKeys = append(sKeys, k)
-	}
-	sort.Strings(sKeys)
-
-	// The map should now contain the latest revisions per subtree
-	for _, k := range sKeys {
-		v := vMap[k]
-		if rebuildInternal {
-			cache.PopulateLogTile(v.subtree, hasher)
-		}
-		if hexKeysFlag {
-			hexKeys(v.subtree)
-		}
-
-		fmt.Fprint(out, of(v.subtree))
-	}
-	return out.String()
-}
-
-func validateFlagsOrDie(summary, recordIO bool) {
-	if summary && recordIO {
-		glog.Fatal("-summary and -recordio are mutually exclusive flags")
-	}
-}
-
-func sequenceLeaves(ls storage.LogStorage, tree *trillian.Tree, treeSize, batchSize int, leafDataFormat string) {
-	glog.Info("Queuing work")
-	for l := 0; l < treeSize; l++ {
-		glog.V(1).Infof("Queuing leaf %d", l)
-
-		leafData := []byte(fmt.Sprintf(leafDataFormat, l))
-		hash := sha256.Sum256(leafData)
-		lh := hash[:]
-		leaf := trillian.LogLeaf{LeafValue: leafData, LeafIdentityHash: lh, MerkleLeafHash: lh}
-		leaves := []*trillian.LogLeaf{&leaf}
-
-		if _, err := ls.QueueLeaves(context.TODO(), tree, leaves, time.Now()); err != nil {
-			glog.Fatalf("QueueLeaves got: %v, want: no err", err)
-		}
-
-		if l > 0 && l%batchSize == 0 {
-			sequence(tree, ls, batchSize, batchSize)
-		}
-	}
-	glog.Info("Finished queueing")
-	// Handle anything left over
-	left := treeSize % batchSize
-	if left == 0 {
-		left = batchSize
-	}
-	sequence(tree, ls, left, batchSize)
-	glog.Info("Finished sequencing")
-}
-
-func traverseTreeStorage(ctx context.Context, ls storage.LogStorage, tt *trillian.Tree, ts int) string {
-	out := new(bytes.Buffer)
-	nodesAtLevel := uint64(ts)
-
-	tx, err := ls.SnapshotForTree(context.TODO(), tt)
-	if err != nil {
-		glog.Fatalf("SnapshotForTree: %v", err)
-	}
-	defer func() {
-		if err := tx.Commit(ctx); err != nil {
-			glog.Fatalf("TX Commit(): %v", err)
-		}
-	}()
-
-	levels := uint(0)
-	n := nodesAtLevel
-	for n > 0 {
-		levels++
-		n = n >> 1
-	}
-
-	// Because of the way we store subtrees omitting internal RHS nodes with one sibling there
-	// is an extra level stored for trees that don't have a number of leaves that is a power
-	// of 2. We account for this here and in the loop below.
-	if !isPerfectTree(int64(ts)) {
-		levels++
-	}
-
-	for level := uint(0); level < levels; level++ {
-		for node := uint64(0); node < nodesAtLevel; node++ {
-			// We're going to request one node at a time, which would normally be slow but we have
-			// the tree in RAM so it's not a real problem.
-			nodeID := compact.NewNodeID(level, node)
-			nodes, err := tx.GetMerkleNodes(context.TODO(), []compact.NodeID{nodeID})
-			if err != nil {
-				glog.Fatalf("GetMerkleNodes: %+v: %v", nodeID, err)
-			}
-			if len(nodes) != 1 {
-				glog.Fatalf("GetMerkleNodes: %+v: want 1 node got: %v", nodeID, nodes)
-			}
-
-			fmt.Fprintf(out, "%6d %6d -> %s\n", level, node, hex.EncodeToString(nodes[0].Hash))
-		}
-
-		nodesAtLevel = nodesAtLevel >> 1
-		fmt.Println()
-		// This handles the extra level in non-perfect trees
-		if nodesAtLevel == 0 {
-			nodesAtLevel = 1
-		}
-	}
-	return out.String()
-}
-
-func dumpLeaves(ctx context.Context, ls storage.LogStorage, tree *trillian.Tree, ts int) string {
-	out := new(bytes.Buffer)
-	tx, err := ls.SnapshotForTree(ctx, tree)
-	if err != nil {
-		glog.Fatalf("SnapshotForTree: %v", err)
-	}
-	defer func() {
-		if err := tx.Commit(ctx); err != nil {
-			glog.Fatalf("TX Commit(): got: %v", err)
-		}
-	}()
-
-	for l := int64(0); l < int64(ts); l++ {
-		leaves, err := tx.GetLeavesByRange(ctx, l, 1)
-		if err != nil {
-			glog.Fatalf("GetLeavesByRange for index %d got: %v", l, err)
-		}
-		fmt.Fprintf(out, "%6d:%s\n", l, leaves[0].LeafValue)
-	}
-	return out.String()
-}
-
-func hexMap(in map[string][]byte) map[string][]byte {
-	m := make(map[string][]byte)
-
-	for k, v := range in {
-		unb64, err := base64.StdEncoding.DecodeString(k)
-		if err != nil {
-			glog.Fatalf("Could not decode key as base 64: %s got: %v", k, err)
-		}
-		m[hex.EncodeToString(unb64)] = v
-	}
-
-	return m
-}
-
-func hexKeys(s *storagepb.SubtreeProto) {
-	s.Leaves = hexMap(s.Leaves)
-	s.InternalNodes = hexMap(s.InternalNodes)
-}
-
-func isPerfectTree(x int64) bool {
-	return x != 0 && (x&(x-1) == 0)
-}
-
-func recordIOHdr() {
-	buf := new(bytes.Buffer)
-	err := binary.Write(buf, binary.BigEndian, recordIOMagic)
-	if err != nil {
-		glog.Fatalf("binary.Write failed: %v", err)
-	}
-	fmt.Print(buf.String())
-}
diff --git a/storage/tools/dump_tree/main.go b/storage/tools/dump_tree/main.go
deleted file mode 100644
index 904e17737c..0000000000
--- a/storage/tools/dump_tree/main.go
+++ /dev/null
@@ -1,77 +0,0 @@
-// Copyright 2017 Google LLC. All Rights Reserved.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//     http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-// See the License for the specific language governing permissions and
-// limitations under the License.
-
-// The dump_tree program uses the in memory storage implementation to create a sequenced
-// log tree of a particular size using known leaf data and then dumps out the resulting
-// SubTree protos for examination and debugging. It does not require any actual storage
-// to be configured.
-//
-// Examples of some usages:
-//
-// Print a summary of the storage protos in a tree of size 1044, rebuilding internal nodes:
-// dump_tree -tree_size 1044 -summary
-//
-// Print all versions of all raw subtree protos for a tree of size 58:
-// dump_tree -tree_size 58 -latest_version=false
-//
-// Print the latest revision of each subtree proto for a tree of size 127 with hex keys:
-// dump_tree -tree_size 127
-//
-// Print out the nodes by level using the NodeReader API for a tree of size 11:
-// dump_tree -tree_size 11 -traverse
-//
-// The format for recordio output is as defined in:
-// https://github.com/google/or-tools/blob/master/ortools/base/recordio.h
-// This program always outputs uncompressed records.
-package main
-
-import (
-	"flag"
-	"fmt"
-
-	"github.com/golang/glog"
-)
-
-var (
-	treeSizeFlag        = flag.Int("tree_size", 871, "The number of leaves to be added to the tree")
-	batchSizeFlag       = flag.Int("batch_size", 50, "The batch size for sequencing")
-	leafDataFormatFlag  = flag.String("leaf_format", "Leaf %d", "The format string for leaf data")
-	latestRevisionFlag  = flag.Bool("latest_version", true, "If true outputs only the latest revision per subtree")
-	summaryFlag         = flag.Bool("summary", false, "If true outputs a brief summary per subtree, false dumps the whole proto")
-	hexKeysFlag         = flag.Bool("hex_keys", false, "If true shows proto keys as hex rather than base64")
-	leafHashesFlag      = flag.Bool("leaf_hashes", false, "If true the summary output includes leaf hashes")
-	recordIOFlag        = flag.Bool("recordio", false, "If true outputs in recordio format")
-	rebuildInternalFlag = flag.Bool("rebuild", true, "If true rebuilds internal nodes + root hash from leaves")
-	traverseFlag        = flag.Bool("traverse", false, "If true dumps a tree traversal via coord space, else raw subtrees")
-	dumpLeavesFlag      = flag.Bool("dump_leaves", false, "If true dumps the leaf data from the tree via the API")
-)
-
-func main() {
-	flag.Parse()
-	defer glog.Flush()
-
-	fmt.Print(Main(Options{
-		TreeSize:       *treeSizeFlag,
-		BatchSize:      *batchSizeFlag,
-		LeafFormat:     *leafDataFormatFlag,
-		LatestRevision: *latestRevisionFlag,
-		Summary:        *summaryFlag,
-		HexKeys:        *hexKeysFlag,
-		LeafHashes:     *leafHashesFlag,
-		RecordIO:       *recordIOFlag,
-		Rebuild:        *rebuildInternalFlag,
-		Traverse:       *traverseFlag,
-		DumpLeaves:     *dumpLeavesFlag,
-	}))
-}