From 0609182fa9a0dbce37e7bb56917e6f2531a0fe11 Mon Sep 17 00:00:00 2001 From: Pavel Kalinnikov <pkalinnikov@google.com> Date: Mon, 25 Apr 2022 12:37:39 +0100 Subject: [PATCH] Remake dumplib into integration test (#2710) dumplib was a tool which created an in-memory log storage of a particular size, and printed its contents with various flags. The useful bit of this tool was a "golden" unit test which checks regressions on the SubtreeProto format in storage. This change removes the tool, refactors the test, and places it in integration/format. --- CHANGELOG.md | 3 + integration/format/format.go | 175 +++++++ .../format}/nochange_test.go | 79 ++-- .../format}/testdata/dump_tree_output_1000 | 0 .../format}/testdata/dump_tree_output_1024 | 0 .../format}/testdata/dump_tree_output_871 | 0 .../format}/testdata/dump_tree_output_96 | 0 storage/tools/dump_tree/dumplib.go | 428 ------------------ storage/tools/dump_tree/main.go | 77 ---- 9 files changed, 205 insertions(+), 557 deletions(-) create mode 100644 integration/format/format.go rename {storage/tools/dump_tree => integration/format}/nochange_test.go (50%) rename {storage/tools/dump_tree => integration/format}/testdata/dump_tree_output_1000 (100%) rename {storage/tools/dump_tree => integration/format}/testdata/dump_tree_output_1024 (100%) rename {storage/tools/dump_tree => integration/format}/testdata/dump_tree_output_871 (100%) rename {storage/tools/dump_tree => integration/format}/testdata/dump_tree_output_96 (100%) delete mode 100644 storage/tools/dump_tree/dumplib.go delete mode 100644 storage/tools/dump_tree/main.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 0dd5523d96..ea518ef255 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,9 @@ * `countFromInformationSchema` function to add support for MySQL 8. ### Removals + + * #2710: Unused `storage/tools/dumplib` was removed. The useful storage format + regression test moved to `integration/format`. * #2711: Unused `storage/tools/hasher` removed. * #2715: Packages under `merkle` are deprecated and to be removed. Use https://github.com/transparency-dev/merkle instead. diff --git a/integration/format/format.go b/integration/format/format.go new file mode 100644 index 0000000000..9d70be68a3 --- /dev/null +++ b/integration/format/format.go @@ -0,0 +1,175 @@ +// Copyright 2017 Google LLC. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package format + +import ( + "bytes" + "context" + "crypto/sha256" + "fmt" + "sort" + "strconv" + "strings" + "time" + + "github.com/google/trillian" + "github.com/google/trillian/log" + "github.com/google/trillian/monitoring" + "github.com/google/trillian/quota" + "github.com/google/trillian/storage" + "github.com/google/trillian/storage/cache" + "github.com/google/trillian/storage/memory" + "github.com/google/trillian/storage/storagepb" + "github.com/google/trillian/types" + "github.com/google/trillian/util/clock" + "github.com/transparency-dev/merkle" + "github.com/transparency-dev/merkle/rfc6962" + "google.golang.org/protobuf/encoding/prototext" + "google.golang.org/protobuf/types/known/durationpb" +) + +func run(treeSize, batchSize int, leafFormat string) (string, error) { + ctx := context.Background() + + ts := memory.NewTreeStorage() + ls := memory.NewLogStorage(ts, monitoring.InertMetricFactory{}) + as := memory.NewAdminStorage(ts) + tree, err := createTree(ctx, as, ls) + if err != nil { + return "", err + } + log.InitMetrics(nil) + + leaves := generateLeaves(treeSize, leafFormat) + if err := sequenceLeaves(ctx, ls, tree, leaves, batchSize); err != nil { + return "", err + } + + // Read the latest LogRoot back. + var root types.LogRootV1 + if err := ls.ReadWriteTransaction(ctx, tree, func(ctx context.Context, tx storage.LogTreeTX) error { + latest, err := tx.LatestSignedLogRoot(ctx) + if err != nil { + return err + } + return root.UnmarshalBinary(latest.LogRoot) + }); err != nil { + return "", fmt.Errorf("ReadWriteTransaction: %v", err) + } + + return latestRevisions(ls, tree.TreeId, rfc6962.DefaultHasher) +} + +func createTree(ctx context.Context, as storage.AdminStorage, ls storage.LogStorage) (*trillian.Tree, error) { + tree, err := storage.CreateTree(ctx, as, &trillian.Tree{ + TreeType: trillian.TreeType_LOG, + TreeState: trillian.TreeState_ACTIVE, + MaxRootDuration: durationpb.New(0 * time.Millisecond), + }) + if err != nil { + return nil, fmt.Errorf("CreateTree: %v", err) + } + + logRoot, err := (&types.LogRootV1{RootHash: rfc6962.DefaultHasher.EmptyRoot()}).MarshalBinary() + if err != nil { + return nil, fmt.Errorf("MarshalBinary: %v", err) + } + + if err = ls.ReadWriteTransaction(ctx, tree, func(ctx context.Context, tx storage.LogTreeTX) error { + return tx.StoreSignedLogRoot(ctx, &trillian.SignedLogRoot{LogRoot: logRoot}) + }); err != nil { + return nil, fmt.Errorf("ReadWriteTransaction: %v", err) + } + + return tree, nil +} + +func generateLeaves(count int, format string) []*trillian.LogLeaf { + leaves := make([]*trillian.LogLeaf, 0, count) + for i := 0; i < count; i++ { + data := []byte(fmt.Sprintf(format, i)) + hash := sha256.Sum256(data) + leaves = append(leaves, &trillian.LogLeaf{ + LeafValue: data, LeafIdentityHash: hash[:], MerkleLeafHash: hash[:], + }) + } + return leaves +} + +func sequenceLeaves(ctx context.Context, ls storage.LogStorage, tree *trillian.Tree, leaves []*trillian.LogLeaf, batchSize int) error { + for i, size := 0, len(leaves); i < size; i += batchSize { + if left := size - i; left < batchSize { + batchSize = left + } + if _, err := ls.QueueLeaves(ctx, tree, leaves[i:i+batchSize], time.Now()); err != nil { + return fmt.Errorf("QueueLeaves: %v", err) + } + + sequenced, err := log.IntegrateBatch(ctx, tree, batchSize, 0, 24*time.Hour, clock.System, ls, quota.Noop()) + if err != nil { + return fmt.Errorf("IntegrateBatch: %v", err) + } + if got, want := sequenced, batchSize; got != want { + return fmt.Errorf("IntegrateBatch: got %d, want %d", got, want) + } + } + return nil +} + +type treeAndRev struct { + subtree *storagepb.SubtreeProto + revision int +} + +func latestRevisions(ls storage.LogStorage, treeID int64, hasher merkle.LogHasher) (string, error) { + // vMap maps subtree prefixes (as strings) to the corresponding subtree proto and its revision + vMap := make(map[string]treeAndRev) + memory.DumpSubtrees(ls, treeID, func(k string, v *storagepb.SubtreeProto) { + // Relies on the btree key space for subtrees being /tree_id/subtree/id/revision. + pieces := strings.Split(k, "/") + if got, want := len(pieces), 5; got != want { + panic(fmt.Sprintf("Btree subtree key segments: got %d, want %d", got, want)) + } + + subID := pieces[3] + rev, err := strconv.Atoi(pieces[4]) + if err != nil { + panic(fmt.Sprintf("Bad subtree key: %v: %v", k, err)) + } + + if rev > vMap[subID].revision { + vMap[subID] = treeAndRev{ + subtree: v, + revision: rev, + } + } + }) + + // Store the keys in sorted order. + keys := make([]string, 0, len(vMap)) + for k := range vMap { + keys = append(keys, k) + } + sort.Strings(keys) + + // The map should now contain the latest revisions per subtree. + out := new(bytes.Buffer) + for _, k := range keys { + subtree := vMap[k].subtree + cache.PopulateLogTile(subtree, hasher) + fmt.Fprintf(out, "%s\n", prototext.Format(subtree)) + } + return out.String(), nil +} diff --git a/storage/tools/dump_tree/nochange_test.go b/integration/format/nochange_test.go similarity index 50% rename from storage/tools/dump_tree/nochange_test.go rename to integration/format/nochange_test.go index b11e0d01a4..0a241b05c7 100644 --- a/storage/tools/dump_tree/nochange_test.go +++ b/integration/format/nochange_test.go @@ -12,7 +12,10 @@ // See the License for the specific language governing permissions and // limitations under the License. -package main +// Package format contains an integration test which builds a log using an +// in-memory storage end-to-end, and makes sure the SubtreeProto storage format +// has no regressions. +package format import ( "io/ioutil" @@ -25,61 +28,33 @@ import ( "google.golang.org/protobuf/testing/protocmp" ) -// TestDBFormatNoChange ensures that the prefix, suffix, and protos stored in the database do not change. -// This test compares the output from dump_tree against a previously saved output. +// TestDBFormatNoChange ensures that SubtreeProto tiles stored in the database +// do not change. This test compares against a previously saved output. func TestDBFormatNoChange(t *testing.T) { for _, tc := range []struct { - desc string - file string - opts Options + name string + size int }{ - { - desc: "tree_size: 96", - file: "testdata/dump_tree_output_96", - opts: Options{ - 96, 50, - "Leaf %d", - true, false, false, false, false, true, false, false, - }, - }, - { - desc: "tree_size: 871", - file: "testdata/dump_tree_output_871", - opts: Options{ - 871, 50, - "Leaf %d", - true, false, false, false, false, true, false, false, - }, - }, - { - desc: "tree_size: 1000", - file: "testdata/dump_tree_output_1000", - opts: Options{ - 1000, 50, - "Leaf %d", - true, false, false, false, false, true, false, false, - }, - }, - { - desc: "tree_size: 1024", - file: "testdata/dump_tree_output_1024", - opts: Options{ - 1024, 50, - "Leaf %d", - true, false, false, false, false, true, false, false, - }, - }, + {name: "dump_tree_output_96", size: 96}, + {name: "dump_tree_output_871", size: 871}, + {name: "dump_tree_output_1000", size: 1000}, + {name: "dump_tree_output_1024", size: 1024}, } { - out := Main(tc.opts) - saved, err := ioutil.ReadFile(tc.file) - if err != nil { - t.Fatalf("ReadFile(%v): %v", tc.file, err) - } - got := parseTiles(t, out) - want := parseTiles(t, string(saved)) - if d := cmp.Diff(want, got, protocmp.Transform()); d != "" { - t.Errorf("Diff(-want,+got):\n%s", d) - } + t.Run(tc.name, func(t *testing.T) { + out, err := run(tc.size, 50, "Leaf %d") + if err != nil { + t.Fatalf("run: %v", err) + } + saved, err := ioutil.ReadFile("testdata/" + tc.name) + if err != nil { + t.Fatalf("ReadFile(%v): %v", tc.name, err) + } + got := parseTiles(t, out) + want := parseTiles(t, string(saved)) + if d := cmp.Diff(want, got, protocmp.Transform()); d != "" { + t.Errorf("Diff(-want,+got):\n%s", d) + } + }) } } diff --git a/storage/tools/dump_tree/testdata/dump_tree_output_1000 b/integration/format/testdata/dump_tree_output_1000 similarity index 100% rename from storage/tools/dump_tree/testdata/dump_tree_output_1000 rename to integration/format/testdata/dump_tree_output_1000 diff --git a/storage/tools/dump_tree/testdata/dump_tree_output_1024 b/integration/format/testdata/dump_tree_output_1024 similarity index 100% rename from storage/tools/dump_tree/testdata/dump_tree_output_1024 rename to integration/format/testdata/dump_tree_output_1024 diff --git a/storage/tools/dump_tree/testdata/dump_tree_output_871 b/integration/format/testdata/dump_tree_output_871 similarity index 100% rename from storage/tools/dump_tree/testdata/dump_tree_output_871 rename to integration/format/testdata/dump_tree_output_871 diff --git a/storage/tools/dump_tree/testdata/dump_tree_output_96 b/integration/format/testdata/dump_tree_output_96 similarity index 100% rename from storage/tools/dump_tree/testdata/dump_tree_output_96 rename to integration/format/testdata/dump_tree_output_96 diff --git a/storage/tools/dump_tree/dumplib.go b/storage/tools/dump_tree/dumplib.go deleted file mode 100644 index 03a0d91f21..0000000000 --- a/storage/tools/dump_tree/dumplib.go +++ /dev/null @@ -1,428 +0,0 @@ -// Copyright 2017 Google LLC. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package main - -import ( - "bytes" - "context" - "crypto/sha256" - "encoding/base64" - "encoding/binary" - "encoding/hex" - "fmt" - "sort" - "strconv" - "strings" - "time" - - "github.com/golang/glog" - "github.com/google/trillian" - "github.com/google/trillian/log" - "github.com/google/trillian/monitoring" - "github.com/google/trillian/quota" - "github.com/google/trillian/storage" - "github.com/google/trillian/storage/cache" - "github.com/google/trillian/storage/memory" - "github.com/google/trillian/storage/storagepb" - "github.com/google/trillian/types" - "github.com/google/trillian/util/clock" - "github.com/transparency-dev/merkle" - "github.com/transparency-dev/merkle/compact" - "github.com/transparency-dev/merkle/rfc6962" - "google.golang.org/protobuf/encoding/prototext" - "google.golang.org/protobuf/proto" - "google.golang.org/protobuf/types/known/durationpb" -) - -// A 32 bit magic number that is written at the start of record io files to identify the format. -const recordIOMagic int32 = 0x3ed7230a - -type treeAndRev struct { - fullKey string - subtree *storagepb.SubtreeProto - revision int -} - -// summarizeProto is an output formatter function that produces a single line summary. -func summarizeProto(leafHashesFlag bool) func(s *storagepb.SubtreeProto) string { - return func(s *storagepb.SubtreeProto) string { - summary := fmt.Sprintf("p: %-20s d: %d lc: %3d ic: %3d\n", - hex.EncodeToString(s.Prefix), - s.Depth, - len(s.Leaves), - s.InternalNodeCount) - - if leafHashesFlag { - for prefix, hash := range s.Leaves { - dp, err := base64.StdEncoding.DecodeString(prefix) - if err != nil { - glog.Fatalf("Failed to decode leaf prefix: %v", err) - } - summary += fmt.Sprintf("%s -> %s\n", hex.EncodeToString(dp), hex.EncodeToString(hash)) - } - } - - return summary - } -} - -// fullProto is an output formatter function that produces a single line in proto text format. -func fullProto(s *storagepb.SubtreeProto) string { - return fmt.Sprintf("%s\n", prototext.Format(s)) -} - -// recordIOProto is an output formatter that produces binary recordio format -func recordIOProto(s *storagepb.SubtreeProto) string { - buf := new(bytes.Buffer) - data, err := proto.Marshal(s) - if err != nil { - glog.Fatalf("Failed to marshal subtree proto: %v", err) - } - dataLen := int64(len(data)) - if err = binary.Write(buf, binary.BigEndian, dataLen); err != nil { - glog.Fatalf("binary.Write failed: %v", err) - } - var compLen int64 - if err = binary.Write(buf, binary.BigEndian, compLen); err != nil { - glog.Fatalf("binary.Write failed: %v", err) - } - // buffer.Write() always returns a nil error - buf.Write(data) - - return buf.String() -} - -func sequence(tree *trillian.Tree, logStorage storage.LogStorage, count, batchSize int) { - glog.Infof("Sequencing batch of size %d", count) - sequenced, err := log.IntegrateBatch(context.TODO(), tree, batchSize, 0, 24*time.Hour, clock.System, logStorage, quota.Noop()) - if err != nil { - glog.Fatalf("IntegrateBatch got: %v, want: no err", err) - } - - if got, want := sequenced, count; got != want { - glog.Fatalf("IntegrateBatch got: %d sequenced, want: %d", got, want) - } -} - -func createTree(as storage.AdminStorage, ls storage.LogStorage) *trillian.Tree { - ctx := context.TODO() - tree := &trillian.Tree{ - TreeType: trillian.TreeType_LOG, - TreeState: trillian.TreeState_ACTIVE, - MaxRootDuration: durationpb.New(0 * time.Millisecond), - } - createdTree, err := storage.CreateTree(ctx, as, tree) - if err != nil { - glog.Fatalf("Create tree: %v", err) - } - - logRoot, err := (&types.LogRootV1{RootHash: rfc6962.DefaultHasher.EmptyRoot()}).MarshalBinary() - if err != nil { - glog.Fatalf("MarshalBinary: %v", err) - } - sthZero := &trillian.SignedLogRoot{LogRoot: logRoot} - - err = ls.ReadWriteTransaction(ctx, createdTree, func(ctx context.Context, tx storage.LogTreeTX) error { - if err := tx.StoreSignedLogRoot(ctx, sthZero); err != nil { - glog.Fatalf("StoreSignedLogRoot: %v", err) - } - return nil - }) - if err != nil { - glog.Fatalf("ReadWriteTransaction: %v", err) - } - - return createdTree -} - -// Options are the commandline arguments one can pass to Main -type Options struct { - TreeSize, BatchSize int - LeafFormat string - LatestRevision, Summary, HexKeys, LeafHashes bool - RecordIO, Rebuild, Traverse, DumpLeaves bool -} - -// Main runs the dump_tree tool -func Main(args Options) string { - ctx := context.Background() - validateFlagsOrDie(args.Summary, args.RecordIO) - - glog.Info("Initializing memory log storage") - ts := memory.NewTreeStorage() - ls := memory.NewLogStorage(ts, monitoring.InertMetricFactory{}) - as := memory.NewAdminStorage(ts) - tree := createTree(as, ls) - - log.InitMetrics(nil) - - // Create the initial tree head at size 0, which is required. And then sequence the leaves. - sequence(tree, ls, 0, args.BatchSize) - sequenceLeaves(ls, tree, args.TreeSize, args.BatchSize, args.LeafFormat) - - // Read the latest STH back - var root types.LogRootV1 - err := ls.ReadWriteTransaction(ctx, tree, func(ctx context.Context, tx storage.LogTreeTX) error { - var err error - sth, err := tx.LatestSignedLogRoot(ctx) - if err != nil { - glog.Fatalf("LatestSignedLogRoot: %v", err) - } - if err := root.UnmarshalBinary(sth.LogRoot); err != nil { - return fmt.Errorf("could not parse current log root: %v", err) - } - - glog.Infof("STH at size %d has hash %s", - root.TreeSize, - hex.EncodeToString(root.RootHash)) - return nil - }) - if err != nil { - glog.Fatalf("ReadWriteTransaction: %v", err) - } - - // All leaves are now sequenced into the tree. The current state is what we need. - glog.Info("Producing output") - - if args.Traverse { - return traverseTreeStorage(ctx, ls, tree, args.TreeSize) - } - - if args.DumpLeaves { - return dumpLeaves(ctx, ls, tree, args.TreeSize) - } - - var formatter func(*storagepb.SubtreeProto) string - switch { - case args.Summary: - formatter = summarizeProto(args.LeafHashes) - case args.RecordIO: - formatter = recordIOProto - recordIOHdr() - default: - formatter = fullProto - } - - if args.LatestRevision { - return latestRevisions(ls, tree.TreeId, rfc6962.DefaultHasher, formatter, args.Rebuild, args.HexKeys) - } - return allRevisions(ls, tree.TreeId, rfc6962.DefaultHasher, formatter, args.Rebuild, args.HexKeys) -} - -func allRevisions(ls storage.LogStorage, treeID int64, hasher merkle.LogHasher, of func(*storagepb.SubtreeProto) string, rebuildInternal, hexKeysFlag bool) string { - out := new(bytes.Buffer) - memory.DumpSubtrees(ls, treeID, func(k string, v *storagepb.SubtreeProto) { - if rebuildInternal { - cache.PopulateLogTile(v, hasher) - } - if hexKeysFlag { - hexKeys(v) - } - fmt.Fprint(out, of(v)) - }) - return out.String() -} - -func latestRevisions(ls storage.LogStorage, treeID int64, hasher merkle.LogHasher, of func(*storagepb.SubtreeProto) string, rebuildInternal, hexKeysFlag bool) string { - out := new(bytes.Buffer) - // vMap maps subtree prefixes (as strings) to the corresponding subtree proto and its revision - vMap := make(map[string]treeAndRev) - memory.DumpSubtrees(ls, treeID, func(k string, v *storagepb.SubtreeProto) { - // Relies on the btree key space for subtrees being /tree_id/subtree/<id>/<revision> - pieces := strings.Split(k, "/") - if got, want := len(pieces), 5; got != want { - glog.Fatalf("Wrong no of Btree subtree key segments. Got: %d, want: %d", got, want) - } - - subID := pieces[3] - subtree := vMap[subID] - rev, err := strconv.Atoi(pieces[4]) - if err != nil { - glog.Fatalf("Bad subtree key: %v", k) - } - - if rev > subtree.revision { - vMap[subID] = treeAndRev{ - fullKey: k, - subtree: v, - revision: rev, - } - } - }) - - // Store the keys in sorted order - var sKeys []string - for k := range vMap { - sKeys = append(sKeys, k) - } - sort.Strings(sKeys) - - // The map should now contain the latest revisions per subtree - for _, k := range sKeys { - v := vMap[k] - if rebuildInternal { - cache.PopulateLogTile(v.subtree, hasher) - } - if hexKeysFlag { - hexKeys(v.subtree) - } - - fmt.Fprint(out, of(v.subtree)) - } - return out.String() -} - -func validateFlagsOrDie(summary, recordIO bool) { - if summary && recordIO { - glog.Fatal("-summary and -recordio are mutually exclusive flags") - } -} - -func sequenceLeaves(ls storage.LogStorage, tree *trillian.Tree, treeSize, batchSize int, leafDataFormat string) { - glog.Info("Queuing work") - for l := 0; l < treeSize; l++ { - glog.V(1).Infof("Queuing leaf %d", l) - - leafData := []byte(fmt.Sprintf(leafDataFormat, l)) - hash := sha256.Sum256(leafData) - lh := hash[:] - leaf := trillian.LogLeaf{LeafValue: leafData, LeafIdentityHash: lh, MerkleLeafHash: lh} - leaves := []*trillian.LogLeaf{&leaf} - - if _, err := ls.QueueLeaves(context.TODO(), tree, leaves, time.Now()); err != nil { - glog.Fatalf("QueueLeaves got: %v, want: no err", err) - } - - if l > 0 && l%batchSize == 0 { - sequence(tree, ls, batchSize, batchSize) - } - } - glog.Info("Finished queueing") - // Handle anything left over - left := treeSize % batchSize - if left == 0 { - left = batchSize - } - sequence(tree, ls, left, batchSize) - glog.Info("Finished sequencing") -} - -func traverseTreeStorage(ctx context.Context, ls storage.LogStorage, tt *trillian.Tree, ts int) string { - out := new(bytes.Buffer) - nodesAtLevel := uint64(ts) - - tx, err := ls.SnapshotForTree(context.TODO(), tt) - if err != nil { - glog.Fatalf("SnapshotForTree: %v", err) - } - defer func() { - if err := tx.Commit(ctx); err != nil { - glog.Fatalf("TX Commit(): %v", err) - } - }() - - levels := uint(0) - n := nodesAtLevel - for n > 0 { - levels++ - n = n >> 1 - } - - // Because of the way we store subtrees omitting internal RHS nodes with one sibling there - // is an extra level stored for trees that don't have a number of leaves that is a power - // of 2. We account for this here and in the loop below. - if !isPerfectTree(int64(ts)) { - levels++ - } - - for level := uint(0); level < levels; level++ { - for node := uint64(0); node < nodesAtLevel; node++ { - // We're going to request one node at a time, which would normally be slow but we have - // the tree in RAM so it's not a real problem. - nodeID := compact.NewNodeID(level, node) - nodes, err := tx.GetMerkleNodes(context.TODO(), []compact.NodeID{nodeID}) - if err != nil { - glog.Fatalf("GetMerkleNodes: %+v: %v", nodeID, err) - } - if len(nodes) != 1 { - glog.Fatalf("GetMerkleNodes: %+v: want 1 node got: %v", nodeID, nodes) - } - - fmt.Fprintf(out, "%6d %6d -> %s\n", level, node, hex.EncodeToString(nodes[0].Hash)) - } - - nodesAtLevel = nodesAtLevel >> 1 - fmt.Println() - // This handles the extra level in non-perfect trees - if nodesAtLevel == 0 { - nodesAtLevel = 1 - } - } - return out.String() -} - -func dumpLeaves(ctx context.Context, ls storage.LogStorage, tree *trillian.Tree, ts int) string { - out := new(bytes.Buffer) - tx, err := ls.SnapshotForTree(ctx, tree) - if err != nil { - glog.Fatalf("SnapshotForTree: %v", err) - } - defer func() { - if err := tx.Commit(ctx); err != nil { - glog.Fatalf("TX Commit(): got: %v", err) - } - }() - - for l := int64(0); l < int64(ts); l++ { - leaves, err := tx.GetLeavesByRange(ctx, l, 1) - if err != nil { - glog.Fatalf("GetLeavesByRange for index %d got: %v", l, err) - } - fmt.Fprintf(out, "%6d:%s\n", l, leaves[0].LeafValue) - } - return out.String() -} - -func hexMap(in map[string][]byte) map[string][]byte { - m := make(map[string][]byte) - - for k, v := range in { - unb64, err := base64.StdEncoding.DecodeString(k) - if err != nil { - glog.Fatalf("Could not decode key as base 64: %s got: %v", k, err) - } - m[hex.EncodeToString(unb64)] = v - } - - return m -} - -func hexKeys(s *storagepb.SubtreeProto) { - s.Leaves = hexMap(s.Leaves) - s.InternalNodes = hexMap(s.InternalNodes) -} - -func isPerfectTree(x int64) bool { - return x != 0 && (x&(x-1) == 0) -} - -func recordIOHdr() { - buf := new(bytes.Buffer) - err := binary.Write(buf, binary.BigEndian, recordIOMagic) - if err != nil { - glog.Fatalf("binary.Write failed: %v", err) - } - fmt.Print(buf.String()) -} diff --git a/storage/tools/dump_tree/main.go b/storage/tools/dump_tree/main.go deleted file mode 100644 index 904e17737c..0000000000 --- a/storage/tools/dump_tree/main.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2017 Google LLC. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// The dump_tree program uses the in memory storage implementation to create a sequenced -// log tree of a particular size using known leaf data and then dumps out the resulting -// SubTree protos for examination and debugging. It does not require any actual storage -// to be configured. -// -// Examples of some usages: -// -// Print a summary of the storage protos in a tree of size 1044, rebuilding internal nodes: -// dump_tree -tree_size 1044 -summary -// -// Print all versions of all raw subtree protos for a tree of size 58: -// dump_tree -tree_size 58 -latest_version=false -// -// Print the latest revision of each subtree proto for a tree of size 127 with hex keys: -// dump_tree -tree_size 127 -// -// Print out the nodes by level using the NodeReader API for a tree of size 11: -// dump_tree -tree_size 11 -traverse -// -// The format for recordio output is as defined in: -// https://github.com/google/or-tools/blob/master/ortools/base/recordio.h -// This program always outputs uncompressed records. -package main - -import ( - "flag" - "fmt" - - "github.com/golang/glog" -) - -var ( - treeSizeFlag = flag.Int("tree_size", 871, "The number of leaves to be added to the tree") - batchSizeFlag = flag.Int("batch_size", 50, "The batch size for sequencing") - leafDataFormatFlag = flag.String("leaf_format", "Leaf %d", "The format string for leaf data") - latestRevisionFlag = flag.Bool("latest_version", true, "If true outputs only the latest revision per subtree") - summaryFlag = flag.Bool("summary", false, "If true outputs a brief summary per subtree, false dumps the whole proto") - hexKeysFlag = flag.Bool("hex_keys", false, "If true shows proto keys as hex rather than base64") - leafHashesFlag = flag.Bool("leaf_hashes", false, "If true the summary output includes leaf hashes") - recordIOFlag = flag.Bool("recordio", false, "If true outputs in recordio format") - rebuildInternalFlag = flag.Bool("rebuild", true, "If true rebuilds internal nodes + root hash from leaves") - traverseFlag = flag.Bool("traverse", false, "If true dumps a tree traversal via coord space, else raw subtrees") - dumpLeavesFlag = flag.Bool("dump_leaves", false, "If true dumps the leaf data from the tree via the API") -) - -func main() { - flag.Parse() - defer glog.Flush() - - fmt.Print(Main(Options{ - TreeSize: *treeSizeFlag, - BatchSize: *batchSizeFlag, - LeafFormat: *leafDataFormatFlag, - LatestRevision: *latestRevisionFlag, - Summary: *summaryFlag, - HexKeys: *hexKeysFlag, - LeafHashes: *leafHashesFlag, - RecordIO: *recordIOFlag, - Rebuild: *rebuildInternalFlag, - Traverse: *traverseFlag, - DumpLeaves: *dumpLeavesFlag, - })) -}