From 9354f31d2bb4f321068b61eb24563b31bb920100 Mon Sep 17 00:00:00 2001 From: chris erway Date: Tue, 27 Sep 2022 12:06:43 -0400 Subject: [PATCH 1/2] tool to dump blocks to directory --- tools/debug/dumpblocks/main.go | 117 +++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 tools/debug/dumpblocks/main.go diff --git a/tools/debug/dumpblocks/main.go b/tools/debug/dumpblocks/main.go new file mode 100644 index 0000000000..e710c89339 --- /dev/null +++ b/tools/debug/dumpblocks/main.go @@ -0,0 +1,117 @@ +// Copyright (C) 2019-2022 Algorand, Inc. +// This file is part of go-algorand +// +// go-algorand is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as +// published by the Free Software Foundation, either version 3 of the +// License, or (at your option) any later version. +// +// go-algorand is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with go-algorand. If not, see . + +package main + +import ( + "database/sql" + "flag" + "fmt" + "math/rand" + "os" + "time" + + _ "github.com/mattn/go-sqlite3" +) + +var blockDBfile = flag.String("blockdb", "", "Block DB filename") +var numBlocks = flag.Int("numblocks", 10000, "Randomly sample this many blocks for training") +var startRound = flag.Int("start", 0, "Sample blocks starting at this round") +var endRound = flag.Int("end", 0, "Sample blocks ending at this round") +var outDir = flag.String("outdir", ".", "Write blocks to this directory") +var randSeed = flag.Int("seed", 0, "Random seed, otherwise will use time") + +func getBlockToFile(db *sql.DB, rnd int64) error { + var buf []byte + err := db.QueryRow("SELECT blkdata FROM blocks WHERE rnd=?", rnd).Scan(&buf) + if err != nil { + return err + } + return os.WriteFile(fmt.Sprintf("%s/%d.block", *outDir, rnd), buf, 0644) +} + +func main() { + flag.Parse() + if *blockDBfile == "" { + fmt.Println("-blockdb=file required") + os.Exit(1) + } + uri := fmt.Sprintf("file:%s?mode=ro", *blockDBfile) + fmt.Println("Opening", uri) + db, err := sql.Open("sqlite3", uri) + if err != nil { + panic(err) + } + defer db.Close() + + err = db.Ping() + if err != nil { + panic(err) + } + + seed := int64(*randSeed) + if seed == 0 { + seed = time.Now().UnixMicro() + } + rand.Seed(seed) + + var minRound, maxRound int64 + if *startRound != 0 { + minRound = int64(*startRound) + } + if *endRound != 0 { + maxRound = int64(*endRound) + } + if maxRound == 0 { + err = db.QueryRow("SELECT MAX(rnd) FROM blocks").Scan(&maxRound) + if err != nil { + panic(err) + } + } + if minRound == 0 { + err := db.QueryRow("SELECT MIN(rnd) FROM blocks").Scan(&minRound) + if err != nil { + panic(err) + } + } + + N := maxRound - minRound + if N <= 0 { + panic("maxRound must be greater than minRound") + } + + if N <= int64(*numBlocks) { + // just get all blocks minRound and maxRound + fmt.Printf("Saving all blocks between round %d and %d\n", minRound, maxRound) + for i := minRound; i < maxRound; i++ { + err = getBlockToFile(db, i) + if err != nil { + panic(err) + } + + } + os.Exit(0) + } + + fmt.Printf("Loading %d random blocks between round %d and %d\n", *numBlocks, minRound, maxRound) + for i := 0; i < *numBlocks; i++ { + round := minRound + rand.Int63n(N) + err = getBlockToFile(db, round) + if err != nil { + panic(err) + } + } +} From 738b6dde4b5fde19fdc149b6a3402485f7759bcc Mon Sep 17 00:00:00 2001 From: chris erway Date: Tue, 27 Sep 2022 13:17:21 -0400 Subject: [PATCH 2/2] upate usage and inclusive --- tools/debug/dumpblocks/main.go | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/tools/debug/dumpblocks/main.go b/tools/debug/dumpblocks/main.go index e710c89339..09698bdb05 100644 --- a/tools/debug/dumpblocks/main.go +++ b/tools/debug/dumpblocks/main.go @@ -29,8 +29,8 @@ import ( var blockDBfile = flag.String("blockdb", "", "Block DB filename") var numBlocks = flag.Int("numblocks", 10000, "Randomly sample this many blocks for training") -var startRound = flag.Int("start", 0, "Sample blocks starting at this round") -var endRound = flag.Int("end", 0, "Sample blocks ending at this round") +var startRound = flag.Int("start", 0, "Sample blocks starting at this round (inclusive)") +var endRound = flag.Int("end", 0, "Sample blocks ending at this round (inclusive)") var outDir = flag.String("outdir", ".", "Write blocks to this directory") var randSeed = flag.Int("seed", 0, "Random seed, otherwise will use time") @@ -43,11 +43,16 @@ func getBlockToFile(db *sql.DB, rnd int64) error { return os.WriteFile(fmt.Sprintf("%s/%d.block", *outDir, rnd), buf, 0644) } +func usage() { + flag.Usage() + os.Exit(1) +} + func main() { flag.Parse() if *blockDBfile == "" { fmt.Println("-blockdb=file required") - os.Exit(1) + usage() } uri := fmt.Sprintf("file:%s?mode=ro", *blockDBfile) fmt.Println("Opening", uri) @@ -94,9 +99,9 @@ func main() { } if N <= int64(*numBlocks) { - // just get all blocks minRound and maxRound + // just get all blocks from minRound to maxRound fmt.Printf("Saving all blocks between round %d and %d\n", minRound, maxRound) - for i := minRound; i < maxRound; i++ { + for i := minRound; i <= maxRound; i++ { err = getBlockToFile(db, i) if err != nil { panic(err) @@ -108,7 +113,7 @@ func main() { fmt.Printf("Loading %d random blocks between round %d and %d\n", *numBlocks, minRound, maxRound) for i := 0; i < *numBlocks; i++ { - round := minRound + rand.Int63n(N) + round := minRound + rand.Int63n(N) + 1 err = getBlockToFile(db, round) if err != nil { panic(err)