historyarchive: add --skip-optional flag

historyarchive/archive.go

@@ -28,12 +28,13 @@ const hexPrefixPat = "/[0-9a-f]{2}/[0-9a-f]{2}/[0-9a-f]{2}/"
 const rootHASPath = ".well-known/stellar-history.json"
 
 type CommandOptions struct {
-	Concurrency int
-	Range       Range
-	DryRun      bool
-	Force       bool
-	Verify      bool
-	Thorough    bool
+	Concurrency  int
+	Range        Range
+	DryRun       bool
+	Force        bool
+	Verify       bool
+	Thorough     bool
+	SkipOptional bool
 }
 
 type ConnectOptions struct {

historyarchive/archive_test.go

@@ -219,8 +219,10 @@ func TestScanSizeSubrangeAllBuckets(t *testing.T) {
 func countMissing(arch *Archive, opts *CommandOptions) int {
 	n := 0
 	arch.Scan(opts)
-	for _, missing := range arch.CheckCheckpointFilesMissing(opts) {
-		n += len(missing)
+	for cat, missing := range arch.CheckCheckpointFilesMissing(opts) {
+		if categoryRequired(cat) || !opts.SkipOptional {
+			n += len(missing)
+		}
 	}
 	n += len(arch.CheckBucketsMissing())
 	return n
@@ -247,6 +249,38 @@ func TestMirror(t *testing.T) {
 	assert.Equal(t, 0, countMissing(dst, opts))
 }
 
+func TestOnlyRequired(t *testing.T) {
+	defer cleanup()
+	optsWithOptional := testOptions()
+
+	optsOnlyRequired := testOptions()
+	optsOnlyRequired.SkipOptional = true
+
+	src := GetRandomPopulatedArchive()
+	dstWithOptional := GetTestArchive()
+	dstOnlyRequired := GetTestArchive()
+
+	// First mirror copies optional files, second does not.
+	Mirror(src, dstWithOptional, optsWithOptional)
+	Mirror(src, dstOnlyRequired, optsOnlyRequired)
+
+	// Scanning the mirror with optional files under either
+	// scan (ignore or require optional) is ok.
+	assert.Equal(t, 0, countMissing(dstWithOptional, optsWithOptional))
+	assert.Equal(t, 0, countMissing(dstWithOptional, optsOnlyRequired))
+
+	// Scanning the mirror without optional files produces
+	// a missing count when requiring optional files.
+	assert.NotEqual(t, 0, countMissing(dstOnlyRequired, optsWithOptional))
+	assert.Equal(t, 0, countMissing(dstOnlyRequired, optsOnlyRequired))
+
+	// First repair ignores optional files, second fills them in.
+	Repair(src, dstOnlyRequired, optsOnlyRequired)
+	assert.NotEqual(t, 0, countMissing(dstOnlyRequired, optsWithOptional))
+	Repair(src, dstOnlyRequired, optsWithOptional)
+	assert.Equal(t, 0, countMissing(dstOnlyRequired, optsWithOptional))
+}
+
 func copyFile(category string, checkpoint uint32, src *Archive, dst *Archive) {
 	pth := CategoryCheckpointPath(category, checkpoint)
 	rdr, err := src.backend.GetFile(pth)

historyarchive/mirror.go

@@ -77,6 +77,9 @@ func Mirror(src *Archive, dst *Archive, opts *CommandOptions) error {
 				}
 
 				for _, cat := range Categories() {
+					if opts.SkipOptional && !categoryRequired(cat) {
+						continue
+					}
 					pth := CategoryCheckpointPath(cat, ix)
 					err = copyPath(src, dst, pth, opts)
 					if err != nil && !categoryRequired(cat) {

historyarchive/repair.go

@@ -6,6 +6,7 @@ package historyarchive
 
 import (
 	"fmt"
+
 	log "github.com/sirupsen/logrus"
 )
 
@@ -28,6 +29,9 @@ func Repair(src *Archive, dst *Archive, opts *CommandOptions) error {
 	repairedHistory := false
 	for cat, missing := range missingCheckpointFiles {
 		for _, chk := range missing {
+			if opts.SkipOptional && !categoryRequired(cat) {
+				continue
+			}
 			pth := CategoryCheckpointPath(cat, chk)
 			exists, err := src.backend.Exists(pth)
 			if err != nil {

historyarchive/scan.go

@@ -7,10 +7,11 @@ package historyarchive
 import (
 	"errors"
 	"fmt"
-	log "github.com/sirupsen/logrus"
 	"strings"
 	"sync"
 	"sync/atomic"
+
+	log "github.com/sirupsen/logrus"
 )
 
 type scanCheckpointFastReq struct {
@@ -58,6 +59,9 @@ func (arch *Archive) ScanCheckpointsSlow(opts *CommandOptions) error {
 	cats := Categories()
 	go func() {
 		for _, cat := range cats {
+			if opts.SkipOptional && !categoryRequired(cat) {
+				continue
+			}
 			for chk := range opts.Range.GenerateCheckpoints(arch.checkpointManager) {
 				req <- scanCheckpointSlowReq{category: cat, checkpoint: chk}
 			}
@@ -117,6 +121,9 @@ func (arch *Archive) ScanCheckpointsFast(opts *CommandOptions) error {
 	cats := Categories()
 	go func() {
 		for _, cat := range cats {
+			if opts.SkipOptional && !categoryRequired(cat) {
+				continue
+			}
 			for _, pth := range RangePaths(opts.Range) {
 				req <- scanCheckpointFastReq{category: cat, pathprefix: pth}
 			}
@@ -397,14 +404,18 @@ func (arch *Archive) ReportMissing(opts *CommandOptions) (bool, error) {
 		if !categoryRequired(cat) {
 			if len(missing) > 0 {
 				s := fmtRangeList(missing, arch.checkpointManager)
-				log.Warnf("Missing non-required %s (%d): %s", cat, len(missing), s)
+				if opts.SkipOptional {
+					log.Warnf("Skipped optional %s files (%d): %s", cat, len(missing), s)
+				} else {
+					log.Warnf("Missing optional %s files (%d): %s", cat, len(missing), s)
+				}
 			}
 			continue
 		}
 		if len(missing) != 0 {
 			s := fmtRangeList(missing, arch.checkpointManager)
 			missingCheckpoints = true
-			log.Errorf("Missing %s (%d): %s", cat, len(missing), s)
+			log.Errorf("Missing %s files (%d): %s", cat, len(missing), s)
 		}
 	}
 

tools/stellar-archivist/CHANGELOG.md

@@ -12,6 +12,8 @@ bumps.  A breaking change will get clearly notified in this log.
 * Dropped support for Go 1.10, 1.11, 1.12.
 * Add `log` command
 * Add `--recent` flag for `mirror` command
+* Improve logging to use structured logging and color, add `--trace`
+* Add `--skip-optional` flag to skip optional (SCP) checkpoint files
 
 ## [v0.1.0] - 2016-08-17
 

tools/stellar-archivist/README.md

@@ -45,6 +45,7 @@ Flags:
   -r, --recent            act on ledger-range difference between achives
       --s3region string   S3 region to connect to (default "us-east-1")
       --s3endpoint string S3 endpoint (default to AWS endpoint for selected region)
+      --skip-optional     skip optional (SCP) checkpoint files
       --thorough          decode and re-encode all buckets
       --verify            verify file contents
 

tools/stellar-archivist/main.go

@@ -6,11 +6,12 @@ package main
 
 import (
 	"fmt"
-	log "github.com/sirupsen/logrus"
 	"net/http"
 	_ "net/http/pprof"
 	"os"
 
+	log "github.com/sirupsen/logrus"
+
 	"github.com/spf13/cobra"
 	"github.com/stellar/go/historyarchive"
 	"github.com/stellar/go/support/errors"
@@ -258,6 +259,13 @@ func main() {
 		"decode and re-encode all buckets",
 	)
 
+	rootCmd.PersistentFlags().BoolVar(
+		&opts.CommandOpts.SkipOptional,
+		"skip-optional",
+		false,
+		"skip optional (SCP) checkpoint files",
+	)
+
 	rootCmd.PersistentFlags().BoolVar(
 		&opts.Profile,
 		"profile",