From 29399aefd63141c286cfd341d4bdc8102067429f Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Sun, 3 May 2020 14:03:27 -0700 Subject: [PATCH 01/31] Separate blockProcessed into blockAdded and blockRemoved --- internal/syncer/base_handler.go | 38 ++++++++++++++++++++--------- internal/syncer/stateful_syncer.go | 9 ++++++- internal/syncer/stateless_syncer.go | 3 +-- internal/syncer/syncer.go | 10 +++++--- 4 files changed, 43 insertions(+), 17 deletions(-) diff --git a/internal/syncer/base_handler.go b/internal/syncer/base_handler.go index 136beaa8..93c73e7e 100644 --- a/internal/syncer/base_handler.go +++ b/internal/syncer/base_handler.go @@ -46,22 +46,17 @@ func NewBaseHandler( } } -// BlockProcessed is called by the syncer after each -// block is processed. -// TODO: refactor to BlockAdded and BlockRemoved -func (h *BaseHandler) BlockProcessed( +// BlockAdded is called by the syncer after a +// block is added. +func (h *BaseHandler) BlockAdded( ctx context.Context, block *types.Block, - reorg bool, balanceChanges []*storage.BalanceChange, ) error { - if !reorg { - log.Printf("Adding block %+v\n", block.BlockIdentifier) - } else { - log.Printf("Orphaning block %+v\n", block.BlockIdentifier) - } + log.Printf("Adding block %+v\n", block.BlockIdentifier) + // Log processed blocks and balance changes - if err := h.logger.BlockStream(ctx, block, reorg); err != nil { + if err := h.logger.BlockStream(ctx, block, false); err != nil { return nil } @@ -74,6 +69,27 @@ func (h *BaseHandler) BlockProcessed( return h.reconciler.QueueChanges(ctx, block.BlockIdentifier, balanceChanges) } +// BlockRemoved is called by the syncer after a +// block is removed. +func (h *BaseHandler) BlockRemoved( + ctx context.Context, + block *types.Block, + balanceChanges []*storage.BalanceChange, +) error { + log.Printf("Orphaning block %+v\n", block.BlockIdentifier) + + // Log processed blocks and balance changes + if err := h.logger.BlockStream(ctx, block, true); err != nil { + return nil + } + + if err := h.logger.BalanceStream(ctx, balanceChanges); err != nil { + return nil + } + + return nil +} + // AccountExempt returns a boolean indicating if the provided // account and currency are exempt from balance tracking and // reconciliation. diff --git a/internal/syncer/stateful_syncer.go b/internal/syncer/stateful_syncer.go index 52d0d44f..82135672 100644 --- a/internal/syncer/stateful_syncer.go +++ b/internal/syncer/stateful_syncer.go @@ -263,6 +263,8 @@ func (s *StatefulSyncer) processBlock( // you don't need to restart validation from genesis. Instead, // you can just restart validation at the block immediately // before any erroneous block. +// +// TODO: batch transactions to orphan blocks func (s *StatefulSyncer) newHeadIndex( ctx context.Context, newHeadIndex int64, @@ -349,7 +351,12 @@ func (s *StatefulSyncer) SyncRange( currIndex = newIndex - if err := s.handler.BlockProcessed(ctx, block, reorg, balanceChanges); err != nil { + if reorg { + err = s.handler.BlockRemoved(ctx, block, balanceChanges) + } else { + err = s.handler.BlockAdded(ctx, block, balanceChanges) + } + if err != nil { return err } } diff --git a/internal/syncer/stateless_syncer.go b/internal/syncer/stateless_syncer.go index 66e5f76c..5fc8f73b 100644 --- a/internal/syncer/stateless_syncer.go +++ b/internal/syncer/stateless_syncer.go @@ -101,10 +101,9 @@ func (s *StatelessSyncer) SyncRange( return err } - err = s.handler.BlockProcessed( + err = s.handler.BlockAdded( ctx, block, - false, changes, ) if err != nil { diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index 1074cf94..3e52b362 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -145,11 +145,15 @@ func Sync( // to handle different events. It is common to write logs or // perform reconciliation in the sync handler. type Handler interface { - // TODO: change to BlockAdded and BlockRemoved - BlockProcessed( + BlockAdded( + ctx context.Context, + block *types.Block, + changes []*storage.BalanceChange, + ) error + + BlockRemoved( ctx context.Context, block *types.Block, - orphan bool, changes []*storage.BalanceChange, ) error From 52da50f16def6538a5b89ec56454ba13ec814ccc Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Sun, 3 May 2020 15:39:36 -0700 Subject: [PATCH 02/31] Single syncer --- cmd/root.go | 55 +++ internal/processor/base_processor.go | 226 +++++++++++ internal/processor/processor.go | 21 + internal/syncer/base_handler.go | 108 ------ internal/syncer/stateful_syncer.go | 401 ------------------- internal/syncer/stateful_syncer_test.go | 495 ------------------------ internal/syncer/stateless_syncer.go | 138 ------- internal/syncer/syncer.go | 334 ++++++++-------- internal/syncer/syncer_test.go | 234 ----------- 9 files changed, 460 insertions(+), 1552 deletions(-) create mode 100644 internal/processor/base_processor.go create mode 100644 internal/processor/processor.go delete mode 100644 internal/syncer/base_handler.go delete mode 100644 internal/syncer/stateful_syncer.go delete mode 100644 internal/syncer/stateful_syncer_test.go delete mode 100644 internal/syncer/stateless_syncer.go delete mode 100644 internal/syncer/syncer_test.go diff --git a/cmd/root.go b/cmd/root.go index 6ea99150..0a595d73 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -15,14 +15,18 @@ package cmd import ( + "context" "encoding/json" "io/ioutil" "log" "path" "time" + "github.com/coinbase/rosetta-cli/internal/logger" + "github.com/coinbase/rosetta-cli/internal/processor" "github.com/coinbase/rosetta-cli/internal/reconciler" + "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/spf13/cobra" ) @@ -213,3 +217,54 @@ func loadAccounts(filePath string) ([]*reconciler.AccountCurrency, error) { return accounts, nil } + +func standardInitialization(ctx context.Context, interestingAccounts []*reconciler.AccountCurrency) { + exemptAccounts, err := loadAccounts(ExemptFile) + if err != nil { + log.Fatal(err) + } + + fetcher := fetcher.New( + ServerURL, + fetcher.WithBlockConcurrency(BlockConcurrency), + fetcher.WithTransactionConcurrency(TransactionConcurrency), + fetcher.WithRetryElapsedTime(ExtendedRetryElapsedTime), + ) + + primaryNetwork, _, err := fetcher.InitializeAsserter(ctx) + if err != nil { + log.Fatal(err) + } + + logger := logger.NewLogger( + DataDir, + LogBlocks, + LogTransactions, + LogBalanceChanges, + LogReconciliations, + ) + + return + + r := reconciler.NewReconciler( + primaryNetwork, + fetcher, + logger, + AccountConcurrency, + HaltOnReconciliationError, + interestingAccounts, + ) + + processor := processor.NewBaseProcessor( + logger, + r, + fetcher.Asserter, + exemptAccounts, + ) + + syncer := syncer.NewSyncer( + primaryNetwork, + fetcher, + processor, + ) +} diff --git a/internal/processor/base_processor.go b/internal/processor/base_processor.go new file mode 100644 index 00000000..d8b79967 --- /dev/null +++ b/internal/processor/base_processor.go @@ -0,0 +1,226 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package processor + +import ( + "context" + "fmt" + "log" + "math/big" + + "github.com/coinbase/rosetta-cli/internal/logger" + "github.com/coinbase/rosetta-cli/internal/reconciler" + "github.com/coinbase/rosetta-cli/internal/storage" + + "github.com/coinbase/rosetta-sdk-go/asserter" + "github.com/coinbase/rosetta-sdk-go/types" +) + +// BaseProcessor logs processed blocks +// and reconciles modified balances. +type BaseProcessor struct { + logger *logger.Logger + reconciler reconciler.Reconciler + asserter *asserter.Asserter + exemptAccounts []*reconciler.AccountCurrency +} + +// NewBaseProcessor constructs a basic Handler. +func NewBaseProcessor( + logger *logger.Logger, + reconciler reconciler.Reconciler, + asserter *asserter.Asserter, + exemptAccounts []*reconciler.AccountCurrency, +) *BaseProcessor { + return &BaseProcessor{ + logger: logger, + reconciler: reconciler, + asserter: asserter, + exemptAccounts: exemptAccounts, + } +} + +// BalanceChanges returns all balance changes for +// a particular block. All balance changes for a +// particular account are summed into a single +// storage.BalanceChanges struct. If a block is being +// orphaned, the opposite of each balance change is +// returned. +func (p *BaseProcessor) BalanceChanges( + ctx context.Context, + block *types.Block, + blockRemoved bool, +) ([]*storage.BalanceChange, error) { + balanceChanges := map[string]*storage.BalanceChange{} + for _, tx := range block.Transactions { + for _, op := range tx.Operations { + skip, err := p.skipOperation( + ctx, + op, + ) + if err != nil { + return nil, err + } + if skip { + continue + } + + amount := op.Amount + blockIdentifier := block.BlockIdentifier + if blockRemoved { + existing, ok := new(big.Int).SetString(amount.Value, 10) + if !ok { + return nil, fmt.Errorf("%s is not an integer", amount.Value) + } + + amount.Value = new(big.Int).Neg(existing).String() + blockIdentifier = block.ParentBlockIdentifier + } + + // Merge values by account and currency + // TODO: change balance key to be this + key := fmt.Sprintf("%s:%s", + storage.GetAccountKey(op.Account), + storage.GetCurrencyKey(op.Amount.Currency), + ) + + val, ok := balanceChanges[key] + if !ok { + balanceChanges[key] = &storage.BalanceChange{ + Account: op.Account, + Currency: op.Amount.Currency, + Difference: amount.Value, + Block: blockIdentifier, + } + continue + } + + newDifference, err := storage.AddStringValues(val.Difference, amount.Value) + if err != nil { + return nil, err + } + val.Difference = newDifference + balanceChanges[key] = val + } + } + + allChanges := []*storage.BalanceChange{} + for _, change := range balanceChanges { + allChanges = append(allChanges, change) + } + + return allChanges, nil +} + +// skipOperation returns a boolean indicating whether +// an operation should be processed. An operation will +// not be processed if it is considered unsuccessful +// or affects an exempt account. +func (p *BaseProcessor) skipOperation( + ctx context.Context, + op *types.Operation, +) (bool, error) { + successful, err := p.asserter.OperationSuccessful(op) + if err != nil { + // Should only occur if responses not validated + return false, err + } + + if !successful { + return true, nil + } + + if op.Account == nil { + return true, nil + } + + // Exempting account in BalanceChanges ensures that storage is not updated + // and that the account is not reconciled. + if p.accountExempt(ctx, op.Account, op.Amount.Currency) { + log.Printf("Skipping exempt account %+v\n", op.Account) + return true, nil + } + + return false, nil +} + +// BlockAdded is called by the syncer after a +// block is added. +func (p *BaseProcessor) BlockAdded( + ctx context.Context, + block *types.Block, +) error { + log.Printf("Adding block %+v\n", block.BlockIdentifier) + + // Log processed blocks and balance changes + if err := p.logger.BlockStream(ctx, block, false); err != nil { + return nil + } + + balanceChanges, err := p.BalanceChanges(ctx, block, false) + if err != nil { + return err + } + + if err := p.logger.BalanceStream(ctx, balanceChanges); err != nil { + return nil + } + + // Mark accounts for reconciliation...this may be + // blocking + return p.reconciler.QueueChanges(ctx, block.BlockIdentifier, balanceChanges) +} + +// BlockRemoved is called by the syncer after a +// block is removed. +func (p *BaseProcessor) BlockRemoved( + ctx context.Context, + block *types.Block, +) error { + log.Printf("Orphaning block %+v\n", block.BlockIdentifier) + + // Log processed blocks and balance changes + if err := p.logger.BlockStream(ctx, block, true); err != nil { + return nil + } + + balanceChanges, err := p.BalanceChanges(ctx, block, true) + if err != nil { + return err + } + + if err := p.logger.BalanceStream(ctx, balanceChanges); err != nil { + return nil + } + + return nil +} + +// accountExempt returns a boolean indicating if the provided +// account and currency are exempt from balance tracking and +// reconciliation. +func (p *BaseProcessor) accountExempt( + ctx context.Context, + account *types.AccountIdentifier, + currency *types.Currency, +) bool { + return reconciler.ContainsAccountCurrency( + p.exemptAccounts, + &reconciler.AccountCurrency{ + Account: account, + Currency: currency, + }, + ) +} diff --git a/internal/processor/processor.go b/internal/processor/processor.go new file mode 100644 index 00000000..3dab097f --- /dev/null +++ b/internal/processor/processor.go @@ -0,0 +1,21 @@ +package processor + +import ( + "context" + "github.com/coinbase/rosetta-sdk-go/types" +) + +// Processor is called at various times during the sync cycle +// to handle different events. It is common to write logs or +// perform reconciliation in the sync processor. +type Processor interface { + BlockAdded( + ctx context.Context, + block *types.Block, + ) error + + BlockRemoved( + ctx context.Context, + block *types.Block, + ) error +} diff --git a/internal/syncer/base_handler.go b/internal/syncer/base_handler.go deleted file mode 100644 index 93c73e7e..00000000 --- a/internal/syncer/base_handler.go +++ /dev/null @@ -1,108 +0,0 @@ -// Copyright 2020 Coinbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package syncer - -import ( - "context" - "log" - - "github.com/coinbase/rosetta-cli/internal/logger" - "github.com/coinbase/rosetta-cli/internal/reconciler" - "github.com/coinbase/rosetta-cli/internal/storage" - - "github.com/coinbase/rosetta-sdk-go/types" -) - -// BaseHandler logs processed blocks -// and reconciles modified balances. -type BaseHandler struct { - logger *logger.Logger - reconciler reconciler.Reconciler - exemptAccounts []*reconciler.AccountCurrency -} - -// NewBaseHandler constructs a basic Handler. -func NewBaseHandler( - logger *logger.Logger, - reconciler reconciler.Reconciler, - exemptAccounts []*reconciler.AccountCurrency, -) Handler { - return &BaseHandler{ - logger: logger, - reconciler: reconciler, - exemptAccounts: exemptAccounts, - } -} - -// BlockAdded is called by the syncer after a -// block is added. -func (h *BaseHandler) BlockAdded( - ctx context.Context, - block *types.Block, - balanceChanges []*storage.BalanceChange, -) error { - log.Printf("Adding block %+v\n", block.BlockIdentifier) - - // Log processed blocks and balance changes - if err := h.logger.BlockStream(ctx, block, false); err != nil { - return nil - } - - if err := h.logger.BalanceStream(ctx, balanceChanges); err != nil { - return nil - } - - // Mark accounts for reconciliation...this may be - // blocking - return h.reconciler.QueueChanges(ctx, block.BlockIdentifier, balanceChanges) -} - -// BlockRemoved is called by the syncer after a -// block is removed. -func (h *BaseHandler) BlockRemoved( - ctx context.Context, - block *types.Block, - balanceChanges []*storage.BalanceChange, -) error { - log.Printf("Orphaning block %+v\n", block.BlockIdentifier) - - // Log processed blocks and balance changes - if err := h.logger.BlockStream(ctx, block, true); err != nil { - return nil - } - - if err := h.logger.BalanceStream(ctx, balanceChanges); err != nil { - return nil - } - - return nil -} - -// AccountExempt returns a boolean indicating if the provided -// account and currency are exempt from balance tracking and -// reconciliation. -func (h *BaseHandler) AccountExempt( - ctx context.Context, - account *types.AccountIdentifier, - currency *types.Currency, -) bool { - return reconciler.ContainsAccountCurrency( - h.exemptAccounts, - &reconciler.AccountCurrency{ - Account: account, - Currency: currency, - }, - ) -} diff --git a/internal/syncer/stateful_syncer.go b/internal/syncer/stateful_syncer.go deleted file mode 100644 index 82135672..00000000 --- a/internal/syncer/stateful_syncer.go +++ /dev/null @@ -1,401 +0,0 @@ -// Copyright 2020 Coinbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package syncer - -import ( - "context" - "errors" - "fmt" - - "github.com/coinbase/rosetta-cli/internal/storage" - - "github.com/coinbase/rosetta-sdk-go/fetcher" - "github.com/coinbase/rosetta-sdk-go/types" -) - -// StatefulSyncer contains the logic that orchestrates -// block fetching, storage, and reconciliation. The -// stateful syncer is useful for creating an application -// where durability and consistency of data is important. -// The stateful syncer supports re-orgs out-of the box. -type StatefulSyncer struct { - network *types.NetworkIdentifier - storage *storage.BlockStorage - fetcher *fetcher.Fetcher - handler Handler - genesisBlock *types.BlockIdentifier -} - -// NewStateful returns a new Syncer. -func NewStateful( - network *types.NetworkIdentifier, - storage *storage.BlockStorage, - fetcher *fetcher.Fetcher, - handler Handler, -) *StatefulSyncer { - return &StatefulSyncer{ - network: network, - storage: storage, - fetcher: fetcher, - handler: handler, - } -} - -// SetStartIndex initializes the genesisBlock -// and attempts to set the newHeadIndex. -func (s *StatefulSyncer) SetStartIndex( - ctx context.Context, - startIndex int64, -) error { - networkStatus, err := s.fetcher.NetworkStatusRetry( - ctx, - s.network, - nil, - ) - if err != nil { - return err - } - - s.genesisBlock = networkStatus.GenesisBlockIdentifier - - if startIndex != -1 { - return s.newHeadIndex(ctx, startIndex) - } - - return nil -} - -// checkReorg determines if the block provided -// has the current head block identifier as its -// parent. If not, it is considered a reorg. -func (s *StatefulSyncer) checkReorg( - ctx context.Context, - tx storage.DatabaseTransaction, - block *types.Block, -) (bool, error) { - head, err := s.storage.GetHeadBlockIdentifier(ctx, tx) - if err == storage.ErrHeadBlockNotFound { - return false, nil - } else if err != nil { - return false, err - } - - if block.ParentBlockIdentifier.Index != head.Index { - return false, fmt.Errorf( - "Got block %d instead of %d", - block.BlockIdentifier.Index, - head.Index+1, - ) - } - - if block.ParentBlockIdentifier.Hash != head.Hash { - return true, nil - } - - return false, nil -} - -// storeBlockBalanceChanges updates the balance -// of each modified account if the operation affecting -// that account is successful. These modified -// accounts are returned to the reconciler -// for active reconciliation. -func (s *StatefulSyncer) storeBlockBalanceChanges( - ctx context.Context, - dbTx storage.DatabaseTransaction, - block *types.Block, - orphan bool, -) ([]*storage.BalanceChange, error) { - balanceChanges := make([]*storage.BalanceChange, 0) - - // Merge all changes for an account:currency - mergedChanges, err := BalanceChanges( - ctx, - s.fetcher.Asserter, - block, - orphan, - s.handler, - ) - if err != nil { - return nil, err - } - - for _, change := range mergedChanges { - balanceChange, err := s.storage.UpdateBalance( - ctx, - dbTx, - change.Account, - &types.Amount{ - Value: change.Difference, - Currency: change.Currency, - }, - change.Block, - ) - if err != nil { - return nil, err - } - - balanceChanges = append(balanceChanges, balanceChange) - } - - return balanceChanges, nil -} - -// orphanBlock removes a block from the database and reverts all its balance -// changes. -func (s *StatefulSyncer) orphanBlock( - ctx context.Context, - tx storage.DatabaseTransaction, - blockIdentifier *types.BlockIdentifier, -) ([]*storage.BalanceChange, error) { - block, err := s.storage.GetBlock(ctx, tx, blockIdentifier) - if err != nil { - return nil, err - } - - err = s.storage.StoreHeadBlockIdentifier(ctx, tx, block.ParentBlockIdentifier) - if err != nil { - return nil, err - } - - balanceChanges, err := s.storeBlockBalanceChanges(ctx, tx, block, true) - if err != nil { - return nil, err - } - - err = s.storage.RemoveBlock(ctx, tx, blockIdentifier) - if err != nil { - return nil, err - } - - return balanceChanges, nil -} - -// addBlock adds a block to the database and stores all balance changes. -func (s *StatefulSyncer) addBlock( - ctx context.Context, - tx storage.DatabaseTransaction, - block *types.Block, -) ([]*storage.BalanceChange, error) { - err := s.storage.StoreBlock(ctx, tx, block) - if err != nil { - return nil, err - } - - err = s.storage.StoreHeadBlockIdentifier(ctx, tx, block.BlockIdentifier) - if err != nil { - return nil, err - } - - balanceChanges, err := s.storeBlockBalanceChanges(ctx, tx, block, false) - if err != nil { - return nil, err - } - - return balanceChanges, nil -} - -// processBlock determines if a block should be added or the current -// head should be orphaned. -func (s *StatefulSyncer) processBlock( - ctx context.Context, - genesisIndex int64, - currIndex int64, - block *types.Block, -) ([]*storage.BalanceChange, int64, bool, error) { - tx := s.storage.NewDatabaseTransaction(ctx, true) - defer tx.Discard(ctx) - - reorg, err := s.checkReorg(ctx, tx, block) - if err != nil { - return nil, currIndex, false, err - } - - var balanceChanges []*storage.BalanceChange - var newIndex int64 - if reorg { - newIndex = currIndex - 1 - if newIndex == genesisIndex { - return nil, 0, false, errors.New("cannot orphan genesis block") - } - - head, err := s.storage.GetHeadBlockIdentifier(ctx, tx) - if err != nil { - return nil, currIndex, false, err - } - - balanceChanges, err = s.orphanBlock(ctx, tx, head) - if err != nil { - return nil, currIndex, false, err - } - } else { - balanceChanges, err = s.addBlock(ctx, tx, block) - if err != nil { - return nil, currIndex, false, err - } - - newIndex = currIndex + 1 - } - - err = tx.Commit(ctx) - if err != nil { - return nil, currIndex, false, err - } - - return balanceChanges, newIndex, reorg, nil -} - -// newHeadIndex reverts all blocks that have -// an index greater than newHeadIndex. This is particularly -// useful when debugging a server implementation because -// you don't need to restart validation from genesis. Instead, -// you can just restart validation at the block immediately -// before any erroneous block. -// -// TODO: batch transactions to orphan blocks -func (s *StatefulSyncer) newHeadIndex( - ctx context.Context, - newHeadIndex int64, -) error { - tx := s.storage.NewDatabaseTransaction(ctx, true) - defer tx.Discard(ctx) - - for { - head, err := s.storage.GetHeadBlockIdentifier(ctx, tx) - if err == storage.ErrHeadBlockNotFound { - return fmt.Errorf( - "cannot start syncing at %d, have not yet processed any blocks", - newHeadIndex, - ) - } else if err != nil { - return err - } - - if head.Index < newHeadIndex { - return fmt.Errorf( - "cannot start syncing at %d, have only processed %d blocks", - newHeadIndex, - head.Index, - ) - } - - if head.Index == newHeadIndex { - break - } - - _, err = s.orphanBlock(ctx, tx, head) - if err != nil { - return err - } - } - - return tx.Commit(ctx) -} - -// SyncRange syncs blocks from startIndex to endIndex, inclusive. -// This function handles re-orgs that may occur while syncing as long -// as the genesisIndex is not orphaned. -func (s *StatefulSyncer) SyncRange( - ctx context.Context, - startIndex int64, - endIndex int64, -) error { - blockMap, err := s.fetcher.BlockRange(ctx, s.network, startIndex, endIndex) - if err != nil { - return err - } - - currIndex := startIndex - for currIndex <= endIndex { - block, ok := blockMap[currIndex] - if !ok { // could happen in a reorg - block, err = s.fetcher.BlockRetry( - ctx, - s.network, - &types.PartialBlockIdentifier{ - Index: &currIndex, - }, - ) - if err != nil { - return err - } - } else { - // Anytime we re-fetch an index, we - // will need to make another call to the node - // as it is likely in a reorg. - delete(blockMap, currIndex) - } - - // Can't return balanceChanges without creating new variable - balanceChanges, newIndex, reorg, err := s.processBlock( - ctx, - s.genesisBlock.Index, - currIndex, - block, - ) - if err != nil { - return err - } - - currIndex = newIndex - - if reorg { - err = s.handler.BlockRemoved(ctx, block, balanceChanges) - } else { - err = s.handler.BlockAdded(ctx, block, balanceChanges) - } - if err != nil { - return err - } - } - - return nil -} - -// CurrentIndex returns the next index to sync. -func (s *StatefulSyncer) CurrentIndex( - ctx context.Context, -) (int64, error) { - tx := s.storage.NewDatabaseTransaction(ctx, false) - defer tx.Discard(ctx) - - var currentIndex int64 - head, err := s.storage.GetHeadBlockIdentifier(ctx, tx) - switch err { - case nil: - currentIndex = head.Index + 1 - case storage.ErrHeadBlockNotFound: - head = s.genesisBlock - currentIndex = head.Index - default: - return -1, err - } - - return currentIndex, nil -} - -// Network returns the syncer network. -func (s *StatefulSyncer) Network( - ctx context.Context, -) *types.NetworkIdentifier { - return s.network -} - -// Fetcher returns the syncer fetcher. -func (s *StatefulSyncer) Fetcher( - ctx context.Context, -) *fetcher.Fetcher { - return s.fetcher -} diff --git a/internal/syncer/stateful_syncer_test.go b/internal/syncer/stateful_syncer_test.go deleted file mode 100644 index ec5218bb..00000000 --- a/internal/syncer/stateful_syncer_test.go +++ /dev/null @@ -1,495 +0,0 @@ -// Copyright 2020 Coinbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package syncer - -import ( - "context" - "fmt" - "testing" - - "github.com/coinbase/rosetta-cli/internal/storage" - - "github.com/coinbase/rosetta-sdk-go/asserter" - "github.com/coinbase/rosetta-sdk-go/fetcher" - "github.com/coinbase/rosetta-sdk-go/types" - - "github.com/stretchr/testify/assert" -) - -var ( - networkIdentifier = &types.NetworkIdentifier{ - Blockchain: "blah", - Network: "testnet", - } - - currency = &types.Currency{ - Symbol: "Blah", - Decimals: 2, - } - - recipient = &types.AccountIdentifier{ - Address: "acct1", - } - - recipientAmount = &types.Amount{ - Value: "100", - Currency: currency, - } - - recipientOperation = &types.Operation{ - OperationIdentifier: &types.OperationIdentifier{ - Index: 0, - }, - Type: "Transfer", - Status: "Success", - Account: recipient, - Amount: recipientAmount, - } - - recipientFailureOperation = &types.Operation{ - OperationIdentifier: &types.OperationIdentifier{ - Index: 1, - }, - Type: "Transfer", - Status: "Failure", - Account: recipient, - Amount: recipientAmount, - } - - recipientTransaction = &types.Transaction{ - TransactionIdentifier: &types.TransactionIdentifier{ - Hash: "tx1", - }, - Operations: []*types.Operation{ - recipientOperation, - recipientFailureOperation, - }, - } - - sender = &types.AccountIdentifier{ - Address: "acct2", - } - - senderAmount = &types.Amount{ - Value: "-100", - Currency: currency, - } - - senderOperation = &types.Operation{ - OperationIdentifier: &types.OperationIdentifier{ - Index: 0, - }, - Type: "Transfer", - Status: "Success", - Account: sender, - Amount: senderAmount, - } - - senderTransaction = &types.Transaction{ - TransactionIdentifier: &types.TransactionIdentifier{ - Hash: "tx2", - }, - Operations: []*types.Operation{ - senderOperation, - }, - } - - orphanGenesis = &types.Block{ - BlockIdentifier: &types.BlockIdentifier{ - Hash: "1", - Index: 1, - }, - ParentBlockIdentifier: &types.BlockIdentifier{ - Hash: "0a", - Index: 0, - }, - Transactions: []*types.Transaction{}, - } - - blockSequence = []*types.Block{ - { // genesis - BlockIdentifier: &types.BlockIdentifier{ - Hash: "0", - Index: 0, - }, - ParentBlockIdentifier: &types.BlockIdentifier{ - Hash: "0", - Index: 0, - }, - }, - { - BlockIdentifier: &types.BlockIdentifier{ - Hash: "1", - Index: 1, - }, - ParentBlockIdentifier: &types.BlockIdentifier{ - Hash: "0", - Index: 0, - }, - Transactions: []*types.Transaction{ - recipientTransaction, - }, - }, - { // reorg - BlockIdentifier: &types.BlockIdentifier{ - Hash: "2", - Index: 2, - }, - ParentBlockIdentifier: &types.BlockIdentifier{ - Hash: "1a", - Index: 1, - }, - }, - { - BlockIdentifier: &types.BlockIdentifier{ - Hash: "1a", - Index: 1, - }, - ParentBlockIdentifier: &types.BlockIdentifier{ - Hash: "0", - Index: 0, - }, - }, - { - BlockIdentifier: &types.BlockIdentifier{ - Hash: "3", - Index: 3, - }, - ParentBlockIdentifier: &types.BlockIdentifier{ - Hash: "2", - Index: 2, - }, - Transactions: []*types.Transaction{ - senderTransaction, - }, - }, - { // invalid block - BlockIdentifier: &types.BlockIdentifier{ - Hash: "5", - Index: 5, - }, - ParentBlockIdentifier: &types.BlockIdentifier{ - Hash: "4", - Index: 4, - }, - }, - } - - operationStatuses = []*types.OperationStatus{ - { - Status: "Success", - Successful: true, - }, - { - Status: "Failure", - Successful: false, - }, - } - - networkStatusResponse = &types.NetworkStatusResponse{ - GenesisBlockIdentifier: &types.BlockIdentifier{ - Index: 0, - Hash: "block 0", - }, - CurrentBlockIdentifier: &types.BlockIdentifier{ - Index: 10000, - Hash: "block 1000", - }, - CurrentBlockTimestamp: asserter.MinUnixEpoch + 1, - Peers: []*types.Peer{ - { - PeerID: "peer 1", - }, - }, - } - - networkOptionsResponse = &types.NetworkOptionsResponse{ - Version: &types.Version{ - RosettaVersion: "1.3.1", - NodeVersion: "1.0", - }, - Allow: &types.Allow{ - OperationStatuses: operationStatuses, - OperationTypes: []string{ - "Transfer", - }, - }, - } -) - -func assertCurrentIndex( - ctx context.Context, - t *testing.T, - syncer *StatefulSyncer, - expectedCurrentIndex int64, -) { - currentIndex, err := syncer.CurrentIndex(ctx) - assert.NoError(t, err) - assert.Equal(t, expectedCurrentIndex, currentIndex) -} - -func TestProcessBlock(t *testing.T) { - ctx := context.Background() - - newDir, err := storage.CreateTempDir() - assert.NoError(t, err) - defer storage.RemoveTempDir(*newDir) - - database, err := storage.NewBadgerStorage(ctx, *newDir) - assert.NoError(t, err) - defer database.Close(ctx) - - blockStorage := storage.NewBlockStorage(ctx, database) - asserter, err := asserter.NewClientWithResponses( - networkIdentifier, - networkStatusResponse, - networkOptionsResponse, - ) - assert.NotNil(t, asserter) - assert.NoError(t, err) - - fetcher := &fetcher.Fetcher{ - Asserter: asserter, - } - syncer := NewStateful(nil, blockStorage, fetcher, nil) - currIndex := int64(0) - genesisIndex := blockSequence[0].BlockIdentifier.Index - - t.Run("No block exists", func(t *testing.T) { - // Add genesis block - balanceChanges, newIndex, reorg, err := syncer.processBlock( - ctx, - genesisIndex, - currIndex, - blockSequence[0], - ) - currIndex = newIndex - assert.False(t, reorg) - assert.Equal(t, int64(1), currIndex) - assert.Equal(t, 0, len(balanceChanges)) - assert.NoError(t, err) - - tx := syncer.storage.NewDatabaseTransaction(ctx, false) - head, err := syncer.storage.GetHeadBlockIdentifier(ctx, tx) - tx.Discard(ctx) - assert.Equal(t, blockSequence[0].BlockIdentifier, head) - assert.NoError(t, err) - - assertCurrentIndex(ctx, t, syncer, currIndex) - }) - - t.Run("Orphan genesis", func(t *testing.T) { - balanceChanges, newIndex, reorg, err := syncer.processBlock( - ctx, - genesisIndex, - currIndex, - orphanGenesis, - ) - - assert.False(t, reorg) - assert.Equal(t, int64(0), newIndex) - assert.Equal(t, 0, len(balanceChanges)) - assert.EqualError(t, err, "cannot orphan genesis block") - - tx := syncer.storage.NewDatabaseTransaction(ctx, false) - head, err := syncer.storage.GetHeadBlockIdentifier(ctx, tx) - tx.Discard(ctx) - assert.Equal(t, blockSequence[0].BlockIdentifier, head) - assert.NoError(t, err) - }) - - t.Run("Block exists, no reorg", func(t *testing.T) { - balanceChanges, newIndex, reorg, err := syncer.processBlock( - ctx, - genesisIndex, - currIndex, - blockSequence[1], - ) - currIndex = newIndex - assert.False(t, reorg) - assert.Equal(t, int64(2), currIndex) - assert.Equal(t, []*storage.BalanceChange{ - { - Account: &types.AccountIdentifier{ - Address: "acct1", - }, - Currency: currency, - Block: blockSequence[1].BlockIdentifier, - Difference: "100", - }, - }, balanceChanges) - assert.NoError(t, err) - - tx := syncer.storage.NewDatabaseTransaction(ctx, false) - head, err := syncer.storage.GetHeadBlockIdentifier(ctx, tx) - assert.Equal(t, blockSequence[1].BlockIdentifier, head) - assert.NoError(t, err) - - amounts, block, err := syncer.storage.GetBalance(ctx, tx, recipient) - tx.Discard(ctx) - assert.Equal(t, map[string]*types.Amount{ - storage.GetCurrencyKey(currency): recipientAmount, - }, amounts) - assert.Equal(t, blockSequence[1].BlockIdentifier, block) - assert.NoError(t, err) - - assertCurrentIndex(ctx, t, syncer, currIndex) - }) - - t.Run("Orphan block", func(t *testing.T) { - // Orphan block - balanceChanges, newIndex, reorg, err := syncer.processBlock( - ctx, - genesisIndex, - currIndex, - blockSequence[2], - ) - currIndex = newIndex - assert.True(t, reorg) - assert.Equal(t, int64(1), currIndex) - assert.Equal(t, []*storage.BalanceChange{ - { - Account: &types.AccountIdentifier{ - Address: "acct1", - }, - Currency: currency, - Block: blockSequence[0].BlockIdentifier, - Difference: "-100", - }, - }, balanceChanges) - assert.NoError(t, err) - assertCurrentIndex(ctx, t, syncer, currIndex) - - // Assert head is back to genesis - tx := syncer.storage.NewDatabaseTransaction(ctx, false) - head, err := syncer.storage.GetHeadBlockIdentifier(ctx, tx) - assert.Equal(t, blockSequence[0].BlockIdentifier, head) - assert.NoError(t, err) - - // Assert that balance change was reverted - // only by the successful operation - zeroAmount := map[string]*types.Amount{ - storage.GetCurrencyKey(currency): { - Value: "0", - Currency: currency, - }, - } - amounts, block, err := syncer.storage.GetBalance(ctx, tx, recipient) - assert.Equal(t, zeroAmount, amounts) - assert.Equal(t, blockSequence[0].BlockIdentifier, block) - assert.NoError(t, err) - - // Assert block is gone - orphanBlock, err := syncer.storage.GetBlock(ctx, tx, blockSequence[1].BlockIdentifier) - assert.Nil(t, orphanBlock) - assert.EqualError(t, err, fmt.Errorf( - "%w %+v", - storage.ErrBlockNotFound, - blockSequence[1].BlockIdentifier, - ).Error()) - tx.Discard(ctx) - - // Process new block - balanceChanges, currIndex, reorg, err = syncer.processBlock( - ctx, - genesisIndex, - currIndex, - blockSequence[3], - ) - assert.False(t, reorg) - assert.Equal(t, int64(2), currIndex) - assert.Equal(t, 0, len(balanceChanges)) - assert.NoError(t, err) - assertCurrentIndex(ctx, t, syncer, currIndex) - - tx = syncer.storage.NewDatabaseTransaction(ctx, false) - head, err = syncer.storage.GetHeadBlockIdentifier(ctx, tx) - tx.Discard(ctx) - assert.Equal(t, blockSequence[3].BlockIdentifier, head) - assert.NoError(t, err) - - balanceChanges, currIndex, reorg, err = syncer.processBlock( - ctx, - genesisIndex, - currIndex, - blockSequence[2], - ) - assert.False(t, reorg) - assert.Equal(t, int64(3), currIndex) - assert.Equal(t, 0, len(balanceChanges)) - assert.NoError(t, err) - - tx = syncer.storage.NewDatabaseTransaction(ctx, false) - head, err = syncer.storage.GetHeadBlockIdentifier(ctx, tx) - assert.Equal(t, blockSequence[2].BlockIdentifier, head) - assert.NoError(t, err) - - amounts, block, err = syncer.storage.GetBalance(ctx, tx, recipient) - tx.Discard(ctx) - assert.Equal(t, zeroAmount, amounts) - assert.Equal(t, blockSequence[0].BlockIdentifier, block) - assert.NoError(t, err) - }) - - t.Run("Block with invalid transaction", func(t *testing.T) { - balanceChanges, currIndex, reorg, err := syncer.processBlock( - ctx, - genesisIndex, - currIndex, - blockSequence[4], - ) - assert.False(t, reorg) - assert.Equal(t, int64(3), currIndex) - assert.Equal(t, 0, len(balanceChanges)) - assert.Contains(t, err.Error(), storage.ErrNegativeBalance.Error()) - - tx := syncer.storage.NewDatabaseTransaction(ctx, false) - head, err := syncer.storage.GetHeadBlockIdentifier(ctx, tx) - tx.Discard(ctx) - assert.NoError(t, err) - assert.Equal(t, blockSequence[2].BlockIdentifier, head) - }) - - t.Run("Out of order block", func(t *testing.T) { - balanceChanges, newIndex, reorg, err := syncer.processBlock( - ctx, - genesisIndex, - currIndex, - blockSequence[5], - ) - currIndex = newIndex - assert.False(t, reorg) - assert.Equal(t, int64(3), currIndex) - assert.Equal(t, 0, len(balanceChanges)) - assert.EqualError(t, err, "Got block 5 instead of 3") - - tx := syncer.storage.NewDatabaseTransaction(ctx, false) - head, err := syncer.storage.GetHeadBlockIdentifier(ctx, tx) - tx.Discard(ctx) - assert.Equal(t, blockSequence[2].BlockIdentifier, head) - assert.NoError(t, err) - }) - - t.Run("Revert all blocks after genesis", func(t *testing.T) { - err := syncer.newHeadIndex(ctx, genesisIndex) - assert.NoError(t, err) - - tx := syncer.storage.NewDatabaseTransaction(ctx, false) - head, err := syncer.storage.GetHeadBlockIdentifier(ctx, tx) - tx.Discard(ctx) - assert.Equal(t, blockSequence[0].BlockIdentifier, head) - assert.NoError(t, err) - }) -} diff --git a/internal/syncer/stateless_syncer.go b/internal/syncer/stateless_syncer.go deleted file mode 100644 index 5fc8f73b..00000000 --- a/internal/syncer/stateless_syncer.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright 2020 Coinbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package syncer - -import ( - "context" - - "github.com/coinbase/rosetta-sdk-go/fetcher" - "github.com/coinbase/rosetta-sdk-go/types" -) - -// StatelessSyncer contains the logic that orchestrates -// stateless block fetching and reconciliation. The stateless -// syncer is useful for performing a quick check over a range of -// blocks without needed to sync all blocks up to the start of -// the range (a common pattern when debugging). It is important -// to note that the stateless syncer does not support reorgs nor -// does it save where it is on restart. -type StatelessSyncer struct { - network *types.NetworkIdentifier - fetcher *fetcher.Fetcher - handler Handler - currentIndex int64 -} - -// NewStateless returns a new Syncer. -func NewStateless( - network *types.NetworkIdentifier, - fetcher *fetcher.Fetcher, - handler Handler, -) *StatelessSyncer { - return &StatelessSyncer{ - network: network, - fetcher: fetcher, - handler: handler, - } -} - -// SetStartIndex initializes the current block index -// with the genesis block index if it is -1. -func (s *StatelessSyncer) SetStartIndex( - ctx context.Context, - startIndex int64, -) error { - if startIndex != -1 { - s.currentIndex = startIndex - return nil - } - - // Sync from genesis + 1 - networkStatus, err := s.fetcher.NetworkStatusRetry( - ctx, - s.network, - nil, - ) - if err != nil { - return err - } - - // Don't sync genesis block because balance lookup will not - // work. - s.currentIndex = networkStatus.GenesisBlockIdentifier.Index + 1 - return nil -} - -// SyncRange syncs blocks from startIndex to endIndex, inclusive. -// This function does NOT handle re-orgs. If you want re-org support, -// checkout the StatefulSyncer. -func (s *StatelessSyncer) SyncRange( - ctx context.Context, - startIndex int64, - endIndex int64, -) error { - blockMap, err := s.fetcher.BlockRange(ctx, s.network, startIndex, endIndex) - if err != nil { - return err - } - - for i := startIndex; i <= endIndex; i++ { - block := blockMap[i] - changes, err := BalanceChanges( - ctx, - s.fetcher.Asserter, - block, - false, - s.handler, - ) - if err != nil { - return err - } - - err = s.handler.BlockAdded( - ctx, - block, - changes, - ) - if err != nil { - return err - } - } - - s.currentIndex = endIndex + 1 - - return nil -} - -// CurrentIndex returns the next index to sync. -func (s *StatelessSyncer) CurrentIndex( - ctx context.Context, -) (int64, error) { - return s.currentIndex, nil -} - -// Network returns the syncer network. -func (s *StatelessSyncer) Network( - ctx context.Context, -) *types.NetworkIdentifier { - return s.network -} - -// Fetcher returns the syncer fetcher. -func (s *StatelessSyncer) Fetcher( - ctx context.Context, -) *fetcher.Fetcher { - return s.fetcher -} diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index 3e52b362..a3926428 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -16,13 +16,13 @@ package syncer import ( "context" + "errors" "fmt" "log" - "math/big" + "reflect" - "github.com/coinbase/rosetta-cli/internal/storage" + "github.com/coinbase/rosetta-cli/internal/processor" - "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/types" ) @@ -33,85 +33,197 @@ const ( maxSync = 1000 ) -// Syncer defines an interface for syncing some -// range of blocks. -type Syncer interface { - SetStartIndex( - ctx context.Context, - startIndex int64, - ) error - - CurrentIndex( - ctx context.Context, - ) (int64, error) - - SyncRange( - ctx context.Context, - rangeStart int64, - rangeEnd int64, - ) error - - Network( - ctx context.Context, - ) *types.NetworkIdentifier - - Fetcher( - ctx context.Context, - ) *fetcher.Fetcher +type Syncer struct { + network *types.NetworkIdentifier + fetcher *fetcher.Fetcher + processor processor.Processor + cancel context.CancelFunc + + // Used to keep track of sync state + genesisBlock *types.BlockIdentifier + currentBlock *types.BlockIdentifier } -// NextSyncableRange returns the next range of indexes to sync +func NewSyncer( + network *types.NetworkIdentifier, + fetcher *fetcher.Fetcher, + processor processor.Processor, + cancel context.CancelFunc, +) *Syncer { + return &Syncer{ + network: network, + fetcher: fetcher, + processor: processor, + cancel: cancel, + } +} + +func (s *Syncer) setStart( + ctx context.Context, + index int64, +) error { + networkStatus, err := s.fetcher.NetworkStatusRetry( + ctx, + s.network, + nil, + ) + if err != nil { + return err + } + + s.genesisBlock = networkStatus.GenesisBlockIdentifier + + if index != -1 { + // Get block at index + block, err := s.fetcher.BlockRetry(ctx, s.network, &types.PartialBlockIdentifier{Index: &index}) + if err != nil { + return err + } + + s.currentBlock = block.BlockIdentifier + return nil + } + + s.currentBlock = networkStatus.GenesisBlockIdentifier + return nil +} + +func (s *Syncer) head( + ctx context.Context, +) (*types.BlockIdentifier, error) { + if s.currentBlock == nil { + return nil, errors.New("start block not set") + } + + return s.currentBlock, nil +} + +// nextSyncableRange returns the next range of indexes to sync // based on what the last processed block in storage is and // the contents of the network status response. -func NextSyncableRange( +func (s *Syncer) nextSyncableRange( ctx context.Context, - s Syncer, endIndex int64, ) (int64, int64, bool, error) { - currentIndex, err := s.CurrentIndex(ctx) + head, err := s.head(ctx) if err != nil { - return -1, -1, false, fmt.Errorf("%w: unable to get current index", err) + return -1, -1, false, fmt.Errorf("%w: unable to get current head", err) } if endIndex == -1 { - networkStatus, err := s.Fetcher(ctx).NetworkStatusRetry( + networkStatus, err := s.fetcher.NetworkStatusRetry( ctx, - s.Network(ctx), + s.network, nil, ) if err != nil { return -1, -1, false, fmt.Errorf("%w: unable to get network status", err) } - return currentIndex, networkStatus.CurrentBlockIdentifier.Index, false, nil + return head.Index, networkStatus.CurrentBlockIdentifier.Index, false, nil } - if currentIndex >= endIndex { + if head.Index >= endIndex { return -1, -1, true, nil } - return currentIndex, endIndex, false, nil + return head.Index, endIndex, false, nil +} + +func (s *Syncer) removeBlock( + ctx context.Context, + block *types.Block, +) (bool, error) { + // Get current block + head, err := s.head(ctx) + if err != nil { + return false, fmt.Errorf("%w: unable to get current head", err) + } + + // Ensure processing correct index + if block.ParentBlockIdentifier.Index != head.Index { + return false, fmt.Errorf( + "Got block %d instead of %d", + block.BlockIdentifier.Index, + head.Index+1, + ) + } + + // Check if block parent is head + if !reflect.DeepEqual(block.ParentBlockIdentifier, head) { + return true, nil + } + + return false, nil +} + +func (s *Syncer) syncRange( + ctx context.Context, + startIndex int64, + endIndex int64, +) error { + blockMap, err := s.fetcher.BlockRange(ctx, s.network, startIndex, endIndex) + if err != nil { + return err + } + + currIndex := startIndex + for currIndex <= endIndex { + block, ok := blockMap[currIndex] + if !ok { // could happen in a reorg + block, err = s.fetcher.BlockRetry( + ctx, + s.network, + &types.PartialBlockIdentifier{ + Index: &currIndex, + }, + ) + if err != nil { + return err + } + } else { + // Anytime we re-fetch an index, we + // will need to make another call to the node + // as it is likely in a reorg. + delete(blockMap, currIndex) + } + + shouldRemove, err := s.removeBlock(ctx, block) + if err != nil { + return err + } + + if shouldRemove { + currIndex-- + err = s.processor.BlockRemoved(ctx, block) + } else { + currIndex++ + err = s.processor.BlockAdded(ctx, block) + } + if err != nil { + return err + } + } + + return nil } // Sync cycles endlessly until there is an error // or the requested range is synced. -func Sync( +func (s *Syncer) Sync( ctx context.Context, - cancel context.CancelFunc, - s Syncer, startIndex int64, endIndex int64, ) error { - defer cancel() + defer s.cancel() - if err := s.SetStartIndex(ctx, startIndex); err != nil { + if err := s.setStart(ctx, startIndex); err != nil { return fmt.Errorf("%w: unable to set start index", err) } for { - rangeStart, rangeEnd, halt, err := NextSyncableRange( + rangeStart, rangeEnd, halt, err := s.nextSyncableRange( ctx, - s, endIndex, ) if err != nil { @@ -127,7 +239,7 @@ func Sync( log.Printf("Syncing %d-%d\n", rangeStart, rangeEnd) - err = s.SyncRange(ctx, rangeStart, rangeEnd) + err = s.syncRange(ctx, rangeStart, rangeEnd) if err != nil { return fmt.Errorf("%w: unable to sync range %d-%d", err, rangeStart, rangeEnd) } @@ -140,133 +252,3 @@ func Sync( log.Printf("Finished syncing %d-%d\n", startIndex, endIndex) return nil } - -// Handler is called at various times during the sync cycle -// to handle different events. It is common to write logs or -// perform reconciliation in the sync handler. -type Handler interface { - BlockAdded( - ctx context.Context, - block *types.Block, - changes []*storage.BalanceChange, - ) error - - BlockRemoved( - ctx context.Context, - block *types.Block, - changes []*storage.BalanceChange, - ) error - - AccountExempt( - ctx context.Context, - account *types.AccountIdentifier, - current *types.Currency, - ) bool -} - -// BalanceChanges returns all balance changes for -// a particular block. All balance changes for a -// particular account are summed into a single -// storage.BalanceChanges struct. If a block is being -// orphaned, the opposite of each balance change is -// returned. -func BalanceChanges( - ctx context.Context, - asserter *asserter.Asserter, - block *types.Block, - orphan bool, - handler Handler, -) ([]*storage.BalanceChange, error) { - balanceChanges := map[string]*storage.BalanceChange{} - for _, tx := range block.Transactions { - for _, op := range tx.Operations { - skip, err := skipOperation( - ctx, - asserter, - handler, - op, - ) - if err != nil { - return nil, err - } - if skip { - continue - } - - amount := op.Amount - blockIdentifier := block.BlockIdentifier - if orphan { - existing, ok := new(big.Int).SetString(amount.Value, 10) - if !ok { - return nil, fmt.Errorf("%s is not an integer", amount.Value) - } - - amount.Value = new(big.Int).Neg(existing).String() - blockIdentifier = block.ParentBlockIdentifier - } - - // Merge values by account and currency - // TODO: change balance key to be this - key := fmt.Sprintf("%s:%s", - storage.GetAccountKey(op.Account), - storage.GetCurrencyKey(op.Amount.Currency), - ) - - val, ok := balanceChanges[key] - if !ok { - balanceChanges[key] = &storage.BalanceChange{ - Account: op.Account, - Currency: op.Amount.Currency, - Difference: amount.Value, - Block: blockIdentifier, - } - continue - } - - newDifference, err := storage.AddStringValues(val.Difference, amount.Value) - if err != nil { - return nil, err - } - val.Difference = newDifference - balanceChanges[key] = val - } - } - - allChanges := []*storage.BalanceChange{} - for _, change := range balanceChanges { - allChanges = append(allChanges, change) - } - - return allChanges, nil -} - -func skipOperation( - ctx context.Context, - asserter *asserter.Asserter, - handler Handler, - op *types.Operation, -) (bool, error) { - successful, err := asserter.OperationSuccessful(op) - if err != nil { - // Should only occur if responses not validated - return false, err - } - - if !successful { - return true, nil - } - - if op.Account == nil { - return true, nil - } - - // Exempting account in BalanceChanges ensures that storage is not updated - // and that the account is not reconciled. If a handler is not provided, - // no account will be marked exempt. - if handler != nil && handler.AccountExempt(ctx, op.Account, op.Amount.Currency) { - log.Printf("Skipping exempt account %+v\n", op.Account) - return true, nil - } - - return false, nil -} diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.go deleted file mode 100644 index 3fa7e92c..00000000 --- a/internal/syncer/syncer_test.go +++ /dev/null @@ -1,234 +0,0 @@ -// Copyright 2020 Coinbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package syncer - -import ( - "context" - "testing" - - "github.com/coinbase/rosetta-cli/internal/reconciler" - "github.com/coinbase/rosetta-cli/internal/storage" - - "github.com/coinbase/rosetta-sdk-go/asserter" - "github.com/coinbase/rosetta-sdk-go/types" - "github.com/stretchr/testify/assert" -) - -func simpleTransactionFactory( - hash string, - address string, - value string, - currency *types.Currency, -) *types.Transaction { - return &types.Transaction{ - TransactionIdentifier: &types.TransactionIdentifier{ - Hash: hash, - }, - Operations: []*types.Operation{ - { - OperationIdentifier: &types.OperationIdentifier{ - Index: 0, - }, - Type: "Transfer", - Status: "Success", - Account: &types.AccountIdentifier{ - Address: address, - }, - Amount: &types.Amount{ - Value: value, - Currency: currency, - }, - }, - }, - } -} - -func TestBalanceChanges(t *testing.T) { - var tests = map[string]struct { - block *types.Block - orphan bool - changes []*storage.BalanceChange - exemptAccounts []*reconciler.AccountCurrency - err error - }{ - "simple block": { - block: &types.Block{ - BlockIdentifier: &types.BlockIdentifier{ - Hash: "1", - Index: 1, - }, - ParentBlockIdentifier: &types.BlockIdentifier{ - Hash: "0", - Index: 0, - }, - Transactions: []*types.Transaction{ - recipientTransaction, - }, - Timestamp: asserter.MinUnixEpoch + 1, - }, - orphan: false, - changes: []*storage.BalanceChange{ - { - Account: recipient, - Currency: currency, - Block: &types.BlockIdentifier{ - Hash: "1", - Index: 1, - }, - Difference: "100", - }, - }, - err: nil, - }, - "simple block account exempt": { - block: &types.Block{ - BlockIdentifier: &types.BlockIdentifier{ - Hash: "1", - Index: 1, - }, - ParentBlockIdentifier: &types.BlockIdentifier{ - Hash: "0", - Index: 0, - }, - Transactions: []*types.Transaction{ - recipientTransaction, - }, - Timestamp: asserter.MinUnixEpoch + 1, - }, - orphan: false, - changes: []*storage.BalanceChange{}, - exemptAccounts: []*reconciler.AccountCurrency{ - { - Account: recipient, - Currency: currency, - }, - }, - err: nil, - }, - "single account sum block": { - block: &types.Block{ - BlockIdentifier: &types.BlockIdentifier{ - Hash: "1", - Index: 1, - }, - ParentBlockIdentifier: &types.BlockIdentifier{ - Hash: "0", - Index: 0, - }, - Transactions: []*types.Transaction{ - simpleTransactionFactory("tx1", "addr1", "100", currency), - simpleTransactionFactory("tx2", "addr1", "150", currency), - simpleTransactionFactory("tx3", "addr2", "150", currency), - }, - Timestamp: asserter.MinUnixEpoch + 1, - }, - orphan: false, - changes: []*storage.BalanceChange{ - { - Account: &types.AccountIdentifier{ - Address: "addr1", - }, - Currency: currency, - Block: &types.BlockIdentifier{ - Hash: "1", - Index: 1, - }, - Difference: "250", - }, - { - Account: &types.AccountIdentifier{ - Address: "addr2", - }, - Currency: currency, - Block: &types.BlockIdentifier{ - Hash: "1", - Index: 1, - }, - Difference: "150", - }, - }, - err: nil, - }, - "single account sum orphan block": { - block: &types.Block{ - BlockIdentifier: &types.BlockIdentifier{ - Hash: "1", - Index: 1, - }, - ParentBlockIdentifier: &types.BlockIdentifier{ - Hash: "0", - Index: 0, - }, - Transactions: []*types.Transaction{ - simpleTransactionFactory("tx1", "addr1", "100", currency), - simpleTransactionFactory("tx2", "addr1", "150", currency), - simpleTransactionFactory("tx3", "addr2", "150", currency), - }, - Timestamp: asserter.MinUnixEpoch + 1, - }, - orphan: true, - changes: []*storage.BalanceChange{ - { - Account: &types.AccountIdentifier{ - Address: "addr1", - }, - Currency: currency, - Block: &types.BlockIdentifier{ - Hash: "0", - Index: 0, - }, - Difference: "-250", - }, - { - Account: &types.AccountIdentifier{ - Address: "addr2", - }, - Currency: currency, - Block: &types.BlockIdentifier{ - Hash: "0", - Index: 0, - }, - Difference: "-150", - }, - }, - err: nil, - }, - } - - ctx := context.Background() - asserter, err := asserter.NewClientWithResponses( - networkIdentifier, - networkStatusResponse, - networkOptionsResponse, - ) - assert.NotNil(t, asserter) - assert.NoError(t, err) - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - handler := NewBaseHandler(nil, nil, test.exemptAccounts) - changes, err := BalanceChanges( - ctx, - asserter, - test.block, - test.orphan, - handler, - ) - - assert.ElementsMatch(t, test.changes, changes) - assert.Equal(t, test.err, err) - }) - } -} From 0b9b7b5433fbb9f02f29dff65723f649e8a08272 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Sun, 3 May 2020 17:10:40 -0700 Subject: [PATCH 03/31] Unifying reconciler --- cmd/check_complete.go | 1 + internal/processor/processor.go | 6 + internal/reconciler/reconciler.go | 575 +++++++++++++++++- internal/reconciler/reconciler_test.go | 184 ------ internal/reconciler/stateful_reconciler.go | 536 ---------------- .../reconciler/stateful_reconciler_test.go | 243 -------- internal/reconciler/stateless_reconciler.go | 238 -------- 7 files changed, 575 insertions(+), 1208 deletions(-) delete mode 100644 internal/reconciler/reconciler_test.go delete mode 100644 internal/reconciler/stateful_reconciler.go delete mode 100644 internal/reconciler/stateful_reconciler_test.go delete mode 100644 internal/reconciler/stateless_reconciler.go diff --git a/cmd/check_complete.go b/cmd/check_complete.go index 190ddc3b..329c1ac4 100644 --- a/cmd/check_complete.go +++ b/cmd/check_complete.go @@ -75,6 +75,7 @@ historical balance lookup should set this to false.`, } func runCheckCompleteCmd(cmd *cobra.Command, args []string) { + // TODO: if no directory passed in, create a temporary one ctx, cancel := context.WithCancel(context.Background()) exemptAccounts, err := loadAccounts(ExemptFile) diff --git a/internal/processor/processor.go b/internal/processor/processor.go index 3dab097f..7e12f8e6 100644 --- a/internal/processor/processor.go +++ b/internal/processor/processor.go @@ -2,13 +2,19 @@ package processor import ( "context" + "github.com/coinbase/rosetta-sdk-go/types" ) // Processor is called at various times during the sync cycle // to handle different events. It is common to write logs or // perform reconciliation in the sync processor. + +// TODO: move back to sync handler...create struct that is SyncHandler and Reconciler Handler type Processor interface { + // TODO: if account appears for the first time, sync its balance at block previous if + // lookup by balance enabled. If not enabled, then warn that must sync from genesis...then + // we can use same logic for all use cases. :fire: BlockAdded( ctx context.Context, block *types.Block, diff --git a/internal/reconciler/reconciler.go b/internal/reconciler/reconciler.go index d8f778f8..471f6a50 100644 --- a/internal/reconciler/reconciler.go +++ b/internal/reconciler/reconciler.go @@ -16,25 +16,586 @@ package reconciler import ( "context" + "errors" "fmt" + "log" "reflect" + "sync" + "time" + "github.com/coinbase/rosetta-cli/internal/logger" "github.com/coinbase/rosetta-cli/internal/storage" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/types" + "golang.org/x/sync/errgroup" ) -// Reconciler defines an interface for comparing -// computed balances with node balances. -type Reconciler interface { - QueueChanges( +const ( + // backlogThreshold is the limit of account lookups + // that can be enqueued to reconcile before new + // requests are dropped. + // TODO: Make configurable + backlogThreshold = 1000 + + // waitToCheckDiff is the syncing difference (live-head) + // to retry instead of exiting. In other words, if the + // processed head is behind the live head by < + // waitToCheckDiff we should try again after sleeping. + // TODO: Make configurable + waitToCheckDiff = 10 + + // waitToCheckDiffSleep is the amount of time to wait + // to check a balance difference if the syncer is within + // waitToCheckDiff from the block a balance was queried at. + waitToCheckDiffSleep = 5 * time.Second + + // activeReconciliation is included in the reconciliation + // error message if reconciliation failed during active + // reconciliation. + activeReconciliation = "ACTIVE" + + // inactiveReconciliation is included in the reconciliation + // error message if reconciliation failed during inactive + // reconciliation. + inactiveReconciliation = "INACTIVE" + + // zeroString is a string of value 0. + zeroString = "0" + + // inactiveReconciliationSleep is used as the time.Duration + // to sleep when there are no seen accounts to reconcile. + inactiveReconciliationSleep = 5 * time.Second + + // inactiveReconciliationRequiredDepth is the minimum + // number of blocks the reconciler should wait between + // inactive reconciliations. + // TODO: make configurable + inactiveReconciliationRequiredDepth = 500 +) + +var ( + // ErrHeadBlockBehindLive is returned when the processed + // head is behind the live head. Sometimes, it is + // preferrable to sleep and wait to catch up when + // we are close to the live head (waitToCheckDiff). + ErrHeadBlockBehindLive = errors.New("head block behind") + + // ErrAccountUpdated is returned when the + // account was updated at a height later than + // the live height (when the account balance was fetched). + ErrAccountUpdated = errors.New("account updated") + + // ErrBlockGone is returned when the processed block + // head is greater than the live head but the block + // does not exist in the store. This likely means + // that the block was orphaned. + ErrBlockGone = errors.New("block gone") +) + +type ReconcilerHandler interface { + BlockExists( ctx context.Context, block *types.BlockIdentifier, - changes []*storage.BalanceChange, - ) error + ) (bool, error) + + CurrentBlock( + ctx context.Context, + ) (*types.BlockIdentifier, error) + + // always compare current balance...just need to set with previous seen if starting + // in middle + // TODO: always pass in head block to do lookup in case we need to fetch balance + // on previous block + AccountBalance( + ctx context.Context, + account *types.AccountIdentifier, + currency *types.Currency, + ) (*types.Amount, *types.BlockIdentifier, error) +} + +// Reconciler contains all logic to reconcile balances of +// types.AccountIdentifiers returned in types.Operations +// by a Rosetta Server. +type Reconciler struct { + network *types.NetworkIdentifier + handler ReconcilerHandler + fetcher *fetcher.Fetcher + logger *logger.Logger + accountConcurrency uint64 + lookupBalanceByBlock bool + haltOnReconciliationError bool + interestingAccounts []*AccountCurrency + changeQueue chan *storage.BalanceChange + + // highWaterMark is used to skip requests when + // we are very far behind the live head. + highWaterMark int64 + + // seenAccts are stored for inactive account + // reconciliation. + seenAccts []*AccountCurrency + inactiveQueue []*storage.BalanceChange + + // inactiveQueueMutex needed because we can't peek at the tip + // of a channel to determine when it is ready to look at. + inactiveQueueMutex sync.Mutex +} + +// NewReconciler creates a new Reconciler. +func NewReconciler( + network *types.NetworkIdentifier, + handler ReconcilerHandler, + fetcher *fetcher.Fetcher, + logger *logger.Logger, + accountConcurrency uint64, + lookupBalanceByBlock bool, + haltOnReconciliationError bool, + interestingAccounts []*AccountCurrency, +) *Reconciler { + r := &Reconciler{ + network: network, + handler: handler, + fetcher: fetcher, + logger: logger, + accountConcurrency: accountConcurrency, + lookupBalanceByBlock: lookupBalanceByBlock, + haltOnReconciliationError: haltOnReconciliationError, + interestingAccounts: interestingAccounts, + highWaterMark: -1, + seenAccts: make([]*AccountCurrency, 0), + inactiveQueue: make([]*storage.BalanceChange, 0), + } + + if lookupBalanceByBlock { + // When lookupBalanceByBlock is enabled, we check + // balance changes synchronously. + r.changeQueue = make(chan *storage.BalanceChange) + } else { + // When lookupBalanceByBlock is disabled, we must check + // balance changes asynchronously. Using a buffered + // channel allows us to add balance changes without blocking. + r.changeQueue = make(chan *storage.BalanceChange, backlogThreshold) + } + + return r +} + +// Reconciliation +// QueueChanges enqueues a slice of *storage.BalanceChanges +// for reconciliation. +func (r *Reconciler) QueueChanges( + ctx context.Context, + // If we pass in parentblock, then we always know what to compare on diff + block *types.BlockIdentifier, + balanceChanges []*storage.BalanceChange, +) error { + // Ensure all interestingAccounts are checked + // TODO: refactor to automatically trigger once an inactive reconciliation error + // is discovered + for _, account := range r.interestingAccounts { + skipAccount := false + // Look through balance changes for account + currency + for _, change := range balanceChanges { + if reflect.DeepEqual(change.Account, account.Account) && + reflect.DeepEqual(change.Currency, account.Currency) { + skipAccount = true + break + } + } + // Account changed on this block + if skipAccount { + continue + } + + // If account + currency not found, add with difference 0 + balanceChanges = append(balanceChanges, &storage.BalanceChange{ + Account: account.Account, + Currency: account.Currency, + Difference: "0", + Block: block, + }) + } + + if !r.lookupBalanceByBlock { + // All changes will have the same block. Return + // if we are too far behind to start reconciling. + if block.Index < r.highWaterMark { + return nil + } + + for _, change := range balanceChanges { + select { + case r.changeQueue <- change: + default: + log.Println("skipping active enqueue because backlog") + } + } + } else { + // Block until all checked for a block or context is Done + for _, change := range balanceChanges { + select { + case r.changeQueue <- change: + case <-ctx.Done(): + return ctx.Err() + } + } + } + + return nil +} + +// CompareBalance checks to see if the computed balance of an account +// is equal to the live balance of an account. This function ensures +// balance is checked correctly in the case of orphaned blocks. +func (r *Reconciler) CompareBalance( + ctx context.Context, + account *types.AccountIdentifier, + currency *types.Currency, + amount string, + liveBlock *types.BlockIdentifier, +) (string, int64, error) { + // Head block should be set before we CompareBalance + head, err := r.handler.CurrentBlock(ctx) + if err != nil { + return zeroString, 0, fmt.Errorf("%w: unable to get current block for reconciliation", err) + } + + // Check if live block is < head (or wait) + if liveBlock.Index > head.Index { + return zeroString, head.Index, fmt.Errorf( + "%w live block %d > head block %d", + ErrHeadBlockBehindLive, + liveBlock.Index, + head.Index, + ) + } + + // Check if live block is in store (ensure not reorged) + _, err = r.handler.BlockExists(ctx, liveBlock) + if err != nil { + return zeroString, head.Index, fmt.Errorf( + "%w %+v", + ErrBlockGone, + liveBlock, + ) + } + + // Check if live block < computed head + cachedBalance, balanceBlock, err := r.handler.AccountBalance(ctx, account, currency) + if err != nil { + return zeroString, head.Index, err + } + + if liveBlock.Index < balanceBlock.Index { + return zeroString, head.Index, fmt.Errorf( + "%w %+v updated at %d", + ErrAccountUpdated, + account, + balanceBlock.Index, + ) + } + + difference, err := storage.SubtractStringValues(cachedBalance.Value, amount) + if err != nil { + return "", -1, err + } + + if difference != zeroString { + return difference, head.Index, nil + } + + return zeroString, head.Index, nil +} + +// getAccountBalance returns the balance for an account +// at either the current block (if lookupBalanceByBlock is +// disabled) or at some historical block. +func (r *Reconciler) bestBalance( + ctx context.Context, + account *types.AccountIdentifier, + currency *types.Currency, + block *types.PartialBlockIdentifier, +) (*types.BlockIdentifier, string, error) { + if !r.lookupBalanceByBlock { + // Use the current balance to reconcile balances when lookupBalanceByBlock + // is disabled. This could be the case when a rosetta server does not + // support historical balance lookups. + block = nil + } + return GetCurrencyBalance( + ctx, + r.fetcher, + r.network, + account, + currency, + block, + ) +} + +// accountReconciliation returns an error if the provided +// AccountAndCurrency's live balance cannot be reconciled +// with the computed balance. +func (r *Reconciler) accountReconciliation( + ctx context.Context, + account *types.AccountIdentifier, + currency *types.Currency, + liveAmount string, + liveBlock *types.BlockIdentifier, + inactive bool, +) error { + accountCurrency := &AccountCurrency{ + Account: account, + Currency: currency, + } + for ctx.Err() == nil { + // If don't have previous balance because stateless, check diff on block + // instead of comparing entire computed balance + difference, headIndex, err := r.CompareBalance( + ctx, + account, + currency, + liveAmount, + liveBlock, + ) + if err != nil { + if errors.Is(err, ErrHeadBlockBehindLive) { + // This error will only occur when lookupBalanceByBlock + // is disabled and the syncer is behind the current block of + // the node. This error should never occur when + // lookupBalanceByBlock is enabled. + diff := liveBlock.Index - headIndex + if diff < waitToCheckDiff { + time.Sleep(waitToCheckDiffSleep) + continue + } + + // Don't wait to check if we are very far behind + log.Printf( + "Skipping reconciliation for %s: %d blocks behind\n", + simpleAccountCurrency(accountCurrency), + diff, + ) + + // Set a highWaterMark to not accept any new + // reconciliation requests unless they happened + // after this new highWaterMark. + r.highWaterMark = liveBlock.Index + break + } + + if errors.Is(err, ErrBlockGone) { + // Either the block has not been processed in a re-org yet + // or the block was orphaned + break + } + + if errors.Is(err, ErrAccountUpdated) { + // If account was updated, it must be + // enqueued again + break + } + + return err + } + + reconciliationType := activeReconciliation + if inactive { + reconciliationType = inactiveReconciliation + } + + if difference != zeroString { + err := r.logger.ReconcileFailureStream( + ctx, + reconciliationType, + accountCurrency.Account, + accountCurrency.Currency, + difference, + liveBlock, + ) + + if err != nil { + return err + } + + if r.haltOnReconciliationError { + return errors.New("reconciliation error") + } + + return nil + } + + r.inactiveAccountQueue(inactive, accountCurrency, liveBlock) + return r.logger.ReconcileSuccessStream( + ctx, + reconciliationType, + accountCurrency.Account, + &types.Amount{ + Value: liveAmount, + Currency: currency, + }, + liveBlock, + ) + } + + return nil +} + +func (r *Reconciler) inactiveAccountQueue( + inactive bool, + accountCurrency *AccountCurrency, + liveBlock *types.BlockIdentifier, +) { + // Only enqueue the first time we see an account on an active reconciliation. + shouldEnqueueInactive := false + if !inactive && !ContainsAccountCurrency(r.seenAccts, accountCurrency) { + r.seenAccts = append(r.seenAccts, accountCurrency) + shouldEnqueueInactive = true + } + + if inactive || shouldEnqueueInactive { + r.inactiveQueueMutex.Lock() + r.inactiveQueue = append(r.inactiveQueue, &storage.BalanceChange{ + Account: accountCurrency.Account, + Currency: accountCurrency.Currency, + Block: liveBlock, + }) + r.inactiveQueueMutex.Unlock() + } +} + +// simpleAccountCurrency returns a string that is a simple +// representation of an AccountCurrency struct. +func simpleAccountCurrency( + accountCurrency *AccountCurrency, +) string { + acctString := accountCurrency.Account.Address + if accountCurrency.Account.SubAccount != nil { + acctString += accountCurrency.Account.SubAccount.Address + } + + acctString += accountCurrency.Currency.Symbol + + return acctString +} + +// reconcileActiveAccounts selects an account +// from the Reconciler account queue and +// reconciles the balance. This is useful +// for detecting if balance changes in operations +// were correct. +func (r *Reconciler) reconcileActiveAccounts( + ctx context.Context, +) error { + for { + select { + case <-ctx.Done(): + return ctx.Err() + case balanceChange := <-r.changeQueue: + if balanceChange.Block.Index < r.highWaterMark { + continue + } + + block, value, err := r.bestBalance( + ctx, + balanceChange.Account, + balanceChange.Currency, + types.ConstructPartialBlockIdentifier(balanceChange.Block), + ) + if err != nil { + return err + } + + err = r.accountReconciliation( + ctx, + balanceChange.Account, + balanceChange.Currency, + value, + block, + false, + ) + if err != nil { + return err + } + } + } +} + +// reconcileInactiveAccounts selects a random account +// from all previously seen accounts and reconciles +// the balance. This is useful for detecting balance +// changes that were not returned in operations. +func (r *Reconciler) reconcileInactiveAccounts( + ctx context.Context, +) error { + for ctx.Err() == nil { + head, err := r.handler.CurrentBlock(ctx) + // When first start syncing, this loop may run before the genesis block is synced. + // If this is the case, we should sleep and try again later instead of exiting. + if errors.Is(err, storage.ErrHeadBlockNotFound) { + log.Println("head block not yet initialized, sleeping...") + time.Sleep(inactiveReconciliationSleep) + continue + } else if err != nil { + return fmt.Errorf("%w: unable to get current block for inactive reconciliation", err) + } + + r.inactiveQueueMutex.Lock() + if len(r.inactiveQueue) > 0 && + r.inactiveQueue[0].Block.Index+inactiveReconciliationRequiredDepth < head.Index { + randAcct := r.inactiveQueue[0] + r.inactiveQueue = r.inactiveQueue[1:] + r.inactiveQueueMutex.Unlock() + + block, amount, err := r.bestBalance( + ctx, + randAcct.Account, + randAcct.Currency, + types.ConstructPartialBlockIdentifier(head), + ) + if err != nil { + return err + } + + err = r.accountReconciliation( + ctx, + randAcct.Account, + randAcct.Currency, + amount, + block, + true, + ) + if err != nil { + return err + } + } else { + r.inactiveQueueMutex.Unlock() + time.Sleep(inactiveReconciliationSleep) + } + } + + return nil +} + +// Reconcile starts the active and inactive Reconciler goroutines. +// If any goroutine errors, the function will return an error. +func (r *Reconciler) Reconcile(ctx context.Context) error { + g, ctx := errgroup.WithContext(ctx) + for j := uint64(0); j < r.accountConcurrency/2; j++ { + g.Go(func() error { + return r.reconcileActiveAccounts(ctx) + }) + + g.Go(func() error { + return r.reconcileInactiveAccounts(ctx) + }) + } + + if err := g.Wait(); err != nil { + return err + } - Reconcile(ctx context.Context) error + return nil } // ExtractAmount returns the types.Amount from a slice of types.Balance diff --git a/internal/reconciler/reconciler_test.go b/internal/reconciler/reconciler_test.go deleted file mode 100644 index 5e2b66e3..00000000 --- a/internal/reconciler/reconciler_test.go +++ /dev/null @@ -1,184 +0,0 @@ -// Copyright 2020 Coinbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package reconciler - -import ( - "fmt" - "testing" - - "github.com/coinbase/rosetta-sdk-go/types" - "github.com/stretchr/testify/assert" -) - -func TestContainsAccountCurrency(t *testing.T) { - currency1 := &types.Currency{ - Symbol: "Blah", - Decimals: 2, - } - currency2 := &types.Currency{ - Symbol: "Blah2", - Decimals: 2, - } - accts := []*AccountCurrency{ - { - Account: &types.AccountIdentifier{ - Address: "test", - }, - Currency: currency1, - }, - { - Account: &types.AccountIdentifier{ - Address: "cool", - SubAccount: &types.SubAccountIdentifier{ - Address: "test2", - }, - }, - Currency: currency1, - }, - { - Account: &types.AccountIdentifier{ - Address: "cool", - SubAccount: &types.SubAccountIdentifier{ - Address: "test2", - Metadata: map[string]interface{}{ - "neat": "stuff", - }, - }, - }, - Currency: currency1, - }, - } - - t.Run("Non-existent account", func(t *testing.T) { - assert.False(t, ContainsAccountCurrency(accts, &AccountCurrency{ - Account: &types.AccountIdentifier{ - Address: "blah", - }, - Currency: currency1, - })) - }) - - t.Run("Basic account", func(t *testing.T) { - assert.True(t, ContainsAccountCurrency(accts, &AccountCurrency{ - Account: &types.AccountIdentifier{ - Address: "test", - }, - Currency: currency1, - })) - }) - - t.Run("Basic account with bad currency", func(t *testing.T) { - assert.False(t, ContainsAccountCurrency(accts, &AccountCurrency{ - Account: &types.AccountIdentifier{ - Address: "test", - }, - Currency: currency2, - })) - }) - - t.Run("Account with subaccount", func(t *testing.T) { - assert.True(t, ContainsAccountCurrency(accts, &AccountCurrency{ - Account: &types.AccountIdentifier{ - Address: "cool", - SubAccount: &types.SubAccountIdentifier{ - Address: "test2", - }, - }, - Currency: currency1, - })) - }) - - t.Run("Account with subaccount and metadata", func(t *testing.T) { - assert.True(t, ContainsAccountCurrency(accts, &AccountCurrency{ - Account: &types.AccountIdentifier{ - Address: "cool", - SubAccount: &types.SubAccountIdentifier{ - Address: "test2", - Metadata: map[string]interface{}{ - "neat": "stuff", - }, - }, - }, - Currency: currency1, - })) - }) - - t.Run("Account with subaccount and unique metadata", func(t *testing.T) { - assert.False(t, ContainsAccountCurrency(accts, &AccountCurrency{ - Account: &types.AccountIdentifier{ - Address: "cool", - SubAccount: &types.SubAccountIdentifier{ - Address: "test2", - Metadata: map[string]interface{}{ - "neater": "stuff", - }, - }, - }, - Currency: currency1, - })) - }) -} - -func TestExtractAmount(t *testing.T) { - var ( - currency1 = &types.Currency{ - Symbol: "curr1", - Decimals: 4, - } - - currency2 = &types.Currency{ - Symbol: "curr2", - Decimals: 7, - } - - amount1 = &types.Amount{ - Value: "100", - Currency: currency1, - } - - amount2 = &types.Amount{ - Value: "200", - Currency: currency2, - } - - balances = []*types.Amount{ - amount1, - amount2, - } - - badCurr = &types.Currency{ - Symbol: "no curr", - Decimals: 100, - } - ) - - t.Run("Non-existent currency", func(t *testing.T) { - result, err := ExtractAmount(balances, badCurr) - assert.Nil(t, result) - assert.EqualError(t, err, fmt.Errorf("could not extract amount for %+v", badCurr).Error()) - }) - - t.Run("Simple account", func(t *testing.T) { - result, err := ExtractAmount(balances, currency1) - assert.Equal(t, amount1, result) - assert.NoError(t, err) - }) - - t.Run("SubAccount", func(t *testing.T) { - result, err := ExtractAmount(balances, currency2) - assert.Equal(t, amount2, result) - assert.NoError(t, err) - }) -} diff --git a/internal/reconciler/stateful_reconciler.go b/internal/reconciler/stateful_reconciler.go deleted file mode 100644 index 61b79160..00000000 --- a/internal/reconciler/stateful_reconciler.go +++ /dev/null @@ -1,536 +0,0 @@ -// Copyright 2020 Coinbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package reconciler - -import ( - "context" - "errors" - "fmt" - "log" - "sync" - "time" - - "github.com/coinbase/rosetta-cli/internal/logger" - "github.com/coinbase/rosetta-cli/internal/storage" - - "github.com/coinbase/rosetta-sdk-go/fetcher" - "github.com/coinbase/rosetta-sdk-go/types" - "golang.org/x/sync/errgroup" -) - -const ( - // backlogThreshold is the limit of account lookups - // that can be enqueued to reconcile before new - // requests are dropped. - // TODO: Make configurable - backlogThreshold = 1000 - - // waitToCheckDiff is the syncing difference (live-head) - // to retry instead of exiting. In other words, if the - // processed head is behind the live head by < - // waitToCheckDiff we should try again after sleeping. - // TODO: Make configurable - waitToCheckDiff = 10 - - // waitToCheckDiffSleep is the amount of time to wait - // to check a balance difference if the syncer is within - // waitToCheckDiff from the block a balance was queried at. - waitToCheckDiffSleep = 5 * time.Second - - // activeReconciliation is included in the reconciliation - // error message if reconciliation failed during active - // reconciliation. - activeReconciliation = "ACTIVE" - - // inactiveReconciliation is included in the reconciliation - // error message if reconciliation failed during inactive - // reconciliation. - inactiveReconciliation = "INACTIVE" - - // zeroString is a string of value 0. - zeroString = "0" - - // inactiveReconciliationSleep is used as the time.Duration - // to sleep when there are no seen accounts to reconcile. - inactiveReconciliationSleep = 5 * time.Second - - // inactiveReconciliationRequiredDepth is the minimum - // number of blocks the reconciler should wait between - // inactive reconciliations. - // TODO: make configurable - inactiveReconciliationRequiredDepth = 500 -) - -var ( - // ErrHeadBlockBehindLive is returned when the processed - // head is behind the live head. Sometimes, it is - // preferrable to sleep and wait to catch up when - // we are close to the live head (waitToCheckDiff). - ErrHeadBlockBehindLive = errors.New("head block behind") - - // ErrAccountUpdated is returned when the - // account was updated at a height later than - // the live height (when the account balance was fetched). - ErrAccountUpdated = errors.New("account updated") - - // ErrBlockGone is returned when the processed block - // head is greater than the live head but the block - // does not exist in the store. This likely means - // that the block was orphaned. - ErrBlockGone = errors.New("block gone") -) - -// StatefulReconciler contains all logic to reconcile balances of -// types.AccountIdentifiers returned in types.Operations -// by a Rosetta Server. -type StatefulReconciler struct { - network *types.NetworkIdentifier - storage *storage.BlockStorage - fetcher *fetcher.Fetcher - logger *logger.Logger - accountConcurrency uint64 - lookupBalanceByBlock bool - haltOnReconciliationError bool - changeQueue chan *storage.BalanceChange - - // highWaterMark is used to skip requests when - // we are very far behind the live head. - highWaterMark int64 - - // seenAccts are stored for inactive account - // reconciliation. - seenAccts []*AccountCurrency - inactiveQueue []*storage.BalanceChange - - // inactiveQueueMutex needed because we can't peek at the tip - // of a channel to determine when it is ready to look at. - inactiveQueueMutex sync.Mutex -} - -// NewStateful creates a new StatefulReconciler. -func NewStateful( - network *types.NetworkIdentifier, - blockStorage *storage.BlockStorage, - fetcher *fetcher.Fetcher, - logger *logger.Logger, - accountConcurrency uint64, - lookupBalanceByBlock bool, - haltOnReconciliationError bool, -) *StatefulReconciler { - return &StatefulReconciler{ - network: network, - storage: blockStorage, - fetcher: fetcher, - logger: logger, - accountConcurrency: accountConcurrency, - lookupBalanceByBlock: lookupBalanceByBlock, - haltOnReconciliationError: haltOnReconciliationError, - changeQueue: make(chan *storage.BalanceChange, backlogThreshold), - highWaterMark: -1, - seenAccts: make([]*AccountCurrency, 0), - inactiveQueue: make([]*storage.BalanceChange, 0), - } -} - -// QueueChanges enqueues a slice of *storage.BalanceChanges -// for reconciliation. -func (r *StatefulReconciler) QueueChanges( - ctx context.Context, - block *types.BlockIdentifier, - balanceChanges []*storage.BalanceChange, -) error { - // All changes will have the same block. Return - // if we are too far behind to start reconciling. - if block.Index < r.highWaterMark { - return nil - } - - // Use a buffered channel so don't need to - // spawn a goroutine to add accounts to channel. - for _, change := range balanceChanges { - select { - case r.changeQueue <- change: - default: - log.Println("skipping active enqueue because backlog") - } - } - - return nil -} - -// CompareBalance checks to see if the computed balance of an account -// is equal to the live balance of an account. This function ensures -// balance is checked correctly in the case of orphaned blocks. -func (r *StatefulReconciler) CompareBalance( - ctx context.Context, - account *types.AccountIdentifier, - currency *types.Currency, - amount string, - liveBlock *types.BlockIdentifier, -) (string, int64, error) { - txn := r.storage.NewDatabaseTransaction(ctx, false) - defer txn.Discard(ctx) - - // Head block should be set before we CompareBalance - head, err := r.storage.GetHeadBlockIdentifier(ctx, txn) - if err != nil { - return zeroString, 0, fmt.Errorf("%w: unable to get current block for reconciliation", err) - } - - // Check if live block is < head (or wait) - if liveBlock.Index > head.Index { - return zeroString, head.Index, fmt.Errorf( - "%w live block %d > head block %d", - ErrHeadBlockBehindLive, - liveBlock.Index, - head.Index, - ) - } - - // Check if live block is in store (ensure not reorged) - _, err = r.storage.GetBlock(ctx, txn, liveBlock) - if err != nil { - return zeroString, head.Index, fmt.Errorf( - "%w %+v", - ErrBlockGone, - liveBlock, - ) - } - - // Check if live block < computed head - amounts, balanceBlock, err := r.storage.GetBalance(ctx, txn, account) - if err != nil { - return zeroString, head.Index, err - } - - if liveBlock.Index < balanceBlock.Index { - return zeroString, head.Index, fmt.Errorf( - "%w %+v updated at %d", - ErrAccountUpdated, - account, - balanceBlock.Index, - ) - } - - // Check balances are equal - computedAmount, ok := amounts[storage.GetCurrencyKey(currency)] - if !ok { - return "", head.Index, fmt.Errorf( - "currency %+v not found", - *currency, - ) - } - - difference, err := storage.SubtractStringValues(computedAmount.Value, amount) - if err != nil { - return "", -1, err - } - - if difference != zeroString { - return difference, head.Index, nil - } - - return zeroString, head.Index, nil -} - -// getAccountBalance returns the balance for an account -// at either the current block (if lookupBalanceByBlock is -// disabled) or at some historical block. -func (r *StatefulReconciler) bestBalance( - ctx context.Context, - account *types.AccountIdentifier, - currency *types.Currency, - block *types.PartialBlockIdentifier, -) (*types.BlockIdentifier, string, error) { - if !r.lookupBalanceByBlock { - // Use the current balance to reconcile balances when lookupBalanceByBlock - // is disabled. This could be the case when a rosetta server does not - // support historical balance lookups. - block = nil - } - return GetCurrencyBalance( - ctx, - r.fetcher, - r.network, - account, - currency, - block, - ) -} - -// accountReconciliation returns an error if the provided -// AccountAndCurrency's live balance cannot be reconciled -// with the computed balance. -func (r *StatefulReconciler) accountReconciliation( - ctx context.Context, - account *types.AccountIdentifier, - currency *types.Currency, - liveAmount string, - liveBlock *types.BlockIdentifier, - inactive bool, -) error { - accountCurrency := &AccountCurrency{ - Account: account, - Currency: currency, - } - for ctx.Err() == nil { - difference, headIndex, err := r.CompareBalance( - ctx, - account, - currency, - liveAmount, - liveBlock, - ) - if err != nil { - if errors.Is(err, ErrHeadBlockBehindLive) { - // This error will only occur when lookupBalanceByBlock - // is disabled and the syncer is behind the current block of - // the node. This error should never occur when - // lookupBalanceByBlock is enabled. - diff := liveBlock.Index - headIndex - if diff < waitToCheckDiff { - time.Sleep(waitToCheckDiffSleep) - continue - } - - // Don't wait to check if we are very far behind - log.Printf( - "Skipping reconciliation for %s: %d blocks behind\n", - simpleAccountCurrency(accountCurrency), - diff, - ) - - // Set a highWaterMark to not accept any new - // reconciliation requests unless they happened - // after this new highWaterMark. - r.highWaterMark = liveBlock.Index - break - } - - if errors.Is(err, ErrBlockGone) { - // Either the block has not been processed in a re-org yet - // or the block was orphaned - break - } - - if errors.Is(err, ErrAccountUpdated) { - // If account was updated, it must be - // enqueued again - break - } - - return err - } - - reconciliationType := activeReconciliation - if inactive { - reconciliationType = inactiveReconciliation - } - - if difference != zeroString { - err := r.logger.ReconcileFailureStream( - ctx, - reconciliationType, - accountCurrency.Account, - accountCurrency.Currency, - difference, - liveBlock, - ) - - if err != nil { - return err - } - - if r.haltOnReconciliationError { - return errors.New("reconciliation error") - } - - return nil - } - - r.inactiveAccountQueue(inactive, accountCurrency, liveBlock) - return r.logger.ReconcileSuccessStream( - ctx, - reconciliationType, - accountCurrency.Account, - &types.Amount{ - Value: liveAmount, - Currency: currency, - }, - liveBlock, - ) - } - - return nil -} - -func (r *StatefulReconciler) inactiveAccountQueue( - inactive bool, - accountCurrency *AccountCurrency, - liveBlock *types.BlockIdentifier, -) { - // Only enqueue the first time we see an account on an active reconciliation. - shouldEnqueueInactive := false - if !inactive && !ContainsAccountCurrency(r.seenAccts, accountCurrency) { - r.seenAccts = append(r.seenAccts, accountCurrency) - shouldEnqueueInactive = true - } - - if inactive || shouldEnqueueInactive { - r.inactiveQueueMutex.Lock() - r.inactiveQueue = append(r.inactiveQueue, &storage.BalanceChange{ - Account: accountCurrency.Account, - Currency: accountCurrency.Currency, - Block: liveBlock, - }) - r.inactiveQueueMutex.Unlock() - } -} - -// simpleAccountCurrency returns a string that is a simple -// representation of an AccountCurrency struct. -func simpleAccountCurrency( - accountCurrency *AccountCurrency, -) string { - acctString := accountCurrency.Account.Address - if accountCurrency.Account.SubAccount != nil { - acctString += accountCurrency.Account.SubAccount.Address - } - - acctString += accountCurrency.Currency.Symbol - - return acctString -} - -// reconcileActiveAccounts selects an account -// from the StatefulReconciler account queue and -// reconciles the balance. This is useful -// for detecting if balance changes in operations -// were correct. -func (r *StatefulReconciler) reconcileActiveAccounts( - ctx context.Context, -) error { - for { - select { - case <-ctx.Done(): - return ctx.Err() - case balanceChange := <-r.changeQueue: - if balanceChange.Block.Index < r.highWaterMark { - continue - } - - block, value, err := r.bestBalance( - ctx, - balanceChange.Account, - balanceChange.Currency, - types.ConstructPartialBlockIdentifier(balanceChange.Block), - ) - if err != nil { - return err - } - - err = r.accountReconciliation( - ctx, - balanceChange.Account, - balanceChange.Currency, - value, - block, - false, - ) - if err != nil { - return err - } - } - } -} - -// reconcileInactiveAccounts selects a random account -// from all previously seen accounts and reconciles -// the balance. This is useful for detecting balance -// changes that were not returned in operations. -func (r *StatefulReconciler) reconcileInactiveAccounts( - ctx context.Context, -) error { - for ctx.Err() == nil { - txn := r.storage.NewDatabaseTransaction(ctx, false) - head, err := r.storage.GetHeadBlockIdentifier(ctx, txn) - txn.Discard(ctx) - // When first start syncing, this loop may run before the genesis block is synced. - // If this is the case, we should sleep and try again later instead of exiting. - if errors.Is(err, storage.ErrHeadBlockNotFound) { - log.Println("head block not yet initialized, sleeping...") - time.Sleep(inactiveReconciliationSleep) - continue - } else if err != nil { - return fmt.Errorf("%w: unable to get current block for inactive reconciliation", err) - } - - r.inactiveQueueMutex.Lock() - if len(r.inactiveQueue) > 0 && - r.inactiveQueue[0].Block.Index+inactiveReconciliationRequiredDepth < head.Index { - randAcct := r.inactiveQueue[0] - r.inactiveQueue = r.inactiveQueue[1:] - r.inactiveQueueMutex.Unlock() - - block, amount, err := r.bestBalance( - ctx, - randAcct.Account, - randAcct.Currency, - types.ConstructPartialBlockIdentifier(head), - ) - if err != nil { - return err - } - - err = r.accountReconciliation( - ctx, - randAcct.Account, - randAcct.Currency, - amount, - block, - true, - ) - if err != nil { - return err - } - } else { - r.inactiveQueueMutex.Unlock() - time.Sleep(inactiveReconciliationSleep) - } - } - - return nil -} - -// Reconcile starts the active and inactive StatefulReconciler goroutines. -// If any goroutine errors, the function will return an error. -func (r *StatefulReconciler) Reconcile(ctx context.Context) error { - g, ctx := errgroup.WithContext(ctx) - for j := uint64(0); j < r.accountConcurrency/2; j++ { - g.Go(func() error { - return r.reconcileActiveAccounts(ctx) - }) - - g.Go(func() error { - return r.reconcileInactiveAccounts(ctx) - }) - } - - if err := g.Wait(); err != nil { - return err - } - - return nil -} diff --git a/internal/reconciler/stateful_reconciler_test.go b/internal/reconciler/stateful_reconciler_test.go deleted file mode 100644 index b05f583a..00000000 --- a/internal/reconciler/stateful_reconciler_test.go +++ /dev/null @@ -1,243 +0,0 @@ -// Copyright 2020 Coinbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package reconciler - -import ( - "context" - "errors" - "fmt" - "testing" - - "github.com/coinbase/rosetta-cli/internal/logger" - "github.com/coinbase/rosetta-cli/internal/storage" - - "github.com/coinbase/rosetta-sdk-go/types" - "github.com/stretchr/testify/assert" -) - -func TestCompareBalance(t *testing.T) { - var ( - account1 = &types.AccountIdentifier{ - Address: "blah", - } - - account2 = &types.AccountIdentifier{ - Address: "blah", - SubAccount: &types.SubAccountIdentifier{ - Address: "sub blah", - }, - } - - currency1 = &types.Currency{ - Symbol: "curr1", - Decimals: 4, - } - - currency2 = &types.Currency{ - Symbol: "curr2", - Decimals: 7, - } - - amount1 = &types.Amount{ - Value: "100", - Currency: currency1, - } - - amount2 = &types.Amount{ - Value: "200", - Currency: currency2, - } - - block0 = &types.BlockIdentifier{ - Hash: "block0", - Index: 0, - } - - block1 = &types.BlockIdentifier{ - Hash: "block1", - Index: 1, - } - - block2 = &types.BlockIdentifier{ - Hash: "block2", - Index: 2, - } - - ctx = context.Background() - ) - - newDir, err := storage.CreateTempDir() - assert.NoError(t, err) - defer storage.RemoveTempDir(*newDir) - - database, err := storage.NewBadgerStorage(ctx, *newDir) - assert.NoError(t, err) - defer database.Close(ctx) - - blockStorage := storage.NewBlockStorage(ctx, database) - logger := logger.NewLogger(*newDir, false, false, false, false) - reconciler := NewStateful(nil, blockStorage, nil, logger, 1, false, true) - - t.Run("No head block yet", func(t *testing.T) { - difference, headIndex, err := reconciler.CompareBalance( - ctx, - account1, - currency1, - amount1.Value, - block1, - ) - assert.Equal(t, "0", difference) - assert.Equal(t, int64(0), headIndex) - assert.True(t, errors.Is(err, storage.ErrHeadBlockNotFound)) - }) - - // Update head block - txn := blockStorage.NewDatabaseTransaction(ctx, true) - err = blockStorage.StoreHeadBlockIdentifier(ctx, txn, block0) - assert.NoError(t, err) - err = txn.Commit(ctx) - assert.NoError(t, err) - - t.Run("Live block is ahead of head block", func(t *testing.T) { - difference, headIndex, err := reconciler.CompareBalance( - ctx, - account1, - currency1, - amount1.Value, - block1, - ) - assert.Equal(t, "0", difference) - assert.Equal(t, int64(0), headIndex) - assert.EqualError(t, err, fmt.Errorf( - "%w live block %d > head block %d", - ErrHeadBlockBehindLive, - 1, - 0, - ).Error()) - }) - - // Update head block - txn = blockStorage.NewDatabaseTransaction(ctx, true) - err = blockStorage.StoreHeadBlockIdentifier(ctx, txn, &types.BlockIdentifier{ - Hash: "hash2", - Index: 2, - }) - assert.NoError(t, err) - err = txn.Commit(ctx) - assert.NoError(t, err) - - t.Run("Live block is not in store", func(t *testing.T) { - difference, headIndex, err := reconciler.CompareBalance( - ctx, - account1, - currency1, - amount1.Value, - block1, - ) - assert.Equal(t, "0", difference) - assert.Equal(t, int64(2), headIndex) - assert.Contains(t, err.Error(), ErrBlockGone.Error()) - }) - - // Add blocks to store behind head - txn = blockStorage.NewDatabaseTransaction(ctx, true) - err = blockStorage.StoreBlock(ctx, txn, &types.Block{ - BlockIdentifier: block0, - ParentBlockIdentifier: block0, - }) - assert.NoError(t, err) - - err = blockStorage.StoreBlock(ctx, txn, &types.Block{ - BlockIdentifier: block1, - ParentBlockIdentifier: block0, - }) - assert.NoError(t, err) - - err = blockStorage.StoreBlock(ctx, txn, &types.Block{ - BlockIdentifier: block2, - ParentBlockIdentifier: block1, - }) - assert.NoError(t, err) - - _, err = blockStorage.UpdateBalance(ctx, txn, account1, amount1, block1) - assert.NoError(t, err) - err = txn.Commit(ctx) - assert.NoError(t, err) - - t.Run("Account updated after live block", func(t *testing.T) { - difference, headIndex, err := reconciler.CompareBalance( - ctx, - account1, - currency1, - amount1.Value, - block0, - ) - assert.Equal(t, "0", difference) - assert.Equal(t, int64(2), headIndex) - assert.Contains(t, err.Error(), ErrAccountUpdated.Error()) - }) - - t.Run("Account balance matches", func(t *testing.T) { - difference, headIndex, err := reconciler.CompareBalance( - ctx, - account1, - currency1, - amount1.Value, - block1, - ) - assert.Equal(t, "0", difference) - assert.Equal(t, int64(2), headIndex) - assert.NoError(t, err) - }) - - t.Run("Account balance matches later live block", func(t *testing.T) { - difference, headIndex, err := reconciler.CompareBalance( - ctx, - account1, - currency1, - amount1.Value, - block2, - ) - assert.Equal(t, "0", difference) - assert.Equal(t, int64(2), headIndex) - assert.NoError(t, err) - }) - - t.Run("Balances are not equal", func(t *testing.T) { - difference, headIndex, err := reconciler.CompareBalance( - ctx, - account1, - currency1, - amount2.Value, - block2, - ) - assert.Equal(t, "-100", difference) - assert.Equal(t, int64(2), headIndex) - assert.NoError(t, err) - }) - - t.Run("Compare balance for non-existent account", func(t *testing.T) { - difference, headIndex, err := reconciler.CompareBalance( - ctx, - account2, - currency1, - amount2.Value, - block2, - ) - assert.Equal(t, "0", difference) - assert.Equal(t, int64(2), headIndex) - assert.Contains(t, err.Error(), storage.ErrAccountNotFound.Error()) - }) -} diff --git a/internal/reconciler/stateless_reconciler.go b/internal/reconciler/stateless_reconciler.go deleted file mode 100644 index d6d9f38f..00000000 --- a/internal/reconciler/stateless_reconciler.go +++ /dev/null @@ -1,238 +0,0 @@ -// Copyright 2020 Coinbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package reconciler - -import ( - "context" - "errors" - "reflect" - - "github.com/coinbase/rosetta-cli/internal/logger" - "github.com/coinbase/rosetta-cli/internal/storage" - - "github.com/coinbase/rosetta-sdk-go/fetcher" - "github.com/coinbase/rosetta-sdk-go/types" - "golang.org/x/sync/errgroup" -) - -// StatelessReconciler compares computed balances with -// the node balance without using any sort of persistent -// state. If it is not possible to lookup a balance by block, -// you must use the StatefulReconciler. If you want to perform -// inactive reconciliation (check for balance changes that -// occurred that were not in blocks), you must also use the -// StatefulReconciler. Lastly, the StatelessReconciler does -// not support re-orgs. -type StatelessReconciler struct { - network *types.NetworkIdentifier - fetcher *fetcher.Fetcher - logger *logger.Logger - accountConcurrency uint64 - haltOnReconciliationError bool - interestingAccounts []*AccountCurrency - changeQueue chan *storage.BalanceChange -} - -// NewStateless returns a new StatelessReconciler. -func NewStateless( - network *types.NetworkIdentifier, - fetcher *fetcher.Fetcher, - logger *logger.Logger, - accountConcurrency uint64, - haltOnReconciliationError bool, - interestingAccounts []*AccountCurrency, -) *StatelessReconciler { - return &StatelessReconciler{ - network: network, - fetcher: fetcher, - logger: logger, - accountConcurrency: accountConcurrency, - haltOnReconciliationError: haltOnReconciliationError, - interestingAccounts: interestingAccounts, - changeQueue: make(chan *storage.BalanceChange), - } -} - -// QueueChanges enqueues a slice of *storage.BalanceChanges -// for reconciliation. -func (r *StatelessReconciler) QueueChanges( - ctx context.Context, - block *types.BlockIdentifier, - balanceChanges []*storage.BalanceChange, -) error { - // Ensure all interestingAccounts are checked - // TODO: refactor to automatically trigger once an inactive reconciliation error - // is discovered - for _, account := range r.interestingAccounts { - skipAccount := false - // Look through balance changes for account + currency - for _, change := range balanceChanges { - if reflect.DeepEqual(change.Account, account.Account) && - reflect.DeepEqual(change.Currency, account.Currency) { - skipAccount = true - break - } - } - - // Account changed on this block - if skipAccount { - continue - } - - // If account + currency not found, add with difference 0 - balanceChanges = append(balanceChanges, &storage.BalanceChange{ - Account: account.Account, - Currency: account.Currency, - Difference: "0", - Block: block, - }) - } - - // Block until all checked for a block - for _, change := range balanceChanges { - select { - case r.changeQueue <- change: - case <-ctx.Done(): - return ctx.Err() - } - } - - return nil -} - -func (r *StatelessReconciler) balanceAtIndex( - ctx context.Context, - account *types.AccountIdentifier, - currency *types.Currency, - index int64, -) (string, error) { - _, value, err := GetCurrencyBalance( - ctx, - r.fetcher, - r.network, - account, - currency, - &types.PartialBlockIdentifier{ - Index: &index, - }, - ) - - return value, err -} - -func (r *StatelessReconciler) reconcileChange( - ctx context.Context, - change *storage.BalanceChange, -) error { - // Get balance at block before change - balanceBefore, err := r.balanceAtIndex( - ctx, - change.Account, - change.Currency, - change.Block.Index-1, - ) - if err != nil { - return err - } - - // Get balance at block with change - balanceAfter, err := r.balanceAtIndex( - ctx, - change.Account, - change.Currency, - change.Block.Index, - ) - if err != nil { - return err - } - - // Get difference between node change and computed change - nodeDifference, err := storage.SubtractStringValues(balanceAfter, balanceBefore) - if err != nil { - return err - } - - difference, err := storage.SubtractStringValues(change.Difference, nodeDifference) - if err != nil { - return err - } - - if difference != zeroString { - err := r.logger.ReconcileFailureStream( - ctx, - activeReconciliation, - change.Account, - change.Currency, - difference, - change.Block, - ) - if err != nil { - return err - } - - if r.haltOnReconciliationError { - return errors.New("reconciliation error") - } - - return nil - } - - return r.logger.ReconcileSuccessStream( - ctx, - activeReconciliation, - change.Account, - &types.Amount{ - Value: balanceAfter, - Currency: change.Currency, - }, - change.Block, - ) -} - -func (r *StatelessReconciler) reconcileAccounts( - ctx context.Context, -) error { - for { - select { - case <-ctx.Done(): - return ctx.Err() - case change := <-r.changeQueue: - err := r.reconcileChange( - ctx, - change, - ) - if err != nil { - return err - } - } - } -} - -// Reconcile starts the active StatelessReconciler goroutines. -// If any goroutine errors, the function will return an error. -func (r *StatelessReconciler) Reconcile(ctx context.Context) error { - g, ctx := errgroup.WithContext(ctx) - for j := uint64(0); j < r.accountConcurrency; j++ { - g.Go(func() error { - return r.reconcileAccounts(ctx) - }) - } - - if err := g.Wait(); err != nil { - return err - } - - return nil -} From 51a4c9f7a8764fcb00f0f5c774706be6919a1934 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 4 May 2020 09:12:48 -0700 Subject: [PATCH 04/31] Cleanup storage --- internal/processor/base_processor.go | 226 --------------- internal/processor/processor.go | 164 +++++++++-- internal/reconciler/reconciler.go | 10 +- internal/storage/block_storage.go | 393 ++++++++++++++++++--------- internal/syncer/syncer.go | 39 ++- internal/{storage => utils}/utils.go | 2 +- 6 files changed, 443 insertions(+), 391 deletions(-) delete mode 100644 internal/processor/base_processor.go rename internal/{storage => utils}/utils.go (99%) diff --git a/internal/processor/base_processor.go b/internal/processor/base_processor.go deleted file mode 100644 index d8b79967..00000000 --- a/internal/processor/base_processor.go +++ /dev/null @@ -1,226 +0,0 @@ -// Copyright 2020 Coinbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package processor - -import ( - "context" - "fmt" - "log" - "math/big" - - "github.com/coinbase/rosetta-cli/internal/logger" - "github.com/coinbase/rosetta-cli/internal/reconciler" - "github.com/coinbase/rosetta-cli/internal/storage" - - "github.com/coinbase/rosetta-sdk-go/asserter" - "github.com/coinbase/rosetta-sdk-go/types" -) - -// BaseProcessor logs processed blocks -// and reconciles modified balances. -type BaseProcessor struct { - logger *logger.Logger - reconciler reconciler.Reconciler - asserter *asserter.Asserter - exemptAccounts []*reconciler.AccountCurrency -} - -// NewBaseProcessor constructs a basic Handler. -func NewBaseProcessor( - logger *logger.Logger, - reconciler reconciler.Reconciler, - asserter *asserter.Asserter, - exemptAccounts []*reconciler.AccountCurrency, -) *BaseProcessor { - return &BaseProcessor{ - logger: logger, - reconciler: reconciler, - asserter: asserter, - exemptAccounts: exemptAccounts, - } -} - -// BalanceChanges returns all balance changes for -// a particular block. All balance changes for a -// particular account are summed into a single -// storage.BalanceChanges struct. If a block is being -// orphaned, the opposite of each balance change is -// returned. -func (p *BaseProcessor) BalanceChanges( - ctx context.Context, - block *types.Block, - blockRemoved bool, -) ([]*storage.BalanceChange, error) { - balanceChanges := map[string]*storage.BalanceChange{} - for _, tx := range block.Transactions { - for _, op := range tx.Operations { - skip, err := p.skipOperation( - ctx, - op, - ) - if err != nil { - return nil, err - } - if skip { - continue - } - - amount := op.Amount - blockIdentifier := block.BlockIdentifier - if blockRemoved { - existing, ok := new(big.Int).SetString(amount.Value, 10) - if !ok { - return nil, fmt.Errorf("%s is not an integer", amount.Value) - } - - amount.Value = new(big.Int).Neg(existing).String() - blockIdentifier = block.ParentBlockIdentifier - } - - // Merge values by account and currency - // TODO: change balance key to be this - key := fmt.Sprintf("%s:%s", - storage.GetAccountKey(op.Account), - storage.GetCurrencyKey(op.Amount.Currency), - ) - - val, ok := balanceChanges[key] - if !ok { - balanceChanges[key] = &storage.BalanceChange{ - Account: op.Account, - Currency: op.Amount.Currency, - Difference: amount.Value, - Block: blockIdentifier, - } - continue - } - - newDifference, err := storage.AddStringValues(val.Difference, amount.Value) - if err != nil { - return nil, err - } - val.Difference = newDifference - balanceChanges[key] = val - } - } - - allChanges := []*storage.BalanceChange{} - for _, change := range balanceChanges { - allChanges = append(allChanges, change) - } - - return allChanges, nil -} - -// skipOperation returns a boolean indicating whether -// an operation should be processed. An operation will -// not be processed if it is considered unsuccessful -// or affects an exempt account. -func (p *BaseProcessor) skipOperation( - ctx context.Context, - op *types.Operation, -) (bool, error) { - successful, err := p.asserter.OperationSuccessful(op) - if err != nil { - // Should only occur if responses not validated - return false, err - } - - if !successful { - return true, nil - } - - if op.Account == nil { - return true, nil - } - - // Exempting account in BalanceChanges ensures that storage is not updated - // and that the account is not reconciled. - if p.accountExempt(ctx, op.Account, op.Amount.Currency) { - log.Printf("Skipping exempt account %+v\n", op.Account) - return true, nil - } - - return false, nil -} - -// BlockAdded is called by the syncer after a -// block is added. -func (p *BaseProcessor) BlockAdded( - ctx context.Context, - block *types.Block, -) error { - log.Printf("Adding block %+v\n", block.BlockIdentifier) - - // Log processed blocks and balance changes - if err := p.logger.BlockStream(ctx, block, false); err != nil { - return nil - } - - balanceChanges, err := p.BalanceChanges(ctx, block, false) - if err != nil { - return err - } - - if err := p.logger.BalanceStream(ctx, balanceChanges); err != nil { - return nil - } - - // Mark accounts for reconciliation...this may be - // blocking - return p.reconciler.QueueChanges(ctx, block.BlockIdentifier, balanceChanges) -} - -// BlockRemoved is called by the syncer after a -// block is removed. -func (p *BaseProcessor) BlockRemoved( - ctx context.Context, - block *types.Block, -) error { - log.Printf("Orphaning block %+v\n", block.BlockIdentifier) - - // Log processed blocks and balance changes - if err := p.logger.BlockStream(ctx, block, true); err != nil { - return nil - } - - balanceChanges, err := p.BalanceChanges(ctx, block, true) - if err != nil { - return err - } - - if err := p.logger.BalanceStream(ctx, balanceChanges); err != nil { - return nil - } - - return nil -} - -// accountExempt returns a boolean indicating if the provided -// account and currency are exempt from balance tracking and -// reconciliation. -func (p *BaseProcessor) accountExempt( - ctx context.Context, - account *types.AccountIdentifier, - currency *types.Currency, -) bool { - return reconciler.ContainsAccountCurrency( - p.exemptAccounts, - &reconciler.AccountCurrency{ - Account: account, - Currency: currency, - }, - ) -} diff --git a/internal/processor/processor.go b/internal/processor/processor.go index 7e12f8e6..cb9d60cb 100644 --- a/internal/processor/processor.go +++ b/internal/processor/processor.go @@ -1,27 +1,155 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package processor import ( "context" + "log" + + "github.com/coinbase/rosetta-cli/internal/logger" + "github.com/coinbase/rosetta-cli/internal/reconciler" + "github.com/coinbase/rosetta-cli/internal/storage" + "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/types" ) -// Processor is called at various times during the sync cycle -// to handle different events. It is common to write logs or -// perform reconciliation in the sync processor. - -// TODO: move back to sync handler...create struct that is SyncHandler and Reconciler Handler -type Processor interface { - // TODO: if account appears for the first time, sync its balance at block previous if - // lookup by balance enabled. If not enabled, then warn that must sync from genesis...then - // we can use same logic for all use cases. :fire: - BlockAdded( - ctx context.Context, - block *types.Block, - ) error - - BlockRemoved( - ctx context.Context, - block *types.Block, - ) error +// Processor is what you write to handle chain data. +type Processor struct { + storage *storage.BlockStorage + logger *logger.Logger + reconciler *reconciler.Reconciler + asserter *asserter.Asserter + exemptAccounts []*reconciler.AccountCurrency +} + +// NewProcessor constructs a basic Handler. +func NewProcessor( + storage *storage.BlockStorage, + logger *logger.Logger, + reconciler *reconciler.Reconciler, + asserter *asserter.Asserter, + exemptAccounts []*reconciler.AccountCurrency, +) *Processor { + return &Processor{ + storage: storage, + logger: logger, + reconciler: reconciler, + asserter: asserter, + exemptAccounts: exemptAccounts, + } +} + +// SkipOperation returns a boolean indicating whether +// an operation should be processed. An operation will +// not be processed if it is considered unsuccessful +// or affects an exempt account. +func (p *Processor) SkipOperation( + ctx context.Context, + op *types.Operation, +) (bool, error) { + successful, err := p.asserter.OperationSuccessful(op) + if err != nil { + // Should only occur if responses not validated + return false, err + } + + if !successful { + return true, nil + } + + if op.Account == nil { + return true, nil + } + + // Exempting account in BalanceChanges ensures that storage is not updated + // and that the account is not reconciled. + if p.accountExempt(ctx, op.Account, op.Amount.Currency) { + log.Printf("Skipping exempt account %+v\n", op.Account) + return true, nil + } + + return false, nil +} + +// BlockAdded is called by the syncer after a +// block is added. +func (p *Processor) BlockAdded( + ctx context.Context, + block *types.Block, +) error { + log.Printf("Adding block %+v\n", block.BlockIdentifier) + + // Log processed blocks and balance changes + if err := p.logger.BlockStream(ctx, block, false); err != nil { + return nil + } + + balanceChanges, err := p.storage.StoreBlock(ctx, block) + if err != nil { + return err + } + + if err := p.logger.BalanceStream(ctx, balanceChanges); err != nil { + return nil + } + + // Mark accounts for reconciliation...this may be + // blocking + return p.reconciler.QueueChanges(ctx, block.BlockIdentifier, balanceChanges) +} + +// BlockRemoved is called by the syncer after a +// block is removed. +func (p *Processor) BlockRemoved( + ctx context.Context, + block *types.Block, +) error { + log.Printf("Orphaning block %+v\n", block.BlockIdentifier) + + // Log processed blocks and balance changes + if err := p.logger.BlockStream(ctx, block, true); err != nil { + return nil + } + + balanceChanges, err := p.storage.RemoveBlock(ctx, block) + if err != nil { + return err + } + + if err := p.logger.BalanceStream(ctx, balanceChanges); err != nil { + return nil + } + + // We only attempt to reconciler changes when blocks are added + return nil +} + +// accountExempt returns a boolean indicating if the provided +// account and currency are exempt from balance tracking and +// reconciliation. +func (p *Processor) accountExempt( + ctx context.Context, + account *types.AccountIdentifier, + currency *types.Currency, +) bool { + return reconciler.ContainsAccountCurrency( + p.exemptAccounts, + &reconciler.AccountCurrency{ + Account: account, + Currency: currency, + }, + ) } diff --git a/internal/reconciler/reconciler.go b/internal/reconciler/reconciler.go index 471f6a50..d100bccf 100644 --- a/internal/reconciler/reconciler.go +++ b/internal/reconciler/reconciler.go @@ -23,8 +23,11 @@ import ( "sync" "time" + // TODO: remove all references to internal packages + // before transitioning to rosetta-sdk-go "github.com/coinbase/rosetta-cli/internal/logger" "github.com/coinbase/rosetta-cli/internal/storage" + "github.com/coinbase/rosetta-cli/internal/utils" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/types" @@ -106,7 +109,8 @@ type ReconcilerHandler interface { // always compare current balance...just need to set with previous seen if starting // in middle // TODO: always pass in head block to do lookup in case we need to fetch balance - // on previous block + // on previous block...this should always be set by the time it gets to the reconciler + // based on storage AccountBalance( ctx context.Context, account *types.AccountIdentifier, @@ -296,7 +300,7 @@ func (r *Reconciler) CompareBalance( ) } - difference, err := storage.SubtractStringValues(cachedBalance.Value, amount) + difference, err := utils.SubtractStringValues(cachedBalance.Value, amount) if err != nil { return "", -1, err } @@ -308,7 +312,7 @@ func (r *Reconciler) CompareBalance( return zeroString, head.Index, nil } -// getAccountBalance returns the balance for an account +// bestBalance returns the balance for an account // at either the current block (if lookupBalanceByBlock is // disabled) or at some historical block. func (r *Reconciler) bestBalance( diff --git a/internal/storage/block_storage.go b/internal/storage/block_storage.go index ce673246..30186cd6 100644 --- a/internal/storage/block_storage.go +++ b/internal/storage/block_storage.go @@ -28,8 +28,9 @@ import ( "path" "strings" - "github.com/coinbase/rosetta-sdk-go/types" + "github.com/coinbase/rosetta-cli/internal/utils" + "github.com/coinbase/rosetta-sdk-go/types" "github.com/davecgh/go-spew/spew" ) @@ -90,9 +91,9 @@ var ( // hashBytes is used to construct a SHA1 // hash to protect against arbitrarily // large key sizes. -func hashBytes(data []byte) []byte { +func hashBytes(data string) []byte { h := sha256.New() - _, err := h.Write(data) + _, err := h.Write([]byte(data)) if err != nil { log.Fatal(err) } @@ -104,73 +105,117 @@ func hashBytes(data []byte) []byte { // hash to protect against arbitrarily // large key sizes. func hashString(data string) string { - return fmt.Sprintf("%x", hashBytes([]byte(data))) + return fmt.Sprintf("%x", hashBytes(data)) } func getHeadBlockKey() []byte { - return hashBytes([]byte(headBlockKey)) + return hashBytes(headBlockKey) } func getBlockKey(blockIdentifier *types.BlockIdentifier) []byte { return hashBytes( - []byte(fmt.Sprintf("%s:%d", blockIdentifier.Hash, blockIdentifier.Index)), + fmt.Sprintf("%s:%d", blockIdentifier.Hash, blockIdentifier.Index), ) } func getHashKey(hash string, isBlock bool) []byte { if isBlock { - return hashBytes([]byte(fmt.Sprintf("%s:%s", blockHashNamespace, hash))) + return hashBytes(fmt.Sprintf("%s:%s", blockHashNamespace, hash)) + } + + return hashBytes(fmt.Sprintf("%s:%s", transactionHashNamespace, hash)) +} + +// GetCurrencyKey is used to identify a *types.Currency +// in an account's map of currencies. It is not feasible +// to create a map of [types.Currency]*types.Amount +// because types.Currency contains a metadata pointer +// that would prevent any equality. +func GetCurrencyKey(currency *types.Currency) string { + if currency.Metadata == nil { + return fmt.Sprintf("%s:%d", currency.Symbol, currency.Decimals) } - return hashBytes([]byte(fmt.Sprintf("%s:%s", transactionHashNamespace, hash))) + // TODO: Handle currency.Metadata + // that has pointer value. + return fmt.Sprintf( + "%s:%d:%v", + currency.Symbol, + currency.Decimals, + currency.Metadata, + ) } // GetAccountKey returns a byte slice representing a *types.AccountIdentifier. // This byte slice automatically handles the existence of *types.SubAccount // detail. -func GetAccountKey(account *types.AccountIdentifier) []byte { +func GetAccountKey(account *types.AccountIdentifier) string { if account.SubAccount == nil { - return hashBytes( - []byte(fmt.Sprintf("%s:%s", balanceNamespace, account.Address)), - ) + return fmt.Sprintf("%s:%s", balanceNamespace, account.Address) } if account.SubAccount.Metadata == nil { - return hashBytes([]byte(fmt.Sprintf( + return fmt.Sprintf( "%s:%s:%s", balanceNamespace, account.Address, account.SubAccount.Address, - ))) + ) } // TODO: handle SubAccount.Metadata // that contains pointer values. - return hashBytes([]byte(fmt.Sprintf( + return fmt.Sprintf( "%s:%s:%s:%v", balanceNamespace, account.Address, account.SubAccount.Address, account.SubAccount.Metadata, - ))) + ) +} + +func GetBalanceKey(account *types.AccountIdentifier, currency *types.Currency) []byte { + return hashBytes( + fmt.Sprintf("%s/%s", GetAccountKey(account), GetCurrencyKey(currency)), + ) +} + +type BlockStorageHelper interface { + AccountBalance( + ctx context.Context, + account *types.AccountIdentifier, + currency *types.Currency, + block *types.BlockIdentifier, + ) (*types.Amount, error) // returns an error if lookupBalanceByBlock disabled + + SkipOperation( + ctx context.Context, + op *types.Operation, + ) (bool, error) } // BlockStorage implements block specific storage methods // on top of a Database and DatabaseTransaction interface. type BlockStorage struct { - db Database + db Database + helper BlockStorageHelper } // NewBlockStorage returns a new BlockStorage. -func NewBlockStorage(ctx context.Context, db Database) *BlockStorage { +func NewBlockStorage( + ctx context.Context, + db Database, + helper BlockStorageHelper, +) *BlockStorage { return &BlockStorage{ - db: db, + db: db, + helper: helper, } } // NewDatabaseTransaction returns a DatabaseTransaction // from the Database that is backing BlockStorage. -func (b *BlockStorage) NewDatabaseTransaction( +func (b *BlockStorage) newDatabaseTransaction( ctx context.Context, write bool, ) DatabaseTransaction { @@ -181,8 +226,10 @@ func (b *BlockStorage) NewDatabaseTransaction( // if it exists. func (b *BlockStorage) GetHeadBlockIdentifier( ctx context.Context, - transaction DatabaseTransaction, ) (*types.BlockIdentifier, error) { + transaction := b.newDatabaseTransaction(ctx, false) + defer transaction.Discard(ctx) + exists, block, err := transaction.Get(ctx, getHeadBlockKey()) if err != nil { return nil, err @@ -221,9 +268,11 @@ func (b *BlockStorage) StoreHeadBlockIdentifier( // GetBlock returns a block, if it exists. func (b *BlockStorage) GetBlock( ctx context.Context, - transaction DatabaseTransaction, blockIdentifier *types.BlockIdentifier, ) (*types.Block, error) { + transaction := b.newDatabaseTransaction(ctx, false) + defer transaction.Discard(ctx) + exists, block, err := transaction.Get(ctx, getBlockKey(blockIdentifier)) if err != nil { return nil, err @@ -279,36 +328,52 @@ func (b *BlockStorage) storeHash( // its transaction hashes for duplicate detection. func (b *BlockStorage) StoreBlock( ctx context.Context, - transaction DatabaseTransaction, block *types.Block, -) error { +) ([]*BalanceChange, error) { + transaction := b.newDatabaseTransaction(ctx, true) + defer transaction.Discard(ctx) buf := new(bytes.Buffer) err := gob.NewEncoder(buf).Encode(block) if err != nil { - return err + return nil, err } // Store block err = transaction.Set(ctx, getBlockKey(block.BlockIdentifier), buf.Bytes()) if err != nil { - return err + return nil, err } // Store block hash err = b.storeHash(ctx, transaction, block.BlockIdentifier.Hash, true) if err != nil { - return err + return nil, err } // Store all transaction hashes for _, txn := range block.Transactions { err = b.storeHash(ctx, transaction, txn.TransactionIdentifier.Hash, false) if err != nil { - return err + return nil, err } } - return nil + changes, err := b.BalanceChanges(ctx, block, false) + if err != nil { + return nil, err + } + + for _, change := range changes { + if err := b.UpdateBalance(ctx, transaction, change); err != nil { + return nil, err + } + } + + if err := transaction.Commit(ctx); err != nil { + return nil, err + } + + return changes, nil } // RemoveBlock removes a block or returns an error. @@ -317,35 +382,47 @@ func (b *BlockStorage) StoreBlock( // detection. This is called within a re-org. func (b *BlockStorage) RemoveBlock( ctx context.Context, - transaction DatabaseTransaction, - block *types.BlockIdentifier, -) error { - // Remove all transaction hashes - blockData, err := b.GetBlock(ctx, transaction, block) + block *types.Block, +) ([]*BalanceChange, error) { + transaction := b.newDatabaseTransaction(ctx, true) + defer transaction.Discard(ctx) + + changes, err := b.BalanceChanges(ctx, block, true) if err != nil { - return err + return nil, err + } + + for _, change := range changes { + if err := b.UpdateBalance(ctx, transaction, change); err != nil { + return nil, err + } } - for _, txn := range blockData.Transactions { + // Remove all transaction hashes + for _, txn := range block.Transactions { err = transaction.Delete(ctx, getHashKey(txn.TransactionIdentifier.Hash, false)) if err != nil { - return err + return nil, err } } // Remove block hash - err = transaction.Delete(ctx, getHashKey(block.Hash, true)) + err = transaction.Delete(ctx, getHashKey(block.BlockIdentifier.Hash, true)) if err != nil { - return err + return nil, err } // Remove block - return transaction.Delete(ctx, getBlockKey(block)) + if err := transaction.Delete(ctx, getBlockKey(block.BlockIdentifier)); err != nil { + return nil, err + } + + return changes, nil } type balanceEntry struct { - Amounts map[string]*types.Amount - Block *types.BlockIdentifier + Amount *types.Amount + Block *types.BlockIdentifier } func serializeBalanceEntry(bal balanceEntry) ([]byte, error) { @@ -369,30 +446,6 @@ func parseBalanceEntry(buf []byte) (*balanceEntry, error) { return &bal, nil } -// GetCurrencyKey is used to identify a *types.Currency -// in an account's map of currencies. It is not feasible -// to create a map of [types.Currency]*types.Amount -// because types.Currency contains a metadata pointer -// that would prevent any equality. -func GetCurrencyKey(currency *types.Currency) string { - if currency.Metadata == nil { - return hashString( - fmt.Sprintf("%s:%d", currency.Symbol, currency.Decimals), - ) - } - - // TODO: Handle currency.Metadata - // that has pointer value. - return hashString( - fmt.Sprintf( - "%s:%d:%v", - currency.Symbol, - currency.Decimals, - currency.Metadata, - ), - ) -} - // BalanceChange represents a balance change that affected // a *types.AccountIdentifier and a *types.Currency. type BalanceChange struct { @@ -402,125 +455,133 @@ type BalanceChange struct { Difference string `json:"difference,omitempty"` } +func (b *BlockStorage) SetBalance( + ctx context.Context, + dbTransaction DatabaseTransaction, + account *types.AccountIdentifier, + amount *types.Amount, + block *types.BlockIdentifier, +) error { + key := GetBalanceKey(account, amount.Currency) + + serialBal, err := serializeBalanceEntry(balanceEntry{ + Amount: amount, + Block: block, + }) + if err != nil { + return err + } + + if err := dbTransaction.Set(ctx, key, serialBal); err != nil { + return err + } + + return nil +} + // UpdateBalance updates a types.AccountIdentifer // by a types.Amount and sets the account's most // recent accessed block. func (b *BlockStorage) UpdateBalance( ctx context.Context, dbTransaction DatabaseTransaction, - account *types.AccountIdentifier, - amount *types.Amount, - block *types.BlockIdentifier, -) (*BalanceChange, error) { - if amount == nil || amount.Currency == nil { - return nil, errors.New("invalid amount") + change *BalanceChange, +) error { + if change.Currency == nil { + return errors.New("invalid currency") } - key := GetAccountKey(account) + key := GetBalanceKey(change.Account, change.Currency) // Get existing balance on key exists, balance, err := dbTransaction.Get(ctx, key) if err != nil { - return nil, err + return err } - // TODO: create a balance key that is the combination - // of account and currency - currencyKey := GetCurrencyKey(amount.Currency) - if !exists { - amountMap := make(map[string]*types.Amount) + // TODO: must be block BEFORE current (should only occur when adding, not removing) + amount, err := b.helper.AccountBalance(ctx, change.Account, change.Currency, nil) + if err != nil { + return fmt.Errorf("%w: unable to get previous account balance", err) + } + newVal, ok := new(big.Int).SetString(amount.Value, 10) if !ok { - return nil, fmt.Errorf("%s is not an integer", amount.Value) + return fmt.Errorf("%s is not an integer", amount.Value) } if newVal.Sign() == -1 { - return nil, fmt.Errorf( + return fmt.Errorf( "%w %+v for %+v at %+v", ErrNegativeBalance, spew.Sdump(amount), - account, - block, + change.Account, + change.Block, ) } - amountMap[currencyKey] = amount serialBal, err := serializeBalanceEntry(balanceEntry{ - Amounts: amountMap, - Block: block, + Amount: amount, + Block: change.Block, }) if err != nil { - return nil, err + return err } if err := dbTransaction.Set(ctx, key, serialBal); err != nil { - return nil, err + return err } - return &BalanceChange{ - Account: account, - Currency: amount.Currency, - Block: block, - Difference: amount.Value, - }, nil + return nil } // Modify balance parseBal, err := parseBalanceEntry(balance) if err != nil { - return nil, err - } - - val, ok := parseBal.Amounts[currencyKey] - if !ok { - parseBal.Amounts[currencyKey] = amount + return err } - oldValue := val.Value - val.Value, err = AddStringValues(amount.Value, oldValue) + oldValue := parseBal.Amount.Value + newVal, err := utils.AddStringValues(change.Difference, oldValue) if err != nil { - return nil, err + return err } - if strings.HasPrefix(val.Value, "-") { - return nil, fmt.Errorf( + if strings.HasPrefix(newVal, "-") { + return fmt.Errorf( "%w %+v for %+v at %+v", ErrNegativeBalance, - spew.Sdump(val), - account, - block, + spew.Sdump(newVal), + change.Account, + change.Block, ) } - parseBal.Amounts[currencyKey] = val - - parseBal.Block = block + parseBal.Amount.Value = newVal + parseBal.Block = change.Block serialBal, err := serializeBalanceEntry(*parseBal) if err != nil { - return nil, err + return err } if err := dbTransaction.Set(ctx, key, serialBal); err != nil { - return nil, err + return err } - return &BalanceChange{ - Account: account, - Currency: amount.Currency, - Block: block, - Difference: amount.Value, - }, nil + return nil } // GetBalance returns all the balances of a types.AccountIdentifier // and the types.BlockIdentifier it was last updated at. -// TODO: change to fetch by account and currency func (b *BlockStorage) GetBalance( ctx context.Context, - transaction DatabaseTransaction, account *types.AccountIdentifier, -) (map[string]*types.Amount, *types.BlockIdentifier, error) { - key := GetAccountKey(account) + currency *types.Currency, +) (*types.Amount, *types.BlockIdentifier, error) { + transaction := b.newDatabaseTransaction(ctx, false) + defer transaction.Discard(ctx) + + key := GetBalanceKey(account, currency) exists, bal, err := transaction.Get(ctx, key) if err != nil { return nil, nil, err @@ -535,12 +596,13 @@ func (b *BlockStorage) GetBalance( return nil, nil, err } - return deserialBal.Amounts, deserialBal.Block, nil + return deserialBal.Amount, deserialBal.Block, nil } // BootstrapBalance represents a balance of // a *types.AccountIdentifier and a *types.Currency in the // genesis block. +// TODO: Must be exported for use type BootstrapBalance struct { Account *types.AccountIdentifier `json:"account_identifier,omitempty"` Currency *types.Currency `json:"currency,omitempty"` @@ -567,15 +629,15 @@ func (b *BlockStorage) BootstrapBalances( return err } - // Update balances in database - dbTransaction := b.NewDatabaseTransaction(ctx, true) - defer dbTransaction.Discard(ctx) - - _, err = b.GetHeadBlockIdentifier(ctx, dbTransaction) + _, err = b.GetHeadBlockIdentifier(ctx) if err != ErrHeadBlockNotFound { return ErrAlreadyStartedSyncing } + // Update balances in database + dbTransaction := b.newDatabaseTransaction(ctx, true) + defer dbTransaction.Discard(ctx) + for _, balance := range balances { // Ensure change.Difference is valid amountValue, ok := new(big.Int).SetString(balance.Value, 10) @@ -594,7 +656,7 @@ func (b *BlockStorage) BootstrapBalances( balance.Currency, ) - _, err = b.UpdateBalance( + err = b.SetBalance( ctx, dbTransaction, balance.Account, @@ -617,3 +679,74 @@ func (b *BlockStorage) BootstrapBalances( log.Printf("%d Balances Bootstrapped\n", len(balances)) return nil } + +// BalanceChanges returns all balance changes for +// a particular block. All balance changes for a +// particular account are summed into a single +// storage.BalanceChanges struct. If a block is being +// orphaned, the opposite of each balance change is +// returned. +func (b *BlockStorage) BalanceChanges( + ctx context.Context, + block *types.Block, + blockRemoved bool, +) ([]*BalanceChange, error) { + balanceChanges := map[string]*BalanceChange{} + for _, tx := range block.Transactions { + for _, op := range tx.Operations { + skip, err := b.helper.SkipOperation( + ctx, + op, + ) + if err != nil { + return nil, err + } + if skip { + continue + } + + amount := op.Amount + blockIdentifier := block.BlockIdentifier + if blockRemoved { + existing, ok := new(big.Int).SetString(amount.Value, 10) + if !ok { + return nil, fmt.Errorf("%s is not an integer", amount.Value) + } + + amount.Value = new(big.Int).Neg(existing).String() + blockIdentifier = block.ParentBlockIdentifier + } + + // Merge values by account and currency + key := fmt.Sprintf( + "%x", + GetBalanceKey(op.Account, op.Amount.Currency), + ) + + val, ok := balanceChanges[key] + if !ok { + balanceChanges[key] = &BalanceChange{ + Account: op.Account, + Currency: op.Amount.Currency, + Difference: amount.Value, + Block: blockIdentifier, + } + continue + } + + newDifference, err := utils.AddStringValues(val.Difference, amount.Value) + if err != nil { + return nil, err + } + val.Difference = newDifference + balanceChanges[key] = val + } + } + + allChanges := []*BalanceChange{} + for _, change := range balanceChanges { + allChanges = append(allChanges, change) + } + + return allChanges, nil +} diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index a3926428..bc1fddfc 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -21,8 +21,6 @@ import ( "log" "reflect" - "github.com/coinbase/rosetta-cli/internal/processor" - "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/types" ) @@ -33,11 +31,26 @@ const ( maxSync = 1000 ) +// SyncHandler is called at various times during the sync cycle +// to handle different events. It is common to write logs or +// perform reconciliation in the sync processor. +type SyncHandler interface { + BlockAdded( + ctx context.Context, + block *types.Block, + ) error + + BlockRemoved( + ctx context.Context, + block *types.Block, + ) error +} + type Syncer struct { - network *types.NetworkIdentifier - fetcher *fetcher.Fetcher - processor processor.Processor - cancel context.CancelFunc + network *types.NetworkIdentifier + fetcher *fetcher.Fetcher + handler SyncHandler + cancel context.CancelFunc // Used to keep track of sync state genesisBlock *types.BlockIdentifier @@ -47,14 +60,14 @@ type Syncer struct { func NewSyncer( network *types.NetworkIdentifier, fetcher *fetcher.Fetcher, - processor processor.Processor, + handler SyncHandler, cancel context.CancelFunc, ) *Syncer { return &Syncer{ - network: network, - fetcher: fetcher, - processor: processor, - cancel: cancel, + network: network, + fetcher: fetcher, + handler: handler, + cancel: cancel, } } @@ -195,10 +208,10 @@ func (s *Syncer) syncRange( if shouldRemove { currIndex-- - err = s.processor.BlockRemoved(ctx, block) + err = s.handler.BlockRemoved(ctx, block) } else { currIndex++ - err = s.processor.BlockAdded(ctx, block) + err = s.handler.BlockAdded(ctx, block) } if err != nil { return err diff --git a/internal/storage/utils.go b/internal/utils/utils.go similarity index 99% rename from internal/storage/utils.go rename to internal/utils/utils.go index d3aed5d5..cb121afb 100644 --- a/internal/storage/utils.go +++ b/internal/utils/utils.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package storage +package utils import ( "fmt" From 54d6d9cc0174f038959a686b547d8f1160741883 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 4 May 2020 11:51:39 -0700 Subject: [PATCH 05/31] Remove logger dependency from reconciler --- internal/reconciler/reconciler.go | 50 ++++++++++++++++++++----------- 1 file changed, 33 insertions(+), 17 deletions(-) diff --git a/internal/reconciler/reconciler.go b/internal/reconciler/reconciler.go index d100bccf..063b73b6 100644 --- a/internal/reconciler/reconciler.go +++ b/internal/reconciler/reconciler.go @@ -25,7 +25,6 @@ import ( // TODO: remove all references to internal packages // before transitioning to rosetta-sdk-go - "github.com/coinbase/rosetta-cli/internal/logger" "github.com/coinbase/rosetta-cli/internal/storage" "github.com/coinbase/rosetta-cli/internal/utils" @@ -96,7 +95,7 @@ var ( ErrBlockGone = errors.New("block gone") ) -type ReconcilerHandler interface { +type ReconcilerHelper interface { BlockExists( ctx context.Context, block *types.BlockIdentifier, @@ -118,14 +117,35 @@ type ReconcilerHandler interface { ) (*types.Amount, *types.BlockIdentifier, error) } +type ReconcilerHandler interface { + ReconciliationFailed( + ctx context.Context, + reconciliationType string, + account *types.AccountIdentifier, + currency *types.Currency, + computedBalance string, + nodeBalance string, + block *types.BlockIdentifier, + ) error + + ReconciliationSucceeded( + ctx context.Context, + reconciliationType string, + account *types.AccountIdentifier, + currency *types.Currency, + balance string, + block *types.BlockIdentifier, + ) error +} + // Reconciler contains all logic to reconcile balances of // types.AccountIdentifiers returned in types.Operations // by a Rosetta Server. type Reconciler struct { network *types.NetworkIdentifier + helper ReconcilerHelper handler ReconcilerHandler fetcher *fetcher.Fetcher - logger *logger.Logger accountConcurrency uint64 lookupBalanceByBlock bool haltOnReconciliationError bool @@ -151,7 +171,6 @@ func NewReconciler( network *types.NetworkIdentifier, handler ReconcilerHandler, fetcher *fetcher.Fetcher, - logger *logger.Logger, accountConcurrency uint64, lookupBalanceByBlock bool, haltOnReconciliationError bool, @@ -161,7 +180,6 @@ func NewReconciler( network: network, handler: handler, fetcher: fetcher, - logger: logger, accountConcurrency: accountConcurrency, lookupBalanceByBlock: lookupBalanceByBlock, haltOnReconciliationError: haltOnReconciliationError, @@ -260,7 +278,7 @@ func (r *Reconciler) CompareBalance( liveBlock *types.BlockIdentifier, ) (string, int64, error) { // Head block should be set before we CompareBalance - head, err := r.handler.CurrentBlock(ctx) + head, err := r.helper.CurrentBlock(ctx) if err != nil { return zeroString, 0, fmt.Errorf("%w: unable to get current block for reconciliation", err) } @@ -276,7 +294,7 @@ func (r *Reconciler) CompareBalance( } // Check if live block is in store (ensure not reorged) - _, err = r.handler.BlockExists(ctx, liveBlock) + _, err = r.helper.BlockExists(ctx, liveBlock) if err != nil { return zeroString, head.Index, fmt.Errorf( "%w %+v", @@ -286,7 +304,7 @@ func (r *Reconciler) CompareBalance( } // Check if live block < computed head - cachedBalance, balanceBlock, err := r.handler.AccountBalance(ctx, account, currency) + cachedBalance, balanceBlock, err := r.helper.AccountBalance(ctx, account, currency) if err != nil { return zeroString, head.Index, err } @@ -409,15 +427,15 @@ func (r *Reconciler) accountReconciliation( } if difference != zeroString { - err := r.logger.ReconcileFailureStream( + err := r.handler.ReconciliationFailed( ctx, reconciliationType, accountCurrency.Account, accountCurrency.Currency, - difference, + "TODO", + liveAmount, liveBlock, ) - if err != nil { return err } @@ -430,14 +448,12 @@ func (r *Reconciler) accountReconciliation( } r.inactiveAccountQueue(inactive, accountCurrency, liveBlock) - return r.logger.ReconcileSuccessStream( + return r.handler.ReconciliationSucceeded( ctx, reconciliationType, accountCurrency.Account, - &types.Amount{ - Value: liveAmount, - Currency: currency, - }, + accountCurrency.Currency, + liveAmount, liveBlock, ) } @@ -533,7 +549,7 @@ func (r *Reconciler) reconcileInactiveAccounts( ctx context.Context, ) error { for ctx.Err() == nil { - head, err := r.handler.CurrentBlock(ctx) + head, err := r.helper.CurrentBlock(ctx) // When first start syncing, this loop may run before the genesis block is synced. // If this is the case, we should sleep and try again later instead of exiting. if errors.Is(err, storage.ErrHeadBlockNotFound) { From 433aa06599e63941959badc7bca213f60cede0fd Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 4 May 2020 12:03:09 -0700 Subject: [PATCH 06/31] Removed reconciler dependency on storage --- internal/logger/logger.go | 4 ++-- internal/reconciler/reconciler.go | 35 ++++++++++++++++++------------- internal/storage/block_storage.go | 24 +++++++-------------- 3 files changed, 30 insertions(+), 33 deletions(-) diff --git a/internal/logger/logger.go b/internal/logger/logger.go index fedd36be..fb762559 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -21,7 +21,7 @@ import ( "os" "path" - "github.com/coinbase/rosetta-cli/internal/storage" + "github.com/coinbase/rosetta-cli/internal/reconciler" "github.com/coinbase/rosetta-sdk-go/types" ) @@ -205,7 +205,7 @@ func (l *Logger) TransactionStream( // to the balanceStreamFile. func (l *Logger) BalanceStream( ctx context.Context, - balanceChanges []*storage.BalanceChange, + balanceChanges []*reconciler.BalanceChange, ) error { if !l.logBalanceChanges { return nil diff --git a/internal/reconciler/reconciler.go b/internal/reconciler/reconciler.go index 063b73b6..40e081fc 100644 --- a/internal/reconciler/reconciler.go +++ b/internal/reconciler/reconciler.go @@ -25,7 +25,6 @@ import ( // TODO: remove all references to internal packages // before transitioning to rosetta-sdk-go - "github.com/coinbase/rosetta-cli/internal/storage" "github.com/coinbase/rosetta-cli/internal/utils" "github.com/coinbase/rosetta-sdk-go/fetcher" @@ -95,6 +94,15 @@ var ( ErrBlockGone = errors.New("block gone") ) +// BalanceChange represents a balance change that affected +// a *types.AccountIdentifier and a *types.Currency. +type BalanceChange struct { + Account *types.AccountIdentifier `json:"account_identifier,omitempty"` + Currency *types.Currency `json:"currency,omitempty"` + Block *types.BlockIdentifier `json:"block_identifier,omitempty"` + Difference string `json:"difference,omitempty"` +} + type ReconcilerHelper interface { BlockExists( ctx context.Context, @@ -150,7 +158,7 @@ type Reconciler struct { lookupBalanceByBlock bool haltOnReconciliationError bool interestingAccounts []*AccountCurrency - changeQueue chan *storage.BalanceChange + changeQueue chan *BalanceChange // highWaterMark is used to skip requests when // we are very far behind the live head. @@ -159,7 +167,7 @@ type Reconciler struct { // seenAccts are stored for inactive account // reconciliation. seenAccts []*AccountCurrency - inactiveQueue []*storage.BalanceChange + inactiveQueue []*BalanceChange // inactiveQueueMutex needed because we can't peek at the tip // of a channel to determine when it is ready to look at. @@ -186,31 +194,31 @@ func NewReconciler( interestingAccounts: interestingAccounts, highWaterMark: -1, seenAccts: make([]*AccountCurrency, 0), - inactiveQueue: make([]*storage.BalanceChange, 0), + inactiveQueue: make([]*BalanceChange, 0), } if lookupBalanceByBlock { // When lookupBalanceByBlock is enabled, we check // balance changes synchronously. - r.changeQueue = make(chan *storage.BalanceChange) + r.changeQueue = make(chan *BalanceChange) } else { // When lookupBalanceByBlock is disabled, we must check // balance changes asynchronously. Using a buffered // channel allows us to add balance changes without blocking. - r.changeQueue = make(chan *storage.BalanceChange, backlogThreshold) + r.changeQueue = make(chan *BalanceChange, backlogThreshold) } return r } // Reconciliation -// QueueChanges enqueues a slice of *storage.BalanceChanges +// QueueChanges enqueues a slice of *BalanceChanges // for reconciliation. func (r *Reconciler) QueueChanges( ctx context.Context, // If we pass in parentblock, then we always know what to compare on diff block *types.BlockIdentifier, - balanceChanges []*storage.BalanceChange, + balanceChanges []*BalanceChange, ) error { // Ensure all interestingAccounts are checked // TODO: refactor to automatically trigger once an inactive reconciliation error @@ -231,7 +239,7 @@ func (r *Reconciler) QueueChanges( } // If account + currency not found, add with difference 0 - balanceChanges = append(balanceChanges, &storage.BalanceChange{ + balanceChanges = append(balanceChanges, &BalanceChange{ Account: account.Account, Currency: account.Currency, Difference: "0", @@ -475,7 +483,7 @@ func (r *Reconciler) inactiveAccountQueue( if inactive || shouldEnqueueInactive { r.inactiveQueueMutex.Lock() - r.inactiveQueue = append(r.inactiveQueue, &storage.BalanceChange{ + r.inactiveQueue = append(r.inactiveQueue, &BalanceChange{ Account: accountCurrency.Account, Currency: accountCurrency.Currency, Block: liveBlock, @@ -552,12 +560,9 @@ func (r *Reconciler) reconcileInactiveAccounts( head, err := r.helper.CurrentBlock(ctx) // When first start syncing, this loop may run before the genesis block is synced. // If this is the case, we should sleep and try again later instead of exiting. - if errors.Is(err, storage.ErrHeadBlockNotFound) { - log.Println("head block not yet initialized, sleeping...") + if err != nil { time.Sleep(inactiveReconciliationSleep) - continue - } else if err != nil { - return fmt.Errorf("%w: unable to get current block for inactive reconciliation", err) + log.Println("%s: unable to get current block for inactive reconciliation", err.Error()) } r.inactiveQueueMutex.Lock() diff --git a/internal/storage/block_storage.go b/internal/storage/block_storage.go index 30186cd6..3cadb3f9 100644 --- a/internal/storage/block_storage.go +++ b/internal/storage/block_storage.go @@ -28,6 +28,7 @@ import ( "path" "strings" + "github.com/coinbase/rosetta-cli/internal/reconciler" "github.com/coinbase/rosetta-cli/internal/utils" "github.com/coinbase/rosetta-sdk-go/types" @@ -329,7 +330,7 @@ func (b *BlockStorage) storeHash( func (b *BlockStorage) StoreBlock( ctx context.Context, block *types.Block, -) ([]*BalanceChange, error) { +) ([]*reconciler.BalanceChange, error) { transaction := b.newDatabaseTransaction(ctx, true) defer transaction.Discard(ctx) buf := new(bytes.Buffer) @@ -383,7 +384,7 @@ func (b *BlockStorage) StoreBlock( func (b *BlockStorage) RemoveBlock( ctx context.Context, block *types.Block, -) ([]*BalanceChange, error) { +) ([]*reconciler.BalanceChange, error) { transaction := b.newDatabaseTransaction(ctx, true) defer transaction.Discard(ctx) @@ -446,15 +447,6 @@ func parseBalanceEntry(buf []byte) (*balanceEntry, error) { return &bal, nil } -// BalanceChange represents a balance change that affected -// a *types.AccountIdentifier and a *types.Currency. -type BalanceChange struct { - Account *types.AccountIdentifier `json:"account_identifier,omitempty"` - Currency *types.Currency `json:"currency,omitempty"` - Block *types.BlockIdentifier `json:"block_identifier,omitempty"` - Difference string `json:"difference,omitempty"` -} - func (b *BlockStorage) SetBalance( ctx context.Context, dbTransaction DatabaseTransaction, @@ -485,7 +477,7 @@ func (b *BlockStorage) SetBalance( func (b *BlockStorage) UpdateBalance( ctx context.Context, dbTransaction DatabaseTransaction, - change *BalanceChange, + change *reconciler.BalanceChange, ) error { if change.Currency == nil { return errors.New("invalid currency") @@ -690,8 +682,8 @@ func (b *BlockStorage) BalanceChanges( ctx context.Context, block *types.Block, blockRemoved bool, -) ([]*BalanceChange, error) { - balanceChanges := map[string]*BalanceChange{} +) ([]*reconciler.BalanceChange, error) { + balanceChanges := map[string]*reconciler.BalanceChange{} for _, tx := range block.Transactions { for _, op := range tx.Operations { skip, err := b.helper.SkipOperation( @@ -725,7 +717,7 @@ func (b *BlockStorage) BalanceChanges( val, ok := balanceChanges[key] if !ok { - balanceChanges[key] = &BalanceChange{ + balanceChanges[key] = &reconciler.BalanceChange{ Account: op.Account, Currency: op.Amount.Currency, Difference: amount.Value, @@ -743,7 +735,7 @@ func (b *BlockStorage) BalanceChanges( } } - allChanges := []*BalanceChange{} + allChanges := []*reconciler.BalanceChange{} for _, change := range balanceChanges { allChanges = append(allChanges, change) } From a2a056f27f6508662569f49363e478443d047270 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 4 May 2020 13:57:08 -0700 Subject: [PATCH 07/31] Resolve most errors in cmd --- cmd/check_account.go | 138 -------------------- cmd/check_complete.go | 66 ++++++---- cmd/check_quick.go | 125 ------------------ cmd/root.go | 57 --------- internal/processor/processor.go | 155 ----------------------- internal/processor/reconciler_handler.go | 33 +++++ internal/processor/reconciler_helper.go | 31 +++++ internal/processor/storage_helper.go | 87 +++++++++++++ internal/processor/sync_handler.go | 92 ++++++++++++++ internal/reconciler/reconciler.go | 43 +++---- internal/syncer/syncer.go | 2 +- 11 files changed, 306 insertions(+), 523 deletions(-) delete mode 100644 cmd/check_account.go delete mode 100644 cmd/check_quick.go delete mode 100644 internal/processor/processor.go create mode 100644 internal/processor/reconciler_handler.go create mode 100644 internal/processor/reconciler_helper.go create mode 100644 internal/processor/storage_helper.go create mode 100644 internal/processor/sync_handler.go diff --git a/cmd/check_account.go b/cmd/check_account.go deleted file mode 100644 index 9dfdb8fb..00000000 --- a/cmd/check_account.go +++ /dev/null @@ -1,138 +0,0 @@ -// Copyright 2020 Coinbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "context" - "log" - - "github.com/coinbase/rosetta-cli/internal/logger" - "github.com/coinbase/rosetta-cli/internal/reconciler" - "github.com/coinbase/rosetta-cli/internal/syncer" - - "github.com/coinbase/rosetta-sdk-go/fetcher" - "github.com/spf13/cobra" - "golang.org/x/sync/errgroup" -) - -var ( - checkAccountCmd = &cobra.Command{ - Use: "check:account", - Short: "Debug inactive reconciliation errors for a group of accounts", - Long: `check:complete identifies accounts with inactive reconciliation -errors (when the balance of an account changes without any operations), however, -it does not identify which block the untracked balance change occurred. This tool -is used for locating exactly which block was missing an operation for a -particular account and currency. - -In the future, this tool will be deprecated as check:complete -will automatically identify the block where the missing operation occurred.`, - Run: runCheckAccountCmd, - } - - accountFile string -) - -func init() { - checkAccountCmd.Flags().StringVar( - &accountFile, - "interesting-accounts", - "", - `Absolute path to a file listing all accounts to check on each block. Look -at the examples directory for an example of how to structure this file.`, - ) - - err := checkAccountCmd.MarkFlagRequired("interesting-accounts") - if err != nil { - log.Fatal(err) - } -} - -func runCheckAccountCmd(cmd *cobra.Command, args []string) { - // TODO: unify startup logic with stateless - ctx, cancel := context.WithCancel(context.Background()) - - interestingAccounts, err := loadAccounts(accountFile) - if err != nil { - log.Fatal(err) - } - - exemptAccounts, err := loadAccounts(ExemptFile) - if err != nil { - log.Fatal(err) - } - - fetcher := fetcher.New( - ServerURL, - fetcher.WithBlockConcurrency(BlockConcurrency), - fetcher.WithTransactionConcurrency(TransactionConcurrency), - fetcher.WithRetryElapsedTime(ExtendedRetryElapsedTime), - ) - - primaryNetwork, _, err := fetcher.InitializeAsserter(ctx) - if err != nil { - log.Fatal(err) - } - - logger := logger.NewLogger( - DataDir, - LogBlocks, - LogTransactions, - LogBalanceChanges, - LogReconciliations, - ) - - g, ctx := errgroup.WithContext(ctx) - - r := reconciler.NewStateless( - primaryNetwork, - fetcher, - logger, - AccountConcurrency, - HaltOnReconciliationError, - interestingAccounts, - ) - - g.Go(func() error { - return r.Reconcile(ctx) - }) - - syncHandler := syncer.NewBaseHandler( - logger, - r, - exemptAccounts, - ) - - statelessSyncer := syncer.NewStateless( - primaryNetwork, - fetcher, - syncHandler, - ) - - g.Go(func() error { - return syncer.Sync( - ctx, - cancel, - statelessSyncer, - StartIndex, - EndIndex, - ) - }) - - err = g.Wait() - if err != nil { - log.Fatal(err) - } -} diff --git a/cmd/check_complete.go b/cmd/check_complete.go index 329c1ac4..728d9bc5 100644 --- a/cmd/check_complete.go +++ b/cmd/check_complete.go @@ -20,6 +20,7 @@ import ( "log" "github.com/coinbase/rosetta-cli/internal/logger" + "github.com/coinbase/rosetta-cli/internal/processor" "github.com/coinbase/rosetta-cli/internal/reconciler" "github.com/coinbase/rosetta-cli/internal/storage" "github.com/coinbase/rosetta-cli/internal/syncer" @@ -54,6 +55,8 @@ index less than the last computed block index.`, // block. Blockchains that do not support historical balance lookup // should set this to false. LookupBalanceByBlock bool + + accountFile string ) func init() { @@ -72,6 +75,13 @@ Populating this value after beginning syncing will return an error.`, change occurred instead of at the current block. Blockchains that do not support historical balance lookup should set this to false.`, ) + checkCompleteCmd.Flags().StringVar( + &accountFile, + "interesting-accounts", + "", + `Absolute path to a file listing all accounts to check on each block. Look +at the examples directory for an example of how to structure this file.`, + ) } func runCheckCompleteCmd(cmd *cobra.Command, args []string) { @@ -83,6 +93,11 @@ func runCheckCompleteCmd(cmd *cobra.Command, args []string) { log.Fatal(fmt.Errorf("%w: unable to load exempt accounts", err)) } + interestingAccounts, err := loadAccounts(accountFile) + if err != nil { + log.Fatal(fmt.Errorf("%w: unable to load interesting accounts", err)) + } + fetcher := fetcher.New( ServerURL, fetcher.WithBlockConcurrency(BlockConcurrency), @@ -101,7 +116,17 @@ func runCheckCompleteCmd(cmd *cobra.Command, args []string) { log.Fatal(fmt.Errorf("%w: unable to initialize data store", err)) } - blockStorage := storage.NewBlockStorage(ctx, localStore) + logger := logger.NewLogger( + DataDir, + LogBlocks, + LogTransactions, + LogBalanceChanges, + LogReconciliations, + ) + + blockStorageHelper := processor.NewBlockStorageHelper(fetcher, exemptAccounts) + + blockStorage := storage.NewBlockStorage(ctx, localStore, blockStorageHelper) if len(BootstrapBalances) > 0 { err = blockStorage.BootstrapBalances( ctx, @@ -113,48 +138,43 @@ func runCheckCompleteCmd(cmd *cobra.Command, args []string) { } } - logger := logger.NewLogger( - DataDir, - LogBlocks, - LogTransactions, - LogBalanceChanges, - LogReconciliations, - ) - - g, ctx := errgroup.WithContext(ctx) + reconcilerHelper := &processor.ReconcilerHelper{} + reconcilerHandler := &processor.ReconcilerHandler{} - r := reconciler.NewStateful( + r := reconciler.NewReconciler( primaryNetwork, - blockStorage, + reconcilerHelper, + reconcilerHandler, fetcher, - logger, AccountConcurrency, LookupBalanceByBlock, - HaltOnReconciliationError, + interestingAccounts, ) - g.Go(func() error { - return r.Reconcile(ctx) - }) - - syncHandler := syncer.NewBaseHandler( + syncHandler := processor.NewSyncHandler( + blockStorage, logger, r, + fetcher, exemptAccounts, ) - statefulSyncer := syncer.NewStateful( + g, ctx := errgroup.WithContext(ctx) + + g.Go(func() error { + return r.Reconcile(ctx) + }) + + syncer := syncer.New( primaryNetwork, - blockStorage, fetcher, syncHandler, + cancel, ) g.Go(func() error { return syncer.Sync( ctx, - cancel, - statefulSyncer, StartIndex, EndIndex, ) diff --git a/cmd/check_quick.go b/cmd/check_quick.go deleted file mode 100644 index 365dde60..00000000 --- a/cmd/check_quick.go +++ /dev/null @@ -1,125 +0,0 @@ -// Copyright 2020 Coinbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "context" - "log" - - "github.com/coinbase/rosetta-cli/internal/logger" - "github.com/coinbase/rosetta-cli/internal/reconciler" - "github.com/coinbase/rosetta-cli/internal/syncer" - - "github.com/coinbase/rosetta-sdk-go/fetcher" - "github.com/spf13/cobra" - "golang.org/x/sync/errgroup" -) - -var ( - checkQuickCmd = &cobra.Command{ - Use: "check:quick", - Short: "Run a simple check of the correctness of a Rosetta server", - Long: `Check all server responses are properly constructed and that -computed balance changes are equal to balance changes reported by the -node. To use check:quick, your server must implement the balance lookup -by block. - -Unlike check:complete, which requires syncing all blocks up -to the blocks you want to check, check:quick allows you to validate -an arbitrary range of blocks (even if earlier blocks weren't synced). -To do this, all you need to do is provide a --start flag and optionally -an --end flag. - -It is important to note that check:quick does not support re-orgs and it -does not check for duplicate blocks and transactions. For these features, -please use check:complete. - -When re-running this command, it will start off from genesis unless you -provide a populated --start flag. If you want to run a stateful validation, -use the check:complete command.`, - Run: runCheckQuickCmd, - } -) - -func runCheckQuickCmd(cmd *cobra.Command, args []string) { - ctx, cancel := context.WithCancel(context.Background()) - - exemptAccounts, err := loadAccounts(ExemptFile) - if err != nil { - log.Fatal(err) - } - - fetcher := fetcher.New( - ServerURL, - fetcher.WithBlockConcurrency(BlockConcurrency), - fetcher.WithTransactionConcurrency(TransactionConcurrency), - fetcher.WithRetryElapsedTime(ExtendedRetryElapsedTime), - ) - - primaryNetwork, _, err := fetcher.InitializeAsserter(ctx) - if err != nil { - log.Fatal(err) - } - - logger := logger.NewLogger( - DataDir, - LogBlocks, - LogTransactions, - LogBalanceChanges, - LogReconciliations, - ) - - g, ctx := errgroup.WithContext(ctx) - - r := reconciler.NewStateless( - primaryNetwork, - fetcher, - logger, - AccountConcurrency, - HaltOnReconciliationError, - nil, - ) - - g.Go(func() error { - return r.Reconcile(ctx) - }) - - syncHandler := syncer.NewBaseHandler( - logger, - r, - exemptAccounts, - ) - - statelessSyncer := syncer.NewStateless( - primaryNetwork, - fetcher, - syncHandler, - ) - - g.Go(func() error { - return syncer.Sync( - ctx, - cancel, - statelessSyncer, - StartIndex, - EndIndex, - ) - }) - - err = g.Wait() - if err != nil { - log.Fatal(err) - } -} diff --git a/cmd/root.go b/cmd/root.go index 0a595d73..e45702f2 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -15,18 +15,14 @@ package cmd import ( - "context" "encoding/json" "io/ioutil" "log" "path" "time" - "github.com/coinbase/rosetta-cli/internal/logger" - "github.com/coinbase/rosetta-cli/internal/processor" "github.com/coinbase/rosetta-cli/internal/reconciler" - "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/spf13/cobra" ) @@ -190,8 +186,6 @@ how to structure this file.`, ) rootCmd.AddCommand(checkCompleteCmd) - rootCmd.AddCommand(checkQuickCmd) - rootCmd.AddCommand(checkAccountCmd) } func loadAccounts(filePath string) ([]*reconciler.AccountCurrency, error) { @@ -217,54 +211,3 @@ func loadAccounts(filePath string) ([]*reconciler.AccountCurrency, error) { return accounts, nil } - -func standardInitialization(ctx context.Context, interestingAccounts []*reconciler.AccountCurrency) { - exemptAccounts, err := loadAccounts(ExemptFile) - if err != nil { - log.Fatal(err) - } - - fetcher := fetcher.New( - ServerURL, - fetcher.WithBlockConcurrency(BlockConcurrency), - fetcher.WithTransactionConcurrency(TransactionConcurrency), - fetcher.WithRetryElapsedTime(ExtendedRetryElapsedTime), - ) - - primaryNetwork, _, err := fetcher.InitializeAsserter(ctx) - if err != nil { - log.Fatal(err) - } - - logger := logger.NewLogger( - DataDir, - LogBlocks, - LogTransactions, - LogBalanceChanges, - LogReconciliations, - ) - - return - - r := reconciler.NewReconciler( - primaryNetwork, - fetcher, - logger, - AccountConcurrency, - HaltOnReconciliationError, - interestingAccounts, - ) - - processor := processor.NewBaseProcessor( - logger, - r, - fetcher.Asserter, - exemptAccounts, - ) - - syncer := syncer.NewSyncer( - primaryNetwork, - fetcher, - processor, - ) -} diff --git a/internal/processor/processor.go b/internal/processor/processor.go deleted file mode 100644 index cb9d60cb..00000000 --- a/internal/processor/processor.go +++ /dev/null @@ -1,155 +0,0 @@ -// Copyright 2020 Coinbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package processor - -import ( - "context" - "log" - - "github.com/coinbase/rosetta-cli/internal/logger" - "github.com/coinbase/rosetta-cli/internal/reconciler" - "github.com/coinbase/rosetta-cli/internal/storage" - - "github.com/coinbase/rosetta-sdk-go/asserter" - "github.com/coinbase/rosetta-sdk-go/types" -) - -// Processor is what you write to handle chain data. -type Processor struct { - storage *storage.BlockStorage - logger *logger.Logger - reconciler *reconciler.Reconciler - asserter *asserter.Asserter - exemptAccounts []*reconciler.AccountCurrency -} - -// NewProcessor constructs a basic Handler. -func NewProcessor( - storage *storage.BlockStorage, - logger *logger.Logger, - reconciler *reconciler.Reconciler, - asserter *asserter.Asserter, - exemptAccounts []*reconciler.AccountCurrency, -) *Processor { - return &Processor{ - storage: storage, - logger: logger, - reconciler: reconciler, - asserter: asserter, - exemptAccounts: exemptAccounts, - } -} - -// SkipOperation returns a boolean indicating whether -// an operation should be processed. An operation will -// not be processed if it is considered unsuccessful -// or affects an exempt account. -func (p *Processor) SkipOperation( - ctx context.Context, - op *types.Operation, -) (bool, error) { - successful, err := p.asserter.OperationSuccessful(op) - if err != nil { - // Should only occur if responses not validated - return false, err - } - - if !successful { - return true, nil - } - - if op.Account == nil { - return true, nil - } - - // Exempting account in BalanceChanges ensures that storage is not updated - // and that the account is not reconciled. - if p.accountExempt(ctx, op.Account, op.Amount.Currency) { - log.Printf("Skipping exempt account %+v\n", op.Account) - return true, nil - } - - return false, nil -} - -// BlockAdded is called by the syncer after a -// block is added. -func (p *Processor) BlockAdded( - ctx context.Context, - block *types.Block, -) error { - log.Printf("Adding block %+v\n", block.BlockIdentifier) - - // Log processed blocks and balance changes - if err := p.logger.BlockStream(ctx, block, false); err != nil { - return nil - } - - balanceChanges, err := p.storage.StoreBlock(ctx, block) - if err != nil { - return err - } - - if err := p.logger.BalanceStream(ctx, balanceChanges); err != nil { - return nil - } - - // Mark accounts for reconciliation...this may be - // blocking - return p.reconciler.QueueChanges(ctx, block.BlockIdentifier, balanceChanges) -} - -// BlockRemoved is called by the syncer after a -// block is removed. -func (p *Processor) BlockRemoved( - ctx context.Context, - block *types.Block, -) error { - log.Printf("Orphaning block %+v\n", block.BlockIdentifier) - - // Log processed blocks and balance changes - if err := p.logger.BlockStream(ctx, block, true); err != nil { - return nil - } - - balanceChanges, err := p.storage.RemoveBlock(ctx, block) - if err != nil { - return err - } - - if err := p.logger.BalanceStream(ctx, balanceChanges); err != nil { - return nil - } - - // We only attempt to reconciler changes when blocks are added - return nil -} - -// accountExempt returns a boolean indicating if the provided -// account and currency are exempt from balance tracking and -// reconciliation. -func (p *Processor) accountExempt( - ctx context.Context, - account *types.AccountIdentifier, - currency *types.Currency, -) bool { - return reconciler.ContainsAccountCurrency( - p.exemptAccounts, - &reconciler.AccountCurrency{ - Account: account, - Currency: currency, - }, - ) -} diff --git a/internal/processor/reconciler_handler.go b/internal/processor/reconciler_handler.go new file mode 100644 index 00000000..c1a1a472 --- /dev/null +++ b/internal/processor/reconciler_handler.go @@ -0,0 +1,33 @@ +package processor + +import ( + "context" + "errors" + + "github.com/coinbase/rosetta-sdk-go/types" +) + +type ReconcilerHandler struct{} + +func (h *ReconcilerHandler) ReconciliationFailed( + ctx context.Context, + reconciliationType string, + account *types.AccountIdentifier, + currency *types.Currency, + computedBalance string, + nodeBalance string, + block *types.BlockIdentifier, +) error { + return errors.New("not implemented") +} + +func (h *ReconcilerHandler) ReconciliationSucceeded( + ctx context.Context, + reconciliationType string, + account *types.AccountIdentifier, + currency *types.Currency, + balance string, + block *types.BlockIdentifier, +) error { + return errors.New("not implemented") +} diff --git a/internal/processor/reconciler_helper.go b/internal/processor/reconciler_helper.go new file mode 100644 index 00000000..fd91adbb --- /dev/null +++ b/internal/processor/reconciler_helper.go @@ -0,0 +1,31 @@ +package processor + +import ( + "context" + "errors" + + "github.com/coinbase/rosetta-sdk-go/types" +) + +type ReconcilerHelper struct{} + +func (h *ReconcilerHelper) BlockExists( + ctx context.Context, + block *types.BlockIdentifier, +) (bool, error) { + return false, errors.New("not implemented") +} + +func (h *ReconcilerHelper) CurrentBlock( + ctx context.Context, +) (*types.BlockIdentifier, error) { + return nil, errors.New("not implemented") +} + +func (h *ReconcilerHelper) AccountBalance( + ctx context.Context, + account *types.AccountIdentifier, + currency *types.Currency, +) (*types.Amount, *types.BlockIdentifier, error) { + return nil, nil, errors.New("not implemented") +} diff --git a/internal/processor/storage_helper.go b/internal/processor/storage_helper.go new file mode 100644 index 00000000..27893f71 --- /dev/null +++ b/internal/processor/storage_helper.go @@ -0,0 +1,87 @@ +package processor + +import ( + "context" + "errors" + "log" + + "github.com/coinbase/rosetta-cli/internal/reconciler" + + "github.com/coinbase/rosetta-sdk-go/fetcher" + "github.com/coinbase/rosetta-sdk-go/types" +) + +type BlockStorageHelper struct { + fetcher *fetcher.Fetcher + + // Configuration settings + exemptAccounts []*reconciler.AccountCurrency +} + +func NewBlockStorageHelper( + fetcher *fetcher.Fetcher, + exemptAccounts []*reconciler.AccountCurrency, +) *BlockStorageHelper { + return &BlockStorageHelper{ + fetcher: fetcher, + exemptAccounts: exemptAccounts, + } +} + +func (h *BlockStorageHelper) AccountBalance( + ctx context.Context, + account *types.AccountIdentifier, + currency *types.Currency, + block *types.BlockIdentifier, +) (*types.Amount, error) { // returns an error if lookupBalanceByBlock disabled + return nil, errors.New("not implemented") +} + +// SkipOperation returns a boolean indicating whether +// an operation should be processed. An operation will +// not be processed if it is considered unsuccessful +// or affects an exempt account. +func (h *BlockStorageHelper) SkipOperation( + ctx context.Context, + op *types.Operation, +) (bool, error) { + successful, err := h.fetcher.Asserter.OperationSuccessful(op) + if err != nil { + // Should only occur if responses not validated + return false, err + } + + if !successful { + return true, nil + } + + if op.Account == nil { + return true, nil + } + + // Exempting account in BalanceChanges ensures that storage is not updated + // and that the account is not reconciled. + if h.accountExempt(ctx, op.Account, op.Amount.Currency) { + log.Printf("Skipping exempt account %+v\n", op.Account) + return true, nil + } + + return false, nil +} + +// accountExempt returns a boolean indicating if the provided +// account and currency are exempt from balance tracking and +// reconciliation. +func (h *BlockStorageHelper) accountExempt( + ctx context.Context, + account *types.AccountIdentifier, + currency *types.Currency, +) bool { + return reconciler.ContainsAccountCurrency( + h.exemptAccounts, + &reconciler.AccountCurrency{ + Account: account, + Currency: currency, + }, + ) +} diff --git a/internal/processor/sync_handler.go b/internal/processor/sync_handler.go new file mode 100644 index 00000000..9f1b45bd --- /dev/null +++ b/internal/processor/sync_handler.go @@ -0,0 +1,92 @@ +package processor + +import ( + "context" + "log" + + "github.com/coinbase/rosetta-cli/internal/logger" + "github.com/coinbase/rosetta-cli/internal/reconciler" + "github.com/coinbase/rosetta-cli/internal/storage" + + "github.com/coinbase/rosetta-sdk-go/fetcher" + "github.com/coinbase/rosetta-sdk-go/types" +) + +type SyncHandler struct { + storage *storage.BlockStorage + logger *logger.Logger + reconciler *reconciler.Reconciler + fetcher *fetcher.Fetcher + + exemptAccounts []*reconciler.AccountCurrency +} + +func NewSyncHandler( + storage *storage.BlockStorage, + logger *logger.Logger, + reconciler *reconciler.Reconciler, + fetcher *fetcher.Fetcher, + exemptAccounts []*reconciler.AccountCurrency, +) *SyncHandler { + return &SyncHandler{ + storage: storage, + logger: logger, + reconciler: reconciler, + fetcher: fetcher, + exemptAccounts: exemptAccounts, + } +} + +// BlockAdded is called by the syncer after a +// block is added. +func (h *SyncHandler) BlockAdded( + ctx context.Context, + block *types.Block, +) error { + log.Printf("Adding block %+v\n", block.BlockIdentifier) + + // Log processed blocks and balance changes + if err := h.logger.BlockStream(ctx, block, false); err != nil { + return nil + } + + balanceChanges, err := h.storage.StoreBlock(ctx, block) + if err != nil { + return err + } + + if err := h.logger.BalanceStream(ctx, balanceChanges); err != nil { + return nil + } + + // Mark accounts for reconciliation...this may be + // blocking + return h.reconciler.QueueChanges(ctx, block.BlockIdentifier, balanceChanges) +} + +// BlockRemoved is called by the syncer after a +// block is removed. +func (h *SyncHandler) BlockRemoved( + ctx context.Context, + block *types.Block, +) error { + log.Printf("Orphaning block %+v\n", block.BlockIdentifier) + + // Log processed blocks and balance changes + if err := h.logger.BlockStream(ctx, block, true); err != nil { + return nil + } + + balanceChanges, err := h.storage.RemoveBlock(ctx, block) + if err != nil { + return err + } + + if err := h.logger.BalanceStream(ctx, balanceChanges); err != nil { + return nil + } + + // We only attempt to reconciler changes when blocks are added, + // not removed + return nil +} diff --git a/internal/reconciler/reconciler.go b/internal/reconciler/reconciler.go index 40e081fc..2bd2f5c0 100644 --- a/internal/reconciler/reconciler.go +++ b/internal/reconciler/reconciler.go @@ -150,15 +150,14 @@ type ReconcilerHandler interface { // types.AccountIdentifiers returned in types.Operations // by a Rosetta Server. type Reconciler struct { - network *types.NetworkIdentifier - helper ReconcilerHelper - handler ReconcilerHandler - fetcher *fetcher.Fetcher - accountConcurrency uint64 - lookupBalanceByBlock bool - haltOnReconciliationError bool - interestingAccounts []*AccountCurrency - changeQueue chan *BalanceChange + network *types.NetworkIdentifier + helper ReconcilerHelper + handler ReconcilerHandler + fetcher *fetcher.Fetcher + accountConcurrency uint64 + lookupBalanceByBlock bool + interestingAccounts []*AccountCurrency + changeQueue chan *BalanceChange // highWaterMark is used to skip requests when // we are very far behind the live head. @@ -177,24 +176,24 @@ type Reconciler struct { // NewReconciler creates a new Reconciler. func NewReconciler( network *types.NetworkIdentifier, + helper ReconcilerHelper, handler ReconcilerHandler, fetcher *fetcher.Fetcher, accountConcurrency uint64, lookupBalanceByBlock bool, - haltOnReconciliationError bool, interestingAccounts []*AccountCurrency, ) *Reconciler { r := &Reconciler{ - network: network, - handler: handler, - fetcher: fetcher, - accountConcurrency: accountConcurrency, - lookupBalanceByBlock: lookupBalanceByBlock, - haltOnReconciliationError: haltOnReconciliationError, - interestingAccounts: interestingAccounts, - highWaterMark: -1, - seenAccts: make([]*AccountCurrency, 0), - inactiveQueue: make([]*BalanceChange, 0), + network: network, + helper: helper, + handler: handler, + fetcher: fetcher, + accountConcurrency: accountConcurrency, + lookupBalanceByBlock: lookupBalanceByBlock, + interestingAccounts: interestingAccounts, + highWaterMark: -1, + seenAccts: make([]*AccountCurrency, 0), + inactiveQueue: make([]*BalanceChange, 0), } if lookupBalanceByBlock { @@ -448,10 +447,6 @@ func (r *Reconciler) accountReconciliation( return err } - if r.haltOnReconciliationError { - return errors.New("reconciliation error") - } - return nil } diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index bc1fddfc..a4b682f7 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -57,7 +57,7 @@ type Syncer struct { currentBlock *types.BlockIdentifier } -func NewSyncer( +func New( network *types.NetworkIdentifier, fetcher *fetcher.Fetcher, handler SyncHandler, From cddba758872635b946baaaa494582c4050079c00 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 4 May 2020 14:29:10 -0700 Subject: [PATCH 08/31] Fix block storage tests --- cmd/check_complete.go | 1 + internal/reconciler/reconciler.go | 6 +- internal/storage/badger_storage_test.go | 10 +- internal/storage/block_storage.go | 78 ++--- internal/storage/block_storage_test.go | 394 ++++++++++++------------ 5 files changed, 240 insertions(+), 249 deletions(-) diff --git a/cmd/check_complete.go b/cmd/check_complete.go index 728d9bc5..4b1a6866 100644 --- a/cmd/check_complete.go +++ b/cmd/check_complete.go @@ -111,6 +111,7 @@ func runCheckCompleteCmd(cmd *cobra.Command, args []string) { log.Fatal(fmt.Errorf("%w: unable to initialize asserter", err)) } + // TODO: if DataDir is empty, use TempDir localStore, err := storage.NewBadgerStorage(ctx, DataDir) if err != nil { log.Fatal(fmt.Errorf("%w: unable to initialize data store", err)) diff --git a/internal/reconciler/reconciler.go b/internal/reconciler/reconciler.go index 2bd2f5c0..451708a4 100644 --- a/internal/reconciler/reconciler.go +++ b/internal/reconciler/reconciler.go @@ -557,7 +557,11 @@ func (r *Reconciler) reconcileInactiveAccounts( // If this is the case, we should sleep and try again later instead of exiting. if err != nil { time.Sleep(inactiveReconciliationSleep) - log.Println("%s: unable to get current block for inactive reconciliation", err.Error()) + log.Printf( + "%s: unable to get current block for inactive reconciliation\n", + err.Error(), + ) + continue } r.inactiveQueueMutex.Lock() diff --git a/internal/storage/badger_storage_test.go b/internal/storage/badger_storage_test.go index aa112752..03e55a62 100644 --- a/internal/storage/badger_storage_test.go +++ b/internal/storage/badger_storage_test.go @@ -18,15 +18,17 @@ import ( "context" "testing" + "github.com/coinbase/rosetta-cli/internal/utils" + "github.com/stretchr/testify/assert" ) func TestDatabase(t *testing.T) { ctx := context.Background() - newDir, err := CreateTempDir() + newDir, err := utils.CreateTempDir() assert.NoError(t, err) - defer RemoveTempDir(*newDir) + defer utils.RemoveTempDir(*newDir) database, err := NewBadgerStorage(ctx, *newDir) assert.NoError(t, err) @@ -55,9 +57,9 @@ func TestDatabase(t *testing.T) { func TestDatabaseTransaction(t *testing.T) { ctx := context.Background() - newDir, err := CreateTempDir() + newDir, err := utils.CreateTempDir() assert.NoError(t, err) - defer RemoveTempDir(*newDir) + defer utils.RemoveTempDir(*newDir) database, err := NewBadgerStorage(ctx, *newDir) assert.NoError(t, err) diff --git a/internal/storage/block_storage.go b/internal/storage/block_storage.go index 3cadb3f9..347a33bb 100644 --- a/internal/storage/block_storage.go +++ b/internal/storage/block_storage.go @@ -26,13 +26,11 @@ import ( "log" "math/big" "path" - "strings" "github.com/coinbase/rosetta-cli/internal/reconciler" "github.com/coinbase/rosetta-cli/internal/utils" "github.com/coinbase/rosetta-sdk-go/types" - "github.com/davecgh/go-spew/spew" ) const ( @@ -418,6 +416,10 @@ func (b *BlockStorage) RemoveBlock( return nil, err } + if err := transaction.Commit(ctx); err != nil { + return nil, err + } + return changes, nil } @@ -490,77 +492,57 @@ func (b *BlockStorage) UpdateBalance( return err } - if !exists { - // TODO: must be block BEFORE current (should only occur when adding, not removing) - amount, err := b.helper.AccountBalance(ctx, change.Account, change.Currency, nil) - if err != nil { - return fmt.Errorf("%w: unable to get previous account balance", err) - } - - newVal, ok := new(big.Int).SetString(amount.Value, 10) - if !ok { - return fmt.Errorf("%s is not an integer", amount.Value) - } - - if newVal.Sign() == -1 { - return fmt.Errorf( - "%w %+v for %+v at %+v", - ErrNegativeBalance, - spew.Sdump(amount), - change.Account, - change.Block, - ) - } - - serialBal, err := serializeBalanceEntry(balanceEntry{ - Amount: amount, - Block: change.Block, - }) + var existingValue string + if exists { + parseBal, err := parseBalanceEntry(balance) if err != nil { return err } - if err := dbTransaction.Set(ctx, key, serialBal); err != nil { - return err + existingValue = parseBal.Amount.Value + } else { + // TODO: must be block BEFORE current (should only occur when adding, not removing) + amount, err := b.helper.AccountBalance(ctx, change.Account, change.Currency, nil) + if err != nil { + return fmt.Errorf("%w: unable to get previous account balance", err) } - return nil + existingValue = amount.Value } - // Modify balance - parseBal, err := parseBalanceEntry(balance) + newVal, err := utils.AddStringValues(change.Difference, existingValue) if err != nil { return err } - oldValue := parseBal.Amount.Value - newVal, err := utils.AddStringValues(change.Difference, oldValue) - if err != nil { - return err + bigNewVal, ok := new(big.Int).SetString(newVal, 10) + if !ok { + return fmt.Errorf("%s is not an integer", newVal) } - if strings.HasPrefix(newVal, "-") { + if bigNewVal.Sign() == -1 { return fmt.Errorf( - "%w %+v for %+v at %+v", + "%w %s:%+v for %+v at %+v", ErrNegativeBalance, - spew.Sdump(newVal), + newVal, + change.Currency, change.Account, change.Block, ) } - parseBal.Amount.Value = newVal - parseBal.Block = change.Block - serialBal, err := serializeBalanceEntry(*parseBal) + serialBal, err := serializeBalanceEntry(balanceEntry{ + Amount: &types.Amount{ + Value: newVal, + Currency: change.Currency, + }, + Block: change.Block, + }) if err != nil { return err } - if err := dbTransaction.Set(ctx, key, serialBal); err != nil { - return err - } - - return nil + return dbTransaction.Set(ctx, key, serialBal) } // GetBalance returns all the balances of a types.AccountIdentifier diff --git a/internal/storage/block_storage_test.go b/internal/storage/block_storage_test.go index bfb71709..21700c9b 100644 --- a/internal/storage/block_storage_test.go +++ b/internal/storage/block_storage_test.go @@ -17,14 +17,17 @@ package storage import ( "context" "encoding/json" + "errors" "fmt" "io/ioutil" "os" "path" "testing" - "github.com/coinbase/rosetta-sdk-go/types" + "github.com/coinbase/rosetta-cli/internal/reconciler" + "github.com/coinbase/rosetta-cli/internal/utils" + "github.com/coinbase/rosetta-sdk-go/types" "github.com/stretchr/testify/assert" ) @@ -42,38 +45,34 @@ func TestHeadBlockIdentifier(t *testing.T) { ctx := context.Background() - newDir, err := CreateTempDir() + newDir, err := utils.CreateTempDir() assert.NoError(t, err) - defer RemoveTempDir(*newDir) + defer utils.RemoveTempDir(*newDir) database, err := NewBadgerStorage(ctx, *newDir) assert.NoError(t, err) defer database.Close(ctx) - storage := NewBlockStorage(ctx, database) + storage := NewBlockStorage(ctx, database, &MockBlockStorageHelper{}) t.Run("No head block set", func(t *testing.T) { - txn := storage.NewDatabaseTransaction(ctx, false) - blockIdentifier, err := storage.GetHeadBlockIdentifier(ctx, txn) - txn.Discard(ctx) + blockIdentifier, err := storage.GetHeadBlockIdentifier(ctx) assert.EqualError(t, err, ErrHeadBlockNotFound.Error()) assert.Nil(t, blockIdentifier) }) t.Run("Set and get head block", func(t *testing.T) { - txn := storage.NewDatabaseTransaction(ctx, true) + txn := storage.newDatabaseTransaction(ctx, true) assert.NoError(t, storage.StoreHeadBlockIdentifier(ctx, txn, newBlockIdentifier)) assert.NoError(t, txn.Commit(ctx)) - txn = storage.NewDatabaseTransaction(ctx, false) - blockIdentifier, err := storage.GetHeadBlockIdentifier(ctx, txn) + blockIdentifier, err := storage.GetHeadBlockIdentifier(ctx) assert.NoError(t, err) - txn.Discard(ctx) assert.Equal(t, newBlockIdentifier, blockIdentifier) }) t.Run("Discard head block update", func(t *testing.T) { - txn := storage.NewDatabaseTransaction(ctx, true) + txn := storage.newDatabaseTransaction(ctx, true) assert.NoError(t, storage.StoreHeadBlockIdentifier(ctx, txn, &types.BlockIdentifier{ Hash: "no blah", @@ -82,20 +81,17 @@ func TestHeadBlockIdentifier(t *testing.T) { ) txn.Discard(ctx) - txn = storage.NewDatabaseTransaction(ctx, false) - blockIdentifier, err := storage.GetHeadBlockIdentifier(ctx, txn) + blockIdentifier, err := storage.GetHeadBlockIdentifier(ctx) assert.NoError(t, err) - txn.Discard(ctx) assert.Equal(t, newBlockIdentifier, blockIdentifier) }) t.Run("Multiple updates to head block", func(t *testing.T) { - txn := storage.NewDatabaseTransaction(ctx, true) + txn := storage.newDatabaseTransaction(ctx, true) assert.NoError(t, storage.StoreHeadBlockIdentifier(ctx, txn, newBlockIdentifier2)) assert.NoError(t, txn.Commit(ctx)) - txn = storage.NewDatabaseTransaction(ctx, false) - blockIdentifier, err := storage.GetHeadBlockIdentifier(ctx, txn) + blockIdentifier, err := storage.GetHeadBlockIdentifier(ctx) assert.NoError(t, err) txn.Discard(ctx) assert.Equal(t, newBlockIdentifier2, blockIdentifier) @@ -155,32 +151,27 @@ func TestBlock(t *testing.T) { ) ctx := context.Background() - newDir, err := CreateTempDir() + newDir, err := utils.CreateTempDir() assert.NoError(t, err) - defer RemoveTempDir(*newDir) + defer utils.RemoveTempDir(*newDir) database, err := NewBadgerStorage(ctx, *newDir) assert.NoError(t, err) defer database.Close(ctx) - storage := NewBlockStorage(ctx, database) + storage := NewBlockStorage(ctx, database, &MockBlockStorageHelper{}) t.Run("Set and get block", func(t *testing.T) { - txn := storage.NewDatabaseTransaction(ctx, true) - assert.NoError(t, storage.StoreBlock(ctx, txn, newBlock)) - assert.NoError(t, txn.Commit(ctx)) + _, err := storage.StoreBlock(ctx, newBlock) + assert.NoError(t, err) - txn = storage.NewDatabaseTransaction(ctx, false) - block, err := storage.GetBlock(ctx, txn, newBlock.BlockIdentifier) - txn.Discard(ctx) + block, err := storage.GetBlock(ctx, newBlock.BlockIdentifier) assert.NoError(t, err) assert.Equal(t, newBlock, block) }) t.Run("Get non-existent block", func(t *testing.T) { - txn := storage.NewDatabaseTransaction(ctx, false) - block, err := storage.GetBlock(ctx, txn, badBlockIdentifier) - txn.Discard(ctx) + block, err := storage.GetBlock(ctx, badBlockIdentifier) assert.EqualError( t, err, @@ -190,35 +181,29 @@ func TestBlock(t *testing.T) { }) t.Run("Set duplicate block hash", func(t *testing.T) { - txn := storage.NewDatabaseTransaction(ctx, true) - err = storage.StoreBlock(ctx, txn, newBlock) + _, err = storage.StoreBlock(ctx, newBlock) assert.EqualError(t, err, fmt.Errorf( "%w %s", ErrDuplicateBlockHash, newBlock.BlockIdentifier.Hash, ).Error()) - txn.Discard(ctx) }) t.Run("Set duplicate transaction hash", func(t *testing.T) { - txn := storage.NewDatabaseTransaction(ctx, true) - err = storage.StoreBlock(ctx, txn, newBlock2) + _, err = storage.StoreBlock(ctx, newBlock2) assert.EqualError(t, err, fmt.Errorf( "%w %s", ErrDuplicateTransactionHash, "blahTx", ).Error()) - txn.Discard(ctx) }) t.Run("Remove block and re-set block of same hash", func(t *testing.T) { - txn := storage.NewDatabaseTransaction(ctx, true) - assert.NoError(t, storage.RemoveBlock(ctx, txn, newBlock.BlockIdentifier)) - assert.NoError(t, txn.Commit(ctx)) + _, err := storage.RemoveBlock(ctx, newBlock) + assert.NoError(t, err) - txn = storage.NewDatabaseTransaction(ctx, true) - assert.NoError(t, storage.StoreBlock(ctx, txn, newBlock)) - assert.NoError(t, txn.Commit(ctx)) + _, err = storage.StoreBlock(ctx, newBlock) + assert.NoError(t, err) }) } @@ -283,7 +268,7 @@ func TestGetAccountKey(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { - assert.Equal(t, hashBytes([]byte(test.key)), GetAccountKey(test.account)) + assert.Equal(t, test.key, GetAccountKey(test.account)) }) } } @@ -296,6 +281,9 @@ func TestBalance(t *testing.T) { account2 = &types.AccountIdentifier{ Address: "blah2", } + account3 = &types.AccountIdentifier{ + Address: "blah3", + } subAccount = &types.AccountIdentifier{ Address: "blah", SubAccount: &types.SubAccountIdentifier{ @@ -352,12 +340,13 @@ func TestBalance(t *testing.T) { Value: "100", Currency: currency, } + amountWithPrevious = &types.Amount{ + Value: "110", + Currency: currency, + } amountNilCurrency = &types.Amount{ Value: "100", } - newAmounts = map[string]*types.Amount{ - GetCurrencyKey(currency): amount, - } newBlock = &types.BlockIdentifier{ Hash: "kdasdj", Index: 123890, @@ -366,11 +355,9 @@ func TestBalance(t *testing.T) { Hash: "pkdasdj", Index: 123890, } - result = map[string]*types.Amount{ - GetCurrencyKey(currency): { - Value: "200", - Currency: currency, - }, + result = &types.Amount{ + Value: "200", + Currency: currency, } newBlock3 = &types.BlockIdentifier{ Hash: "pkdgdj", @@ -380,246 +367,231 @@ func TestBalance(t *testing.T) { Value: "-1000", Currency: currency, } + mockHelper = &MockBlockStorageHelper{} ) ctx := context.Background() - newDir, err := CreateTempDir() + newDir, err := utils.CreateTempDir() assert.NoError(t, err) - defer RemoveTempDir(*newDir) + defer utils.RemoveTempDir(*newDir) database, err := NewBadgerStorage(ctx, *newDir) assert.NoError(t, err) defer database.Close(ctx) - storage := NewBlockStorage(ctx, database) + storage := NewBlockStorage(ctx, database, mockHelper) t.Run("Get unset balance", func(t *testing.T) { - txn := storage.NewDatabaseTransaction(ctx, false) - amounts, block, err := storage.GetBalance(ctx, txn, account) - txn.Discard(ctx) - assert.Nil(t, amounts) + amount, block, err := storage.GetBalance(ctx, account, currency) + assert.Nil(t, amount) assert.Nil(t, block) assert.EqualError(t, err, fmt.Errorf("%w %+v", ErrAccountNotFound, account).Error()) }) t.Run("Set and get balance", func(t *testing.T) { - txn := storage.NewDatabaseTransaction(ctx, true) - balanceChange, err := storage.UpdateBalance( + txn := storage.newDatabaseTransaction(ctx, true) + err := storage.UpdateBalance( ctx, txn, - account, - amount, - newBlock, + &reconciler.BalanceChange{ + Account: account, + Currency: currency, + Block: newBlock, + Difference: amount.Value, + }, ) - assert.Equal(t, &BalanceChange{ - Account: &types.AccountIdentifier{ - Address: "blah", - }, - Currency: currency, - Block: newBlock, - Difference: "100", - }, balanceChange) assert.NoError(t, err) assert.NoError(t, txn.Commit(ctx)) - txn = storage.NewDatabaseTransaction(ctx, false) - amounts, block, err := storage.GetBalance(ctx, txn, account) - txn.Discard(ctx) + retrievedAmount, block, err := storage.GetBalance(ctx, account, currency) assert.NoError(t, err) - assert.Equal(t, newAmounts, amounts) + assert.Equal(t, amount, retrievedAmount) assert.Equal(t, newBlock, block) }) + t.Run("Set and get balance with storage helper", func(t *testing.T) { + mockHelper.AccountBalanceAmount = "10" + txn := storage.newDatabaseTransaction(ctx, true) + err := storage.UpdateBalance( + ctx, + txn, + &reconciler.BalanceChange{ + Account: account3, + Currency: currency, + Block: newBlock, + Difference: amount.Value, + }, + ) + assert.NoError(t, err) + assert.NoError(t, txn.Commit(ctx)) + + retrievedAmount, block, err := storage.GetBalance(ctx, account3, currency) + assert.NoError(t, err) + assert.Equal(t, amountWithPrevious, retrievedAmount) + assert.Equal(t, newBlock, block) + + mockHelper.AccountBalanceAmount = "" + }) + t.Run("Set balance with nil currency", func(t *testing.T) { - txn := storage.NewDatabaseTransaction(ctx, true) - balanceChange, err := storage.UpdateBalance( + txn := storage.newDatabaseTransaction(ctx, true) + err := storage.UpdateBalance( ctx, txn, - account, - amountNilCurrency, - newBlock, + &reconciler.BalanceChange{ + Account: account, + Currency: nil, + Block: newBlock, + Difference: amountNilCurrency.Value, + }, ) - assert.Nil(t, balanceChange) - assert.EqualError(t, err, "invalid amount") + assert.EqualError(t, err, "invalid currency") txn.Discard(ctx) - txn = storage.NewDatabaseTransaction(ctx, false) - amounts, block, err := storage.GetBalance(ctx, txn, account) - txn.Discard(ctx) + retrievedAmount, block, err := storage.GetBalance(ctx, account, currency) assert.NoError(t, err) - assert.Equal(t, newAmounts, amounts) + assert.Equal(t, amount, retrievedAmount) assert.Equal(t, newBlock, block) }) t.Run("Modify existing balance", func(t *testing.T) { - txn := storage.NewDatabaseTransaction(ctx, true) - balanceChange, err := storage.UpdateBalance( + txn := storage.newDatabaseTransaction(ctx, true) + err := storage.UpdateBalance( ctx, txn, - account, - amount, - newBlock2, + &reconciler.BalanceChange{ + Account: account, + Currency: currency, + Block: newBlock2, + Difference: amount.Value, + }, ) - assert.Equal(t, &BalanceChange{ - Account: &types.AccountIdentifier{ - Address: "blah", - }, - Currency: currency, - Block: newBlock2, - Difference: "100", - }, balanceChange) assert.NoError(t, err) assert.NoError(t, txn.Commit(ctx)) - txn = storage.NewDatabaseTransaction(ctx, true) - amounts, block, err := storage.GetBalance(ctx, txn, account) - txn.Discard(ctx) + retrievedAmount, block, err := storage.GetBalance(ctx, account, currency) assert.NoError(t, err) - assert.Equal(t, result, amounts) + assert.Equal(t, result, retrievedAmount) assert.Equal(t, newBlock2, block) }) t.Run("Discard transaction", func(t *testing.T) { - txn := storage.NewDatabaseTransaction(ctx, true) - balanceChange, err := storage.UpdateBalance( + txn := storage.newDatabaseTransaction(ctx, true) + err := storage.UpdateBalance( ctx, txn, - account, - amount, - newBlock3, + &reconciler.BalanceChange{ + Account: account, + Currency: currency, + Block: newBlock3, + Difference: amount.Value, + }, ) - assert.Equal(t, &BalanceChange{ - Account: &types.AccountIdentifier{ - Address: "blah", - }, - Currency: currency, - Block: newBlock3, - Difference: "100", - }, balanceChange) assert.NoError(t, err) // Get balance during transaction - txn2 := storage.NewDatabaseTransaction(ctx, false) - amounts, block, err := storage.GetBalance(ctx, txn2, account) - txn2.Discard(ctx) + retrievedAmount, block, err := storage.GetBalance(ctx, account, currency) assert.NoError(t, err) - assert.Equal(t, result, amounts) + assert.Equal(t, result, retrievedAmount) assert.Equal(t, newBlock2, block) txn.Discard(ctx) - - txn = storage.NewDatabaseTransaction(ctx, false) - amounts, block, err = storage.GetBalance(ctx, txn, account) - txn.Discard(ctx) - assert.NoError(t, err) - assert.Equal(t, result, amounts) - assert.Equal(t, newBlock2, block) }) t.Run("Attempt modification to push balance negative on existing account", func(t *testing.T) { - txn := storage.NewDatabaseTransaction(ctx, true) - balanceChange, err := storage.UpdateBalance( + txn := storage.newDatabaseTransaction(ctx, true) + err := storage.UpdateBalance( ctx, txn, - account, - largeDeduction, - newBlock2, + &reconciler.BalanceChange{ + Account: account, + Currency: largeDeduction.Currency, + Block: newBlock3, + Difference: largeDeduction.Value, + }, ) - assert.Nil(t, balanceChange) - assert.Contains(t, err.Error(), ErrNegativeBalance.Error()) + assert.True(t, errors.Is(err, ErrNegativeBalance)) txn.Discard(ctx) }) t.Run("Attempt modification to push balance negative on new acct", func(t *testing.T) { - txn := storage.NewDatabaseTransaction(ctx, true) - balanceChange, err := storage.UpdateBalance( + txn := storage.newDatabaseTransaction(ctx, true) + err := storage.UpdateBalance( ctx, txn, - account2, - largeDeduction, - newBlock2, + &reconciler.BalanceChange{ + Account: account2, + Currency: largeDeduction.Currency, + Block: newBlock2, + Difference: largeDeduction.Value, + }, ) - assert.Nil(t, balanceChange) - assert.Contains(t, err.Error(), ErrNegativeBalance.Error()) + assert.Error(t, err) + assert.True(t, errors.Is(err, ErrNegativeBalance)) txn.Discard(ctx) }) t.Run("sub account set and get balance", func(t *testing.T) { - txn := storage.NewDatabaseTransaction(ctx, true) - balanceChange, err := storage.UpdateBalance( + txn := storage.newDatabaseTransaction(ctx, true) + err := storage.UpdateBalance( ctx, txn, - subAccount, - amount, - newBlock, + &reconciler.BalanceChange{ + Account: subAccount, + Currency: amount.Currency, + Block: newBlock, + Difference: amount.Value, + }, ) - assert.Equal(t, &BalanceChange{ - Account: subAccount, - Currency: currency, - Block: newBlock, - Difference: "100", - }, balanceChange) assert.NoError(t, err) assert.NoError(t, txn.Commit(ctx)) - txn = storage.NewDatabaseTransaction(ctx, false) - amounts, block, err := storage.GetBalance(ctx, txn, subAccountNewPointer) - txn.Discard(ctx) + retrievedAmount, block, err := storage.GetBalance(ctx, subAccountNewPointer, amount.Currency) assert.NoError(t, err) - assert.Equal(t, newAmounts, amounts) + assert.Equal(t, amount, retrievedAmount) assert.Equal(t, newBlock, block) }) t.Run("sub account metadata set and get balance", func(t *testing.T) { - txn := storage.NewDatabaseTransaction(ctx, true) - balanceChange, err := storage.UpdateBalance( + txn := storage.newDatabaseTransaction(ctx, true) + err := storage.UpdateBalance( ctx, txn, - subAccountMetadata, - amount, - newBlock, + &reconciler.BalanceChange{ + Account: subAccountMetadata, + Currency: amount.Currency, + Block: newBlock, + Difference: amount.Value, + }, ) - assert.Equal(t, &BalanceChange{ - Account: subAccountMetadata, - Currency: currency, - Block: newBlock, - Difference: "100", - }, balanceChange) assert.NoError(t, err) assert.NoError(t, txn.Commit(ctx)) - txn = storage.NewDatabaseTransaction(ctx, false) - amounts, block, err := storage.GetBalance(ctx, txn, subAccountMetadataNewPointer) - txn.Discard(ctx) + retrievedAmount, block, err := storage.GetBalance(ctx, subAccountMetadataNewPointer, amount.Currency) assert.NoError(t, err) - assert.Equal(t, newAmounts, amounts) + assert.Equal(t, amount, retrievedAmount) assert.Equal(t, newBlock, block) }) t.Run("sub account unique metadata set and get balance", func(t *testing.T) { - txn := storage.NewDatabaseTransaction(ctx, true) - balanceChange, err := storage.UpdateBalance( + txn := storage.newDatabaseTransaction(ctx, true) + err := storage.UpdateBalance( ctx, txn, - subAccountMetadata2, - amount, - newBlock, + &reconciler.BalanceChange{ + Account: subAccountMetadata2, + Currency: amount.Currency, + Block: newBlock, + Difference: amount.Value, + }, ) - assert.Equal(t, &BalanceChange{ - Account: subAccountMetadata2, - Currency: currency, - Block: newBlock, - Difference: "100", - }, balanceChange) assert.NoError(t, err) assert.NoError(t, txn.Commit(ctx)) - txn = storage.NewDatabaseTransaction(ctx, false) - amounts, block, err := storage.GetBalance(ctx, txn, subAccountMetadata2NewPointer) - txn.Discard(ctx) + retrievedAmount, block, err := storage.GetBalance(ctx, subAccountMetadata2NewPointer, amount.Currency) assert.NoError(t, err) - assert.Equal(t, newAmounts, amounts) + assert.Equal(t, amount, retrievedAmount) assert.Equal(t, newBlock, block) }) } @@ -671,7 +643,7 @@ func TestGetCurrencyKey(t *testing.T) { for name, test := range tests { t.Run(name, func(t *testing.T) { - assert.Equal(t, hashString(test.key), GetCurrencyKey(test.currency)) + assert.Equal(t, test.key, GetCurrencyKey(test.currency)) }) } } @@ -691,15 +663,15 @@ func TestBootstrapBalances(t *testing.T) { ctx := context.Background() - newDir, err := CreateTempDir() + newDir, err := utils.CreateTempDir() assert.NoError(t, err) - defer RemoveTempDir(*newDir) + defer utils.RemoveTempDir(*newDir) database, err := NewBadgerStorage(ctx, *newDir) assert.NoError(t, err) defer database.Close(ctx) - storage := NewBlockStorage(ctx, database) + storage := NewBlockStorage(ctx, database, &MockBlockStorageHelper{}) bootstrapBalancesFile := path.Join(*newDir, "balances.csv") t.Run("File doesn't exist", func(t *testing.T) { @@ -741,15 +713,13 @@ func TestBootstrapBalances(t *testing.T) { ) assert.NoError(t, err) - tx := storage.NewDatabaseTransaction(ctx, false) - amountMap, blockIdentifier, err := storage.GetBalance( + retrievedAmount, blockIdentifier, err := storage.GetBalance( ctx, - tx, account, + amount.Currency, ) - tx.Discard(ctx) - assert.Equal(t, amount, amountMap[GetCurrencyKey(amount.Currency)]) + assert.Equal(t, amount, retrievedAmount) assert.Equal(t, genesisBlockIdentifier, blockIdentifier) assert.NoError(t, err) }) @@ -822,7 +792,7 @@ func TestBootstrapBalances(t *testing.T) { }) t.Run("Head block identifier already set", func(t *testing.T) { - tx := storage.NewDatabaseTransaction(ctx, true) + tx := storage.newDatabaseTransaction(ctx, true) err := storage.StoreHeadBlockIdentifier(ctx, tx, genesisBlockIdentifier) assert.NoError(t, err) assert.NoError(t, tx.Commit(ctx)) @@ -832,3 +802,35 @@ func TestBootstrapBalances(t *testing.T) { assert.EqualError(t, err, ErrAlreadyStartedSyncing.Error()) }) } + +type MockBlockStorageHelper struct { + AccountBalanceAmount string +} + +func (h *MockBlockStorageHelper) AccountBalance( + ctx context.Context, + account *types.AccountIdentifier, + currency *types.Currency, + block *types.BlockIdentifier, +) (*types.Amount, error) { + value := "0" + if len(h.AccountBalanceAmount) > 0 { + value = h.AccountBalanceAmount + } + + return &types.Amount{ + Value: value, + Currency: currency, + }, nil +} + +func (h *MockBlockStorageHelper) SkipOperation( + ctx context.Context, + op *types.Operation, +) (bool, error) { + if op.Account == nil || op.Amount == nil { + return true, nil + } + + return false, nil +} From 86043e12f9549905c994363d0dc277afb9e66afa Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 4 May 2020 20:46:43 -0700 Subject: [PATCH 09/31] Implement all processors --- cmd/check_complete.go | 19 +++++++-- internal/logger/logger.go | 20 +++++---- internal/processor/reconciler_handler.go | 53 ++++++++++++++++++++++-- internal/processor/reconciler_helper.go | 29 +++++++++++-- internal/processor/storage_helper.go | 42 ++++++++++++++++--- internal/reconciler/reconciler.go | 6 ++- internal/storage/block_storage.go | 8 ++-- internal/storage/block_storage_test.go | 10 +++++ 8 files changed, 157 insertions(+), 30 deletions(-) diff --git a/cmd/check_complete.go b/cmd/check_complete.go index 4b1a6866..abee31e4 100644 --- a/cmd/check_complete.go +++ b/cmd/check_complete.go @@ -85,7 +85,6 @@ at the examples directory for an example of how to structure this file.`, } func runCheckCompleteCmd(cmd *cobra.Command, args []string) { - // TODO: if no directory passed in, create a temporary one ctx, cancel := context.WithCancel(context.Background()) exemptAccounts, err := loadAccounts(ExemptFile) @@ -125,7 +124,12 @@ func runCheckCompleteCmd(cmd *cobra.Command, args []string) { LogReconciliations, ) - blockStorageHelper := processor.NewBlockStorageHelper(fetcher, exemptAccounts) + blockStorageHelper := processor.NewBlockStorageHelper( + primaryNetwork, + fetcher, + LookupBalanceByBlock, + exemptAccounts, + ) blockStorage := storage.NewBlockStorage(ctx, localStore, blockStorageHelper) if len(BootstrapBalances) > 0 { @@ -139,8 +143,15 @@ func runCheckCompleteCmd(cmd *cobra.Command, args []string) { } } - reconcilerHelper := &processor.ReconcilerHelper{} - reconcilerHandler := &processor.ReconcilerHandler{} + reconcilerHelper := processor.NewReconcilerHelper( + blockStorage, + ) + + reconcilerHandler := processor.NewReconcilerHandler( + cancel, + logger, + HaltOnReconciliationError, + ) r := reconciler.NewReconciler( primaryNetwork, diff --git a/internal/logger/logger.go b/internal/logger/logger.go index fb762559..b4294d22 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -244,7 +244,8 @@ func (l *Logger) ReconcileSuccessStream( ctx context.Context, reconciliationType string, account *types.AccountIdentifier, - balance *types.Amount, + currency *types.Currency, + balance string, block *types.BlockIdentifier, ) error { if !l.logReconciliation { @@ -272,8 +273,8 @@ func (l *Logger) ReconcileSuccessStream( "Type:%s Account: %s Currency: %s Balance: %s Block: %d:%s\n", reconciliationType, account.Address, - balance.Currency.Symbol, - balance.Value, + currency.Symbol, + balance, block.Index, block.Hash, )) @@ -291,16 +292,18 @@ func (l *Logger) ReconcileFailureStream( reconciliationType string, account *types.AccountIdentifier, currency *types.Currency, - difference string, + computedBalance string, + nodeBalance string, block *types.BlockIdentifier, ) error { // Always print out reconciliation failures log.Printf( - "%s Reconciliation failed for %+v at %d by %s (computed-node)\n", + "%s Reconciliation failed for %+v at %d computed: %s node: %s\n", reconciliationType, account, block.Index, - difference, + computedBalance, + nodeBalance, ) if !l.logReconciliation { @@ -318,13 +321,14 @@ func (l *Logger) ReconcileFailureStream( defer f.Close() _, err = f.WriteString(fmt.Sprintf( - "Type:%s Account: %+v Currency: %+v Block: %s:%d Difference(computed-node):%s\n", + "Type:%s Account: %+v Currency: %+v Block: %s:%d computed: %s node: %s\n", reconciliationType, account, currency, block.Hash, block.Index, - difference, + computedBalance, + nodeBalance, )) if err != nil { return err diff --git a/internal/processor/reconciler_handler.go b/internal/processor/reconciler_handler.go index c1a1a472..488f41fe 100644 --- a/internal/processor/reconciler_handler.go +++ b/internal/processor/reconciler_handler.go @@ -4,10 +4,28 @@ import ( "context" "errors" + "github.com/coinbase/rosetta-cli/internal/logger" + "github.com/coinbase/rosetta-sdk-go/types" ) -type ReconcilerHandler struct{} +type ReconcilerHandler struct { + cancel context.CancelFunc + logger *logger.Logger + haltOnReconciliationError bool +} + +func NewReconcilerHandler( + cancel context.CancelFunc, + logger *logger.Logger, + haltOnReconciliationError bool, +) *ReconcilerHandler { + return &ReconcilerHandler{ + cancel: cancel, + logger: logger, + haltOnReconciliationError: haltOnReconciliationError, + } +} func (h *ReconcilerHandler) ReconciliationFailed( ctx context.Context, @@ -18,7 +36,29 @@ func (h *ReconcilerHandler) ReconciliationFailed( nodeBalance string, block *types.BlockIdentifier, ) error { - return errors.New("not implemented") + err := h.logger.ReconcileFailureStream( + ctx, + reconciliationType, + account, + currency, + computedBalance, + nodeBalance, + block, + ) + if err != nil { + return err + } + + if h.haltOnReconciliationError { + h.cancel() + return errors.New("halting due to reconciliation error") + } + + // TODO: automatically find block with missing operation + // if inactive reconciliation error. Can do this by asserting + // the impacted address has a balance change of 0 on all blocks + // it is not active. + return nil } func (h *ReconcilerHandler) ReconciliationSucceeded( @@ -29,5 +69,12 @@ func (h *ReconcilerHandler) ReconciliationSucceeded( balance string, block *types.BlockIdentifier, ) error { - return errors.New("not implemented") + return h.logger.ReconcileSuccessStream( + ctx, + reconciliationType, + account, + currency, + balance, + block, + ) } diff --git a/internal/processor/reconciler_helper.go b/internal/processor/reconciler_helper.go index fd91adbb..bd1d2bf8 100644 --- a/internal/processor/reconciler_helper.go +++ b/internal/processor/reconciler_helper.go @@ -4,22 +4,43 @@ import ( "context" "errors" + "github.com/coinbase/rosetta-cli/internal/storage" + "github.com/coinbase/rosetta-sdk-go/types" ) -type ReconcilerHelper struct{} +type ReconcilerHelper struct { + storage *storage.BlockStorage +} + +func NewReconcilerHelper( + storage *storage.BlockStorage, +) *ReconcilerHelper { + return &ReconcilerHelper{ + storage: storage, + } +} func (h *ReconcilerHelper) BlockExists( ctx context.Context, block *types.BlockIdentifier, ) (bool, error) { - return false, errors.New("not implemented") + _, err := h.storage.GetBlock(ctx, block) + if err == nil { + return true, nil + } + + if errors.Is(err, storage.ErrBlockNotFound) { + return false, nil + } + + return false, err } func (h *ReconcilerHelper) CurrentBlock( ctx context.Context, ) (*types.BlockIdentifier, error) { - return nil, errors.New("not implemented") + return h.storage.GetHeadBlockIdentifier(ctx) } func (h *ReconcilerHelper) AccountBalance( @@ -27,5 +48,5 @@ func (h *ReconcilerHelper) AccountBalance( account *types.AccountIdentifier, currency *types.Currency, ) (*types.Amount, *types.BlockIdentifier, error) { - return nil, nil, errors.New("not implemented") + return h.storage.GetBalance(ctx, account, currency) } diff --git a/internal/processor/storage_helper.go b/internal/processor/storage_helper.go index 27893f71..7db715e7 100644 --- a/internal/processor/storage_helper.go +++ b/internal/processor/storage_helper.go @@ -2,7 +2,6 @@ package processor import ( "context" - "errors" "log" "github.com/coinbase/rosetta-cli/internal/reconciler" @@ -12,19 +11,25 @@ import ( ) type BlockStorageHelper struct { + network *types.NetworkIdentifier fetcher *fetcher.Fetcher // Configuration settings - exemptAccounts []*reconciler.AccountCurrency + lookupBalanceByBlock bool + exemptAccounts []*reconciler.AccountCurrency } func NewBlockStorageHelper( + network *types.NetworkIdentifier, fetcher *fetcher.Fetcher, + lookupBalanceByBlock bool, exemptAccounts []*reconciler.AccountCurrency, ) *BlockStorageHelper { return &BlockStorageHelper{ - fetcher: fetcher, - exemptAccounts: exemptAccounts, + network: network, + fetcher: fetcher, + lookupBalanceByBlock: lookupBalanceByBlock, + exemptAccounts: exemptAccounts, } } @@ -33,8 +38,33 @@ func (h *BlockStorageHelper) AccountBalance( account *types.AccountIdentifier, currency *types.Currency, block *types.BlockIdentifier, -) (*types.Amount, error) { // returns an error if lookupBalanceByBlock disabled - return nil, errors.New("not implemented") +) (*types.Amount, error) { + if !h.lookupBalanceByBlock { + return &types.Amount{ + Value: "0", + Currency: currency, + }, nil + } + + // In the case that we are syncing from arbitrary height, + // we may need to recover the balance of an account to + // perform validations. + _, value, err := reconciler.GetCurrencyBalance( + ctx, + h.fetcher, + h.network, + account, + currency, + types.ConstructPartialBlockIdentifier(block), + ) + if err != nil { + return nil, err + } + + return &types.Amount{ + Value: value, + Currency: currency, + }, nil } // SkipOperation returns a boolean indicating whether diff --git a/internal/reconciler/reconciler.go b/internal/reconciler/reconciler.go index 451708a4..f598efca 100644 --- a/internal/reconciler/reconciler.go +++ b/internal/reconciler/reconciler.go @@ -311,7 +311,11 @@ func (r *Reconciler) CompareBalance( } // Check if live block < computed head - cachedBalance, balanceBlock, err := r.helper.AccountBalance(ctx, account, currency) + cachedBalance, balanceBlock, err := r.helper.AccountBalance( + ctx, + account, + currency, + ) if err != nil { return zeroString, head.Index, err } diff --git a/internal/storage/block_storage.go b/internal/storage/block_storage.go index 347a33bb..6ce9855e 100644 --- a/internal/storage/block_storage.go +++ b/internal/storage/block_storage.go @@ -363,7 +363,7 @@ func (b *BlockStorage) StoreBlock( } for _, change := range changes { - if err := b.UpdateBalance(ctx, transaction, change); err != nil { + if err := b.UpdateBalance(ctx, transaction, change, block.ParentBlockIdentifier); err != nil { return nil, err } } @@ -392,7 +392,7 @@ func (b *BlockStorage) RemoveBlock( } for _, change := range changes { - if err := b.UpdateBalance(ctx, transaction, change); err != nil { + if err := b.UpdateBalance(ctx, transaction, change, block.BlockIdentifier); err != nil { return nil, err } } @@ -480,6 +480,7 @@ func (b *BlockStorage) UpdateBalance( ctx context.Context, dbTransaction DatabaseTransaction, change *reconciler.BalanceChange, + parentBlock *types.BlockIdentifier, ) error { if change.Currency == nil { return errors.New("invalid currency") @@ -501,8 +502,7 @@ func (b *BlockStorage) UpdateBalance( existingValue = parseBal.Amount.Value } else { - // TODO: must be block BEFORE current (should only occur when adding, not removing) - amount, err := b.helper.AccountBalance(ctx, change.Account, change.Currency, nil) + amount, err := b.helper.AccountBalance(ctx, change.Account, change.Currency, parentBlock) if err != nil { return fmt.Errorf("%w: unable to get previous account balance", err) } diff --git a/internal/storage/block_storage_test.go b/internal/storage/block_storage_test.go index 21700c9b..98055faa 100644 --- a/internal/storage/block_storage_test.go +++ b/internal/storage/block_storage_test.go @@ -400,6 +400,7 @@ func TestBalance(t *testing.T) { Block: newBlock, Difference: amount.Value, }, + nil, ) assert.NoError(t, err) assert.NoError(t, txn.Commit(ctx)) @@ -422,6 +423,7 @@ func TestBalance(t *testing.T) { Block: newBlock, Difference: amount.Value, }, + nil, ) assert.NoError(t, err) assert.NoError(t, txn.Commit(ctx)) @@ -445,6 +447,7 @@ func TestBalance(t *testing.T) { Block: newBlock, Difference: amountNilCurrency.Value, }, + nil, ) assert.EqualError(t, err, "invalid currency") txn.Discard(ctx) @@ -466,6 +469,7 @@ func TestBalance(t *testing.T) { Block: newBlock2, Difference: amount.Value, }, + nil, ) assert.NoError(t, err) assert.NoError(t, txn.Commit(ctx)) @@ -487,6 +491,7 @@ func TestBalance(t *testing.T) { Block: newBlock3, Difference: amount.Value, }, + nil, ) assert.NoError(t, err) @@ -510,6 +515,7 @@ func TestBalance(t *testing.T) { Block: newBlock3, Difference: largeDeduction.Value, }, + nil, ) assert.True(t, errors.Is(err, ErrNegativeBalance)) txn.Discard(ctx) @@ -526,6 +532,7 @@ func TestBalance(t *testing.T) { Block: newBlock2, Difference: largeDeduction.Value, }, + nil, ) assert.Error(t, err) assert.True(t, errors.Is(err, ErrNegativeBalance)) @@ -543,6 +550,7 @@ func TestBalance(t *testing.T) { Block: newBlock, Difference: amount.Value, }, + nil, ) assert.NoError(t, err) assert.NoError(t, txn.Commit(ctx)) @@ -564,6 +572,7 @@ func TestBalance(t *testing.T) { Block: newBlock, Difference: amount.Value, }, + nil, ) assert.NoError(t, err) assert.NoError(t, txn.Commit(ctx)) @@ -585,6 +594,7 @@ func TestBalance(t *testing.T) { Block: newBlock, Difference: amount.Value, }, + nil, ) assert.NoError(t, err) assert.NoError(t, txn.Commit(ctx)) From 5f557228b3cd35b2365129b67d30d7bbc6846505 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 4 May 2020 20:57:22 -0700 Subject: [PATCH 10/31] Cleanup CMD --- cmd/check.go | 385 ++++++++++++++++++++++++ cmd/check_complete.go | 199 ------------ cmd/root.go | 179 +---------- internal/storage/badger_storage_test.go | 8 +- internal/storage/block_storage_test.go | 18 +- internal/utils/utils.go | 8 +- 6 files changed, 403 insertions(+), 394 deletions(-) create mode 100644 cmd/check.go delete mode 100644 cmd/check_complete.go diff --git a/cmd/check.go b/cmd/check.go new file mode 100644 index 00000000..781287b7 --- /dev/null +++ b/cmd/check.go @@ -0,0 +1,385 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "log" + "path" + "time" + + "github.com/coinbase/rosetta-cli/internal/logger" + "github.com/coinbase/rosetta-cli/internal/processor" + "github.com/coinbase/rosetta-cli/internal/reconciler" + "github.com/coinbase/rosetta-cli/internal/storage" + "github.com/coinbase/rosetta-cli/internal/syncer" + "github.com/coinbase/rosetta-cli/internal/utils" + + "github.com/coinbase/rosetta-sdk-go/fetcher" + "github.com/spf13/cobra" + "golang.org/x/sync/errgroup" +) + +const ( + // ExtendedRetryElapsedTime is used to override the default fetcher + // retry elapsed time. In practice, extending the retry elapsed time + // has prevented retry exhaustion errors when many goroutines are + // used to fetch data from the Rosetta server. + // + // TODO: make configurable + ExtendedRetryElapsedTime = 5 * time.Minute +) + +var ( + checkCmd = &cobra.Command{ + Use: "check", + Short: "Run a full check of the correctness of a Rosetta server", + Long: `Check all server responses are properly constructed, that +there are no duplicate blocks and transactions, that blocks can be processed +from genesis to the current block (re-orgs handled automatically), and that +computed balance changes are equal to balance changes reported by the node. + +When re-running this command, it will start where it left off. If you want +to discard some number of blocks populate the --start flag with some block +index less than the last computed block index.`, + Run: runCheckCmd, + } + + // BootstrapBalances is a path to a file used to bootstrap + // balances before starting syncing. Populating this value + // after beginning syncing will return an error. + BootstrapBalances string + + // LookupBalanceByBlock determines if balances are looked up + // at the block where a balance change occurred instead of at the current + // block. Blockchains that do not support historical balance lookup + // should set this to false. + LookupBalanceByBlock bool + + // DataDir is a folder used to store logs + // and any data used to perform validation. + DataDir string + + // ServerURL is the base URL for a Rosetta + // server to validate. + ServerURL string + + // StartIndex is the block index to start syncing. + StartIndex int64 + + // EndIndex is the block index to stop syncing. + EndIndex int64 + + // BlockConcurrency is the concurrency to use + // while fetching blocks. + BlockConcurrency uint64 + + // TransactionConcurrency is the concurrency to use + // while fetching transactions (if required). + TransactionConcurrency uint64 + + // AccountConcurrency is the concurrency to use + // while fetching accounts during reconciliation. + AccountConcurrency uint64 + + // LogBlocks determines if blocks are + // logged. + LogBlocks bool + + // LogTransactions determines if transactions are + // logged. + LogTransactions bool + + // LogBalanceChanges determines if balance changes are + // logged. + LogBalanceChanges bool + + // LogReconciliations determines if reconciliations are + // logged. + LogReconciliations bool + + // HaltOnReconciliationError determines if processing + // should stop when encountering a reconciliation error. + // It can be beneficial to collect all reconciliation errors + // during development. + HaltOnReconciliationError bool + + // ExemptFile is an absolute path to a file listing all accounts + // to exempt from balance tracking and reconciliation. + ExemptFile string + + // InterestingFile is an absolute path to a file listing all accounts + // to actively reconcile on each block (if there are no operations + // present for the account, the reconciler asserts a balance change of 0). + InterestingFile string +) + +func loadAccounts(filePath string) ([]*reconciler.AccountCurrency, error) { + if len(filePath) == 0 { + return []*reconciler.AccountCurrency{}, nil + } + + accountsRaw, err := ioutil.ReadFile(path.Clean(filePath)) + if err != nil { + return nil, err + } + + accounts := []*reconciler.AccountCurrency{} + if err := json.Unmarshal(accountsRaw, &accounts); err != nil { + return nil, err + } + + prettyAccounts, err := json.MarshalIndent(accounts, "", " ") + if err != nil { + return nil, err + } + log.Printf("Found %d accounts at %s: %s\n", len(accounts), filePath, string(prettyAccounts)) + + return accounts, nil +} + +func init() { + checkCmd.Flags().StringVar( + &DataDir, + "data-dir", + "", + "folder used to store logs and any data used to perform validation", + ) + checkCmd.Flags().StringVar( + &ServerURL, + "server-url", + "http://localhost:8080", + "base URL for a Rosetta server to validate", + ) + checkCmd.Flags().Int64Var( + &StartIndex, + "start", + -1, + "block index to start syncing", + ) + checkCmd.Flags().Int64Var( + &EndIndex, + "end", + -1, + "block index to stop syncing", + ) + checkCmd.Flags().Uint64Var( + &BlockConcurrency, + "block-concurrency", + 8, + "concurrency to use while fetching blocks", + ) + checkCmd.Flags().Uint64Var( + &TransactionConcurrency, + "transaction-concurrency", + 16, + "concurrency to use while fetching transactions (if required)", + ) + checkCmd.Flags().Uint64Var( + &AccountConcurrency, + "account-concurrency", + 8, + "concurrency to use while fetching accounts during reconciliation", + ) + checkCmd.Flags().BoolVar( + &LogBlocks, + "log-blocks", + false, + "log processed blocks", + ) + checkCmd.Flags().BoolVar( + &LogTransactions, + "log-transactions", + false, + "log processed transactions", + ) + checkCmd.Flags().BoolVar( + &LogBalanceChanges, + "log-balance-changes", + false, + "log balance changes", + ) + checkCmd.Flags().BoolVar( + &LogReconciliations, + "log-reconciliations", + false, + "log balance reconciliations", + ) + checkCmd.Flags().BoolVar( + &HaltOnReconciliationError, + "halt-on-reconciliation-error", + true, + `Determines if block processing should halt on a reconciliation +error. It can be beneficial to collect all reconciliation errors or silence +reconciliation errors during development.`, + ) + checkCmd.Flags().StringVar( + &ExemptFile, + "exempt-accounts", + "", + `Absolute path to a file listing all accounts to exempt from balance +tracking and reconciliation. Look at the examples directory for an example of +how to structure this file.`, + ) + checkCmd.Flags().StringVar( + &BootstrapBalances, + "bootstrap-balances", + "", + `Absolute path to a file used to bootstrap balances before starting syncing. +Populating this value after beginning syncing will return an error.`, + ) + checkCmd.Flags().BoolVar( + &LookupBalanceByBlock, + "lookup-balance-by-block", + true, + `When set to true, balances are looked up at the block where a balance +change occurred instead of at the current block. Blockchains that do not support +historical balance lookup should set this to false.`, + ) + checkCmd.Flags().StringVar( + &InterestingFile, + "interesting-accounts", + "", + `Absolute path to a file listing all accounts to check on each block. Look +at the examples directory for an example of how to structure this file.`, + ) +} + +func runCheckCmd(cmd *cobra.Command, args []string) { + ctx, cancel := context.WithCancel(context.Background()) + + exemptAccounts, err := loadAccounts(ExemptFile) + if err != nil { + log.Fatal(fmt.Errorf("%w: unable to load exempt accounts", err)) + } + + interestingAccounts, err := loadAccounts(InterestingFile) + if err != nil { + log.Fatal(fmt.Errorf("%w: unable to load interesting accounts", err)) + } + + fetcher := fetcher.New( + ServerURL, + fetcher.WithBlockConcurrency(BlockConcurrency), + fetcher.WithTransactionConcurrency(TransactionConcurrency), + fetcher.WithRetryElapsedTime(ExtendedRetryElapsedTime), + ) + + // TODO: sync and reconcile on subnetworks, if they exist. + primaryNetwork, networkStatus, err := fetcher.InitializeAsserter(ctx) + if err != nil { + log.Fatal(fmt.Errorf("%w: unable to initialize asserter", err)) + } + + // If data directory is not specified, we use a temporary directory + // and delete its contents when execution is complete. + if len(DataDir) == 0 { + tmpDir, err := utils.CreateTempDir() + if err != nil { + log.Fatal(fmt.Errorf("%w: unable to create temporary directory", err)) + } + defer utils.RemoveTempDir(tmpDir) + + DataDir = tmpDir + } + localStore, err := storage.NewBadgerStorage(ctx, DataDir) + if err != nil { + log.Fatal(fmt.Errorf("%w: unable to initialize data store", err)) + } + + logger := logger.NewLogger( + DataDir, + LogBlocks, + LogTransactions, + LogBalanceChanges, + LogReconciliations, + ) + + blockStorageHelper := processor.NewBlockStorageHelper( + primaryNetwork, + fetcher, + LookupBalanceByBlock, + exemptAccounts, + ) + + blockStorage := storage.NewBlockStorage(ctx, localStore, blockStorageHelper) + if len(BootstrapBalances) > 0 { + err = blockStorage.BootstrapBalances( + ctx, + BootstrapBalances, + networkStatus.GenesisBlockIdentifier, + ) + if err != nil { + log.Fatal(fmt.Errorf("%w: unable to bootstrap balances", err)) + } + } + + reconcilerHelper := processor.NewReconcilerHelper( + blockStorage, + ) + + reconcilerHandler := processor.NewReconcilerHandler( + cancel, + logger, + HaltOnReconciliationError, + ) + + r := reconciler.NewReconciler( + primaryNetwork, + reconcilerHelper, + reconcilerHandler, + fetcher, + AccountConcurrency, + LookupBalanceByBlock, + interestingAccounts, + ) + + syncHandler := processor.NewSyncHandler( + blockStorage, + logger, + r, + fetcher, + exemptAccounts, + ) + + g, ctx := errgroup.WithContext(ctx) + + g.Go(func() error { + return r.Reconcile(ctx) + }) + + syncer := syncer.New( + primaryNetwork, + fetcher, + syncHandler, + cancel, + ) + + g.Go(func() error { + return syncer.Sync( + ctx, + StartIndex, + EndIndex, + ) + }) + + err = g.Wait() + if err != nil { + log.Fatal(err) + } +} diff --git a/cmd/check_complete.go b/cmd/check_complete.go deleted file mode 100644 index abee31e4..00000000 --- a/cmd/check_complete.go +++ /dev/null @@ -1,199 +0,0 @@ -// Copyright 2020 Coinbase, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package cmd - -import ( - "context" - "fmt" - "log" - - "github.com/coinbase/rosetta-cli/internal/logger" - "github.com/coinbase/rosetta-cli/internal/processor" - "github.com/coinbase/rosetta-cli/internal/reconciler" - "github.com/coinbase/rosetta-cli/internal/storage" - "github.com/coinbase/rosetta-cli/internal/syncer" - - "github.com/coinbase/rosetta-sdk-go/fetcher" - "github.com/spf13/cobra" - "golang.org/x/sync/errgroup" -) - -var ( - checkCompleteCmd = &cobra.Command{ - Use: "check:complete", - Short: "Run a full check of the correctness of a Rosetta server", - Long: `Check all server responses are properly constructed, that -there are no duplicate blocks and transactions, that blocks can be processed -from genesis to the current block (re-orgs handled automatically), and that -computed balance changes are equal to balance changes reported by the node. - -When re-running this command, it will start where it left off. If you want -to discard some number of blocks populate the --start flag with some block -index less than the last computed block index.`, - Run: runCheckCompleteCmd, - } - - // BootstrapBalances is a path to a file used to bootstrap - // balances before starting syncing. Populating this value - // after beginning syncing will return an error. - BootstrapBalances string - - // LookupBalanceByBlock determines if balances are looked up - // at the block where a balance change occurred instead of at the current - // block. Blockchains that do not support historical balance lookup - // should set this to false. - LookupBalanceByBlock bool - - accountFile string -) - -func init() { - checkCompleteCmd.Flags().StringVar( - &BootstrapBalances, - "bootstrap-balances", - "", - `Absolute path to a file used to bootstrap balances before starting syncing. -Populating this value after beginning syncing will return an error.`, - ) - checkCompleteCmd.Flags().BoolVar( - &LookupBalanceByBlock, - "lookup-balance-by-block", - true, - `When set to true, balances are looked up at the block where a balance -change occurred instead of at the current block. Blockchains that do not support -historical balance lookup should set this to false.`, - ) - checkCompleteCmd.Flags().StringVar( - &accountFile, - "interesting-accounts", - "", - `Absolute path to a file listing all accounts to check on each block. Look -at the examples directory for an example of how to structure this file.`, - ) -} - -func runCheckCompleteCmd(cmd *cobra.Command, args []string) { - ctx, cancel := context.WithCancel(context.Background()) - - exemptAccounts, err := loadAccounts(ExemptFile) - if err != nil { - log.Fatal(fmt.Errorf("%w: unable to load exempt accounts", err)) - } - - interestingAccounts, err := loadAccounts(accountFile) - if err != nil { - log.Fatal(fmt.Errorf("%w: unable to load interesting accounts", err)) - } - - fetcher := fetcher.New( - ServerURL, - fetcher.WithBlockConcurrency(BlockConcurrency), - fetcher.WithTransactionConcurrency(TransactionConcurrency), - fetcher.WithRetryElapsedTime(ExtendedRetryElapsedTime), - ) - - // TODO: sync and reconcile on subnetworks, if they exist. - primaryNetwork, networkStatus, err := fetcher.InitializeAsserter(ctx) - if err != nil { - log.Fatal(fmt.Errorf("%w: unable to initialize asserter", err)) - } - - // TODO: if DataDir is empty, use TempDir - localStore, err := storage.NewBadgerStorage(ctx, DataDir) - if err != nil { - log.Fatal(fmt.Errorf("%w: unable to initialize data store", err)) - } - - logger := logger.NewLogger( - DataDir, - LogBlocks, - LogTransactions, - LogBalanceChanges, - LogReconciliations, - ) - - blockStorageHelper := processor.NewBlockStorageHelper( - primaryNetwork, - fetcher, - LookupBalanceByBlock, - exemptAccounts, - ) - - blockStorage := storage.NewBlockStorage(ctx, localStore, blockStorageHelper) - if len(BootstrapBalances) > 0 { - err = blockStorage.BootstrapBalances( - ctx, - BootstrapBalances, - networkStatus.GenesisBlockIdentifier, - ) - if err != nil { - log.Fatal(fmt.Errorf("%w: unable to bootstrap balances", err)) - } - } - - reconcilerHelper := processor.NewReconcilerHelper( - blockStorage, - ) - - reconcilerHandler := processor.NewReconcilerHandler( - cancel, - logger, - HaltOnReconciliationError, - ) - - r := reconciler.NewReconciler( - primaryNetwork, - reconcilerHelper, - reconcilerHandler, - fetcher, - AccountConcurrency, - LookupBalanceByBlock, - interestingAccounts, - ) - - syncHandler := processor.NewSyncHandler( - blockStorage, - logger, - r, - fetcher, - exemptAccounts, - ) - - g, ctx := errgroup.WithContext(ctx) - - g.Go(func() error { - return r.Reconcile(ctx) - }) - - syncer := syncer.New( - primaryNetwork, - fetcher, - syncHandler, - cancel, - ) - - g.Go(func() error { - return syncer.Sync( - ctx, - StartIndex, - EndIndex, - ) - }) - - err = g.Wait() - if err != nil { - log.Fatal(err) - } -} diff --git a/cmd/root.go b/cmd/root.go index e45702f2..590b2398 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -15,84 +15,14 @@ package cmd import ( - "encoding/json" - "io/ioutil" - "log" - "path" - "time" - - "github.com/coinbase/rosetta-cli/internal/reconciler" - "github.com/spf13/cobra" ) -const ( - // ExtendedRetryElapsedTime is used to override the default fetcher - // retry elapsed time. In practice, extending the retry elapsed time - // has prevented retry exhaustion errors when many goroutines are - // used to fetch data from the Rosetta server. - // - // TODO: make configurable - ExtendedRetryElapsedTime = 5 * time.Minute -) - var ( rootCmd = &cobra.Command{ Use: "rosetta-cli", Short: "CLI for the Rosetta API", } - - // DataDir is a folder used to store logs - // and any data used to perform validation. - DataDir string - - // ServerURL is the base URL for a Rosetta - // server to validate. - ServerURL string - - // StartIndex is the block index to start syncing. - StartIndex int64 - - // EndIndex is the block index to stop syncing. - EndIndex int64 - - // BlockConcurrency is the concurrency to use - // while fetching blocks. - BlockConcurrency uint64 - - // TransactionConcurrency is the concurrency to use - // while fetching transactions (if required). - TransactionConcurrency uint64 - - // AccountConcurrency is the concurrency to use - // while fetching accounts during reconciliation. - AccountConcurrency uint64 - - // LogBlocks determines if blocks are - // logged. - LogBlocks bool - - // LogTransactions determines if transactions are - // logged. - LogTransactions bool - - // LogBalanceChanges determines if balance changes are - // logged. - LogBalanceChanges bool - - // LogReconciliations determines if reconciliations are - // logged. - LogReconciliations bool - - // HaltOnReconciliationError determines if processing - // should stop when encountering a reconciliation error. - // It can be beneficial to collect all reconciliation errors - // during development. - HaltOnReconciliationError bool - - // ExemptFile is an absolute path to a file listing all accounts - // to exempt from balance tracking and reconciliation. - ExemptFile string ) // Execute handles all invocations of the @@ -102,112 +32,5 @@ func Execute() error { } func init() { - rootCmd.PersistentFlags().StringVar( - &DataDir, - "data-dir", - "./validator-data", - "folder used to store logs and any data used to perform validation", - ) - rootCmd.PersistentFlags().StringVar( - &ServerURL, - "server-url", - "http://localhost:8080", - "base URL for a Rosetta server to validate", - ) - rootCmd.PersistentFlags().Int64Var( - &StartIndex, - "start", - -1, - "block index to start syncing", - ) - rootCmd.PersistentFlags().Int64Var( - &EndIndex, - "end", - -1, - "block index to stop syncing", - ) - rootCmd.PersistentFlags().Uint64Var( - &BlockConcurrency, - "block-concurrency", - 8, - "concurrency to use while fetching blocks", - ) - rootCmd.PersistentFlags().Uint64Var( - &TransactionConcurrency, - "transaction-concurrency", - 16, - "concurrency to use while fetching transactions (if required)", - ) - rootCmd.PersistentFlags().Uint64Var( - &AccountConcurrency, - "account-concurrency", - 8, - "concurrency to use while fetching accounts during reconciliation", - ) - rootCmd.PersistentFlags().BoolVar( - &LogBlocks, - "log-blocks", - false, - "log processed blocks", - ) - rootCmd.PersistentFlags().BoolVar( - &LogTransactions, - "log-transactions", - false, - "log processed transactions", - ) - rootCmd.PersistentFlags().BoolVar( - &LogBalanceChanges, - "log-balance-changes", - false, - "log balance changes", - ) - rootCmd.PersistentFlags().BoolVar( - &LogReconciliations, - "log-reconciliations", - false, - "log balance reconciliations", - ) - rootCmd.PersistentFlags().BoolVar( - &HaltOnReconciliationError, - "halt-on-reconciliation-error", - true, - `Determines if block processing should halt on a reconciliation -error. It can be beneficial to collect all reconciliation errors or silence -reconciliation errors during development.`, - ) - rootCmd.PersistentFlags().StringVar( - &ExemptFile, - "exempt-accounts", - "", - `Absolute path to a file listing all accounts to exempt from balance -tracking and reconciliation. Look at the examples directory for an example of -how to structure this file.`, - ) - - rootCmd.AddCommand(checkCompleteCmd) -} - -func loadAccounts(filePath string) ([]*reconciler.AccountCurrency, error) { - if len(filePath) == 0 { - return []*reconciler.AccountCurrency{}, nil - } - - accountsRaw, err := ioutil.ReadFile(path.Clean(filePath)) - if err != nil { - return nil, err - } - - accounts := []*reconciler.AccountCurrency{} - if err := json.Unmarshal(accountsRaw, &accounts); err != nil { - return nil, err - } - - prettyAccounts, err := json.MarshalIndent(accounts, "", " ") - if err != nil { - return nil, err - } - log.Printf("Found %d accounts at %s: %s\n", len(accounts), filePath, string(prettyAccounts)) - - return accounts, nil + rootCmd.AddCommand(checkCmd) } diff --git a/internal/storage/badger_storage_test.go b/internal/storage/badger_storage_test.go index 03e55a62..9abdce14 100644 --- a/internal/storage/badger_storage_test.go +++ b/internal/storage/badger_storage_test.go @@ -28,9 +28,9 @@ func TestDatabase(t *testing.T) { newDir, err := utils.CreateTempDir() assert.NoError(t, err) - defer utils.RemoveTempDir(*newDir) + defer utils.RemoveTempDir(newDir) - database, err := NewBadgerStorage(ctx, *newDir) + database, err := NewBadgerStorage(ctx, newDir) assert.NoError(t, err) defer database.Close(ctx) @@ -59,9 +59,9 @@ func TestDatabaseTransaction(t *testing.T) { newDir, err := utils.CreateTempDir() assert.NoError(t, err) - defer utils.RemoveTempDir(*newDir) + defer utils.RemoveTempDir(newDir) - database, err := NewBadgerStorage(ctx, *newDir) + database, err := NewBadgerStorage(ctx, newDir) assert.NoError(t, err) defer database.Close(ctx) diff --git a/internal/storage/block_storage_test.go b/internal/storage/block_storage_test.go index 98055faa..4f7689c2 100644 --- a/internal/storage/block_storage_test.go +++ b/internal/storage/block_storage_test.go @@ -47,9 +47,9 @@ func TestHeadBlockIdentifier(t *testing.T) { newDir, err := utils.CreateTempDir() assert.NoError(t, err) - defer utils.RemoveTempDir(*newDir) + defer utils.RemoveTempDir(newDir) - database, err := NewBadgerStorage(ctx, *newDir) + database, err := NewBadgerStorage(ctx, newDir) assert.NoError(t, err) defer database.Close(ctx) @@ -153,9 +153,9 @@ func TestBlock(t *testing.T) { newDir, err := utils.CreateTempDir() assert.NoError(t, err) - defer utils.RemoveTempDir(*newDir) + defer utils.RemoveTempDir(newDir) - database, err := NewBadgerStorage(ctx, *newDir) + database, err := NewBadgerStorage(ctx, newDir) assert.NoError(t, err) defer database.Close(ctx) @@ -374,9 +374,9 @@ func TestBalance(t *testing.T) { newDir, err := utils.CreateTempDir() assert.NoError(t, err) - defer utils.RemoveTempDir(*newDir) + defer utils.RemoveTempDir(newDir) - database, err := NewBadgerStorage(ctx, *newDir) + database, err := NewBadgerStorage(ctx, newDir) assert.NoError(t, err) defer database.Close(ctx) @@ -675,14 +675,14 @@ func TestBootstrapBalances(t *testing.T) { newDir, err := utils.CreateTempDir() assert.NoError(t, err) - defer utils.RemoveTempDir(*newDir) + defer utils.RemoveTempDir(newDir) - database, err := NewBadgerStorage(ctx, *newDir) + database, err := NewBadgerStorage(ctx, newDir) assert.NoError(t, err) defer database.Close(ctx) storage := NewBlockStorage(ctx, database, &MockBlockStorageHelper{}) - bootstrapBalancesFile := path.Join(*newDir, "balances.csv") + bootstrapBalancesFile := path.Join(newDir, "balances.csv") t.Run("File doesn't exist", func(t *testing.T) { err = storage.BootstrapBalances( diff --git a/internal/utils/utils.go b/internal/utils/utils.go index cb121afb..7c68c3c4 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -24,13 +24,13 @@ import ( // CreateTempDir creates a directory in // /tmp for usage within testing. -func CreateTempDir() (*string, error) { - storageDir, err := ioutil.TempDir("", "rosetta-worker") +func CreateTempDir() (string, error) { + storageDir, err := ioutil.TempDir("", "rosetta-cli") if err != nil { - return nil, err + return "", err } - return &storageDir, nil + return storageDir, nil } // RemoveTempDir deletes a directory at From 124cdd5fe3503ff37f21fca636f196e9cd66576a Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 4 May 2020 21:30:56 -0700 Subject: [PATCH 11/31] Reconciler tests --- internal/reconciler/reconciler.go | 34 +- internal/reconciler/reconciler_test.go | 438 +++++++++++++++++++++++++ 2 files changed, 452 insertions(+), 20 deletions(-) create mode 100644 internal/reconciler/reconciler_test.go diff --git a/internal/reconciler/reconciler.go b/internal/reconciler/reconciler.go index f598efca..062537f6 100644 --- a/internal/reconciler/reconciler.go +++ b/internal/reconciler/reconciler.go @@ -113,11 +113,6 @@ type ReconcilerHelper interface { ctx context.Context, ) (*types.BlockIdentifier, error) - // always compare current balance...just need to set with previous seen if starting - // in middle - // TODO: always pass in head block to do lookup in case we need to fetch balance - // on previous block...this should always be set by the time it gets to the reconciler - // based on storage AccountBalance( ctx context.Context, account *types.AccountIdentifier, @@ -283,16 +278,16 @@ func (r *Reconciler) CompareBalance( currency *types.Currency, amount string, liveBlock *types.BlockIdentifier, -) (string, int64, error) { +) (string, string, int64, error) { // Head block should be set before we CompareBalance head, err := r.helper.CurrentBlock(ctx) if err != nil { - return zeroString, 0, fmt.Errorf("%w: unable to get current block for reconciliation", err) + return zeroString, "", 0, fmt.Errorf("%w: unable to get current block for reconciliation", err) } // Check if live block is < head (or wait) if liveBlock.Index > head.Index { - return zeroString, head.Index, fmt.Errorf( + return zeroString, "", head.Index, fmt.Errorf( "%w live block %d > head block %d", ErrHeadBlockBehindLive, liveBlock.Index, @@ -301,9 +296,12 @@ func (r *Reconciler) CompareBalance( } // Check if live block is in store (ensure not reorged) - _, err = r.helper.BlockExists(ctx, liveBlock) + exists, err := r.helper.BlockExists(ctx, liveBlock) if err != nil { - return zeroString, head.Index, fmt.Errorf( + return zeroString, "", 0, fmt.Errorf("%w: unable to check if block exists: %+v", err, liveBlock) + } + if !exists { + return zeroString, "", head.Index, fmt.Errorf( "%w %+v", ErrBlockGone, liveBlock, @@ -317,11 +315,11 @@ func (r *Reconciler) CompareBalance( currency, ) if err != nil { - return zeroString, head.Index, err + return zeroString, "", head.Index, fmt.Errorf("%w: unable to get cached balance for %+v:%+v", err, account, currency) } if liveBlock.Index < balanceBlock.Index { - return zeroString, head.Index, fmt.Errorf( + return zeroString, "", head.Index, fmt.Errorf( "%w %+v updated at %d", ErrAccountUpdated, account, @@ -331,14 +329,10 @@ func (r *Reconciler) CompareBalance( difference, err := utils.SubtractStringValues(cachedBalance.Value, amount) if err != nil { - return "", -1, err - } - - if difference != zeroString { - return difference, head.Index, nil + return "", "", -1, err } - return zeroString, head.Index, nil + return difference, cachedBalance.Value, head.Index, nil } // bestBalance returns the balance for an account @@ -384,7 +378,7 @@ func (r *Reconciler) accountReconciliation( for ctx.Err() == nil { // If don't have previous balance because stateless, check diff on block // instead of comparing entire computed balance - difference, headIndex, err := r.CompareBalance( + difference, cachedBalance, headIndex, err := r.CompareBalance( ctx, account, currency, @@ -443,7 +437,7 @@ func (r *Reconciler) accountReconciliation( reconciliationType, accountCurrency.Account, accountCurrency.Currency, - "TODO", + cachedBalance, liveAmount, liveBlock, ) diff --git a/internal/reconciler/reconciler_test.go b/internal/reconciler/reconciler_test.go new file mode 100644 index 00000000..2fdc575b --- /dev/null +++ b/internal/reconciler/reconciler_test.go @@ -0,0 +1,438 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package reconciler + +import ( + "context" + "errors" + "fmt" + "reflect" + "testing" + + "github.com/coinbase/rosetta-sdk-go/types" + "github.com/stretchr/testify/assert" +) + +func TestContainsAccountCurrency(t *testing.T) { + currency1 := &types.Currency{ + Symbol: "Blah", + Decimals: 2, + } + currency2 := &types.Currency{ + Symbol: "Blah2", + Decimals: 2, + } + accts := []*AccountCurrency{ + { + Account: &types.AccountIdentifier{ + Address: "test", + }, + Currency: currency1, + }, + { + Account: &types.AccountIdentifier{ + Address: "cool", + SubAccount: &types.SubAccountIdentifier{ + Address: "test2", + }, + }, + Currency: currency1, + }, + { + Account: &types.AccountIdentifier{ + Address: "cool", + SubAccount: &types.SubAccountIdentifier{ + Address: "test2", + Metadata: map[string]interface{}{ + "neat": "stuff", + }, + }, + }, + Currency: currency1, + }, + } + + t.Run("Non-existent account", func(t *testing.T) { + assert.False(t, ContainsAccountCurrency(accts, &AccountCurrency{ + Account: &types.AccountIdentifier{ + Address: "blah", + }, + Currency: currency1, + })) + }) + + t.Run("Basic account", func(t *testing.T) { + assert.True(t, ContainsAccountCurrency(accts, &AccountCurrency{ + Account: &types.AccountIdentifier{ + Address: "test", + }, + Currency: currency1, + })) + }) + + t.Run("Basic account with bad currency", func(t *testing.T) { + assert.False(t, ContainsAccountCurrency(accts, &AccountCurrency{ + Account: &types.AccountIdentifier{ + Address: "test", + }, + Currency: currency2, + })) + }) + + t.Run("Account with subaccount", func(t *testing.T) { + assert.True(t, ContainsAccountCurrency(accts, &AccountCurrency{ + Account: &types.AccountIdentifier{ + Address: "cool", + SubAccount: &types.SubAccountIdentifier{ + Address: "test2", + }, + }, + Currency: currency1, + })) + }) + + t.Run("Account with subaccount and metadata", func(t *testing.T) { + assert.True(t, ContainsAccountCurrency(accts, &AccountCurrency{ + Account: &types.AccountIdentifier{ + Address: "cool", + SubAccount: &types.SubAccountIdentifier{ + Address: "test2", + Metadata: map[string]interface{}{ + "neat": "stuff", + }, + }, + }, + Currency: currency1, + })) + }) + + t.Run("Account with subaccount and unique metadata", func(t *testing.T) { + assert.False(t, ContainsAccountCurrency(accts, &AccountCurrency{ + Account: &types.AccountIdentifier{ + Address: "cool", + SubAccount: &types.SubAccountIdentifier{ + Address: "test2", + Metadata: map[string]interface{}{ + "neater": "stuff", + }, + }, + }, + Currency: currency1, + })) + }) +} + +func TestExtractAmount(t *testing.T) { + var ( + currency1 = &types.Currency{ + Symbol: "curr1", + Decimals: 4, + } + + currency2 = &types.Currency{ + Symbol: "curr2", + Decimals: 7, + } + + amount1 = &types.Amount{ + Value: "100", + Currency: currency1, + } + + amount2 = &types.Amount{ + Value: "200", + Currency: currency2, + } + + balances = []*types.Amount{ + amount1, + amount2, + } + + badCurr = &types.Currency{ + Symbol: "no curr", + Decimals: 100, + } + ) + + t.Run("Non-existent currency", func(t *testing.T) { + result, err := ExtractAmount(balances, badCurr) + assert.Nil(t, result) + assert.EqualError(t, err, fmt.Errorf("could not extract amount for %+v", badCurr).Error()) + }) + + t.Run("Simple account", func(t *testing.T) { + result, err := ExtractAmount(balances, currency1) + assert.Equal(t, amount1, result) + assert.NoError(t, err) + }) + + t.Run("SubAccount", func(t *testing.T) { + result, err := ExtractAmount(balances, currency2) + assert.Equal(t, amount2, result) + assert.NoError(t, err) + }) +} + +func TestCompareBalance(t *testing.T) { + var ( + account1 = &types.AccountIdentifier{ + Address: "blah", + } + + account2 = &types.AccountIdentifier{ + Address: "blah", + SubAccount: &types.SubAccountIdentifier{ + Address: "sub blah", + }, + } + + currency1 = &types.Currency{ + Symbol: "curr1", + Decimals: 4, + } + + currency2 = &types.Currency{ + Symbol: "curr2", + Decimals: 7, + } + + amount1 = &types.Amount{ + Value: "100", + Currency: currency1, + } + + amount2 = &types.Amount{ + Value: "200", + Currency: currency2, + } + + block0 = &types.BlockIdentifier{ + Hash: "block0", + Index: 0, + } + + block1 = &types.BlockIdentifier{ + Hash: "block1", + Index: 1, + } + + block2 = &types.BlockIdentifier{ + Hash: "block2", + Index: 2, + } + + ctx = context.Background() + + mh = &MockReconcilerHelper{} + ) + + reconciler := NewReconciler( + nil, + mh, + nil, + nil, + 1, + false, + []*AccountCurrency{}, + ) + + t.Run("No head block yet", func(t *testing.T) { + difference, cachedBalance, headIndex, err := reconciler.CompareBalance( + ctx, + account1, + currency1, + amount1.Value, + block1, + ) + assert.Equal(t, "0", difference) + assert.Equal(t, "", cachedBalance) + assert.Equal(t, int64(0), headIndex) + assert.Error(t, err) + }) + + // Update head block + mh.HeadBlock = block0 + + t.Run("Live block is ahead of head block", func(t *testing.T) { + difference, cachedBalance, headIndex, err := reconciler.CompareBalance( + ctx, + account1, + currency1, + amount1.Value, + block1, + ) + assert.Equal(t, "0", difference) + assert.Equal(t, "", cachedBalance) + assert.Equal(t, int64(0), headIndex) + assert.EqualError(t, err, fmt.Errorf( + "%w live block %d > head block %d", + ErrHeadBlockBehindLive, + 1, + 0, + ).Error()) + }) + + // Update head block + mh.HeadBlock = &types.BlockIdentifier{ + Hash: "hash2", + Index: 2, + } + + t.Run("Live block is not in store", func(t *testing.T) { + difference, cachedBalance, headIndex, err := reconciler.CompareBalance( + ctx, + account1, + currency1, + amount1.Value, + block1, + ) + assert.Equal(t, "0", difference) + assert.Equal(t, "", cachedBalance) + assert.Equal(t, int64(2), headIndex) + assert.Contains(t, err.Error(), ErrBlockGone.Error()) + }) + + // Add blocks to store behind head + mh.StoredBlocks = map[string]*types.Block{} + mh.StoredBlocks[block0.Hash] = &types.Block{ + BlockIdentifier: block0, + ParentBlockIdentifier: block0, + } + mh.StoredBlocks[block1.Hash] = &types.Block{ + BlockIdentifier: block1, + ParentBlockIdentifier: block0, + } + mh.StoredBlocks[block2.Hash] = &types.Block{ + BlockIdentifier: block2, + ParentBlockIdentifier: block1, + } + mh.BalanceAccount = account1 + mh.BalanceAmount = amount1 + mh.BalanceBlock = block1 + + t.Run("Account updated after live block", func(t *testing.T) { + difference, cachedBalance, headIndex, err := reconciler.CompareBalance( + ctx, + account1, + currency1, + amount1.Value, + block0, + ) + assert.Equal(t, "0", difference) + assert.Equal(t, "", cachedBalance) + assert.Equal(t, int64(2), headIndex) + assert.Contains(t, err.Error(), ErrAccountUpdated.Error()) + }) + + t.Run("Account balance matches", func(t *testing.T) { + difference, cachedBalance, headIndex, err := reconciler.CompareBalance( + ctx, + account1, + currency1, + amount1.Value, + block1, + ) + assert.Equal(t, "0", difference) + assert.Equal(t, amount1.Value, cachedBalance) + assert.Equal(t, int64(2), headIndex) + assert.NoError(t, err) + }) + + t.Run("Account balance matches later live block", func(t *testing.T) { + difference, cachedBalance, headIndex, err := reconciler.CompareBalance( + ctx, + account1, + currency1, + amount1.Value, + block2, + ) + assert.Equal(t, "0", difference) + assert.Equal(t, amount1.Value, cachedBalance) + assert.Equal(t, int64(2), headIndex) + assert.NoError(t, err) + }) + + t.Run("Balances are not equal", func(t *testing.T) { + difference, cachedBalance, headIndex, err := reconciler.CompareBalance( + ctx, + account1, + currency1, + amount2.Value, + block2, + ) + assert.Equal(t, "-100", difference) + assert.Equal(t, amount1.Value, cachedBalance) + assert.Equal(t, int64(2), headIndex) + assert.NoError(t, err) + }) + + t.Run("Compare balance for non-existent account", func(t *testing.T) { + difference, cachedBalance, headIndex, err := reconciler.CompareBalance( + ctx, + account2, + currency1, + amount2.Value, + block2, + ) + assert.Equal(t, "0", difference) + assert.Equal(t, "", cachedBalance) + assert.Equal(t, int64(2), headIndex) + assert.Error(t, err) + }) +} + +type MockReconcilerHelper struct { + HeadBlock *types.BlockIdentifier + StoredBlocks map[string]*types.Block + + BalanceAccount *types.AccountIdentifier + BalanceAmount *types.Amount + BalanceBlock *types.BlockIdentifier +} + +func (h *MockReconcilerHelper) BlockExists( + ctx context.Context, + block *types.BlockIdentifier, +) (bool, error) { + _, ok := h.StoredBlocks[block.Hash] + if !ok { + return false, nil + } + + return true, nil +} + +func (h *MockReconcilerHelper) CurrentBlock( + ctx context.Context, +) (*types.BlockIdentifier, error) { + if h.HeadBlock == nil { + return nil, errors.New("head block is nil") + } + + return h.HeadBlock, nil +} + +func (h *MockReconcilerHelper) AccountBalance( + ctx context.Context, + account *types.AccountIdentifier, + currency *types.Currency, +) (*types.Amount, *types.BlockIdentifier, error) { + if h.BalanceAccount == nil || !reflect.DeepEqual(account, h.BalanceAccount) { + return nil, nil, errors.New("account does not exist") + } + + return h.BalanceAmount, h.BalanceBlock, nil +} From f9fb25edb6d2d5a919da688f78638766f9eb4b88 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 4 May 2020 21:41:49 -0700 Subject: [PATCH 12/31] Add BalanceChanges test --- internal/storage/block_storage_test.go | 268 +++++++++++++++++++++++++ 1 file changed, 268 insertions(+) diff --git a/internal/storage/block_storage_test.go b/internal/storage/block_storage_test.go index 4f7689c2..94d24f33 100644 --- a/internal/storage/block_storage_test.go +++ b/internal/storage/block_storage_test.go @@ -27,6 +27,7 @@ import ( "github.com/coinbase/rosetta-cli/internal/reconciler" "github.com/coinbase/rosetta-cli/internal/utils" + "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/types" "github.com/stretchr/testify/assert" ) @@ -813,8 +814,264 @@ func TestBootstrapBalances(t *testing.T) { }) } +func simpleTransactionFactory( + hash string, + address string, + value string, + currency *types.Currency, +) *types.Transaction { + return &types.Transaction{ + TransactionIdentifier: &types.TransactionIdentifier{ + Hash: hash, + }, + Operations: []*types.Operation{ + { + OperationIdentifier: &types.OperationIdentifier{ + Index: 0, + }, + Type: "Transfer", + Status: "Success", + Account: &types.AccountIdentifier{ + Address: address, + }, + Amount: &types.Amount{ + Value: value, + Currency: currency, + }, + }, + }, + } +} + +func TestBalanceChanges(t *testing.T) { + var ( + currency = &types.Currency{ + Symbol: "Blah", + Decimals: 2, + } + + recipient = &types.AccountIdentifier{ + Address: "acct1", + } + + recipientAmount = &types.Amount{ + Value: "100", + Currency: currency, + } + + recipientOperation = &types.Operation{ + OperationIdentifier: &types.OperationIdentifier{ + Index: 0, + }, + Type: "Transfer", + Status: "Success", + Account: recipient, + Amount: recipientAmount, + } + + recipientFailureOperation = &types.Operation{ + OperationIdentifier: &types.OperationIdentifier{ + Index: 1, + }, + Type: "Transfer", + Status: "Failure", + Account: recipient, + Amount: recipientAmount, + } + + recipientTransaction = &types.Transaction{ + TransactionIdentifier: &types.TransactionIdentifier{ + Hash: "tx1", + }, + Operations: []*types.Operation{ + recipientOperation, + recipientFailureOperation, + }, + } + ) + + var tests = map[string]struct { + block *types.Block + orphan bool + changes []*reconciler.BalanceChange + exemptAccounts []*reconciler.AccountCurrency + err error + }{ + "simple block": { + block: &types.Block{ + BlockIdentifier: &types.BlockIdentifier{ + Hash: "1", + Index: 1, + }, + ParentBlockIdentifier: &types.BlockIdentifier{ + Hash: "0", + Index: 0, + }, + Transactions: []*types.Transaction{ + recipientTransaction, + }, + Timestamp: asserter.MinUnixEpoch + 1, + }, + orphan: false, + changes: []*reconciler.BalanceChange{ + { + Account: recipient, + Currency: currency, + Block: &types.BlockIdentifier{ + Hash: "1", + Index: 1, + }, + Difference: "100", + }, + }, + err: nil, + }, + "simple block account exempt": { + block: &types.Block{ + BlockIdentifier: &types.BlockIdentifier{ + Hash: "1", + Index: 1, + }, + ParentBlockIdentifier: &types.BlockIdentifier{ + Hash: "0", + Index: 0, + }, + Transactions: []*types.Transaction{ + recipientTransaction, + }, + Timestamp: asserter.MinUnixEpoch + 1, + }, + orphan: false, + changes: []*reconciler.BalanceChange{}, + exemptAccounts: []*reconciler.AccountCurrency{ + { + Account: recipient, + Currency: currency, + }, + }, + err: nil, + }, + "single account sum block": { + block: &types.Block{ + BlockIdentifier: &types.BlockIdentifier{ + Hash: "1", + Index: 1, + }, + ParentBlockIdentifier: &types.BlockIdentifier{ + Hash: "0", + Index: 0, + }, + Transactions: []*types.Transaction{ + simpleTransactionFactory("tx1", "addr1", "100", currency), + simpleTransactionFactory("tx2", "addr1", "150", currency), + simpleTransactionFactory("tx3", "addr2", "150", currency), + }, + Timestamp: asserter.MinUnixEpoch + 1, + }, + orphan: false, + changes: []*reconciler.BalanceChange{ + { + Account: &types.AccountIdentifier{ + Address: "addr1", + }, + Currency: currency, + Block: &types.BlockIdentifier{ + Hash: "1", + Index: 1, + }, + Difference: "250", + }, + { + Account: &types.AccountIdentifier{ + Address: "addr2", + }, + Currency: currency, + Block: &types.BlockIdentifier{ + Hash: "1", + Index: 1, + }, + Difference: "150", + }, + }, + err: nil, + }, + "single account sum orphan block": { + block: &types.Block{ + BlockIdentifier: &types.BlockIdentifier{ + Hash: "1", + Index: 1, + }, + ParentBlockIdentifier: &types.BlockIdentifier{ + Hash: "0", + Index: 0, + }, + Transactions: []*types.Transaction{ + simpleTransactionFactory("tx1", "addr1", "100", currency), + simpleTransactionFactory("tx2", "addr1", "150", currency), + simpleTransactionFactory("tx3", "addr2", "150", currency), + }, + Timestamp: asserter.MinUnixEpoch + 1, + }, + orphan: true, + changes: []*reconciler.BalanceChange{ + { + Account: &types.AccountIdentifier{ + Address: "addr1", + }, + Currency: currency, + Block: &types.BlockIdentifier{ + Hash: "0", + Index: 0, + }, + Difference: "-250", + }, + { + Account: &types.AccountIdentifier{ + Address: "addr2", + }, + Currency: currency, + Block: &types.BlockIdentifier{ + Hash: "0", + Index: 0, + }, + Difference: "-150", + }, + }, + err: nil, + }, + } + + ctx := context.Background() + + newDir, err := utils.CreateTempDir() + assert.NoError(t, err) + defer utils.RemoveTempDir(newDir) + + database, err := NewBadgerStorage(ctx, newDir) + assert.NoError(t, err) + defer database.Close(ctx) + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + storage := NewBlockStorage(ctx, database, &MockBlockStorageHelper{ + ExemptAccounts: test.exemptAccounts, + }) + + changes, err := storage.BalanceChanges( + ctx, + test.block, + test.orphan, + ) + + assert.ElementsMatch(t, test.changes, changes) + assert.Equal(t, test.err, err) + }) + } +} + type MockBlockStorageHelper struct { AccountBalanceAmount string + ExemptAccounts []*reconciler.AccountCurrency } func (h *MockBlockStorageHelper) AccountBalance( @@ -842,5 +1099,16 @@ func (h *MockBlockStorageHelper) SkipOperation( return true, nil } + if op.Status == "Failure" { + return true, nil + } + + if reconciler.ContainsAccountCurrency(h.ExemptAccounts, &reconciler.AccountCurrency{ + Account: op.Account, + Currency: op.Amount.Currency, + }) { + return true, nil + } + return false, nil } From 38edbbb2578a54aa5a5e50dce948b1dad649ef0c Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Mon, 4 May 2020 22:50:11 -0700 Subject: [PATCH 13/31] Add syncer tests --- cmd/check.go | 6 + internal/storage/block_storage.go | 29 +++ internal/syncer/syncer.go | 152 ++++++++------- internal/syncer/syncer_test.go | 307 ++++++++++++++++++++++++++++++ 4 files changed, 422 insertions(+), 72 deletions(-) create mode 100644 internal/syncer/syncer_test.go diff --git a/cmd/check.go b/cmd/check.go index 781287b7..e37812be 100644 --- a/cmd/check.go +++ b/cmd/check.go @@ -329,6 +329,12 @@ func runCheckCmd(cmd *cobra.Command, args []string) { } } + if StartIndex != -1 { + if err = blockStorage.SetNewStartIndex(ctx, StartIndex); err != nil { + log.Fatal(fmt.Errorf("%w: unable to set new start index", err)) + } + } + reconcilerHelper := processor.NewReconcilerHelper( blockStorage, ) diff --git a/internal/storage/block_storage.go b/internal/storage/block_storage.go index 6ce9855e..ec25057d 100644 --- a/internal/storage/block_storage.go +++ b/internal/storage/block_storage.go @@ -449,6 +449,35 @@ func parseBalanceEntry(buf []byte) (*balanceEntry, error) { return &bal, nil } +func (b *BlockStorage) SetNewStartIndex( + ctx context.Context, + startIndex int64, +) error { + head, err := b.GetHeadBlockIdentifier(ctx) + if errors.Is(err, ErrHeadBlockNotFound) { + return nil + } + if err != nil { + return err + } + + currBlock := head + for currBlock.Index > startIndex { + block, err := b.GetBlock(ctx, head) + if err != nil { + return err + } + + if _, err = b.RemoveBlock(ctx, block); err != nil { + return err + } + + currBlock = block.ParentBlockIdentifier + } + + return nil +} + func (b *BlockStorage) SetBalance( ctx context.Context, dbTransaction DatabaseTransaction, diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index a4b682f7..b630ed69 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -28,7 +28,9 @@ import ( const ( // maxSync is the maximum number of blocks // to try and sync in a given SyncCycle. - maxSync = 1000 + maxSync = 999 + + reorgCache = 20 ) // SyncHandler is called at various times during the sync cycle @@ -54,7 +56,8 @@ type Syncer struct { // Used to keep track of sync state genesisBlock *types.BlockIdentifier - currentBlock *types.BlockIdentifier + nextIndex int64 + blockCache []*types.Block } func New( @@ -64,10 +67,11 @@ func New( cancel context.CancelFunc, ) *Syncer { return &Syncer{ - network: network, - fetcher: fetcher, - handler: handler, - cancel: cancel, + network: network, + fetcher: fetcher, + handler: handler, + cancel: cancel, + blockCache: make([]*types.Block, reorgCache), } } @@ -87,40 +91,23 @@ func (s *Syncer) setStart( s.genesisBlock = networkStatus.GenesisBlockIdentifier if index != -1 { - // Get block at index - block, err := s.fetcher.BlockRetry(ctx, s.network, &types.PartialBlockIdentifier{Index: &index}) - if err != nil { - return err - } - - s.currentBlock = block.BlockIdentifier + s.nextIndex = index return nil } - s.currentBlock = networkStatus.GenesisBlockIdentifier + s.nextIndex = networkStatus.GenesisBlockIdentifier.Index return nil } -func (s *Syncer) head( - ctx context.Context, -) (*types.BlockIdentifier, error) { - if s.currentBlock == nil { - return nil, errors.New("start block not set") - } - - return s.currentBlock, nil -} - // nextSyncableRange returns the next range of indexes to sync // based on what the last processed block in storage is and // the contents of the network status response. func (s *Syncer) nextSyncableRange( ctx context.Context, endIndex int64, -) (int64, int64, bool, error) { - head, err := s.head(ctx) - if err != nil { - return -1, -1, false, fmt.Errorf("%w: unable to get current head", err) +) (int64, bool, error) { + if s.nextIndex == -1 { + return -1, false, errors.New("unable to get current head") } if endIndex == -1 { @@ -130,65 +117,102 @@ func (s *Syncer) nextSyncableRange( nil, ) if err != nil { - return -1, -1, false, fmt.Errorf("%w: unable to get network status", err) + return -1, false, fmt.Errorf("%w: unable to get network status", err) } - return head.Index, networkStatus.CurrentBlockIdentifier.Index, false, nil + endIndex = networkStatus.CurrentBlockIdentifier.Index + } + + if s.nextIndex >= endIndex { + return -1, true, nil } - if head.Index >= endIndex { - return -1, -1, true, nil + if endIndex-s.nextIndex > maxSync { + endIndex = s.nextIndex + maxSync } - return head.Index, endIndex, false, nil + return endIndex, false, nil } func (s *Syncer) removeBlock( ctx context.Context, block *types.Block, -) (bool, error) { - // Get current block - head, err := s.head(ctx) - if err != nil { - return false, fmt.Errorf("%w: unable to get current head", err) +) (bool, *types.Block, error) { + if len(s.blockCache) == 0 { + return false, nil, nil } // Ensure processing correct index - if block.ParentBlockIdentifier.Index != head.Index { - return false, fmt.Errorf( + if block.BlockIdentifier.Index != s.nextIndex { + return false, nil, fmt.Errorf( "Got block %d instead of %d", block.BlockIdentifier.Index, - head.Index+1, + s.nextIndex, ) } // Check if block parent is head - if !reflect.DeepEqual(block.ParentBlockIdentifier, head) { - return true, nil + lastBlock := s.blockCache[len(s.blockCache)-1] + if !reflect.DeepEqual(block.ParentBlockIdentifier, lastBlock.BlockIdentifier) { + if reflect.DeepEqual(s.genesisBlock, lastBlock.BlockIdentifier) { + return false, nil, fmt.Errorf("cannot remove genesis block") + } + + return true, lastBlock, nil } - return false, nil + return false, lastBlock, nil +} + +func (s *Syncer) processBlock( + ctx context.Context, + block *types.Block, +) error { + shouldRemove, lastBlock, err := s.removeBlock(ctx, block) + if err != nil { + return err + } + + if shouldRemove { + err = s.handler.BlockRemoved(ctx, lastBlock) + if err != nil { + return err + } + s.blockCache = s.blockCache[:len(s.blockCache)-1] + s.nextIndex = lastBlock.BlockIdentifier.Index + return nil + } + + err = s.handler.BlockAdded(ctx, block) + if err != nil { + return err + } + + s.blockCache = append(s.blockCache, block) + if len(s.blockCache) > reorgCache { + s.blockCache = s.blockCache[1:] + } + s.nextIndex = block.BlockIdentifier.Index + 1 + return nil } func (s *Syncer) syncRange( ctx context.Context, - startIndex int64, endIndex int64, ) error { - blockMap, err := s.fetcher.BlockRange(ctx, s.network, startIndex, endIndex) + blockMap, err := s.fetcher.BlockRange(ctx, s.network, s.nextIndex, endIndex) if err != nil { return err } - currIndex := startIndex - for currIndex <= endIndex { - block, ok := blockMap[currIndex] + for s.nextIndex <= endIndex { + block, ok := blockMap[s.nextIndex] if !ok { // could happen in a reorg block, err = s.fetcher.BlockRetry( ctx, s.network, &types.PartialBlockIdentifier{ - Index: &currIndex, + Index: &s.nextIndex, }, ) if err != nil { @@ -198,22 +222,10 @@ func (s *Syncer) syncRange( // Anytime we re-fetch an index, we // will need to make another call to the node // as it is likely in a reorg. - delete(blockMap, currIndex) + delete(blockMap, s.nextIndex) } - shouldRemove, err := s.removeBlock(ctx, block) - if err != nil { - return err - } - - if shouldRemove { - currIndex-- - err = s.handler.BlockRemoved(ctx, block) - } else { - currIndex++ - err = s.handler.BlockAdded(ctx, block) - } - if err != nil { + if err = s.processBlock(ctx, block); err != nil { return err } } @@ -235,7 +247,7 @@ func (s *Syncer) Sync( } for { - rangeStart, rangeEnd, halt, err := s.nextSyncableRange( + rangeEnd, halt, err := s.nextSyncableRange( ctx, endIndex, ) @@ -246,15 +258,11 @@ func (s *Syncer) Sync( break } - if rangeEnd-rangeStart > maxSync { - rangeEnd = rangeStart + maxSync - } - - log.Printf("Syncing %d-%d\n", rangeStart, rangeEnd) + log.Printf("Syncing %d-%d\n", s.nextIndex, rangeEnd) - err = s.syncRange(ctx, rangeStart, rangeEnd) + err = s.syncRange(ctx, rangeEnd) if err != nil { - return fmt.Errorf("%w: unable to sync range %d-%d", err, rangeStart, rangeEnd) + return fmt.Errorf("%w: unable to sync to %d", err, rangeEnd) } if ctx.Err() != nil { diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.go new file mode 100644 index 00000000..7db222d4 --- /dev/null +++ b/internal/syncer/syncer_test.go @@ -0,0 +1,307 @@ +package syncer + +import ( + "context" + "testing" + + "github.com/coinbase/rosetta-sdk-go/asserter" + "github.com/coinbase/rosetta-sdk-go/types" + + "github.com/stretchr/testify/assert" +) + +var ( + networkIdentifier = &types.NetworkIdentifier{ + Blockchain: "blah", + Network: "testnet", + } + + currency = &types.Currency{ + Symbol: "Blah", + Decimals: 2, + } + + recipient = &types.AccountIdentifier{ + Address: "acct1", + } + + recipientAmount = &types.Amount{ + Value: "100", + Currency: currency, + } + + recipientOperation = &types.Operation{ + OperationIdentifier: &types.OperationIdentifier{ + Index: 0, + }, + Type: "Transfer", + Status: "Success", + Account: recipient, + Amount: recipientAmount, + } + + recipientFailureOperation = &types.Operation{ + OperationIdentifier: &types.OperationIdentifier{ + Index: 1, + }, + Type: "Transfer", + Status: "Failure", + Account: recipient, + Amount: recipientAmount, + } + + recipientTransaction = &types.Transaction{ + TransactionIdentifier: &types.TransactionIdentifier{ + Hash: "tx1", + }, + Operations: []*types.Operation{ + recipientOperation, + recipientFailureOperation, + }, + } + + sender = &types.AccountIdentifier{ + Address: "acct2", + } + + senderAmount = &types.Amount{ + Value: "-100", + Currency: currency, + } + + senderOperation = &types.Operation{ + OperationIdentifier: &types.OperationIdentifier{ + Index: 0, + }, + Type: "Transfer", + Status: "Success", + Account: sender, + Amount: senderAmount, + } + + senderTransaction = &types.Transaction{ + TransactionIdentifier: &types.TransactionIdentifier{ + Hash: "tx2", + }, + Operations: []*types.Operation{ + senderOperation, + }, + } + + orphanGenesis = &types.Block{ + BlockIdentifier: &types.BlockIdentifier{ + Hash: "1", + Index: 1, + }, + ParentBlockIdentifier: &types.BlockIdentifier{ + Hash: "0a", + Index: 0, + }, + Transactions: []*types.Transaction{}, + } + + blockSequence = []*types.Block{ + { // genesis + BlockIdentifier: &types.BlockIdentifier{ + Hash: "0", + Index: 0, + }, + ParentBlockIdentifier: &types.BlockIdentifier{ + Hash: "0", + Index: 0, + }, + }, + { + BlockIdentifier: &types.BlockIdentifier{ + Hash: "1", + Index: 1, + }, + ParentBlockIdentifier: &types.BlockIdentifier{ + Hash: "0", + Index: 0, + }, + Transactions: []*types.Transaction{ + recipientTransaction, + }, + }, + { // reorg + BlockIdentifier: &types.BlockIdentifier{ + Hash: "2", + Index: 2, + }, + ParentBlockIdentifier: &types.BlockIdentifier{ + Hash: "1a", + Index: 1, + }, + }, + { + BlockIdentifier: &types.BlockIdentifier{ + Hash: "1a", + Index: 1, + }, + ParentBlockIdentifier: &types.BlockIdentifier{ + Hash: "0", + Index: 0, + }, + }, + { + BlockIdentifier: &types.BlockIdentifier{ + Hash: "3", + Index: 3, + }, + ParentBlockIdentifier: &types.BlockIdentifier{ + Hash: "2", + Index: 2, + }, + Transactions: []*types.Transaction{ + senderTransaction, + }, + }, + { // invalid block + BlockIdentifier: &types.BlockIdentifier{ + Hash: "5", + Index: 5, + }, + ParentBlockIdentifier: &types.BlockIdentifier{ + Hash: "4", + Index: 4, + }, + }, + } + + operationStatuses = []*types.OperationStatus{ + { + Status: "Success", + Successful: true, + }, + { + Status: "Failure", + Successful: false, + }, + } + + networkStatusResponse = &types.NetworkStatusResponse{ + GenesisBlockIdentifier: &types.BlockIdentifier{ + Index: 0, + Hash: "block 0", + }, + CurrentBlockIdentifier: &types.BlockIdentifier{ + Index: 10000, + Hash: "block 1000", + }, + CurrentBlockTimestamp: asserter.MinUnixEpoch + 1, + Peers: []*types.Peer{ + { + PeerID: "peer 1", + }, + }, + } + + networkOptionsResponse = &types.NetworkOptionsResponse{ + Version: &types.Version{ + RosettaVersion: "1.3.1", + NodeVersion: "1.0", + }, + Allow: &types.Allow{ + OperationStatuses: operationStatuses, + OperationTypes: []string{ + "Transfer", + }, + }, + } +) + +func lastBlockIdentifier(syncer *Syncer) *types.BlockIdentifier { + return syncer.blockCache[len(syncer.blockCache)-1].BlockIdentifier +} + +func TestProcessBlock(t *testing.T) { + ctx := context.Background() + + syncer := New(nil, nil, &MockSyncHandler{}, nil) + syncer.genesisBlock = blockSequence[0].BlockIdentifier + syncer.blockCache = []*types.Block{} + + t.Run("No block exists", func(t *testing.T) { + err := syncer.processBlock( + ctx, + blockSequence[0], + ) + assert.NoError(t, err) + assert.Equal(t, int64(1), syncer.nextIndex) + assert.Equal(t, blockSequence[0].BlockIdentifier, lastBlockIdentifier(syncer)) + }) + + t.Run("Orphan genesis", func(t *testing.T) { + err := syncer.processBlock( + ctx, + orphanGenesis, + ) + + assert.EqualError(t, err, "cannot remove genesis block") + assert.Equal(t, int64(1), syncer.nextIndex) + assert.Equal(t, blockSequence[0].BlockIdentifier, lastBlockIdentifier(syncer)) + }) + + t.Run("Block exists, no reorg", func(t *testing.T) { + err := syncer.processBlock( + ctx, + blockSequence[1], + ) + assert.NoError(t, err) + assert.Equal(t, int64(2), syncer.nextIndex) + assert.Equal(t, blockSequence[1].BlockIdentifier, lastBlockIdentifier(syncer)) + }) + + t.Run("Orphan block", func(t *testing.T) { + err := syncer.processBlock( + ctx, + blockSequence[2], + ) + assert.NoError(t, err) + assert.Equal(t, int64(1), syncer.nextIndex) + assert.Equal(t, blockSequence[0].BlockIdentifier, lastBlockIdentifier(syncer)) + + err = syncer.processBlock( + ctx, + blockSequence[3], + ) + assert.NoError(t, err) + assert.Equal(t, int64(2), syncer.nextIndex) + assert.Equal(t, blockSequence[3].BlockIdentifier, lastBlockIdentifier(syncer)) + + err = syncer.processBlock( + ctx, + blockSequence[2], + ) + assert.NoError(t, err) + assert.Equal(t, int64(3), syncer.nextIndex) + assert.Equal(t, blockSequence[2].BlockIdentifier, lastBlockIdentifier(syncer)) + }) + + t.Run("Out of order block", func(t *testing.T) { + err := syncer.processBlock( + ctx, + blockSequence[5], + ) + assert.EqualError(t, err, "Got block 5 instead of 3") + assert.Equal(t, int64(3), syncer.nextIndex) + assert.Equal(t, blockSequence[2].BlockIdentifier, lastBlockIdentifier(syncer)) + }) +} + +type MockSyncHandler struct{} + +func (h *MockSyncHandler) BlockAdded( + ctx context.Context, + block *types.Block, +) error { + return nil +} + +func (h *MockSyncHandler) BlockRemoved( + ctx context.Context, + block *types.Block, +) error { + return nil +} From 6cf8a0a740f89475870727c830274bf5c2267585 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 5 May 2020 08:23:58 -0700 Subject: [PATCH 14/31] nits --- cmd/check.go | 7 ++++- internal/processor/storage_helper.go | 3 +- internal/storage/block_storage.go | 7 ----- internal/syncer/syncer.go | 14 +++++---- internal/syncer/syncer_test.go | 44 +--------------------------- 5 files changed, 17 insertions(+), 58 deletions(-) diff --git a/cmd/check.go b/cmd/check.go index e37812be..0186c1d5 100644 --- a/cmd/check.go +++ b/cmd/check.go @@ -329,10 +329,15 @@ func runCheckCmd(cmd *cobra.Command, args []string) { } } - if StartIndex != -1 { + if StartIndex != -1 { // attempt to remove blocks from storage (without handling) if err = blockStorage.SetNewStartIndex(ctx, StartIndex); err != nil { log.Fatal(fmt.Errorf("%w: unable to set new start index", err)) } + } else { // attempt to load last processed index + head, err := blockStorage.GetHeadBlockIdentifier(ctx) + if err == nil { + StartIndex = head.Index + 1 + } } reconcilerHelper := processor.NewReconcilerHelper( diff --git a/internal/processor/storage_helper.go b/internal/processor/storage_helper.go index 7db715e7..7e170edb 100644 --- a/internal/processor/storage_helper.go +++ b/internal/processor/storage_helper.go @@ -91,7 +91,7 @@ func (h *BlockStorageHelper) SkipOperation( // Exempting account in BalanceChanges ensures that storage is not updated // and that the account is not reconciled. - if h.accountExempt(ctx, op.Account, op.Amount.Currency) { + if h.accountExempt(op.Account, op.Amount.Currency) { log.Printf("Skipping exempt account %+v\n", op.Account) return true, nil } @@ -103,7 +103,6 @@ func (h *BlockStorageHelper) SkipOperation( // account and currency are exempt from balance tracking and // reconciliation. func (h *BlockStorageHelper) accountExempt( - ctx context.Context, account *types.AccountIdentifier, currency *types.Currency, ) bool { diff --git a/internal/storage/block_storage.go b/internal/storage/block_storage.go index ec25057d..4a59423a 100644 --- a/internal/storage/block_storage.go +++ b/internal/storage/block_storage.go @@ -100,13 +100,6 @@ func hashBytes(data string) []byte { return h.Sum(nil) } -// hashString is used to construct a SHA1 -// hash to protect against arbitrarily -// large key sizes. -func hashString(data string) string { - return fmt.Sprintf("%x", hashBytes(data)) -} - func getHeadBlockKey() []byte { return hashBytes(headBlockKey) } diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index b630ed69..97c97120 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -57,7 +57,12 @@ type Syncer struct { // Used to keep track of sync state genesisBlock *types.BlockIdentifier nextIndex int64 - blockCache []*types.Block + + // TODO: to ensure reorgs are handled correctly, it should be possible + // to pass in recently processed blocks to the syncer. Without this, the + // syncer may process an index that is not connected to previously added + // blocks (ParentBlockIdentifier != lastProcessedBlock.BlockIdentifier). + blockCache []*types.Block } func New( @@ -71,7 +76,7 @@ func New( fetcher: fetcher, handler: handler, cancel: cancel, - blockCache: make([]*types.Block, reorgCache), + blockCache: []*types.Block{}, } } @@ -134,8 +139,7 @@ func (s *Syncer) nextSyncableRange( return endIndex, false, nil } -func (s *Syncer) removeBlock( - ctx context.Context, +func (s *Syncer) checkRemove( block *types.Block, ) (bool, *types.Block, error) { if len(s.blockCache) == 0 { @@ -168,7 +172,7 @@ func (s *Syncer) processBlock( ctx context.Context, block *types.Block, ) error { - shouldRemove, lastBlock, err := s.removeBlock(ctx, block) + shouldRemove, lastBlock, err := s.checkRemove(block) if err != nil { return err } diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.go index 7db222d4..776e43b1 100644 --- a/internal/syncer/syncer_test.go +++ b/internal/syncer/syncer_test.go @@ -4,7 +4,6 @@ import ( "context" "testing" - "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/types" "github.com/stretchr/testify/assert" @@ -168,47 +167,6 @@ var ( }, }, } - - operationStatuses = []*types.OperationStatus{ - { - Status: "Success", - Successful: true, - }, - { - Status: "Failure", - Successful: false, - }, - } - - networkStatusResponse = &types.NetworkStatusResponse{ - GenesisBlockIdentifier: &types.BlockIdentifier{ - Index: 0, - Hash: "block 0", - }, - CurrentBlockIdentifier: &types.BlockIdentifier{ - Index: 10000, - Hash: "block 1000", - }, - CurrentBlockTimestamp: asserter.MinUnixEpoch + 1, - Peers: []*types.Peer{ - { - PeerID: "peer 1", - }, - }, - } - - networkOptionsResponse = &types.NetworkOptionsResponse{ - Version: &types.Version{ - RosettaVersion: "1.3.1", - NodeVersion: "1.0", - }, - Allow: &types.Allow{ - OperationStatuses: operationStatuses, - OperationTypes: []string{ - "Transfer", - }, - }, - } ) func lastBlockIdentifier(syncer *Syncer) *types.BlockIdentifier { @@ -218,7 +176,7 @@ func lastBlockIdentifier(syncer *Syncer) *types.BlockIdentifier { func TestProcessBlock(t *testing.T) { ctx := context.Background() - syncer := New(nil, nil, &MockSyncHandler{}, nil) + syncer := New(networkIdentifier, nil, &MockSyncHandler{}, nil) syncer.genesisBlock = blockSequence[0].BlockIdentifier syncer.blockCache = []*types.Block{} From 4fcbad6fa1167d344550a96aa309f489e1300d96 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 5 May 2020 08:34:37 -0700 Subject: [PATCH 15/31] Set head block on store/remove --- internal/storage/block_storage.go | 8 ++++++++ internal/storage/block_storage_test.go | 22 +++++++++++++++++++++- 2 files changed, 29 insertions(+), 1 deletion(-) diff --git a/internal/storage/block_storage.go b/internal/storage/block_storage.go index 4a59423a..04bcfec7 100644 --- a/internal/storage/block_storage.go +++ b/internal/storage/block_storage.go @@ -361,6 +361,10 @@ func (b *BlockStorage) StoreBlock( } } + if err = b.StoreHeadBlockIdentifier(ctx, transaction, block.BlockIdentifier); err != nil { + return nil, err + } + if err := transaction.Commit(ctx); err != nil { return nil, err } @@ -409,6 +413,10 @@ func (b *BlockStorage) RemoveBlock( return nil, err } + if err = b.StoreHeadBlockIdentifier(ctx, transaction, block.ParentBlockIdentifier); err != nil { + return nil, err + } + if err := transaction.Commit(ctx); err != nil { return nil, err } diff --git a/internal/storage/block_storage_test.go b/internal/storage/block_storage_test.go index 94d24f33..76e400e2 100644 --- a/internal/storage/block_storage_test.go +++ b/internal/storage/block_storage_test.go @@ -103,7 +103,11 @@ func TestBlock(t *testing.T) { var ( newBlock = &types.Block{ BlockIdentifier: &types.BlockIdentifier{ - Hash: "blah", + Hash: "blah 1", + Index: 1, + }, + ParentBlockIdentifier: &types.BlockIdentifier{ + Hash: "blah 0", Index: 0, }, Timestamp: 1, @@ -131,6 +135,10 @@ func TestBlock(t *testing.T) { newBlock2 = &types.Block{ BlockIdentifier: &types.BlockIdentifier{ Hash: "blah 2", + Index: 2, + }, + ParentBlockIdentifier: &types.BlockIdentifier{ + Hash: "blah 1", Index: 1, }, Timestamp: 1, @@ -169,6 +177,10 @@ func TestBlock(t *testing.T) { block, err := storage.GetBlock(ctx, newBlock.BlockIdentifier) assert.NoError(t, err) assert.Equal(t, newBlock, block) + + head, err := storage.GetHeadBlockIdentifier(ctx) + assert.NoError(t, err) + assert.Equal(t, newBlock.BlockIdentifier, head) }) t.Run("Get non-existent block", func(t *testing.T) { @@ -197,12 +209,20 @@ func TestBlock(t *testing.T) { ErrDuplicateTransactionHash, "blahTx", ).Error()) + + head, err := storage.GetHeadBlockIdentifier(ctx) + assert.NoError(t, err) + assert.Equal(t, newBlock.BlockIdentifier, head) }) t.Run("Remove block and re-set block of same hash", func(t *testing.T) { _, err := storage.RemoveBlock(ctx, newBlock) assert.NoError(t, err) + head, err := storage.GetHeadBlockIdentifier(ctx) + assert.NoError(t, err) + assert.Equal(t, newBlock.ParentBlockIdentifier, head) + _, err = storage.StoreBlock(ctx, newBlock) assert.NoError(t, err) }) From 0425ad7707b5b528330654027722ab845f9f554d Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 5 May 2020 09:38:34 -0700 Subject: [PATCH 16/31] Handle GetAccount when syncing from arbitrary height --- internal/processor/reconciler_helper.go | 3 ++- internal/processor/storage_helper.go | 3 ++- internal/reconciler/reconciler.go | 2 ++ internal/reconciler/reconciler_test.go | 1 + internal/storage/block_storage.go | 30 ++++++++++++++++++++++++- internal/storage/block_storage_test.go | 28 +++++++++++++---------- 6 files changed, 52 insertions(+), 15 deletions(-) diff --git a/internal/processor/reconciler_helper.go b/internal/processor/reconciler_helper.go index bd1d2bf8..09d84575 100644 --- a/internal/processor/reconciler_helper.go +++ b/internal/processor/reconciler_helper.go @@ -47,6 +47,7 @@ func (h *ReconcilerHelper) AccountBalance( ctx context.Context, account *types.AccountIdentifier, currency *types.Currency, + headBlock *types.BlockIdentifier, ) (*types.Amount, *types.BlockIdentifier, error) { - return h.storage.GetBalance(ctx, account, currency) + return h.storage.GetBalance(ctx, account, currency, headBlock) } diff --git a/internal/processor/storage_helper.go b/internal/processor/storage_helper.go index 7e170edb..8abbfa58 100644 --- a/internal/processor/storage_helper.go +++ b/internal/processor/storage_helper.go @@ -2,6 +2,7 @@ package processor import ( "context" + "fmt" "log" "github.com/coinbase/rosetta-cli/internal/reconciler" @@ -58,7 +59,7 @@ func (h *BlockStorageHelper) AccountBalance( types.ConstructPartialBlockIdentifier(block), ) if err != nil { - return nil, err + return nil, fmt.Errorf("%w: unable to get currency balance in storage helper", err) } return &types.Amount{ diff --git a/internal/reconciler/reconciler.go b/internal/reconciler/reconciler.go index 062537f6..d3187570 100644 --- a/internal/reconciler/reconciler.go +++ b/internal/reconciler/reconciler.go @@ -117,6 +117,7 @@ type ReconcilerHelper interface { ctx context.Context, account *types.AccountIdentifier, currency *types.Currency, + headBlock *types.BlockIdentifier, ) (*types.Amount, *types.BlockIdentifier, error) } @@ -313,6 +314,7 @@ func (r *Reconciler) CompareBalance( ctx, account, currency, + head, ) if err != nil { return zeroString, "", head.Index, fmt.Errorf("%w: unable to get cached balance for %+v:%+v", err, account, currency) diff --git a/internal/reconciler/reconciler_test.go b/internal/reconciler/reconciler_test.go index 2fdc575b..3532b9b6 100644 --- a/internal/reconciler/reconciler_test.go +++ b/internal/reconciler/reconciler_test.go @@ -429,6 +429,7 @@ func (h *MockReconcilerHelper) AccountBalance( ctx context.Context, account *types.AccountIdentifier, currency *types.Currency, + headBlock *types.BlockIdentifier, ) (*types.Amount, *types.BlockIdentifier, error) { if h.BalanceAccount == nil || !reflect.DeepEqual(account, h.BalanceAccount) { return nil, nil, errors.New("account does not exist") diff --git a/internal/storage/block_storage.go b/internal/storage/block_storage.go index 04bcfec7..afcd3be0 100644 --- a/internal/storage/block_storage.go +++ b/internal/storage/block_storage.go @@ -581,6 +581,7 @@ func (b *BlockStorage) GetBalance( ctx context.Context, account *types.AccountIdentifier, currency *types.Currency, + headBlock *types.BlockIdentifier, ) (*types.Amount, *types.BlockIdentifier, error) { transaction := b.newDatabaseTransaction(ctx, false) defer transaction.Discard(ctx) @@ -591,8 +592,35 @@ func (b *BlockStorage) GetBalance( return nil, nil, err } + // When beginning syncing from an arbitrary height, an account may + // not yet have a cached balance when requested. If this is the case, + // we fetch the balance from the node for the given height and persist + // it. This is particularly useful when monitoring interesting accounts. if !exists { - return nil, nil, fmt.Errorf("%w %+v", ErrAccountNotFound, account) + amount, err := b.helper.AccountBalance(ctx, account, currency, headBlock) + if err != nil { + return nil, nil, fmt.Errorf("%w: unable to get account balance from helper", err) + } + + writeTransaction := b.newDatabaseTransaction(ctx, true) + defer writeTransaction.Discard(ctx) + err = b.SetBalance( + ctx, + writeTransaction, + account, + amount, + headBlock, + ) + if err != nil { + return nil, nil, fmt.Errorf("%w: unable to set account balance", err) + } + + err = writeTransaction.Commit(ctx) + if err != nil { + return nil, nil, fmt.Errorf("%w: unable to commit account balance transaction", err) + } + + return amount, headBlock, nil } deserialBal, err := parseBalanceEntry(bal) diff --git a/internal/storage/block_storage_test.go b/internal/storage/block_storage_test.go index 76e400e2..cd38d684 100644 --- a/internal/storage/block_storage_test.go +++ b/internal/storage/block_storage_test.go @@ -404,10 +404,13 @@ func TestBalance(t *testing.T) { storage := NewBlockStorage(ctx, database, mockHelper) t.Run("Get unset balance", func(t *testing.T) { - amount, block, err := storage.GetBalance(ctx, account, currency) - assert.Nil(t, amount) - assert.Nil(t, block) - assert.EqualError(t, err, fmt.Errorf("%w %+v", ErrAccountNotFound, account).Error()) + amount, block, err := storage.GetBalance(ctx, account, currency, newBlock) + assert.NoError(t, err) + assert.Equal(t, &types.Amount{ + Value: "0", + Currency: currency, + }, amount) + assert.Equal(t, newBlock, block) }) t.Run("Set and get balance", func(t *testing.T) { @@ -426,7 +429,7 @@ func TestBalance(t *testing.T) { assert.NoError(t, err) assert.NoError(t, txn.Commit(ctx)) - retrievedAmount, block, err := storage.GetBalance(ctx, account, currency) + retrievedAmount, block, err := storage.GetBalance(ctx, account, currency, newBlock) assert.NoError(t, err) assert.Equal(t, amount, retrievedAmount) assert.Equal(t, newBlock, block) @@ -449,7 +452,7 @@ func TestBalance(t *testing.T) { assert.NoError(t, err) assert.NoError(t, txn.Commit(ctx)) - retrievedAmount, block, err := storage.GetBalance(ctx, account3, currency) + retrievedAmount, block, err := storage.GetBalance(ctx, account3, currency, newBlock) assert.NoError(t, err) assert.Equal(t, amountWithPrevious, retrievedAmount) assert.Equal(t, newBlock, block) @@ -473,7 +476,7 @@ func TestBalance(t *testing.T) { assert.EqualError(t, err, "invalid currency") txn.Discard(ctx) - retrievedAmount, block, err := storage.GetBalance(ctx, account, currency) + retrievedAmount, block, err := storage.GetBalance(ctx, account, currency, newBlock) assert.NoError(t, err) assert.Equal(t, amount, retrievedAmount) assert.Equal(t, newBlock, block) @@ -495,7 +498,7 @@ func TestBalance(t *testing.T) { assert.NoError(t, err) assert.NoError(t, txn.Commit(ctx)) - retrievedAmount, block, err := storage.GetBalance(ctx, account, currency) + retrievedAmount, block, err := storage.GetBalance(ctx, account, currency, newBlock2) assert.NoError(t, err) assert.Equal(t, result, retrievedAmount) assert.Equal(t, newBlock2, block) @@ -517,7 +520,7 @@ func TestBalance(t *testing.T) { assert.NoError(t, err) // Get balance during transaction - retrievedAmount, block, err := storage.GetBalance(ctx, account, currency) + retrievedAmount, block, err := storage.GetBalance(ctx, account, currency, newBlock2) assert.NoError(t, err) assert.Equal(t, result, retrievedAmount) assert.Equal(t, newBlock2, block) @@ -576,7 +579,7 @@ func TestBalance(t *testing.T) { assert.NoError(t, err) assert.NoError(t, txn.Commit(ctx)) - retrievedAmount, block, err := storage.GetBalance(ctx, subAccountNewPointer, amount.Currency) + retrievedAmount, block, err := storage.GetBalance(ctx, subAccountNewPointer, amount.Currency, newBlock) assert.NoError(t, err) assert.Equal(t, amount, retrievedAmount) assert.Equal(t, newBlock, block) @@ -598,7 +601,7 @@ func TestBalance(t *testing.T) { assert.NoError(t, err) assert.NoError(t, txn.Commit(ctx)) - retrievedAmount, block, err := storage.GetBalance(ctx, subAccountMetadataNewPointer, amount.Currency) + retrievedAmount, block, err := storage.GetBalance(ctx, subAccountMetadataNewPointer, amount.Currency, newBlock) assert.NoError(t, err) assert.Equal(t, amount, retrievedAmount) assert.Equal(t, newBlock, block) @@ -620,7 +623,7 @@ func TestBalance(t *testing.T) { assert.NoError(t, err) assert.NoError(t, txn.Commit(ctx)) - retrievedAmount, block, err := storage.GetBalance(ctx, subAccountMetadata2NewPointer, amount.Currency) + retrievedAmount, block, err := storage.GetBalance(ctx, subAccountMetadata2NewPointer, amount.Currency, newBlock) assert.NoError(t, err) assert.Equal(t, amount, retrievedAmount) assert.Equal(t, newBlock, block) @@ -748,6 +751,7 @@ func TestBootstrapBalances(t *testing.T) { ctx, account, amount.Currency, + genesisBlockIdentifier, ) assert.Equal(t, amount, retrievedAmount) From 75945cbeba9435a899a873685c61aa42611665ba Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 5 May 2020 10:03:50 -0700 Subject: [PATCH 17/31] Fix set new start index --- internal/reconciler/reconciler.go | 5 +---- internal/storage/block_storage.go | 5 +++-- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/internal/reconciler/reconciler.go b/internal/reconciler/reconciler.go index d3187570..c7162e5a 100644 --- a/internal/reconciler/reconciler.go +++ b/internal/reconciler/reconciler.go @@ -178,6 +178,7 @@ func NewReconciler( accountConcurrency uint64, lookupBalanceByBlock bool, interestingAccounts []*AccountCurrency, + // TODO: load seenAccts and inactiveQueue from prior run (if exists) ) *Reconciler { r := &Reconciler{ network: network, @@ -206,18 +207,14 @@ func NewReconciler( return r } -// Reconciliation // QueueChanges enqueues a slice of *BalanceChanges // for reconciliation. func (r *Reconciler) QueueChanges( ctx context.Context, - // If we pass in parentblock, then we always know what to compare on diff block *types.BlockIdentifier, balanceChanges []*BalanceChange, ) error { // Ensure all interestingAccounts are checked - // TODO: refactor to automatically trigger once an inactive reconciliation error - // is discovered for _, account := range r.interestingAccounts { skipAccount := false // Look through balance changes for account + currency diff --git a/internal/storage/block_storage.go b/internal/storage/block_storage.go index afcd3be0..aed502d0 100644 --- a/internal/storage/block_storage.go +++ b/internal/storage/block_storage.go @@ -463,8 +463,9 @@ func (b *BlockStorage) SetNewStartIndex( } currBlock := head - for currBlock.Index > startIndex { - block, err := b.GetBlock(ctx, head) + for currBlock.Index >= startIndex { + log.Printf("Removing block %+v\n", currBlock) + block, err := b.GetBlock(ctx, currBlock) if err != nil { return err } From d260ff2b139740d1eee22fc6e855c79f2accbe34 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 5 May 2020 10:38:29 -0700 Subject: [PATCH 18/31] Add util for account string --- Makefile | 7 +- internal/logger/logger.go | 32 +++---- internal/storage/block_storage.go | 50 +--------- internal/storage/block_storage_test.go | 118 ----------------------- internal/utils/utils.go | 48 ++++++++++ internal/utils/utils_test.go | 126 +++++++++++++++++++++++++ 6 files changed, 192 insertions(+), 189 deletions(-) create mode 100644 internal/utils/utils_test.go diff --git a/Makefile b/Makefile index f8e3b1f5..ddb8b8fe 100644 --- a/Makefile +++ b/Makefile @@ -3,14 +3,15 @@ watch-transactions watch-balances watch-reconciliations \ view-block-benchmarks view-account-benchmarks LICENCE_SCRIPT=addlicense -c "Coinbase, Inc." -l "apache" -v +GO_INSTALL=GO111MODULE=off go get TEST_SCRIPT=go test -v ./internal/... deps: go get ./... go get github.com/stretchr/testify - go get github.com/google/addlicense - go get github.com/segmentio/golines - go get github.com/mattn/goveralls + ${GO_INSTALL} github.com/google/addlicense + ${GO_INSTALL} github.com/segmentio/golines + ${GO_INSTALL} github.com/mattn/goveralls lint: golangci-lint run -v \ diff --git a/internal/logger/logger.go b/internal/logger/logger.go index b4294d22..ea9de34c 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -22,6 +22,7 @@ import ( "path" "github.com/coinbase/rosetta-cli/internal/reconciler" + "github.com/coinbase/rosetta-cli/internal/utils" "github.com/coinbase/rosetta-sdk-go/types" ) @@ -167,7 +168,7 @@ func (l *Logger) TransactionStream( } participant := "" if op.Account != nil { - participant = op.Account.Address + participant = utils.AccountString(op.Account) } networkIndex := op.OperationIdentifier.Index @@ -188,13 +189,6 @@ func (l *Logger) TransactionStream( if err != nil { return err } - - if op.Account != nil && op.Account.Metadata != nil { - _, err = f.WriteString(fmt.Sprintf("Account Metadata: %+v\n", op.Account.Metadata)) - if err != nil { - return err - } - } } } @@ -223,10 +217,10 @@ func (l *Logger) BalanceStream( for _, balanceChange := range balanceChanges { balanceLog := fmt.Sprintf( - "Account: %s Change: %s%s Block: %d:%s", + "Account: %s Change: %s:%s Block: %d:%s", balanceChange.Account.Address, balanceChange.Difference, - balanceChange.Currency.Symbol, + utils.CurrencyString(balanceChange.Currency), balanceChange.Block.Index, balanceChange.Block.Hash, ) @@ -263,17 +257,17 @@ func (l *Logger) ReconcileSuccessStream( defer f.Close() log.Printf( - "%s Reconciled %+v at %d\n", + "%s Reconciled %s at %d\n", reconciliationType, - account, + utils.AccountString(account), block.Index, ) _, err = f.WriteString(fmt.Sprintf( "Type:%s Account: %s Currency: %s Balance: %s Block: %d:%s\n", reconciliationType, - account.Address, - currency.Symbol, + utils.AccountString(account), + utils.CurrencyString(currency), balance, block.Index, block.Hash, @@ -298,9 +292,9 @@ func (l *Logger) ReconcileFailureStream( ) error { // Always print out reconciliation failures log.Printf( - "%s Reconciliation failed for %+v at %d computed: %s node: %s\n", + "%s Reconciliation failed for %s at %d computed: %s node: %s\n", reconciliationType, - account, + utils.AccountString(account), block.Index, computedBalance, nodeBalance, @@ -321,10 +315,10 @@ func (l *Logger) ReconcileFailureStream( defer f.Close() _, err = f.WriteString(fmt.Sprintf( - "Type:%s Account: %+v Currency: %+v Block: %s:%d computed: %s node: %s\n", + "Type:%s Account: %s Currency: %s Block: %s:%d computed: %s node: %s\n", reconciliationType, - account, - currency, + utils.AccountString(account), + utils.CurrencyString(currency), block.Hash, block.Index, computedBalance, diff --git a/internal/storage/block_storage.go b/internal/storage/block_storage.go index aed502d0..a0470578 100644 --- a/internal/storage/block_storage.go +++ b/internal/storage/block_storage.go @@ -118,57 +118,9 @@ func getHashKey(hash string, isBlock bool) []byte { return hashBytes(fmt.Sprintf("%s:%s", transactionHashNamespace, hash)) } -// GetCurrencyKey is used to identify a *types.Currency -// in an account's map of currencies. It is not feasible -// to create a map of [types.Currency]*types.Amount -// because types.Currency contains a metadata pointer -// that would prevent any equality. -func GetCurrencyKey(currency *types.Currency) string { - if currency.Metadata == nil { - return fmt.Sprintf("%s:%d", currency.Symbol, currency.Decimals) - } - - // TODO: Handle currency.Metadata - // that has pointer value. - return fmt.Sprintf( - "%s:%d:%v", - currency.Symbol, - currency.Decimals, - currency.Metadata, - ) -} - -// GetAccountKey returns a byte slice representing a *types.AccountIdentifier. -// This byte slice automatically handles the existence of *types.SubAccount -// detail. -func GetAccountKey(account *types.AccountIdentifier) string { - if account.SubAccount == nil { - return fmt.Sprintf("%s:%s", balanceNamespace, account.Address) - } - - if account.SubAccount.Metadata == nil { - return fmt.Sprintf( - "%s:%s:%s", - balanceNamespace, - account.Address, - account.SubAccount.Address, - ) - } - - // TODO: handle SubAccount.Metadata - // that contains pointer values. - return fmt.Sprintf( - "%s:%s:%s:%v", - balanceNamespace, - account.Address, - account.SubAccount.Address, - account.SubAccount.Metadata, - ) -} - func GetBalanceKey(account *types.AccountIdentifier, currency *types.Currency) []byte { return hashBytes( - fmt.Sprintf("%s/%s", GetAccountKey(account), GetCurrencyKey(currency)), + fmt.Sprintf("%s/%s/%s", balanceNamespace, utils.AccountString(account), utils.CurrencyString(currency)), ) } diff --git a/internal/storage/block_storage_test.go b/internal/storage/block_storage_test.go index cd38d684..3fc0343a 100644 --- a/internal/storage/block_storage_test.go +++ b/internal/storage/block_storage_test.go @@ -228,72 +228,6 @@ func TestBlock(t *testing.T) { }) } -func TestGetAccountKey(t *testing.T) { - var tests = map[string]struct { - account *types.AccountIdentifier - key string - }{ - "simple account": { - account: &types.AccountIdentifier{ - Address: "hello", - }, - key: "balance:hello", - }, - "subaccount": { - account: &types.AccountIdentifier{ - Address: "hello", - SubAccount: &types.SubAccountIdentifier{ - Address: "stake", - }, - }, - key: "balance:hello:stake", - }, - "subaccount with string metadata": { - account: &types.AccountIdentifier{ - Address: "hello", - SubAccount: &types.SubAccountIdentifier{ - Address: "stake", - Metadata: map[string]interface{}{ - "cool": "neat", - }, - }, - }, - key: "balance:hello:stake:map[cool:neat]", - }, - "subaccount with number metadata": { - account: &types.AccountIdentifier{ - Address: "hello", - SubAccount: &types.SubAccountIdentifier{ - Address: "stake", - Metadata: map[string]interface{}{ - "cool": 1, - }, - }, - }, - key: "balance:hello:stake:map[cool:1]", - }, - "subaccount with complex metadata": { - account: &types.AccountIdentifier{ - Address: "hello", - SubAccount: &types.SubAccountIdentifier{ - Address: "stake", - Metadata: map[string]interface{}{ - "cool": 1, - "awesome": "neat", - }, - }, - }, - key: "balance:hello:stake:map[awesome:neat cool:1]", - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - assert.Equal(t, test.key, GetAccountKey(test.account)) - }) - } -} - func TestBalance(t *testing.T) { var ( account = &types.AccountIdentifier{ @@ -630,58 +564,6 @@ func TestBalance(t *testing.T) { }) } -func TestGetCurrencyKey(t *testing.T) { - var tests = map[string]struct { - currency *types.Currency - key string - }{ - "simple currency": { - currency: &types.Currency{ - Symbol: "BTC", - Decimals: 8, - }, - key: "BTC:8", - }, - "currency with string metadata": { - currency: &types.Currency{ - Symbol: "BTC", - Decimals: 8, - Metadata: map[string]interface{}{ - "issuer": "satoshi", - }, - }, - key: "BTC:8:map[issuer:satoshi]", - }, - "currency with number metadata": { - currency: &types.Currency{ - Symbol: "BTC", - Decimals: 8, - Metadata: map[string]interface{}{ - "issuer": 1, - }, - }, - key: "BTC:8:map[issuer:1]", - }, - "currency with complex metadata": { - currency: &types.Currency{ - Symbol: "BTC", - Decimals: 8, - Metadata: map[string]interface{}{ - "issuer": "satoshi", - "count": 10, - }, - }, - key: "BTC:8:map[count:10 issuer:satoshi]", - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - assert.Equal(t, test.key, GetCurrencyKey(test.currency)) - }) - } -} - func TestBootstrapBalances(t *testing.T) { var ( fileMode = os.FileMode(0600) diff --git a/internal/utils/utils.go b/internal/utils/utils.go index 7c68c3c4..c6a5d449 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -20,6 +20,8 @@ import ( "log" "math/big" "os" + + "github.com/coinbase/rosetta-sdk-go/types" ) // CreateTempDir creates a directory in @@ -80,3 +82,49 @@ func SubtractStringValues( newVal := new(big.Int).Sub(aVal, bVal) return newVal.String(), nil } + +// AccountString returns a byte slice representing a *types.AccountIdentifier. +// This byte slice automatically handles the existence of *types.SubAccount +// detail. +func AccountString(account *types.AccountIdentifier) string { + if account.SubAccount == nil { + return account.Address + } + + if account.SubAccount.Metadata == nil { + return fmt.Sprintf( + "%s:%s", + account.Address, + account.SubAccount.Address, + ) + } + + // TODO: handle SubAccount.Metadata + // that contains pointer values. + return fmt.Sprintf( + "%s:%s:%v", + account.Address, + account.SubAccount.Address, + account.SubAccount.Metadata, + ) +} + +// CurrencyString is used to identify a *types.Currency +// in an account's map of currencies. It is not feasible +// to create a map of [types.Currency]*types.Amount +// because types.Currency contains a metadata pointer +// that would prevent any equality. +func CurrencyString(currency *types.Currency) string { + if currency.Metadata == nil { + return fmt.Sprintf("%s:%d", currency.Symbol, currency.Decimals) + } + + // TODO: Handle currency.Metadata + // that has pointer value. + return fmt.Sprintf( + "%s:%d:%v", + currency.Symbol, + currency.Decimals, + currency.Metadata, + ) +} diff --git a/internal/utils/utils_test.go b/internal/utils/utils_test.go new file mode 100644 index 00000000..7ff2ca25 --- /dev/null +++ b/internal/utils/utils_test.go @@ -0,0 +1,126 @@ +package utils + +import ( + "testing" + + "github.com/coinbase/rosetta-sdk-go/types" + "github.com/stretchr/testify/assert" +) + +func TestGetAccountString(t *testing.T) { + var tests = map[string]struct { + account *types.AccountIdentifier + key string + }{ + "simple account": { + account: &types.AccountIdentifier{ + Address: "hello", + }, + key: "hello", + }, + "subaccount": { + account: &types.AccountIdentifier{ + Address: "hello", + SubAccount: &types.SubAccountIdentifier{ + Address: "stake", + }, + }, + key: "hello:stake", + }, + "subaccount with string metadata": { + account: &types.AccountIdentifier{ + Address: "hello", + SubAccount: &types.SubAccountIdentifier{ + Address: "stake", + Metadata: map[string]interface{}{ + "cool": "neat", + }, + }, + }, + key: "hello:stake:map[cool:neat]", + }, + "subaccount with number metadata": { + account: &types.AccountIdentifier{ + Address: "hello", + SubAccount: &types.SubAccountIdentifier{ + Address: "stake", + Metadata: map[string]interface{}{ + "cool": 1, + }, + }, + }, + key: "hello:stake:map[cool:1]", + }, + "subaccount with complex metadata": { + account: &types.AccountIdentifier{ + Address: "hello", + SubAccount: &types.SubAccountIdentifier{ + Address: "stake", + Metadata: map[string]interface{}{ + "cool": 1, + "awesome": "neat", + }, + }, + }, + key: "hello:stake:map[awesome:neat cool:1]", + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + assert.Equal(t, test.key, AccountString(test.account)) + }) + } +} + +func TestCurrencyString(t *testing.T) { + var tests = map[string]struct { + currency *types.Currency + key string + }{ + "simple currency": { + currency: &types.Currency{ + Symbol: "BTC", + Decimals: 8, + }, + key: "BTC:8", + }, + "currency with string metadata": { + currency: &types.Currency{ + Symbol: "BTC", + Decimals: 8, + Metadata: map[string]interface{}{ + "issuer": "satoshi", + }, + }, + key: "BTC:8:map[issuer:satoshi]", + }, + "currency with number metadata": { + currency: &types.Currency{ + Symbol: "BTC", + Decimals: 8, + Metadata: map[string]interface{}{ + "issuer": 1, + }, + }, + key: "BTC:8:map[issuer:1]", + }, + "currency with complex metadata": { + currency: &types.Currency{ + Symbol: "BTC", + Decimals: 8, + Metadata: map[string]interface{}{ + "issuer": "satoshi", + "count": 10, + }, + }, + key: "BTC:8:map[count:10 issuer:satoshi]", + }, + } + + for name, test := range tests { + t.Run(name, func(t *testing.T) { + assert.Equal(t, test.key, CurrencyString(test.currency)) + }) + } +} From 59179e1f9a1608f40ff9b3f8416a82ffd47053bf Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 5 May 2020 10:57:20 -0700 Subject: [PATCH 19/31] Stage rest of changes for PR --- README.md | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/README.md b/README.md index 640b4fa2..448e100b 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,14 @@ in this specification will enable exchanges, block explorers, and wallets to integrate with much less communication overhead and network-specific work. + +## TODO: +* Load block cache on restart from storage (to ensure reorgs are handled correctly) +* Persist seen account balances and account balance queue across restarts +* Add ability to view a block (view:block) +* Exempt account types from reconciliation (hopefully from JSON initialization...ex: lockedGld balance in Celo) + + ## Install ``` go get github.com/coinbase/rosetta-cli From aa9ba2589d24455e652f99bbf0aaccbe807c37d4 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 5 May 2020 12:22:52 -0700 Subject: [PATCH 20/31] [WIP] Add view commands --- README.md | 7 +++- cmd/check.go | 16 +------- cmd/root.go | 13 +++++++ cmd/view_account.go | 82 +++++++++++++++++++++++++++++++++++++++++ cmd/view_block.go | 73 ++++++++++++++++++++++++++++++++++++ internal/utils/utils.go | 18 ++++++++- 6 files changed, 190 insertions(+), 19 deletions(-) create mode 100644 cmd/view_account.go create mode 100644 cmd/view_block.go diff --git a/README.md b/README.md index 448e100b..9ec9c463 100644 --- a/README.md +++ b/README.md @@ -20,9 +20,12 @@ and network-specific work. ## TODO: * Load block cache on restart from storage (to ensure reorgs are handled correctly) -* Persist seen account balances and account balance queue across restarts -* Add ability to view a block (view:block) +* Persist seen account balances and account balance queue across restarts (this allows inactive reconciliation to continue across restarts) +! Add ability to view a block (view:block, view:account) + * add examples in README +* Test that account balance works without partial block identifier (returns current block) * Exempt account types from reconciliation (hopefully from JSON initialization...ex: lockedGld balance in Celo) + * goes hand in hand with changing SubAccount to type ## Install diff --git a/cmd/check.go b/cmd/check.go index 0186c1d5..4f3239cf 100644 --- a/cmd/check.go +++ b/cmd/check.go @@ -75,10 +75,6 @@ index less than the last computed block index.`, // and any data used to perform validation. DataDir string - // ServerURL is the base URL for a Rosetta - // server to validate. - ServerURL string - // StartIndex is the block index to start syncing. StartIndex int64 @@ -144,11 +140,7 @@ func loadAccounts(filePath string) ([]*reconciler.AccountCurrency, error) { return nil, err } - prettyAccounts, err := json.MarshalIndent(accounts, "", " ") - if err != nil { - return nil, err - } - log.Printf("Found %d accounts at %s: %s\n", len(accounts), filePath, string(prettyAccounts)) + log.Printf("Found %d accounts at %s: %s\n", len(accounts), filePath, utils.PrettyPrintStruct(accounts)) return accounts, nil } @@ -160,12 +152,6 @@ func init() { "", "folder used to store logs and any data used to perform validation", ) - checkCmd.Flags().StringVar( - &ServerURL, - "server-url", - "http://localhost:8080", - "base URL for a Rosetta server to validate", - ) checkCmd.Flags().Int64Var( &StartIndex, "start", diff --git a/cmd/root.go b/cmd/root.go index 590b2398..3464e612 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -23,6 +23,10 @@ var ( Use: "rosetta-cli", Short: "CLI for the Rosetta API", } + + // ServerURL is the base URL for a Rosetta + // server to validate. + ServerURL string ) // Execute handles all invocations of the @@ -32,5 +36,14 @@ func Execute() error { } func init() { + rootCmd.PersistentFlags().StringVar( + &ServerURL, + "server-url", + "http://localhost:8080", + "base URL for a Rosetta server", + ) + rootCmd.AddCommand(checkCmd) + rootCmd.AddCommand(viewBlockCmd) + rootCmd.AddCommand(viewAccountCmd) } diff --git a/cmd/view_account.go b/cmd/view_account.go new file mode 100644 index 00000000..5e7e9a70 --- /dev/null +++ b/cmd/view_account.go @@ -0,0 +1,82 @@ +package cmd + +import ( + "context" + "encoding/json" + "fmt" + "log" + "strconv" + + "github.com/coinbase/rosetta-cli/internal/utils" + + "github.com/coinbase/rosetta-sdk-go/asserter" + "github.com/coinbase/rosetta-sdk-go/fetcher" + "github.com/coinbase/rosetta-sdk-go/types" + "github.com/spf13/cobra" +) + +var ( + viewAccountCmd = &cobra.Command{ + Use: "view:account", + Short: "", + Long: ``, + Run: runViewAccountCmd, + Args: cobra.MinimumNArgs(1), + } +) + +func runViewAccountCmd(cmd *cobra.Command, args []string) { + ctx := context.Background() + + account := &types.AccountIdentifier{} + if err := json.Unmarshal([]byte(args[0]), account); err != nil { + log.Fatal(fmt.Errorf("%w: unable to unmarshal account %s", err, args[0])) + } + + if err := asserter.AccountIdentifier(account); err != nil { + log.Fatal(fmt.Errorf("%w: invalid account identifier %s", err, utils.AccountString(account))) + } + + // Create a new fetcher + newFetcher := fetcher.New( + ServerURL, + ) + + // Initialize the fetcher's asserter + // + // Behind the scenes this makes a call to get the + // network status and uses the response to inform + // the asserter what are valid responses. + primaryNetwork, _, err := newFetcher.InitializeAsserter(ctx) + if err != nil { + log.Fatal(err) + } + + // Print the primary network and network status + // TODO: support specifying which network to get block from + log.Printf("Primary Network: %s\n", utils.PrettyPrintStruct(primaryNetwork)) + + var lookupBlock *types.PartialBlockIdentifier + if len(args) > 1 { + index, err := strconv.ParseInt(args[1], 10, 64) + if err != nil { + log.Fatal(fmt.Errorf("%w: unable to parse index %s", err, args[0])) + } + + lookupBlock = &types.PartialBlockIdentifier{Index: &index} + } + + block, amounts, metadata, err := newFetcher.AccountBalanceRetry( + ctx, + primaryNetwork, + account, + lookupBlock, + ) + if err != nil { + log.Fatal(fmt.Errorf("%w: unable to fetch account %s", err, utils.AccountString(account))) + } + + log.Printf("Amounts: %s\n", utils.PrettyPrintStruct(amounts)) + log.Printf("Metadata: %s\n", utils.PrettyPrintStruct(metadata)) + log.Printf("Balance Fetched At: %s\n", utils.PrettyPrintStruct(block)) +} diff --git a/cmd/view_block.go b/cmd/view_block.go new file mode 100644 index 00000000..c284d3b2 --- /dev/null +++ b/cmd/view_block.go @@ -0,0 +1,73 @@ +package cmd + +import ( + "context" + "fmt" + "log" + "strconv" + + "github.com/coinbase/rosetta-cli/internal/utils" + + "github.com/coinbase/rosetta-sdk-go/fetcher" + "github.com/coinbase/rosetta-sdk-go/types" + "github.com/spf13/cobra" +) + +var ( + viewBlockCmd = &cobra.Command{ + Use: "view:block", + Short: "", + Long: ``, + Run: runViewBlockCmd, + Args: cobra.ExactArgs(1), + } +) + +func runViewBlockCmd(cmd *cobra.Command, args []string) { + ctx := context.Background() + index, err := strconv.ParseInt(args[0], 10, 64) + if err != nil { + log.Fatal(fmt.Errorf("%w: unable to parse index %s", err, args[0])) + } + + // Create a new fetcher + newFetcher := fetcher.New( + ServerURL, + ) + + // Initialize the fetcher's asserter + // + // Behind the scenes this makes a call to get the + // network status and uses the response to inform + // the asserter what are valid responses. + primaryNetwork, _, err := newFetcher.InitializeAsserter(ctx) + if err != nil { + log.Fatal(err) + } + + // Print the primary network and network status + // TODO: support specifying which network to get block from + log.Printf("Primary Network: %s\n", utils.PrettyPrintStruct(primaryNetwork)) + + // Fetch the specified block with retries (automatically + // asserted for correctness) + // + // On another note, notice that fetcher.BlockRetry + // automatically fetches all transactions that are + // returned in BlockResponse.OtherTransactions. If you use + // the client directly, you will need to implement a mechanism + // to fully populate the block by fetching all these + // transactions. + block, err := newFetcher.BlockRetry( + ctx, + primaryNetwork, + &types.PartialBlockIdentifier{ + Index: &index, + }, + ) + if err != nil { + log.Fatal(fmt.Errorf("%w: unable to fetch block", err)) + } + + log.Printf("Current Block: %s\n", utils.PrettyPrintStruct(block)) +} diff --git a/internal/utils/utils.go b/internal/utils/utils.go index c6a5d449..a4593e8e 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -15,6 +15,7 @@ package utils import ( + "encoding/json" "fmt" "io/ioutil" "log" @@ -102,7 +103,7 @@ func AccountString(account *types.AccountIdentifier) string { // TODO: handle SubAccount.Metadata // that contains pointer values. return fmt.Sprintf( - "%s:%s:%v", + "%s:%s:%+v", account.Address, account.SubAccount.Address, account.SubAccount.Metadata, @@ -122,9 +123,22 @@ func CurrencyString(currency *types.Currency) string { // TODO: Handle currency.Metadata // that has pointer value. return fmt.Sprintf( - "%s:%d:%v", + "%s:%d:%+v", currency.Symbol, currency.Decimals, currency.Metadata, ) } + +func PrettyPrintStruct(val interface{}) string { + prettyStruct, err := json.MarshalIndent( + val, + "", + " ", + ) + if err != nil { + log.Fatal(err) + } + + return string(prettyStruct) +} From 9ab99d7c65de67e22cba13944231a434dfd6e4b9 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 5 May 2020 14:04:55 -0700 Subject: [PATCH 21/31] Add create spec command --- README.md | 5 ++-- cmd/create_spec.go | 61 ++++++++++++++++++++++++++++++++++++++++++++++ cmd/root.go | 1 + 3 files changed, 65 insertions(+), 2 deletions(-) create mode 100644 cmd/create_spec.go diff --git a/README.md b/README.md index 9ec9c463..6cd4b57d 100644 --- a/README.md +++ b/README.md @@ -23,10 +23,11 @@ and network-specific work. * Persist seen account balances and account balance queue across restarts (this allows inactive reconciliation to continue across restarts) ! Add ability to view a block (view:block, view:account) * add examples in README -* Test that account balance works without partial block identifier (returns current block) +* Ensure basic endpoints are working as expected + * Test that account balance works without partial block identifier (returns current block) * Exempt account types from reconciliation (hopefully from JSON initialization...ex: lockedGld balance in Celo) * goes hand in hand with changing SubAccount to type - +! Create a spec file command ## Install ``` diff --git a/cmd/create_spec.go b/cmd/create_spec.go new file mode 100644 index 00000000..bfee4f1f --- /dev/null +++ b/cmd/create_spec.go @@ -0,0 +1,61 @@ +package cmd + +import ( + "context" + "io/ioutil" + "log" + "os" + "path" + + "github.com/coinbase/rosetta-cli/internal/utils" + + "github.com/coinbase/rosetta-sdk-go/asserter" + "github.com/coinbase/rosetta-sdk-go/fetcher" + "github.com/spf13/cobra" +) + +var ( + createSpecCmd = &cobra.Command{ + Use: "create:spec", + Short: "", + Long: ``, + Run: runCreateSpecCmd, + Args: cobra.ExactArgs(1), + } +) + +func runCreateSpecCmd(cmd *cobra.Command, args []string) { + ctx := context.Background() + + // Create a new fetcher + newFetcher := fetcher.New( + ServerURL, + ) + + // Initialize the fetcher's asserter + _, _, err := newFetcher.InitializeAsserter(ctx) + if err != nil { + log.Fatal(err) + } + + network, genesisBlock, operationTypes, operationStatuses, errors, err := newFetcher.Asserter.ClientConfiguration() + if err != nil { + log.Fatal(err) + } + + specFileContents := asserter.FileConfiguration{ + NetworkIdentifier: network, + GenesisBlockIdentifier: genesisBlock, + AllowedOperationTypes: operationTypes, + AllowedOperationStatuses: operationStatuses, + AllowedErrors: errors, + } + + specString := utils.PrettyPrintStruct(specFileContents) + log.Printf("Spec File: %s\n", specString) + + if err := ioutil.WriteFile(path.Clean(args[0]), []byte(specString), os.FileMode(0600)); err != nil { + log.Fatal(err) + } + +} diff --git a/cmd/root.go b/cmd/root.go index 3464e612..cde0910c 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -46,4 +46,5 @@ func init() { rootCmd.AddCommand(checkCmd) rootCmd.AddCommand(viewBlockCmd) rootCmd.AddCommand(viewAccountCmd) + rootCmd.AddCommand(createSpecCmd) } From d280df99e0783d4c87c12018671d9f1eb75353a6 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 6 May 2020 14:45:46 -0700 Subject: [PATCH 22/31] Update to rosetta-sdk-go v0.1.7 --- go.mod | 2 +- go.sum | 2 + internal/logger/logger.go | 45 +++++++-- internal/reconciler/reconciler.go | 15 ++- internal/reconciler/reconciler_test.go | 19 ++-- internal/storage/block_storage.go | 7 +- internal/storage/block_storage_test.go | 24 ++--- internal/syncer/syncer.go | 11 ++- internal/utils/utils.go | 104 +------------------- internal/utils/utils_test.go | 126 ------------------------- 10 files changed, 80 insertions(+), 275 deletions(-) delete mode 100644 internal/utils/utils_test.go diff --git a/go.mod b/go.mod index 5ff33a96..e9faa3c2 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.1.6 + github.com/coinbase/rosetta-sdk-go v0.1.7 github.com/davecgh/go-spew v1.1.1 github.com/dgraph-io/badger v1.6.0 github.com/pkg/errors v0.8.1 diff --git a/go.sum b/go.sum index 09f88727..b540e9ca 100644 --- a/go.sum +++ b/go.sum @@ -8,6 +8,8 @@ github.com/coinbase/rosetta-sdk-go v0.1.5 h1:fkYLDs8f7RuwKDJiaZv4qtVRntCOAcSm6Ve github.com/coinbase/rosetta-sdk-go v0.1.5/go.mod h1:lbmGsBpBiSZWP4WQqEW2CuTweGcU/ioQHwZ8CYo6yO8= github.com/coinbase/rosetta-sdk-go v0.1.6 h1:l6vyt+Gad7TWIy2Tf7giXO8jniBcLPhhW6021QFgIq0= github.com/coinbase/rosetta-sdk-go v0.1.6/go.mod h1:lbmGsBpBiSZWP4WQqEW2CuTweGcU/ioQHwZ8CYo6yO8= +github.com/coinbase/rosetta-sdk-go v0.1.7 h1:4NRMHWPSpmFNohtzQ/Nv9wsPLBEbgccNBbjwZTG1Owk= +github.com/coinbase/rosetta-sdk-go v0.1.7/go.mod h1:lbmGsBpBiSZWP4WQqEW2CuTweGcU/ioQHwZ8CYo6yO8= github.com/coinbase/rosetta-validator v0.1.2 h1:d1i/XuZu3tPEnc/HKLzixK0yXbMyg5UEzS2qba7tv5I= github.com/coinbase/rosetta-validator v0.1.2/go.mod h1:SgJPFHi1Ikr+tC5MkUhkrnLQKLT0Qzf7g+s1rnZ3kGo= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= diff --git a/internal/logger/logger.go b/internal/logger/logger.go index ea9de34c..d48e1dd3 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -22,7 +22,6 @@ import ( "path" "github.com/coinbase/rosetta-cli/internal/reconciler" - "github.com/coinbase/rosetta-cli/internal/utils" "github.com/coinbase/rosetta-sdk-go/types" ) @@ -168,7 +167,10 @@ func (l *Logger) TransactionStream( } participant := "" if op.Account != nil { - participant = utils.AccountString(op.Account) + participant, err = types.AccountString(op.Account) + if err != nil { + return err + } } networkIndex := op.OperationIdentifier.Index @@ -216,11 +218,16 @@ func (l *Logger) BalanceStream( defer f.Close() for _, balanceChange := range balanceChanges { + currencyString, err := types.CurrencyString(balanceChange.Currency) + if err != nil { + return err + } + balanceLog := fmt.Sprintf( "Account: %s Change: %s:%s Block: %d:%s", balanceChange.Account.Address, balanceChange.Difference, - utils.CurrencyString(balanceChange.Currency), + currencyString, balanceChange.Block.Index, balanceChange.Block.Hash, ) @@ -256,18 +263,28 @@ func (l *Logger) ReconcileSuccessStream( } defer f.Close() + accountString, err := types.AccountString(account) + if err != nil { + return err + } + log.Printf( "%s Reconciled %s at %d\n", reconciliationType, - utils.AccountString(account), + accountString, block.Index, ) + currencyString, err := types.CurrencyString(currency) + if err != nil { + return err + } + _, err = f.WriteString(fmt.Sprintf( "Type:%s Account: %s Currency: %s Balance: %s Block: %d:%s\n", reconciliationType, - utils.AccountString(account), - utils.CurrencyString(currency), + accountString, + currencyString, balance, block.Index, block.Hash, @@ -291,10 +308,15 @@ func (l *Logger) ReconcileFailureStream( block *types.BlockIdentifier, ) error { // Always print out reconciliation failures + accountString, err := types.AccountString(account) + if err != nil { + return err + } + log.Printf( "%s Reconciliation failed for %s at %d computed: %s node: %s\n", reconciliationType, - utils.AccountString(account), + accountString, block.Index, computedBalance, nodeBalance, @@ -314,11 +336,16 @@ func (l *Logger) ReconcileFailureStream( } defer f.Close() + currencyString, err := types.CurrencyString(currency) + if err != nil { + return err + } + _, err = f.WriteString(fmt.Sprintf( "Type:%s Account: %s Currency: %s Block: %s:%d computed: %s node: %s\n", reconciliationType, - utils.AccountString(account), - utils.CurrencyString(currency), + accountString, + currencyString, block.Hash, block.Index, computedBalance, diff --git a/internal/reconciler/reconciler.go b/internal/reconciler/reconciler.go index c7162e5a..6b0c867c 100644 --- a/internal/reconciler/reconciler.go +++ b/internal/reconciler/reconciler.go @@ -19,12 +19,9 @@ import ( "errors" "fmt" "log" - "reflect" "sync" "time" - // TODO: remove all references to internal packages - // before transitioning to rosetta-sdk-go "github.com/coinbase/rosetta-cli/internal/utils" "github.com/coinbase/rosetta-sdk-go/fetcher" @@ -219,8 +216,8 @@ func (r *Reconciler) QueueChanges( skipAccount := false // Look through balance changes for account + currency for _, change := range balanceChanges { - if reflect.DeepEqual(change.Account, account.Account) && - reflect.DeepEqual(change.Currency, account.Currency) { + if utils.Equal(change.Account, account.Account) && + utils.Equal(change.Currency, account.Currency) { skipAccount = true break } @@ -326,7 +323,7 @@ func (r *Reconciler) CompareBalance( ) } - difference, err := utils.SubtractStringValues(cachedBalance.Value, amount) + difference, err := types.SubtractValues(cachedBalance.Value, amount) if err != nil { return "", "", -1, err } @@ -626,7 +623,7 @@ func ExtractAmount( currency *types.Currency, ) (*types.Amount, error) { for _, b := range balances { - if !reflect.DeepEqual(b.Currency, currency) { + if !utils.Equal(b.Currency, currency) { continue } @@ -651,8 +648,8 @@ func ContainsAccountCurrency( change *AccountCurrency, ) bool { for _, a := range arr { - if reflect.DeepEqual(a.Account, change.Account) && - reflect.DeepEqual(a.Currency, change.Currency) { + if utils.Equal(a.Account, change.Account) && + utils.Equal(a.Currency, change.Currency) { return true } } diff --git a/internal/reconciler/reconciler_test.go b/internal/reconciler/reconciler_test.go index 3532b9b6..f2ff4549 100644 --- a/internal/reconciler/reconciler_test.go +++ b/internal/reconciler/reconciler_test.go @@ -16,6 +16,7 @@ package reconciler import ( "context" + "encoding/json" "errors" "fmt" "reflect" @@ -55,9 +56,9 @@ func TestContainsAccountCurrency(t *testing.T) { Address: "cool", SubAccount: &types.SubAccountIdentifier{ Address: "test2", - Metadata: map[string]interface{}{ - "neat": "stuff", - }, + Metadata: json.RawMessage(`{ + "neat": "stuff" + }`), }, }, Currency: currency1, @@ -109,9 +110,9 @@ func TestContainsAccountCurrency(t *testing.T) { Address: "cool", SubAccount: &types.SubAccountIdentifier{ Address: "test2", - Metadata: map[string]interface{}{ - "neat": "stuff", - }, + Metadata: json.RawMessage(`{ + "neat": "stuff" + }`), }, }, Currency: currency1, @@ -124,9 +125,9 @@ func TestContainsAccountCurrency(t *testing.T) { Address: "cool", SubAccount: &types.SubAccountIdentifier{ Address: "test2", - Metadata: map[string]interface{}{ - "neater": "stuff", - }, + Metadata: json.RawMessage(`{ + "neater": "stuff" + }`), }, }, Currency: currency1, diff --git a/internal/storage/block_storage.go b/internal/storage/block_storage.go index a0470578..d04eb1a0 100644 --- a/internal/storage/block_storage.go +++ b/internal/storage/block_storage.go @@ -28,7 +28,6 @@ import ( "path" "github.com/coinbase/rosetta-cli/internal/reconciler" - "github.com/coinbase/rosetta-cli/internal/utils" "github.com/coinbase/rosetta-sdk-go/types" ) @@ -120,7 +119,7 @@ func getHashKey(hash string, isBlock bool) []byte { func GetBalanceKey(account *types.AccountIdentifier, currency *types.Currency) []byte { return hashBytes( - fmt.Sprintf("%s/%s/%s", balanceNamespace, utils.AccountString(account), utils.CurrencyString(currency)), + fmt.Sprintf("%s/%s/%s", balanceNamespace, types.Hash(account), types.Hash(currency)), ) } @@ -493,7 +492,7 @@ func (b *BlockStorage) UpdateBalance( existingValue = amount.Value } - newVal, err := utils.AddStringValues(change.Difference, existingValue) + newVal, err := types.AddValues(change.Difference, existingValue) if err != nil { return err } @@ -719,7 +718,7 @@ func (b *BlockStorage) BalanceChanges( continue } - newDifference, err := utils.AddStringValues(val.Difference, amount.Value) + newDifference, err := types.AddValues(val.Difference, amount.Value) if err != nil { return nil, err } diff --git a/internal/storage/block_storage_test.go b/internal/storage/block_storage_test.go index 3fc0343a..840e043f 100644 --- a/internal/storage/block_storage_test.go +++ b/internal/storage/block_storage_test.go @@ -255,36 +255,36 @@ func TestBalance(t *testing.T) { Address: "blah", SubAccount: &types.SubAccountIdentifier{ Address: "stake", - Metadata: map[string]interface{}{ - "cool": "hello", - }, + Metadata: json.RawMessage(`{ + "cool": "hello" + }`), }, } subAccountMetadataNewPointer = &types.AccountIdentifier{ Address: "blah", SubAccount: &types.SubAccountIdentifier{ Address: "stake", - Metadata: map[string]interface{}{ - "cool": "hello", - }, + Metadata: json.RawMessage(`{ + "cool": "hello" + }`), }, } subAccountMetadata2 = &types.AccountIdentifier{ Address: "blah", SubAccount: &types.SubAccountIdentifier{ Address: "stake", - Metadata: map[string]interface{}{ - "cool": 10, - }, + Metadata: json.RawMessage(`{ + "cool": 10 + }`), }, } subAccountMetadata2NewPointer = &types.AccountIdentifier{ Address: "blah", SubAccount: &types.SubAccountIdentifier{ Address: "stake", - Metadata: map[string]interface{}{ - "cool": 10, - }, + Metadata: json.RawMessage(`{ + "cool": 10 + }`), }, } currency = &types.Currency{ diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index 97c97120..43609efc 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -19,7 +19,8 @@ import ( "errors" "fmt" "log" - "reflect" + + "github.com/coinbase/rosetta-cli/internal/utils" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/types" @@ -30,7 +31,7 @@ const ( // to try and sync in a given SyncCycle. maxSync = 999 - reorgCache = 20 + ReorgCache = 20 ) // SyncHandler is called at various times during the sync cycle @@ -157,8 +158,8 @@ func (s *Syncer) checkRemove( // Check if block parent is head lastBlock := s.blockCache[len(s.blockCache)-1] - if !reflect.DeepEqual(block.ParentBlockIdentifier, lastBlock.BlockIdentifier) { - if reflect.DeepEqual(s.genesisBlock, lastBlock.BlockIdentifier) { + if !utils.Equal(block.ParentBlockIdentifier, lastBlock.BlockIdentifier) { + if utils.Equal(s.genesisBlock, lastBlock.BlockIdentifier) { return false, nil, fmt.Errorf("cannot remove genesis block") } @@ -193,7 +194,7 @@ func (s *Syncer) processBlock( } s.blockCache = append(s.blockCache, block) - if len(s.blockCache) > reorgCache { + if len(s.blockCache) > ReorgCache { s.blockCache = s.blockCache[1:] } s.nextIndex = block.BlockIdentifier.Index + 1 diff --git a/internal/utils/utils.go b/internal/utils/utils.go index a4593e8e..69879bfc 100644 --- a/internal/utils/utils.go +++ b/internal/utils/utils.go @@ -15,11 +15,8 @@ package utils import ( - "encoding/json" - "fmt" "io/ioutil" "log" - "math/big" "os" "github.com/coinbase/rosetta-sdk-go/types" @@ -44,101 +41,8 @@ func RemoveTempDir(dir string) { } } -// AddStringValues adds string amounts using -// big.Int. -func AddStringValues( - a string, - b string, -) (string, error) { - aVal, ok := new(big.Int).SetString(a, 10) - if !ok { - return "", fmt.Errorf("%s is not an integer", a) - } - - bVal, ok := new(big.Int).SetString(b, 10) - if !ok { - return "", fmt.Errorf("%s is not an integer", b) - } - - newVal := new(big.Int).Add(aVal, bVal) - return newVal.String(), nil -} - -// SubtractStringValues subtracts a-b using -// big.Int. -func SubtractStringValues( - a string, - b string, -) (string, error) { - aVal, ok := new(big.Int).SetString(a, 10) - if !ok { - return "", fmt.Errorf("%s is not an integer", a) - } - - bVal, ok := new(big.Int).SetString(b, 10) - if !ok { - return "", fmt.Errorf("%s is not an integer", b) - } - - newVal := new(big.Int).Sub(aVal, bVal) - return newVal.String(), nil -} - -// AccountString returns a byte slice representing a *types.AccountIdentifier. -// This byte slice automatically handles the existence of *types.SubAccount -// detail. -func AccountString(account *types.AccountIdentifier) string { - if account.SubAccount == nil { - return account.Address - } - - if account.SubAccount.Metadata == nil { - return fmt.Sprintf( - "%s:%s", - account.Address, - account.SubAccount.Address, - ) - } - - // TODO: handle SubAccount.Metadata - // that contains pointer values. - return fmt.Sprintf( - "%s:%s:%+v", - account.Address, - account.SubAccount.Address, - account.SubAccount.Metadata, - ) -} - -// CurrencyString is used to identify a *types.Currency -// in an account's map of currencies. It is not feasible -// to create a map of [types.Currency]*types.Amount -// because types.Currency contains a metadata pointer -// that would prevent any equality. -func CurrencyString(currency *types.Currency) string { - if currency.Metadata == nil { - return fmt.Sprintf("%s:%d", currency.Symbol, currency.Decimals) - } - - // TODO: Handle currency.Metadata - // that has pointer value. - return fmt.Sprintf( - "%s:%d:%+v", - currency.Symbol, - currency.Decimals, - currency.Metadata, - ) -} - -func PrettyPrintStruct(val interface{}) string { - prettyStruct, err := json.MarshalIndent( - val, - "", - " ", - ) - if err != nil { - log.Fatal(err) - } - - return string(prettyStruct) +// Equal returns a boolean indicating if two +// interfaces are equal. +func Equal(a interface{}, b interface{}) bool { + return types.Hash(a) == types.Hash(b) } diff --git a/internal/utils/utils_test.go b/internal/utils/utils_test.go deleted file mode 100644 index 7ff2ca25..00000000 --- a/internal/utils/utils_test.go +++ /dev/null @@ -1,126 +0,0 @@ -package utils - -import ( - "testing" - - "github.com/coinbase/rosetta-sdk-go/types" - "github.com/stretchr/testify/assert" -) - -func TestGetAccountString(t *testing.T) { - var tests = map[string]struct { - account *types.AccountIdentifier - key string - }{ - "simple account": { - account: &types.AccountIdentifier{ - Address: "hello", - }, - key: "hello", - }, - "subaccount": { - account: &types.AccountIdentifier{ - Address: "hello", - SubAccount: &types.SubAccountIdentifier{ - Address: "stake", - }, - }, - key: "hello:stake", - }, - "subaccount with string metadata": { - account: &types.AccountIdentifier{ - Address: "hello", - SubAccount: &types.SubAccountIdentifier{ - Address: "stake", - Metadata: map[string]interface{}{ - "cool": "neat", - }, - }, - }, - key: "hello:stake:map[cool:neat]", - }, - "subaccount with number metadata": { - account: &types.AccountIdentifier{ - Address: "hello", - SubAccount: &types.SubAccountIdentifier{ - Address: "stake", - Metadata: map[string]interface{}{ - "cool": 1, - }, - }, - }, - key: "hello:stake:map[cool:1]", - }, - "subaccount with complex metadata": { - account: &types.AccountIdentifier{ - Address: "hello", - SubAccount: &types.SubAccountIdentifier{ - Address: "stake", - Metadata: map[string]interface{}{ - "cool": 1, - "awesome": "neat", - }, - }, - }, - key: "hello:stake:map[awesome:neat cool:1]", - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - assert.Equal(t, test.key, AccountString(test.account)) - }) - } -} - -func TestCurrencyString(t *testing.T) { - var tests = map[string]struct { - currency *types.Currency - key string - }{ - "simple currency": { - currency: &types.Currency{ - Symbol: "BTC", - Decimals: 8, - }, - key: "BTC:8", - }, - "currency with string metadata": { - currency: &types.Currency{ - Symbol: "BTC", - Decimals: 8, - Metadata: map[string]interface{}{ - "issuer": "satoshi", - }, - }, - key: "BTC:8:map[issuer:satoshi]", - }, - "currency with number metadata": { - currency: &types.Currency{ - Symbol: "BTC", - Decimals: 8, - Metadata: map[string]interface{}{ - "issuer": 1, - }, - }, - key: "BTC:8:map[issuer:1]", - }, - "currency with complex metadata": { - currency: &types.Currency{ - Symbol: "BTC", - Decimals: 8, - Metadata: map[string]interface{}{ - "issuer": "satoshi", - "count": 10, - }, - }, - key: "BTC:8:map[count:10 issuer:satoshi]", - }, - } - - for name, test := range tests { - t.Run(name, func(t *testing.T) { - assert.Equal(t, test.key, CurrencyString(test.currency)) - }) - } -} From 4b6f978a8e4d7c88e16fcf036682cbf31b33fe39 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 6 May 2020 15:20:41 -0700 Subject: [PATCH 23/31] Fill syncer cache on restart (if possible) --- README.md | 8 +- cmd/check.go | 16 ++- cmd/create_spec.go | 20 +--- cmd/view_account.go | 14 ++- cmd/view_block.go | 6 +- internal/storage/block_storage.go | 25 +++++ internal/storage/block_storage_test.go | 141 ++++++++++++++++--------- internal/syncer/syncer.go | 15 ++- internal/syncer/syncer_test.go | 42 +++++++- 9 files changed, 200 insertions(+), 87 deletions(-) diff --git a/README.md b/README.md index 6cd4b57d..06704758 100644 --- a/README.md +++ b/README.md @@ -19,15 +19,11 @@ and network-specific work. ## TODO: -* Load block cache on restart from storage (to ensure reorgs are handled correctly) -* Persist seen account balances and account balance queue across restarts (this allows inactive reconciliation to continue across restarts) +! Load block cache on restart from storage (to ensure reorgs are handled correctly) ! Add ability to view a block (view:block, view:account) * add examples in README -* Ensure basic endpoints are working as expected - * Test that account balance works without partial block identifier (returns current block) -* Exempt account types from reconciliation (hopefully from JSON initialization...ex: lockedGld balance in Celo) - * goes hand in hand with changing SubAccount to type ! Create a spec file command + * add examples in README ## Install ``` diff --git a/cmd/check.go b/cmd/check.go index 4f3239cf..7fb2766f 100644 --- a/cmd/check.go +++ b/cmd/check.go @@ -31,6 +31,7 @@ import ( "github.com/coinbase/rosetta-cli/internal/utils" "github.com/coinbase/rosetta-sdk-go/fetcher" + "github.com/coinbase/rosetta-sdk-go/types" "github.com/spf13/cobra" "golang.org/x/sync/errgroup" ) @@ -140,7 +141,7 @@ func loadAccounts(filePath string) ([]*reconciler.AccountCurrency, error) { return nil, err } - log.Printf("Found %d accounts at %s: %s\n", len(accounts), filePath, utils.PrettyPrintStruct(accounts)) + log.Printf("Found %d accounts at %s: %s\n", len(accounts), filePath, types.PrettyPrintStruct(accounts)) return accounts, nil } @@ -304,6 +305,7 @@ func runCheckCmd(cmd *cobra.Command, args []string) { ) blockStorage := storage.NewBlockStorage(ctx, localStore, blockStorageHelper) + // Bootstrap balances if provided if len(BootstrapBalances) > 0 { err = blockStorage.BootstrapBalances( ctx, @@ -315,6 +317,7 @@ func runCheckCmd(cmd *cobra.Command, args []string) { } } + // Ensure storage is in correct state for starting at index if StartIndex != -1 { // attempt to remove blocks from storage (without handling) if err = blockStorage.SetNewStartIndex(ctx, StartIndex); err != nil { log.Fatal(fmt.Errorf("%w: unable to set new start index", err)) @@ -360,11 +363,22 @@ func runCheckCmd(cmd *cobra.Command, args []string) { return r.Reconcile(ctx) }) + // Load in previous blocks into syncer cache to handle reorgs. + // If previously processed blocks exist in storage, they are fetched. + // Otherwise, none are provided to the cache (the syncer will not attempt + // a reorg if the cache is empty). + blockCache := []*types.Block{} + if StartIndex != -1 { + // This is the case if blocks already in storage or if stateless start + blockCache = blockStorage.CreateBlockCache(ctx) + } + syncer := syncer.New( primaryNetwork, fetcher, syncHandler, cancel, + blockCache, ) g.Go(func() error { diff --git a/cmd/create_spec.go b/cmd/create_spec.go index bfee4f1f..a9ab7145 100644 --- a/cmd/create_spec.go +++ b/cmd/create_spec.go @@ -2,15 +2,14 @@ package cmd import ( "context" + "fmt" "io/ioutil" "log" "os" "path" - "github.com/coinbase/rosetta-cli/internal/utils" - - "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/fetcher" + "github.com/coinbase/rosetta-sdk-go/types" "github.com/spf13/cobra" ) @@ -38,24 +37,15 @@ func runCreateSpecCmd(cmd *cobra.Command, args []string) { log.Fatal(err) } - network, genesisBlock, operationTypes, operationStatuses, errors, err := newFetcher.Asserter.ClientConfiguration() + configuration, err := newFetcher.Asserter.ClientConfiguration() if err != nil { - log.Fatal(err) + log.Fatal(fmt.Errorf("%w: unable to generate spec", err)) } - specFileContents := asserter.FileConfiguration{ - NetworkIdentifier: network, - GenesisBlockIdentifier: genesisBlock, - AllowedOperationTypes: operationTypes, - AllowedOperationStatuses: operationStatuses, - AllowedErrors: errors, - } - - specString := utils.PrettyPrintStruct(specFileContents) + specString := types.PrettyPrintStruct(configuration) log.Printf("Spec File: %s\n", specString) if err := ioutil.WriteFile(path.Clean(args[0]), []byte(specString), os.FileMode(0600)); err != nil { log.Fatal(err) } - } diff --git a/cmd/view_account.go b/cmd/view_account.go index 5e7e9a70..af4a19df 100644 --- a/cmd/view_account.go +++ b/cmd/view_account.go @@ -7,8 +7,6 @@ import ( "log" "strconv" - "github.com/coinbase/rosetta-cli/internal/utils" - "github.com/coinbase/rosetta-sdk-go/asserter" "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/types" @@ -34,7 +32,7 @@ func runViewAccountCmd(cmd *cobra.Command, args []string) { } if err := asserter.AccountIdentifier(account); err != nil { - log.Fatal(fmt.Errorf("%w: invalid account identifier %s", err, utils.AccountString(account))) + log.Fatal(fmt.Errorf("%w: invalid account identifier %+v", err, account)) } // Create a new fetcher @@ -54,7 +52,7 @@ func runViewAccountCmd(cmd *cobra.Command, args []string) { // Print the primary network and network status // TODO: support specifying which network to get block from - log.Printf("Primary Network: %s\n", utils.PrettyPrintStruct(primaryNetwork)) + log.Printf("Primary Network: %s\n", types.PrettyPrintStruct(primaryNetwork)) var lookupBlock *types.PartialBlockIdentifier if len(args) > 1 { @@ -73,10 +71,10 @@ func runViewAccountCmd(cmd *cobra.Command, args []string) { lookupBlock, ) if err != nil { - log.Fatal(fmt.Errorf("%w: unable to fetch account %s", err, utils.AccountString(account))) + log.Fatal(fmt.Errorf("%w: unable to fetch account %+v", err, account)) } - log.Printf("Amounts: %s\n", utils.PrettyPrintStruct(amounts)) - log.Printf("Metadata: %s\n", utils.PrettyPrintStruct(metadata)) - log.Printf("Balance Fetched At: %s\n", utils.PrettyPrintStruct(block)) + log.Printf("Amounts: %s\n", types.PrettyPrintStruct(amounts)) + log.Printf("Metadata: %s\n", types.PrettyPrintStruct(metadata)) + log.Printf("Balance Fetched At: %s\n", types.PrettyPrintStruct(block)) } diff --git a/cmd/view_block.go b/cmd/view_block.go index c284d3b2..c0a2030e 100644 --- a/cmd/view_block.go +++ b/cmd/view_block.go @@ -6,8 +6,6 @@ import ( "log" "strconv" - "github.com/coinbase/rosetta-cli/internal/utils" - "github.com/coinbase/rosetta-sdk-go/fetcher" "github.com/coinbase/rosetta-sdk-go/types" "github.com/spf13/cobra" @@ -47,7 +45,7 @@ func runViewBlockCmd(cmd *cobra.Command, args []string) { // Print the primary network and network status // TODO: support specifying which network to get block from - log.Printf("Primary Network: %s\n", utils.PrettyPrintStruct(primaryNetwork)) + log.Printf("Primary Network: %s\n", types.PrettyPrintStruct(primaryNetwork)) // Fetch the specified block with retries (automatically // asserted for correctness) @@ -69,5 +67,5 @@ func runViewBlockCmd(cmd *cobra.Command, args []string) { log.Fatal(fmt.Errorf("%w: unable to fetch block", err)) } - log.Printf("Current Block: %s\n", utils.PrettyPrintStruct(block)) + log.Printf("Current Block: %s\n", types.PrettyPrintStruct(block)) } diff --git a/internal/storage/block_storage.go b/internal/storage/block_storage.go index d04eb1a0..a4e94d4d 100644 --- a/internal/storage/block_storage.go +++ b/internal/storage/block_storage.go @@ -28,6 +28,7 @@ import ( "path" "github.com/coinbase/rosetta-cli/internal/reconciler" + "github.com/coinbase/rosetta-cli/internal/syncer" "github.com/coinbase/rosetta-sdk-go/types" ) @@ -734,3 +735,27 @@ func (b *BlockStorage) BalanceChanges( return allChanges, nil } + +// CreateBlockCache populates a slice of blocks with the most recent +// ones in storage. +func (b *BlockStorage) CreateBlockCache(ctx context.Context) []*types.Block { + cache := []*types.Block{} + head, err := b.GetHeadBlockIdentifier(ctx) + if err != nil { + return cache + } + + for len(cache) < syncer.ReorgCache { + block, err := b.GetBlock(ctx, head) + if err != nil { + return cache + } + + log.Printf("Added %+v to cache\n", block.BlockIdentifier) + + cache = append([]*types.Block{block}, cache...) + head = block.ParentBlockIdentifier + } + + return cache +} diff --git a/internal/storage/block_storage_test.go b/internal/storage/block_storage_test.go index 840e043f..d1cacceb 100644 --- a/internal/storage/block_storage_test.go +++ b/internal/storage/block_storage_test.go @@ -99,65 +99,78 @@ func TestHeadBlockIdentifier(t *testing.T) { }) } -func TestBlock(t *testing.T) { - var ( - newBlock = &types.Block{ - BlockIdentifier: &types.BlockIdentifier{ - Hash: "blah 1", - Index: 1, - }, - ParentBlockIdentifier: &types.BlockIdentifier{ - Hash: "blah 0", - Index: 0, - }, - Timestamp: 1, - Transactions: []*types.Transaction{ - { - TransactionIdentifier: &types.TransactionIdentifier{ - Hash: "blahTx", - }, - Operations: []*types.Operation{ - { - OperationIdentifier: &types.OperationIdentifier{ - Index: 0, - }, +var ( + newBlock = &types.Block{ + BlockIdentifier: &types.BlockIdentifier{ + Hash: "blah 1", + Index: 1, + }, + ParentBlockIdentifier: &types.BlockIdentifier{ + Hash: "blah 0", + Index: 0, + }, + Timestamp: 1, + Transactions: []*types.Transaction{ + { + TransactionIdentifier: &types.TransactionIdentifier{ + Hash: "blahTx", + }, + Operations: []*types.Operation{ + { + OperationIdentifier: &types.OperationIdentifier{ + Index: 0, }, }, }, }, - } + }, + } - badBlockIdentifier = &types.BlockIdentifier{ - Hash: "missing blah", - Index: 0, - } + badBlockIdentifier = &types.BlockIdentifier{ + Hash: "missing blah", + Index: 0, + } - newBlock2 = &types.Block{ - BlockIdentifier: &types.BlockIdentifier{ - Hash: "blah 2", - Index: 2, - }, - ParentBlockIdentifier: &types.BlockIdentifier{ - Hash: "blah 1", - Index: 1, - }, - Timestamp: 1, - Transactions: []*types.Transaction{ - { - TransactionIdentifier: &types.TransactionIdentifier{ - Hash: "blahTx", - }, - Operations: []*types.Operation{ - { - OperationIdentifier: &types.OperationIdentifier{ - Index: 0, - }, + newBlock2 = &types.Block{ + BlockIdentifier: &types.BlockIdentifier{ + Hash: "blah 2", + Index: 2, + }, + ParentBlockIdentifier: &types.BlockIdentifier{ + Hash: "blah 1", + Index: 1, + }, + Timestamp: 1, + Transactions: []*types.Transaction{ + { + TransactionIdentifier: &types.TransactionIdentifier{ + Hash: "blahTx", + }, + Operations: []*types.Operation{ + { + OperationIdentifier: &types.OperationIdentifier{ + Index: 0, }, }, }, }, - } - ) + }, + } + + newBlock3 = &types.Block{ + BlockIdentifier: &types.BlockIdentifier{ + Hash: "blah 2", + Index: 2, + }, + ParentBlockIdentifier: &types.BlockIdentifier{ + Hash: "blah 1", + Index: 1, + }, + Timestamp: 1, + } +) + +func TestBlock(t *testing.T) { ctx := context.Background() newDir, err := utils.CreateTempDir() @@ -975,6 +988,36 @@ func TestBalanceChanges(t *testing.T) { } } +func TestCreateBlockCache(t *testing.T) { + ctx := context.Background() + + newDir, err := utils.CreateTempDir() + assert.NoError(t, err) + defer utils.RemoveTempDir(newDir) + + database, err := NewBadgerStorage(ctx, newDir) + assert.NoError(t, err) + defer database.Close(ctx) + + storage := NewBlockStorage(ctx, database, &MockBlockStorageHelper{}) + + t.Run("no blocks processed", func(t *testing.T) { + assert.Equal(t, []*types.Block{}, storage.CreateBlockCache(ctx)) + }) + + t.Run("1 block processed", func(t *testing.T) { + _, err = storage.StoreBlock(ctx, newBlock) + assert.NoError(t, err) + assert.Equal(t, []*types.Block{newBlock}, storage.CreateBlockCache(ctx)) + }) + + t.Run("2 blocks processed", func(t *testing.T) { + _, err = storage.StoreBlock(ctx, newBlock3) + assert.NoError(t, err) + assert.Equal(t, []*types.Block{newBlock, newBlock3}, storage.CreateBlockCache(ctx)) + }) +} + type MockBlockStorageHelper struct { AccountBalanceAmount string ExemptAccounts []*reconciler.AccountCurrency diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index 43609efc..bb2888c5 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -59,10 +59,13 @@ type Syncer struct { genesisBlock *types.BlockIdentifier nextIndex int64 - // TODO: to ensure reorgs are handled correctly, it should be possible - // to pass in recently processed blocks to the syncer. Without this, the + // To ensure reorgs are handled correctly, the syncer must be able + // to observe blocks it has previously processed. Without this, the // syncer may process an index that is not connected to previously added // blocks (ParentBlockIdentifier != lastProcessedBlock.BlockIdentifier). + // + // If a blockchain does not have reorgs, it is not necessary to populate + // the blockCache on creation. blockCache []*types.Block } @@ -71,13 +74,19 @@ func New( fetcher *fetcher.Fetcher, handler SyncHandler, cancel context.CancelFunc, + blockCache []*types.Block, ) *Syncer { + cache := blockCache + if cache == nil { + cache = []*types.Block{} + } + return &Syncer{ network: network, fetcher: fetcher, handler: handler, cancel: cancel, - blockCache: []*types.Block{}, + blockCache: cache, } } diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.go index 776e43b1..293e6e91 100644 --- a/internal/syncer/syncer_test.go +++ b/internal/syncer/syncer_test.go @@ -176,11 +176,16 @@ func lastBlockIdentifier(syncer *Syncer) *types.BlockIdentifier { func TestProcessBlock(t *testing.T) { ctx := context.Background() - syncer := New(networkIdentifier, nil, &MockSyncHandler{}, nil) + syncer := New(networkIdentifier, nil, &MockSyncHandler{}, nil, nil) syncer.genesisBlock = blockSequence[0].BlockIdentifier syncer.blockCache = []*types.Block{} t.Run("No block exists", func(t *testing.T) { + assert.Equal( + t, + []*types.Block{}, + syncer.blockCache, + ) err := syncer.processBlock( ctx, blockSequence[0], @@ -188,6 +193,11 @@ func TestProcessBlock(t *testing.T) { assert.NoError(t, err) assert.Equal(t, int64(1), syncer.nextIndex) assert.Equal(t, blockSequence[0].BlockIdentifier, lastBlockIdentifier(syncer)) + assert.Equal( + t, + []*types.Block{blockSequence[0]}, + syncer.blockCache, + ) }) t.Run("Orphan genesis", func(t *testing.T) { @@ -199,6 +209,11 @@ func TestProcessBlock(t *testing.T) { assert.EqualError(t, err, "cannot remove genesis block") assert.Equal(t, int64(1), syncer.nextIndex) assert.Equal(t, blockSequence[0].BlockIdentifier, lastBlockIdentifier(syncer)) + assert.Equal( + t, + []*types.Block{blockSequence[0]}, + syncer.blockCache, + ) }) t.Run("Block exists, no reorg", func(t *testing.T) { @@ -209,6 +224,11 @@ func TestProcessBlock(t *testing.T) { assert.NoError(t, err) assert.Equal(t, int64(2), syncer.nextIndex) assert.Equal(t, blockSequence[1].BlockIdentifier, lastBlockIdentifier(syncer)) + assert.Equal( + t, + []*types.Block{blockSequence[0], blockSequence[1]}, + syncer.blockCache, + ) }) t.Run("Orphan block", func(t *testing.T) { @@ -219,6 +239,11 @@ func TestProcessBlock(t *testing.T) { assert.NoError(t, err) assert.Equal(t, int64(1), syncer.nextIndex) assert.Equal(t, blockSequence[0].BlockIdentifier, lastBlockIdentifier(syncer)) + assert.Equal( + t, + []*types.Block{blockSequence[0]}, + syncer.blockCache, + ) err = syncer.processBlock( ctx, @@ -227,6 +252,11 @@ func TestProcessBlock(t *testing.T) { assert.NoError(t, err) assert.Equal(t, int64(2), syncer.nextIndex) assert.Equal(t, blockSequence[3].BlockIdentifier, lastBlockIdentifier(syncer)) + assert.Equal( + t, + []*types.Block{blockSequence[0], blockSequence[3]}, + syncer.blockCache, + ) err = syncer.processBlock( ctx, @@ -235,6 +265,11 @@ func TestProcessBlock(t *testing.T) { assert.NoError(t, err) assert.Equal(t, int64(3), syncer.nextIndex) assert.Equal(t, blockSequence[2].BlockIdentifier, lastBlockIdentifier(syncer)) + assert.Equal( + t, + []*types.Block{blockSequence[0], blockSequence[3], blockSequence[2]}, + syncer.blockCache, + ) }) t.Run("Out of order block", func(t *testing.T) { @@ -245,6 +280,11 @@ func TestProcessBlock(t *testing.T) { assert.EqualError(t, err, "Got block 5 instead of 3") assert.Equal(t, int64(3), syncer.nextIndex) assert.Equal(t, blockSequence[2].BlockIdentifier, lastBlockIdentifier(syncer)) + assert.Equal( + t, + []*types.Block{blockSequence[0], blockSequence[3], blockSequence[2]}, + syncer.blockCache, + ) }) } From dac1e3598e1549ce49a3dcdd634ff046fdf7a9cb Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 6 May 2020 15:49:06 -0700 Subject: [PATCH 24/31] Use BlockIdentifier for remove block --- cmd/check.go | 6 +-- internal/logger/logger.go | 51 +++++++++++++++++++------- internal/processor/sync_handler.go | 10 ++--- internal/storage/block_storage.go | 15 +++++--- internal/storage/block_storage_test.go | 16 ++++++-- internal/syncer/syncer.go | 34 ++++++++--------- internal/syncer/syncer_test.go | 51 ++++++++++++++++---------- 7 files changed, 117 insertions(+), 66 deletions(-) diff --git a/cmd/check.go b/cmd/check.go index 7fb2766f..d01f7454 100644 --- a/cmd/check.go +++ b/cmd/check.go @@ -367,10 +367,10 @@ func runCheckCmd(cmd *cobra.Command, args []string) { // If previously processed blocks exist in storage, they are fetched. // Otherwise, none are provided to the cache (the syncer will not attempt // a reorg if the cache is empty). - blockCache := []*types.Block{} + pastBlocks := []*types.BlockIdentifier{} if StartIndex != -1 { // This is the case if blocks already in storage or if stateless start - blockCache = blockStorage.CreateBlockCache(ctx) + pastBlocks = blockStorage.CreateBlockCache(ctx) } syncer := syncer.New( @@ -378,7 +378,7 @@ func runCheckCmd(cmd *cobra.Command, args []string) { fetcher, syncHandler, cancel, - blockCache, + pastBlocks, ) g.Go(func() error { diff --git a/internal/logger/logger.go b/internal/logger/logger.go index d48e1dd3..8c3d8549 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -84,12 +84,11 @@ func NewLogger( } } -// BlockStream writes the next processed block to the end of the +// AddBlockStream writes the next processed block to the end of the // blockStreamFile output file. -func (l *Logger) BlockStream( +func (l *Logger) AddBlockStream( ctx context.Context, block *types.Block, - orphan bool, ) error { if !l.logBlocks { return nil @@ -105,14 +104,9 @@ func (l *Logger) BlockStream( } defer f.Close() - verb := addEvent - if orphan { - verb = removeEvent - } - _, err = f.WriteString(fmt.Sprintf( "%s Block %d:%s with Parent Block %d:%s\n", - verb, + addEvent, block.BlockIdentifier.Index, block.BlockIdentifier.Hash, block.ParentBlockIdentifier.Index, @@ -122,7 +116,40 @@ func (l *Logger) BlockStream( return err } - return l.TransactionStream(ctx, block, verb) + return l.TransactionStream(ctx, block) +} + +// RemoveBlockStream writes the next processed block to the end of the +// blockStreamFile output file. +func (l *Logger) RemoveBlockStream( + ctx context.Context, + block *types.BlockIdentifier, +) error { + if !l.logBlocks { + return nil + } + + f, err := os.OpenFile( + path.Join(l.logDir, blockStreamFile), + os.O_APPEND|os.O_CREATE|os.O_WRONLY, + logFilePermissions, + ) + if err != nil { + return err + } + defer f.Close() + + _, err = f.WriteString(fmt.Sprintf( + "%s Block %d:%s\n", + removeEvent, + block.Index, + block.Hash, + )) + if err != nil { + return err + } + + return nil } // TransactionStream writes the next processed block's transactions @@ -130,7 +157,6 @@ func (l *Logger) BlockStream( func (l *Logger) TransactionStream( ctx context.Context, block *types.Block, - verb string, ) error { if !l.logTransactions { return nil @@ -148,8 +174,7 @@ func (l *Logger) TransactionStream( for _, tx := range block.Transactions { _, err = f.WriteString(fmt.Sprintf( - "%s Transaction %s at Block %d:%s\n", - verb, + "Transaction %s at Block %d:%s\n", tx.TransactionIdentifier.Hash, block.BlockIdentifier.Index, block.BlockIdentifier.Hash, diff --git a/internal/processor/sync_handler.go b/internal/processor/sync_handler.go index 9f1b45bd..c8e73099 100644 --- a/internal/processor/sync_handler.go +++ b/internal/processor/sync_handler.go @@ -46,7 +46,7 @@ func (h *SyncHandler) BlockAdded( log.Printf("Adding block %+v\n", block.BlockIdentifier) // Log processed blocks and balance changes - if err := h.logger.BlockStream(ctx, block, false); err != nil { + if err := h.logger.AddBlockStream(ctx, block); err != nil { return nil } @@ -68,16 +68,16 @@ func (h *SyncHandler) BlockAdded( // block is removed. func (h *SyncHandler) BlockRemoved( ctx context.Context, - block *types.Block, + blockIdentifier *types.BlockIdentifier, ) error { - log.Printf("Orphaning block %+v\n", block.BlockIdentifier) + log.Printf("Orphaning block %+v\n", blockIdentifier) // Log processed blocks and balance changes - if err := h.logger.BlockStream(ctx, block, true); err != nil { + if err := h.logger.RemoveBlockStream(ctx, blockIdentifier); err != nil { return nil } - balanceChanges, err := h.storage.RemoveBlock(ctx, block) + balanceChanges, err := h.storage.RemoveBlock(ctx, blockIdentifier) if err != nil { return err } diff --git a/internal/storage/block_storage.go b/internal/storage/block_storage.go index a4e94d4d..f7f270bf 100644 --- a/internal/storage/block_storage.go +++ b/internal/storage/block_storage.go @@ -330,8 +330,13 @@ func (b *BlockStorage) StoreBlock( // detection. This is called within a re-org. func (b *BlockStorage) RemoveBlock( ctx context.Context, - block *types.Block, + blockIdentifier *types.BlockIdentifier, ) ([]*reconciler.BalanceChange, error) { + block, err := b.GetBlock(ctx, blockIdentifier) + if err != nil { + return nil, err + } + transaction := b.newDatabaseTransaction(ctx, true) defer transaction.Discard(ctx) @@ -422,7 +427,7 @@ func (b *BlockStorage) SetNewStartIndex( return err } - if _, err = b.RemoveBlock(ctx, block); err != nil { + if _, err = b.RemoveBlock(ctx, block.BlockIdentifier); err != nil { return err } @@ -738,8 +743,8 @@ func (b *BlockStorage) BalanceChanges( // CreateBlockCache populates a slice of blocks with the most recent // ones in storage. -func (b *BlockStorage) CreateBlockCache(ctx context.Context) []*types.Block { - cache := []*types.Block{} +func (b *BlockStorage) CreateBlockCache(ctx context.Context) []*types.BlockIdentifier { + cache := []*types.BlockIdentifier{} head, err := b.GetHeadBlockIdentifier(ctx) if err != nil { return cache @@ -753,7 +758,7 @@ func (b *BlockStorage) CreateBlockCache(ctx context.Context) []*types.Block { log.Printf("Added %+v to cache\n", block.BlockIdentifier) - cache = append([]*types.Block{block}, cache...) + cache = append([]*types.BlockIdentifier{block.BlockIdentifier}, cache...) head = block.ParentBlockIdentifier } diff --git a/internal/storage/block_storage_test.go b/internal/storage/block_storage_test.go index d1cacceb..fc9f6257 100644 --- a/internal/storage/block_storage_test.go +++ b/internal/storage/block_storage_test.go @@ -229,7 +229,7 @@ func TestBlock(t *testing.T) { }) t.Run("Remove block and re-set block of same hash", func(t *testing.T) { - _, err := storage.RemoveBlock(ctx, newBlock) + _, err := storage.RemoveBlock(ctx, newBlock.BlockIdentifier) assert.NoError(t, err) head, err := storage.GetHeadBlockIdentifier(ctx) @@ -1002,19 +1002,27 @@ func TestCreateBlockCache(t *testing.T) { storage := NewBlockStorage(ctx, database, &MockBlockStorageHelper{}) t.Run("no blocks processed", func(t *testing.T) { - assert.Equal(t, []*types.Block{}, storage.CreateBlockCache(ctx)) + assert.Equal(t, []*types.BlockIdentifier{}, storage.CreateBlockCache(ctx)) }) t.Run("1 block processed", func(t *testing.T) { _, err = storage.StoreBlock(ctx, newBlock) assert.NoError(t, err) - assert.Equal(t, []*types.Block{newBlock}, storage.CreateBlockCache(ctx)) + assert.Equal( + t, + []*types.BlockIdentifier{newBlock.BlockIdentifier}, + storage.CreateBlockCache(ctx), + ) }) t.Run("2 blocks processed", func(t *testing.T) { _, err = storage.StoreBlock(ctx, newBlock3) assert.NoError(t, err) - assert.Equal(t, []*types.Block{newBlock, newBlock3}, storage.CreateBlockCache(ctx)) + assert.Equal( + t, + []*types.BlockIdentifier{newBlock.BlockIdentifier, newBlock3.BlockIdentifier}, + storage.CreateBlockCache(ctx), + ) }) } diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index bb2888c5..182f38db 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -45,7 +45,7 @@ type SyncHandler interface { BlockRemoved( ctx context.Context, - block *types.Block, + block *types.BlockIdentifier, ) error } @@ -66,7 +66,7 @@ type Syncer struct { // // If a blockchain does not have reorgs, it is not necessary to populate // the blockCache on creation. - blockCache []*types.Block + pastBlocks []*types.BlockIdentifier } func New( @@ -74,11 +74,11 @@ func New( fetcher *fetcher.Fetcher, handler SyncHandler, cancel context.CancelFunc, - blockCache []*types.Block, + pastBlocks []*types.BlockIdentifier, ) *Syncer { - cache := blockCache - if cache == nil { - cache = []*types.Block{} + past := pastBlocks + if past == nil { + past = []*types.BlockIdentifier{} } return &Syncer{ @@ -86,7 +86,7 @@ func New( fetcher: fetcher, handler: handler, cancel: cancel, - blockCache: cache, + pastBlocks: past, } } @@ -151,8 +151,8 @@ func (s *Syncer) nextSyncableRange( func (s *Syncer) checkRemove( block *types.Block, -) (bool, *types.Block, error) { - if len(s.blockCache) == 0 { +) (bool, *types.BlockIdentifier, error) { + if len(s.pastBlocks) == 0 { return false, nil, nil } @@ -166,9 +166,9 @@ func (s *Syncer) checkRemove( } // Check if block parent is head - lastBlock := s.blockCache[len(s.blockCache)-1] - if !utils.Equal(block.ParentBlockIdentifier, lastBlock.BlockIdentifier) { - if utils.Equal(s.genesisBlock, lastBlock.BlockIdentifier) { + lastBlock := s.pastBlocks[len(s.pastBlocks)-1] + if !utils.Equal(block.ParentBlockIdentifier, lastBlock) { + if utils.Equal(s.genesisBlock, lastBlock) { return false, nil, fmt.Errorf("cannot remove genesis block") } @@ -192,8 +192,8 @@ func (s *Syncer) processBlock( if err != nil { return err } - s.blockCache = s.blockCache[:len(s.blockCache)-1] - s.nextIndex = lastBlock.BlockIdentifier.Index + s.pastBlocks = s.pastBlocks[:len(s.pastBlocks)-1] + s.nextIndex = lastBlock.Index return nil } @@ -202,9 +202,9 @@ func (s *Syncer) processBlock( return err } - s.blockCache = append(s.blockCache, block) - if len(s.blockCache) > ReorgCache { - s.blockCache = s.blockCache[1:] + s.pastBlocks = append(s.pastBlocks, block.BlockIdentifier) + if len(s.pastBlocks) > ReorgCache { + s.pastBlocks = s.pastBlocks[1:] } s.nextIndex = block.BlockIdentifier.Index + 1 return nil diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.go index 293e6e91..bbbb780e 100644 --- a/internal/syncer/syncer_test.go +++ b/internal/syncer/syncer_test.go @@ -170,7 +170,7 @@ var ( ) func lastBlockIdentifier(syncer *Syncer) *types.BlockIdentifier { - return syncer.blockCache[len(syncer.blockCache)-1].BlockIdentifier + return syncer.pastBlocks[len(syncer.pastBlocks)-1] } func TestProcessBlock(t *testing.T) { @@ -178,13 +178,12 @@ func TestProcessBlock(t *testing.T) { syncer := New(networkIdentifier, nil, &MockSyncHandler{}, nil, nil) syncer.genesisBlock = blockSequence[0].BlockIdentifier - syncer.blockCache = []*types.Block{} t.Run("No block exists", func(t *testing.T) { assert.Equal( t, - []*types.Block{}, - syncer.blockCache, + []*types.BlockIdentifier{}, + syncer.pastBlocks, ) err := syncer.processBlock( ctx, @@ -195,8 +194,8 @@ func TestProcessBlock(t *testing.T) { assert.Equal(t, blockSequence[0].BlockIdentifier, lastBlockIdentifier(syncer)) assert.Equal( t, - []*types.Block{blockSequence[0]}, - syncer.blockCache, + []*types.BlockIdentifier{blockSequence[0].BlockIdentifier}, + syncer.pastBlocks, ) }) @@ -211,8 +210,8 @@ func TestProcessBlock(t *testing.T) { assert.Equal(t, blockSequence[0].BlockIdentifier, lastBlockIdentifier(syncer)) assert.Equal( t, - []*types.Block{blockSequence[0]}, - syncer.blockCache, + []*types.BlockIdentifier{blockSequence[0].BlockIdentifier}, + syncer.pastBlocks, ) }) @@ -226,8 +225,11 @@ func TestProcessBlock(t *testing.T) { assert.Equal(t, blockSequence[1].BlockIdentifier, lastBlockIdentifier(syncer)) assert.Equal( t, - []*types.Block{blockSequence[0], blockSequence[1]}, - syncer.blockCache, + []*types.BlockIdentifier{ + blockSequence[0].BlockIdentifier, + blockSequence[1].BlockIdentifier, + }, + syncer.pastBlocks, ) }) @@ -241,8 +243,8 @@ func TestProcessBlock(t *testing.T) { assert.Equal(t, blockSequence[0].BlockIdentifier, lastBlockIdentifier(syncer)) assert.Equal( t, - []*types.Block{blockSequence[0]}, - syncer.blockCache, + []*types.BlockIdentifier{blockSequence[0].BlockIdentifier}, + syncer.pastBlocks, ) err = syncer.processBlock( @@ -254,8 +256,11 @@ func TestProcessBlock(t *testing.T) { assert.Equal(t, blockSequence[3].BlockIdentifier, lastBlockIdentifier(syncer)) assert.Equal( t, - []*types.Block{blockSequence[0], blockSequence[3]}, - syncer.blockCache, + []*types.BlockIdentifier{ + blockSequence[0].BlockIdentifier, + blockSequence[3].BlockIdentifier, + }, + syncer.pastBlocks, ) err = syncer.processBlock( @@ -267,8 +272,12 @@ func TestProcessBlock(t *testing.T) { assert.Equal(t, blockSequence[2].BlockIdentifier, lastBlockIdentifier(syncer)) assert.Equal( t, - []*types.Block{blockSequence[0], blockSequence[3], blockSequence[2]}, - syncer.blockCache, + []*types.BlockIdentifier{ + blockSequence[0].BlockIdentifier, + blockSequence[3].BlockIdentifier, + blockSequence[2].BlockIdentifier, + }, + syncer.pastBlocks, ) }) @@ -282,8 +291,12 @@ func TestProcessBlock(t *testing.T) { assert.Equal(t, blockSequence[2].BlockIdentifier, lastBlockIdentifier(syncer)) assert.Equal( t, - []*types.Block{blockSequence[0], blockSequence[3], blockSequence[2]}, - syncer.blockCache, + []*types.BlockIdentifier{ + blockSequence[0].BlockIdentifier, + blockSequence[3].BlockIdentifier, + blockSequence[2].BlockIdentifier, + }, + syncer.pastBlocks, ) }) } @@ -299,7 +312,7 @@ func (h *MockSyncHandler) BlockAdded( func (h *MockSyncHandler) BlockRemoved( ctx context.Context, - block *types.Block, + block *types.BlockIdentifier, ) error { return nil } From 360392dcf422d72270ad39b02af92c0440fd08da Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 6 May 2020 16:06:41 -0700 Subject: [PATCH 25/31] Return an error if trying to start at block ahead of state store --- internal/storage/block_storage.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/internal/storage/block_storage.go b/internal/storage/block_storage.go index f7f270bf..a4268e4b 100644 --- a/internal/storage/block_storage.go +++ b/internal/storage/block_storage.go @@ -419,6 +419,10 @@ func (b *BlockStorage) SetNewStartIndex( return err } + if head.Index < startIndex { + return fmt.Errorf("last processed block %d is less than start index %d", head.Index, startIndex) + } + currBlock := head for currBlock.Index >= startIndex { log.Printf("Removing block %+v\n", currBlock) From 7c5d1993e9e36a02683ad78d6b7688355c18629b Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 6 May 2020 16:12:09 -0700 Subject: [PATCH 26/31] nits --- cmd/check.go | 7 +- cmd/create_configuration.go | 69 +++++++++++++++++++ cmd/create_spec.go | 51 -------------- cmd/root.go | 2 +- cmd/view_account.go | 14 ++++ cmd/view_block.go | 14 ++++ internal/processor/reconciler_handler.go | 14 ++++ internal/processor/reconciler_helper.go | 14 ++++ internal/processor/storage_helper.go | 14 ++++ .../{sync_handler.go => syncer_handler.go} | 14 ++++ internal/reconciler/reconciler.go | 30 +++++--- internal/storage/block_storage.go | 12 ++-- internal/storage/block_storage_test.go | 21 +++++- internal/syncer/syncer.go | 6 +- internal/syncer/syncer_test.go | 14 ++++ 15 files changed, 224 insertions(+), 72 deletions(-) create mode 100644 cmd/create_configuration.go delete mode 100644 cmd/create_spec.go rename internal/processor/{sync_handler.go => syncer_handler.go} (78%) diff --git a/cmd/check.go b/cmd/check.go index d01f7454..2e2413e4 100644 --- a/cmd/check.go +++ b/cmd/check.go @@ -141,7 +141,12 @@ func loadAccounts(filePath string) ([]*reconciler.AccountCurrency, error) { return nil, err } - log.Printf("Found %d accounts at %s: %s\n", len(accounts), filePath, types.PrettyPrintStruct(accounts)) + log.Printf( + "Found %d accounts at %s: %s\n", + len(accounts), + filePath, + types.PrettyPrintStruct(accounts), + ) return accounts, nil } diff --git a/cmd/create_configuration.go b/cmd/create_configuration.go new file mode 100644 index 00000000..9e5cbdde --- /dev/null +++ b/cmd/create_configuration.go @@ -0,0 +1,69 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cmd + +import ( + "context" + "fmt" + "io/ioutil" + "log" + "os" + "path" + + "github.com/coinbase/rosetta-sdk-go/fetcher" + "github.com/coinbase/rosetta-sdk-go/types" + "github.com/spf13/cobra" +) + +const ( + fileMode = 0600 +) + +var ( + createConfigurationCmd = &cobra.Command{ + Use: "create:configuration", + Short: "", + Long: ``, + Run: runCreateConfigurationCmd, + Args: cobra.ExactArgs(1), + } +) + +func runCreateConfigurationCmd(cmd *cobra.Command, args []string) { + ctx := context.Background() + + // Create a new fetcher + newFetcher := fetcher.New( + ServerURL, + ) + + // Initialize the fetcher's asserter + _, _, err := newFetcher.InitializeAsserter(ctx) + if err != nil { + log.Fatal(err) + } + + configuration, err := newFetcher.Asserter.ClientConfiguration() + if err != nil { + log.Fatal(fmt.Errorf("%w: unable to generate spec", err)) + } + + specString := types.PrettyPrintStruct(configuration) + log.Printf("Spec File: %s\n", specString) + + if err := ioutil.WriteFile(path.Clean(args[0]), []byte(specString), os.FileMode(fileMode)); err != nil { + log.Fatal(err) + } +} diff --git a/cmd/create_spec.go b/cmd/create_spec.go deleted file mode 100644 index a9ab7145..00000000 --- a/cmd/create_spec.go +++ /dev/null @@ -1,51 +0,0 @@ -package cmd - -import ( - "context" - "fmt" - "io/ioutil" - "log" - "os" - "path" - - "github.com/coinbase/rosetta-sdk-go/fetcher" - "github.com/coinbase/rosetta-sdk-go/types" - "github.com/spf13/cobra" -) - -var ( - createSpecCmd = &cobra.Command{ - Use: "create:spec", - Short: "", - Long: ``, - Run: runCreateSpecCmd, - Args: cobra.ExactArgs(1), - } -) - -func runCreateSpecCmd(cmd *cobra.Command, args []string) { - ctx := context.Background() - - // Create a new fetcher - newFetcher := fetcher.New( - ServerURL, - ) - - // Initialize the fetcher's asserter - _, _, err := newFetcher.InitializeAsserter(ctx) - if err != nil { - log.Fatal(err) - } - - configuration, err := newFetcher.Asserter.ClientConfiguration() - if err != nil { - log.Fatal(fmt.Errorf("%w: unable to generate spec", err)) - } - - specString := types.PrettyPrintStruct(configuration) - log.Printf("Spec File: %s\n", specString) - - if err := ioutil.WriteFile(path.Clean(args[0]), []byte(specString), os.FileMode(0600)); err != nil { - log.Fatal(err) - } -} diff --git a/cmd/root.go b/cmd/root.go index cde0910c..b7c88363 100644 --- a/cmd/root.go +++ b/cmd/root.go @@ -46,5 +46,5 @@ func init() { rootCmd.AddCommand(checkCmd) rootCmd.AddCommand(viewBlockCmd) rootCmd.AddCommand(viewAccountCmd) - rootCmd.AddCommand(createSpecCmd) + rootCmd.AddCommand(createConfigurationCmd) } diff --git a/cmd/view_account.go b/cmd/view_account.go index af4a19df..cb01da3e 100644 --- a/cmd/view_account.go +++ b/cmd/view_account.go @@ -1,3 +1,17 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cmd import ( diff --git a/cmd/view_block.go b/cmd/view_block.go index c0a2030e..12633e8b 100644 --- a/cmd/view_block.go +++ b/cmd/view_block.go @@ -1,3 +1,17 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package cmd import ( diff --git a/internal/processor/reconciler_handler.go b/internal/processor/reconciler_handler.go index 488f41fe..a2a7f890 100644 --- a/internal/processor/reconciler_handler.go +++ b/internal/processor/reconciler_handler.go @@ -1,3 +1,17 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package processor import ( diff --git a/internal/processor/reconciler_helper.go b/internal/processor/reconciler_helper.go index 09d84575..356c6612 100644 --- a/internal/processor/reconciler_helper.go +++ b/internal/processor/reconciler_helper.go @@ -1,3 +1,17 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package processor import ( diff --git a/internal/processor/storage_helper.go b/internal/processor/storage_helper.go index 8abbfa58..1a3bb666 100644 --- a/internal/processor/storage_helper.go +++ b/internal/processor/storage_helper.go @@ -1,3 +1,17 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package processor import ( diff --git a/internal/processor/sync_handler.go b/internal/processor/syncer_handler.go similarity index 78% rename from internal/processor/sync_handler.go rename to internal/processor/syncer_handler.go index c8e73099..f836a66a 100644 --- a/internal/processor/sync_handler.go +++ b/internal/processor/syncer_handler.go @@ -1,3 +1,17 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package processor import ( diff --git a/internal/reconciler/reconciler.go b/internal/reconciler/reconciler.go index 6b0c867c..cb20b8c8 100644 --- a/internal/reconciler/reconciler.go +++ b/internal/reconciler/reconciler.go @@ -100,7 +100,7 @@ type BalanceChange struct { Difference string `json:"difference,omitempty"` } -type ReconcilerHelper interface { +type Helper interface { BlockExists( ctx context.Context, block *types.BlockIdentifier, @@ -118,7 +118,7 @@ type ReconcilerHelper interface { ) (*types.Amount, *types.BlockIdentifier, error) } -type ReconcilerHandler interface { +type Handler interface { ReconciliationFailed( ctx context.Context, reconciliationType string, @@ -144,8 +144,8 @@ type ReconcilerHandler interface { // by a Rosetta Server. type Reconciler struct { network *types.NetworkIdentifier - helper ReconcilerHelper - handler ReconcilerHandler + helper Helper + handler Handler fetcher *fetcher.Fetcher accountConcurrency uint64 lookupBalanceByBlock bool @@ -169,8 +169,8 @@ type Reconciler struct { // NewReconciler creates a new Reconciler. func NewReconciler( network *types.NetworkIdentifier, - helper ReconcilerHelper, - handler ReconcilerHandler, + helper Helper, + handler Handler, fetcher *fetcher.Fetcher, accountConcurrency uint64, lookupBalanceByBlock bool, @@ -277,7 +277,10 @@ func (r *Reconciler) CompareBalance( // Head block should be set before we CompareBalance head, err := r.helper.CurrentBlock(ctx) if err != nil { - return zeroString, "", 0, fmt.Errorf("%w: unable to get current block for reconciliation", err) + return zeroString, "", 0, fmt.Errorf( + "%w: unable to get current block for reconciliation", + err, + ) } // Check if live block is < head (or wait) @@ -293,7 +296,11 @@ func (r *Reconciler) CompareBalance( // Check if live block is in store (ensure not reorged) exists, err := r.helper.BlockExists(ctx, liveBlock) if err != nil { - return zeroString, "", 0, fmt.Errorf("%w: unable to check if block exists: %+v", err, liveBlock) + return zeroString, "", 0, fmt.Errorf( + "%w: unable to check if block exists: %+v", + err, + liveBlock, + ) } if !exists { return zeroString, "", head.Index, fmt.Errorf( @@ -311,7 +318,12 @@ func (r *Reconciler) CompareBalance( head, ) if err != nil { - return zeroString, "", head.Index, fmt.Errorf("%w: unable to get cached balance for %+v:%+v", err, account, currency) + return zeroString, "", head.Index, fmt.Errorf( + "%w: unable to get cached balance for %+v:%+v", + err, + account, + currency, + ) } if liveBlock.Index < balanceBlock.Index { diff --git a/internal/storage/block_storage.go b/internal/storage/block_storage.go index a4268e4b..ded27a48 100644 --- a/internal/storage/block_storage.go +++ b/internal/storage/block_storage.go @@ -124,7 +124,7 @@ func GetBalanceKey(account *types.AccountIdentifier, currency *types.Currency) [ ) } -type BlockStorageHelper interface { +type Helper interface { AccountBalance( ctx context.Context, account *types.AccountIdentifier, @@ -142,14 +142,14 @@ type BlockStorageHelper interface { // on top of a Database and DatabaseTransaction interface. type BlockStorage struct { db Database - helper BlockStorageHelper + helper Helper } // NewBlockStorage returns a new BlockStorage. func NewBlockStorage( ctx context.Context, db Database, - helper BlockStorageHelper, + helper Helper, ) *BlockStorage { return &BlockStorage{ db: db, @@ -420,7 +420,11 @@ func (b *BlockStorage) SetNewStartIndex( } if head.Index < startIndex { - return fmt.Errorf("last processed block %d is less than start index %d", head.Index, startIndex) + return fmt.Errorf( + "last processed block %d is less than start index %d", + head.Index, + startIndex, + ) } currBlock := head diff --git a/internal/storage/block_storage_test.go b/internal/storage/block_storage_test.go index fc9f6257..6b865f58 100644 --- a/internal/storage/block_storage_test.go +++ b/internal/storage/block_storage_test.go @@ -526,7 +526,12 @@ func TestBalance(t *testing.T) { assert.NoError(t, err) assert.NoError(t, txn.Commit(ctx)) - retrievedAmount, block, err := storage.GetBalance(ctx, subAccountNewPointer, amount.Currency, newBlock) + retrievedAmount, block, err := storage.GetBalance( + ctx, + subAccountNewPointer, + amount.Currency, + newBlock, + ) assert.NoError(t, err) assert.Equal(t, amount, retrievedAmount) assert.Equal(t, newBlock, block) @@ -548,7 +553,12 @@ func TestBalance(t *testing.T) { assert.NoError(t, err) assert.NoError(t, txn.Commit(ctx)) - retrievedAmount, block, err := storage.GetBalance(ctx, subAccountMetadataNewPointer, amount.Currency, newBlock) + retrievedAmount, block, err := storage.GetBalance( + ctx, + subAccountMetadataNewPointer, + amount.Currency, + newBlock, + ) assert.NoError(t, err) assert.Equal(t, amount, retrievedAmount) assert.Equal(t, newBlock, block) @@ -570,7 +580,12 @@ func TestBalance(t *testing.T) { assert.NoError(t, err) assert.NoError(t, txn.Commit(ctx)) - retrievedAmount, block, err := storage.GetBalance(ctx, subAccountMetadata2NewPointer, amount.Currency, newBlock) + retrievedAmount, block, err := storage.GetBalance( + ctx, + subAccountMetadata2NewPointer, + amount.Currency, + newBlock, + ) assert.NoError(t, err) assert.Equal(t, amount, retrievedAmount) assert.Equal(t, newBlock, block) diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index 182f38db..a0125a31 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -37,7 +37,7 @@ const ( // SyncHandler is called at various times during the sync cycle // to handle different events. It is common to write logs or // perform reconciliation in the sync processor. -type SyncHandler interface { +type Handler interface { BlockAdded( ctx context.Context, block *types.Block, @@ -52,7 +52,7 @@ type SyncHandler interface { type Syncer struct { network *types.NetworkIdentifier fetcher *fetcher.Fetcher - handler SyncHandler + handler Handler cancel context.CancelFunc // Used to keep track of sync state @@ -72,7 +72,7 @@ type Syncer struct { func New( network *types.NetworkIdentifier, fetcher *fetcher.Fetcher, - handler SyncHandler, + handler Handler, cancel context.CancelFunc, pastBlocks []*types.BlockIdentifier, ) *Syncer { diff --git a/internal/syncer/syncer_test.go b/internal/syncer/syncer_test.go index bbbb780e..c7d8fcef 100644 --- a/internal/syncer/syncer_test.go +++ b/internal/syncer/syncer_test.go @@ -1,3 +1,17 @@ +// Copyright 2020 Coinbase, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package syncer import ( From 49d95dc7335dda8f3549edf3f14a967d50a1f805 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Wed, 6 May 2020 21:08:07 -0700 Subject: [PATCH 27/31] Update to use rosetta-sdk-go v0.1.8 --- go.mod | 2 +- go.sum | 4 +++ internal/logger/logger.go | 44 +++++--------------------- internal/reconciler/reconciler_test.go | 19 ++++++----- internal/storage/block_storage_test.go | 24 +++++++------- 5 files changed, 34 insertions(+), 59 deletions(-) diff --git a/go.mod b/go.mod index e9faa3c2..55ec8543 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/coinbase/rosetta-cli go 1.13 require ( - github.com/coinbase/rosetta-sdk-go v0.1.7 + github.com/coinbase/rosetta-sdk-go v0.1.8 github.com/davecgh/go-spew v1.1.1 github.com/dgraph-io/badger v1.6.0 github.com/pkg/errors v0.8.1 diff --git a/go.sum b/go.sum index b540e9ca..714c8fdd 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,8 @@ github.com/coinbase/rosetta-sdk-go v0.1.6 h1:l6vyt+Gad7TWIy2Tf7giXO8jniBcLPhhW60 github.com/coinbase/rosetta-sdk-go v0.1.6/go.mod h1:lbmGsBpBiSZWP4WQqEW2CuTweGcU/ioQHwZ8CYo6yO8= github.com/coinbase/rosetta-sdk-go v0.1.7 h1:4NRMHWPSpmFNohtzQ/Nv9wsPLBEbgccNBbjwZTG1Owk= github.com/coinbase/rosetta-sdk-go v0.1.7/go.mod h1:lbmGsBpBiSZWP4WQqEW2CuTweGcU/ioQHwZ8CYo6yO8= +github.com/coinbase/rosetta-sdk-go v0.1.8 h1:YJf5Us7Pl4G8CRBZ5+LEUIV4k9ZI/18nFloPaHBySFk= +github.com/coinbase/rosetta-sdk-go v0.1.8/go.mod h1:y1wXRc1wod4fEp3jhW+9D6MSFtnm/9X5yNxJv92Cv2E= github.com/coinbase/rosetta-validator v0.1.2 h1:d1i/XuZu3tPEnc/HKLzixK0yXbMyg5UEzS2qba7tv5I= github.com/coinbase/rosetta-validator v0.1.2/go.mod h1:SgJPFHi1Ikr+tC5MkUhkrnLQKLT0Qzf7g+s1rnZ3kGo= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -35,6 +37,8 @@ github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANyt github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= +github.com/mitchellh/mapstructure v1.3.0 h1:iDwIio/3gk2QtLLEsqU5lInaMzos0hDTz8a6lazSFVw= +github.com/mitchellh/mapstructure v1.3.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= diff --git a/internal/logger/logger.go b/internal/logger/logger.go index 8c3d8549..2afe02e4 100644 --- a/internal/logger/logger.go +++ b/internal/logger/logger.go @@ -192,10 +192,7 @@ func (l *Logger) TransactionStream( } participant := "" if op.Account != nil { - participant, err = types.AccountString(op.Account) - if err != nil { - return err - } + participant = types.AccountString(op.Account) } networkIndex := op.OperationIdentifier.Index @@ -243,16 +240,11 @@ func (l *Logger) BalanceStream( defer f.Close() for _, balanceChange := range balanceChanges { - currencyString, err := types.CurrencyString(balanceChange.Currency) - if err != nil { - return err - } - balanceLog := fmt.Sprintf( "Account: %s Change: %s:%s Block: %d:%s", balanceChange.Account.Address, balanceChange.Difference, - currencyString, + types.CurrencyString(balanceChange.Currency), balanceChange.Block.Index, balanceChange.Block.Hash, ) @@ -288,28 +280,18 @@ func (l *Logger) ReconcileSuccessStream( } defer f.Close() - accountString, err := types.AccountString(account) - if err != nil { - return err - } - log.Printf( "%s Reconciled %s at %d\n", reconciliationType, - accountString, + types.AccountString(account), block.Index, ) - currencyString, err := types.CurrencyString(currency) - if err != nil { - return err - } - _, err = f.WriteString(fmt.Sprintf( "Type:%s Account: %s Currency: %s Balance: %s Block: %d:%s\n", reconciliationType, - accountString, - currencyString, + types.AccountString(account), + types.CurrencyString(currency), balance, block.Index, block.Hash, @@ -333,15 +315,10 @@ func (l *Logger) ReconcileFailureStream( block *types.BlockIdentifier, ) error { // Always print out reconciliation failures - accountString, err := types.AccountString(account) - if err != nil { - return err - } - log.Printf( "%s Reconciliation failed for %s at %d computed: %s node: %s\n", reconciliationType, - accountString, + types.AccountString(account), block.Index, computedBalance, nodeBalance, @@ -361,16 +338,11 @@ func (l *Logger) ReconcileFailureStream( } defer f.Close() - currencyString, err := types.CurrencyString(currency) - if err != nil { - return err - } - _, err = f.WriteString(fmt.Sprintf( "Type:%s Account: %s Currency: %s Block: %s:%d computed: %s node: %s\n", reconciliationType, - accountString, - currencyString, + types.AccountString(account), + types.CurrencyString(currency), block.Hash, block.Index, computedBalance, diff --git a/internal/reconciler/reconciler_test.go b/internal/reconciler/reconciler_test.go index f2ff4549..3532b9b6 100644 --- a/internal/reconciler/reconciler_test.go +++ b/internal/reconciler/reconciler_test.go @@ -16,7 +16,6 @@ package reconciler import ( "context" - "encoding/json" "errors" "fmt" "reflect" @@ -56,9 +55,9 @@ func TestContainsAccountCurrency(t *testing.T) { Address: "cool", SubAccount: &types.SubAccountIdentifier{ Address: "test2", - Metadata: json.RawMessage(`{ - "neat": "stuff" - }`), + Metadata: map[string]interface{}{ + "neat": "stuff", + }, }, }, Currency: currency1, @@ -110,9 +109,9 @@ func TestContainsAccountCurrency(t *testing.T) { Address: "cool", SubAccount: &types.SubAccountIdentifier{ Address: "test2", - Metadata: json.RawMessage(`{ - "neat": "stuff" - }`), + Metadata: map[string]interface{}{ + "neat": "stuff", + }, }, }, Currency: currency1, @@ -125,9 +124,9 @@ func TestContainsAccountCurrency(t *testing.T) { Address: "cool", SubAccount: &types.SubAccountIdentifier{ Address: "test2", - Metadata: json.RawMessage(`{ - "neater": "stuff" - }`), + Metadata: map[string]interface{}{ + "neater": "stuff", + }, }, }, Currency: currency1, diff --git a/internal/storage/block_storage_test.go b/internal/storage/block_storage_test.go index 6b865f58..6e757ab0 100644 --- a/internal/storage/block_storage_test.go +++ b/internal/storage/block_storage_test.go @@ -268,36 +268,36 @@ func TestBalance(t *testing.T) { Address: "blah", SubAccount: &types.SubAccountIdentifier{ Address: "stake", - Metadata: json.RawMessage(`{ - "cool": "hello" - }`), + Metadata: map[string]interface{}{ + "cool": "hello", + }, }, } subAccountMetadataNewPointer = &types.AccountIdentifier{ Address: "blah", SubAccount: &types.SubAccountIdentifier{ Address: "stake", - Metadata: json.RawMessage(`{ - "cool": "hello" - }`), + Metadata: map[string]interface{}{ + "cool": "hello", + }, }, } subAccountMetadata2 = &types.AccountIdentifier{ Address: "blah", SubAccount: &types.SubAccountIdentifier{ Address: "stake", - Metadata: json.RawMessage(`{ - "cool": 10 - }`), + Metadata: map[string]interface{}{ + "cool": 10, + }, }, } subAccountMetadata2NewPointer = &types.AccountIdentifier{ Address: "blah", SubAccount: &types.SubAccountIdentifier{ Address: "stake", - Metadata: json.RawMessage(`{ - "cool": 10 - }`), + Metadata: map[string]interface{}{ + "cool": 10, + }, }, } currency = &types.Currency{ From e679b8dfb094e79024e5d02d49c447d2222fd5ba Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 7 May 2020 08:31:27 -0700 Subject: [PATCH 28/31] Cleanup comments --- README.md | 129 ++++------------------- cmd/check.go | 4 +- internal/processor/reconciler_handler.go | 6 ++ internal/processor/reconciler_helper.go | 13 +++ internal/processor/storage_helper.go | 7 ++ internal/processor/syncer_handler.go | 14 +-- internal/reconciler/reconciler.go | 8 ++ internal/storage/block_storage.go | 12 ++- internal/syncer/syncer.go | 20 +++- 9 files changed, 91 insertions(+), 122 deletions(-) diff --git a/README.md b/README.md index 06704758..f8a01f5e 100644 --- a/README.md +++ b/README.md @@ -17,7 +17,6 @@ in this specification will enable exchanges, block explorers, and wallets to integrate with much less communication overhead and network-specific work. - ## TODO: ! Load block cache on restart from storage (to ensure reorgs are handled correctly) ! Add ability to view a block (view:block, view:account) @@ -32,125 +31,18 @@ go get github.com/coinbase/rosetta-cli ## Usage ``` -CLI for the Rosetta API - -Usage: - rosetta-cli [command] - -Available Commands: - check:account Debug inactive reconciliation errors for a group of accounts - check:complete Run a full check of the correctness of a Rosetta server - check:quick Run a simple check of the correctness of a Rosetta server - help Help about any command - -Flags: - --account-concurrency uint concurrency to use while fetching accounts during reconciliation (default 8) - --block-concurrency uint concurrency to use while fetching blocks (default 8) - --data-dir string folder used to store logs and any data used to perform validation (default "./validator-data") - --end int block index to stop syncing (default -1) - --exempt-accounts string Absolute path to a file listing all accounts to exempt from balance - tracking and reconciliation. Look at the examples directory for an example of - how to structure this file. - --halt-on-reconciliation-error Determines if block processing should halt on a reconciliation - error. It can be beneficial to collect all reconciliation errors or silence - reconciliation errors during development. (default true) - -h, --help help for rosetta-cli - --log-balance-changes log balance changes - --log-blocks log processed blocks - --log-reconciliations log balance reconciliations - --log-transactions log processed transactions - --server-url string base URL for a Rosetta server to validate (default "http://localhost:8080") - --start int block index to start syncing (default -1) - --transaction-concurrency uint concurrency to use while fetching transactions (if required) (default 16) - -Use "rosetta-cli [command] --help" for more information about a command. ``` ### check:complete ``` -Check all server responses are properly constructed, that -there are no duplicate blocks and transactions, that blocks can be processed -from genesis to the current block (re-orgs handled automatically), and that -computed balance changes are equal to balance changes reported by the node. - -When re-running this command, it will start where it left off. If you want -to discard some number of blocks populate the --start flag with some block -index less than the last computed block index. - -Usage: - rosetta-cli check:complete [flags] - -Flags: - --bootstrap-balances string Absolute path to a file used to bootstrap balances before starting syncing. - Populating this value after beginning syncing will return an error. - -h, --help help for check:complete - --lookup-balance-by-block When set to true, balances are looked up at the block where a balance - change occurred instead of at the current block. Blockchains that do not support - historical balance lookup should set this to false. (default true) ``` ### check:quick ``` -Check all server responses are properly constructed and that -computed balance changes are equal to balance changes reported by the -node. To use check:quick, your server must implement the balance lookup -by block. - -Unlike check:complete, which requires syncing all blocks up -to the blocks you want to check, check:quick allows you to validate -an arbitrary range of blocks (even if earlier blocks weren't synced). -To do this, all you need to do is provide a --start flag and optionally -an --end flag. - -It is important to note that check:quick does not support re-orgs and it -does not check for duplicate blocks and transactions. For these features, -please use check:complete. - -When re-running this command, it will start off from genesis unless you -provide a populated --start flag. If you want to run a stateful validation, -use the check:complete command. - -Usage: - rosetta-cli check:quick [flags] - -Flags: - -h, --help help for check:quick ``` ### check:account ``` -check:complete identifies accounts with inactive reconciliation -errors (when the balance of an account changes without any operations), however, -it does not identify which block the untracked balance change occurred. This tool -is used for locating exactly which block was missing an operation for a -particular account and currency. - -In the future, this tool will be deprecated as check:complete -will automatically identify the block where the missing operation occurred. - -Usage: - rosetta-cli check:account [flags] - -Flags: - -h, --help help for check:account - --interesting-accounts string Absolute path to a file listing all accounts to check on each block. Look - at the examples directory for an example of how to structure this file. - -Global Flags: - --account-concurrency uint concurrency to use while fetching accounts during reconciliation (default 8) - --block-concurrency uint concurrency to use while fetching blocks (default 8) - --data-dir string folder used to store logs and any data used to perform validation (default "./validator-data") - --end int block index to stop syncing (default -1) - --halt-on-reconciliation-error Determines if block processing should halt on a reconciliation - error. It can be beneficial to collect all reconciliation errors or silence - reconciliation errors during development. (default true) - --log-balance-changes log balance changes - --log-blocks log processed blocks - --log-reconciliations log balance reconciliations - --log-transactions log processed transactions - --server-url string base URL for a Rosetta server to validate (default "http://localhost:8080") - --start int block index to start syncing (default -1) - --transaction-concurrency uint concurrency to use while fetching transactions (if required) (default 16) ``` ## Development @@ -159,6 +51,25 @@ Global Flags: * `make lint` to lint the source code (included generated code) * `make release` to run one last check before opening a PR +### Helper/Handler +Many of the internal packages use a `Helper/Handler` interface pattern to acquire +required information or to send events to some client implementation. An example +of this is in the `internal/reconciler` package where a `Helper` is used to get +the account balance and the `Handler` is called to incidate whether the +reconciliation of an account was successful. + +### Repo Structure +``` +cmd +internal + logger // logic to write syncing information to stdout/files + processor // Helper/Handler implementations for reconciler, storage, and syncer + reconciler // checks for equality between computed balance and node balance + storage // persists block to temporary storage and allows for querying balances + syncer // coordinates block syncing (inlcuding re-orgs) + utils // useful functions +``` + ## Correctness Checks This tool performs a variety of correctness checks using the Rosetta Server. If any correctness check fails, the validator will exit and print out a detailed @@ -188,7 +99,7 @@ exit. The validator randomly checks the balances of accounts that aren't involved in any transactions. The balances of accounts could change on the blockchain node without being included in an operation -returned by the Rosetta Server. Recall that **ALL** balance-changing +returned by the Rosetta Server. Recall that all balance-changing operations must be returned by the Rosetta Server. ## Future Work diff --git a/cmd/check.go b/cmd/check.go index 2e2413e4..eb45ad1d 100644 --- a/cmd/check.go +++ b/cmd/check.go @@ -354,7 +354,7 @@ func runCheckCmd(cmd *cobra.Command, args []string) { interestingAccounts, ) - syncHandler := processor.NewSyncHandler( + syncerHandler := processor.NewSyncerHandler( blockStorage, logger, r, @@ -381,7 +381,7 @@ func runCheckCmd(cmd *cobra.Command, args []string) { syncer := syncer.New( primaryNetwork, fetcher, - syncHandler, + syncerHandler, cancel, pastBlocks, ) diff --git a/internal/processor/reconciler_handler.go b/internal/processor/reconciler_handler.go index a2a7f890..70e13eb8 100644 --- a/internal/processor/reconciler_handler.go +++ b/internal/processor/reconciler_handler.go @@ -23,12 +23,14 @@ import ( "github.com/coinbase/rosetta-sdk-go/types" ) +// ReconcilerHandler implements the Reconciler.Handler interface. type ReconcilerHandler struct { cancel context.CancelFunc logger *logger.Logger haltOnReconciliationError bool } +// NewReconcilerHandler creates a new ReconcilerHandler. func NewReconcilerHandler( cancel context.CancelFunc, logger *logger.Logger, @@ -41,6 +43,9 @@ func NewReconcilerHandler( } } +// ReconciliationFailed is called each time a reconciliation fails. +// In this Handler implementation, we halt if haltOnReconciliationError +// was set to true. We also cancel the context. func (h *ReconcilerHandler) ReconciliationFailed( ctx context.Context, reconciliationType string, @@ -75,6 +80,7 @@ func (h *ReconcilerHandler) ReconciliationFailed( return nil } +// ReconciliationSucceeded is called each time a reconciliation succeeds. func (h *ReconcilerHandler) ReconciliationSucceeded( ctx context.Context, reconciliationType string, diff --git a/internal/processor/reconciler_helper.go b/internal/processor/reconciler_helper.go index 356c6612..d9dfbb4e 100644 --- a/internal/processor/reconciler_helper.go +++ b/internal/processor/reconciler_helper.go @@ -23,10 +23,13 @@ import ( "github.com/coinbase/rosetta-sdk-go/types" ) +// ReconcilerHelper implements the Reconciler.Helper +// interface. type ReconcilerHelper struct { storage *storage.BlockStorage } +// NewReconcilerHelper returns a new ReconcilerHelper. func NewReconcilerHelper( storage *storage.BlockStorage, ) *ReconcilerHelper { @@ -35,6 +38,10 @@ func NewReconcilerHelper( } } +// BlockExists returns a boolean indicating if block_storage +// contains a block. This is necessary to reconcile across +// reorgs. If the block returned on an account balance fetch +// does not exist, reconciliation will be skipped. func (h *ReconcilerHelper) BlockExists( ctx context.Context, block *types.BlockIdentifier, @@ -51,12 +58,18 @@ func (h *ReconcilerHelper) BlockExists( return false, err } +// CurrentBlock returns the last processed block and is used +// to determine which block to check account balances at during +// inactive reconciliation. func (h *ReconcilerHelper) CurrentBlock( ctx context.Context, ) (*types.BlockIdentifier, error) { return h.storage.GetHeadBlockIdentifier(ctx) } +// AccountBalance returns the balance of an account in block storage. +// It is necessary to perform this check outside of the Reconciler +// package to allow for separation from a default storage backend. func (h *ReconcilerHelper) AccountBalance( ctx context.Context, account *types.AccountIdentifier, diff --git a/internal/processor/storage_helper.go b/internal/processor/storage_helper.go index 1a3bb666..ce3d2597 100644 --- a/internal/processor/storage_helper.go +++ b/internal/processor/storage_helper.go @@ -25,6 +25,8 @@ import ( "github.com/coinbase/rosetta-sdk-go/types" ) +// BlockStorageHelper implements the storage.Helper +// interface. type BlockStorageHelper struct { network *types.NetworkIdentifier fetcher *fetcher.Fetcher @@ -34,6 +36,7 @@ type BlockStorageHelper struct { exemptAccounts []*reconciler.AccountCurrency } +// NewBlockStorageHelper returns a new BlockStorageHelper. func NewBlockStorageHelper( network *types.NetworkIdentifier, fetcher *fetcher.Fetcher, @@ -48,6 +51,10 @@ func NewBlockStorageHelper( } } +// AccountBalance attempts to fetch the balance +// for a missing account in storage. This is necessary +// for running the "check" command at an arbitrary height +// instead of syncing from genesis. func (h *BlockStorageHelper) AccountBalance( ctx context.Context, account *types.AccountIdentifier, diff --git a/internal/processor/syncer_handler.go b/internal/processor/syncer_handler.go index f836a66a..93357aa6 100644 --- a/internal/processor/syncer_handler.go +++ b/internal/processor/syncer_handler.go @@ -26,7 +26,8 @@ import ( "github.com/coinbase/rosetta-sdk-go/types" ) -type SyncHandler struct { +// SyncerHandler implements the syncer.Handler interface. +type SyncerHandler struct { storage *storage.BlockStorage logger *logger.Logger reconciler *reconciler.Reconciler @@ -35,14 +36,15 @@ type SyncHandler struct { exemptAccounts []*reconciler.AccountCurrency } -func NewSyncHandler( +// NewSyncerHandler returns a new SyncerHandler. +func NewSyncerHandler( storage *storage.BlockStorage, logger *logger.Logger, reconciler *reconciler.Reconciler, fetcher *fetcher.Fetcher, exemptAccounts []*reconciler.AccountCurrency, -) *SyncHandler { - return &SyncHandler{ +) *SyncerHandler { + return &SyncerHandler{ storage: storage, logger: logger, reconciler: reconciler, @@ -53,7 +55,7 @@ func NewSyncHandler( // BlockAdded is called by the syncer after a // block is added. -func (h *SyncHandler) BlockAdded( +func (h *SyncerHandler) BlockAdded( ctx context.Context, block *types.Block, ) error { @@ -80,7 +82,7 @@ func (h *SyncHandler) BlockAdded( // BlockRemoved is called by the syncer after a // block is removed. -func (h *SyncHandler) BlockRemoved( +func (h *SyncerHandler) BlockRemoved( ctx context.Context, blockIdentifier *types.BlockIdentifier, ) error { diff --git a/internal/reconciler/reconciler.go b/internal/reconciler/reconciler.go index cb20b8c8..e20ac44d 100644 --- a/internal/reconciler/reconciler.go +++ b/internal/reconciler/reconciler.go @@ -100,6 +100,11 @@ type BalanceChange struct { Difference string `json:"difference,omitempty"` } +// Helper functions are used by Reconciler to compare +// computed balances from a block with the balance calculated +// by the node. Defining an interface allows the client to determine +// what sort of storage layer they want to use to provide the required +// information. type Helper interface { BlockExists( ctx context.Context, @@ -118,6 +123,9 @@ type Helper interface { ) (*types.Amount, *types.BlockIdentifier, error) } +// Handler is called by Reconciler after a reconciliation +// is performed. When a reconciliation failure is observed, +// it is up to the client to halt syncing or log the result. type Handler interface { ReconciliationFailed( ctx context.Context, diff --git a/internal/storage/block_storage.go b/internal/storage/block_storage.go index ded27a48..cac9ac59 100644 --- a/internal/storage/block_storage.go +++ b/internal/storage/block_storage.go @@ -118,19 +118,23 @@ func getHashKey(hash string, isBlock bool) []byte { return hashBytes(fmt.Sprintf("%s:%s", transactionHashNamespace, hash)) } +// GetBalanceKey returns a deterministic hash of an types.Account + types.Currency. func GetBalanceKey(account *types.AccountIdentifier, currency *types.Currency) []byte { return hashBytes( fmt.Sprintf("%s/%s/%s", balanceNamespace, types.Hash(account), types.Hash(currency)), ) } +// Helper functions are used by BlockStorage to process blocks. Defining an +// interface allows the client to determine if they wish to query the node for +// certain information or use another datastore. type Helper interface { AccountBalance( ctx context.Context, account *types.AccountIdentifier, currency *types.Currency, block *types.BlockIdentifier, - ) (*types.Amount, error) // returns an error if lookupBalanceByBlock disabled + ) (*types.Amount, error) SkipOperation( ctx context.Context, @@ -407,6 +411,8 @@ func parseBalanceEntry(buf []byte) (*balanceEntry, error) { return &bal, nil } +// SetNewStartIndex attempts to remove all blocks +// greater than or equal to the startIndex. func (b *BlockStorage) SetNewStartIndex( ctx context.Context, startIndex int64, @@ -445,6 +451,8 @@ func (b *BlockStorage) SetNewStartIndex( return nil } +// SetBalance allows a client to set the balance of an account in a database +// transaction. This is particularly useful for bootstrapping balances. func (b *BlockStorage) SetBalance( ctx context.Context, dbTransaction DatabaseTransaction, @@ -758,7 +766,7 @@ func (b *BlockStorage) CreateBlockCache(ctx context.Context) []*types.BlockIdent return cache } - for len(cache) < syncer.ReorgCache { + for len(cache) < syncer.PastBlockSize { block, err := b.GetBlock(ctx, head) if err != nil { return cache diff --git a/internal/syncer/syncer.go b/internal/syncer/syncer.go index a0125a31..f4b4e8f9 100644 --- a/internal/syncer/syncer.go +++ b/internal/syncer/syncer.go @@ -31,10 +31,16 @@ const ( // to try and sync in a given SyncCycle. maxSync = 999 - ReorgCache = 20 + // PastBlockSize is the maximum number of previously + // processed blocks we keep in the syncer to handle + // reorgs correctly. If there is a reorg greater than + // PastBlockSize, it will not be handled correctly. + // + // TODO: make configurable + PastBlockSize = 20 ) -// SyncHandler is called at various times during the sync cycle +// Handler is called at various times during the sync cycle // to handle different events. It is common to write logs or // perform reconciliation in the sync processor. type Handler interface { @@ -49,6 +55,12 @@ type Handler interface { ) error } +// Syncer coordinates blockchain syncing without relying on +// a storage interface. Instead, it calls a provided Handler +// whenever a block is added or removed. This provides the client +// the opportunity to define the logic used to handle each new block. +// In the rosetta-cli, we handle reconciliation, state storage, and +// logging in the handler. type Syncer struct { network *types.NetworkIdentifier fetcher *fetcher.Fetcher @@ -69,6 +81,8 @@ type Syncer struct { pastBlocks []*types.BlockIdentifier } +// New creates a new Syncer. If pastBlocks is left nil, it will +// be set to an empty slice. func New( network *types.NetworkIdentifier, fetcher *fetcher.Fetcher, @@ -203,7 +217,7 @@ func (s *Syncer) processBlock( } s.pastBlocks = append(s.pastBlocks, block.BlockIdentifier) - if len(s.pastBlocks) > ReorgCache { + if len(s.pastBlocks) > PastBlockSize { s.pastBlocks = s.pastBlocks[1:] } s.nextIndex = block.BlockIdentifier.Index + 1 From 1da24db72b0428e19dace390ce53cceaf4245533 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 7 May 2020 08:56:06 -0700 Subject: [PATCH 29/31] Update README --- README.md | 150 ++++++++++++++++++++++++++++-- cmd/check.go | 29 +++++- cmd/create_configuration.go | 19 +++- cmd/view_account.go | 19 ++-- cmd/view_block.go | 16 +++- internal/reconciler/reconciler.go | 2 +- 6 files changed, 204 insertions(+), 31 deletions(-) diff --git a/README.md b/README.md index f8a01f5e..848925f2 100644 --- a/README.md +++ b/README.md @@ -17,13 +17,6 @@ in this specification will enable exchanges, block explorers, and wallets to integrate with much less communication overhead and network-specific work. -## TODO: -! Load block cache on restart from storage (to ensure reorgs are handled correctly) -! Add ability to view a block (view:block, view:account) - * add examples in README -! Create a spec file command - * add examples in README - ## Install ``` go get github.com/coinbase/rosetta-cli @@ -31,18 +24,154 @@ go get github.com/coinbase/rosetta-cli ## Usage ``` +CLI for the Rosetta API + +Usage: + rosetta-cli [command] + +Available Commands: + check Check the correctness of a Rosetta Node API Server + create:configuration Generate a static configuration file for the Asserter + help Help about any command + view:account View an account balance + view:block View a block + +Flags: + -h, --help help for rosetta-cli + --server-url string base URL for a Rosetta server (default "http://localhost:8080") + +Use "rosetta-cli [command] --help" for more information about a command. +``` + +### check +``` +Check all server responses are properly constructed, that +there are no duplicate blocks and transactions, that blocks can be processed +from genesis to the current block (re-orgs handled automatically), and that +computed balance changes are equal to balance changes reported by the node. + +When re-running this command, it will start where it left off if you specify +some --data-dir. Otherwise, it will create a new temporary directory and start +again from the genesis block. If you want to discard some number of blocks +populate the --start flag with some block index. Starting from a given index +can be useful to debug a small range of blocks for issues but it is highly +recommended you sync from start to finish to ensure all correctness checks +are performed. + +By default, account balances are looked up at specific heights (instead of +only at the current block). If your node does not support this functionality +set --lookup-balance-by-block to false. This will make reconciliation much +less efficient but it will still work. + +To debug an INACTIVE account reconciliation error, set the +--interesting-accounts flag to the absolute path of a JSON file containing +accounts that will be actively checked for balance changes at each block. This +will return an error at the block where a balance change occurred with no +corresponding operations. + +If your blockchain has a genesis allocation of funds and you set +--lookup-balance-by-block to false, you must provide an +absolute path to a JSON file containing initial balances with the +--bootstrap-balances flag. You can look at the examples folder for an example +of what one of these files looks like. + +Usage: + rosetta-cli check [flags] + +Flags: + --account-concurrency uint concurrency to use while fetching accounts during reconciliation (default 8) + --block-concurrency uint concurrency to use while fetching blocks (default 8) + --bootstrap-balances string Absolute path to a file used to bootstrap balances before starting syncing. + Populating this value after beginning syncing will return an error. + --data-dir string folder used to store logs and any data used to perform validation + --end int block index to stop syncing (default -1) + --exempt-accounts string Absolute path to a file listing all accounts to exempt from balance + tracking and reconciliation. Look at the examples directory for an example of + how to structure this file. + --halt-on-reconciliation-error Determines if block processing should halt on a reconciliation + error. It can be beneficial to collect all reconciliation errors or silence + reconciliation errors during development. (default true) + -h, --help help for check + --interesting-accounts string Absolute path to a file listing all accounts to check on each block. Look + at the examples directory for an example of how to structure this file. + --log-balance-changes log balance changes + --log-blocks log processed blocks + --log-reconciliations log balance reconciliations + --log-transactions log processed transactions + --lookup-balance-by-block When set to true, balances are looked up at the block where a balance + change occurred instead of at the current block. Blockchains that do not support + historical balance lookup should set this to false. (default true) + --start int block index to start syncing (default -1) + --transaction-concurrency uint concurrency to use while fetching transactions (if required) (default 16) + +Global Flags: + --server-url string base URL for a Rosetta server (default "http://localhost:8080") ``` -### check:complete +### create:configuration ``` +In production deployments, it is useful to initialize the response +Asserter (https://github.com/coinbase/rosetta-sdk-go/tree/master/asserter) using +a static configuration instead of intializing a configuration dynamically +from the node. This allows a client to error on new types/statuses that may +have been added in an update instead of silently erroring. + +To use this command, simply provide an absolute path as the argument for where +the configuration file should be saved (in JSON). Populate the optional +--server-url flag with the url of the server to generate the configuration +from. + +Usage: + rosetta-cli create:configuration [flags] + +Flags: + -h, --help help for create:configuration + +Global Flags: + --server-url string base URL for a Rosetta server (default "http://localhost:8080") ``` -### check:quick +### view:account ``` +While debugging, it is often useful to inspect the state +of an account at a certain block. This command allows you to look up +any account by providing a JSON representation of a types.AccountIdentifier +(and optionally a height to perform the query). + +For example, you could run view:account '{"address":"interesting address"}' 1000 +to lookup the balance of an interesting address at block 1000. Allowing the +address to specified as JSON allows for querying by SubAccountIdentifier. + +Usage: + rosetta-cli view:account [flags] + +Flags: + -h, --help help for view:account + +Global Flags: + --server-url string base URL for a Rosetta server (default "http://localhost:8080") ``` -### check:account +### view:block ``` +While debugging a Node API implementation, it can be very +useful to inspect block contents. This command allows you to fetch any +block by index to inspect its contents. It uses the +fetcher (https://github.com/coinbase/rosetta-sdk-go/tree/master/fetcher) package +to automatically get all transactions in the block and assert the format +of the block is correct before printing. + +If this command errors, it is likely because the block you are trying to +fetch is formatted incorrectly. + +Usage: + rosetta-cli view:block [flags] + +Flags: + -h, --help help for view:block + +Global Flags: + --server-url string base URL for a Rosetta server (default "http://localhost:8080") ``` ## Development @@ -61,6 +190,7 @@ reconciliation of an account was successful. ### Repo Structure ``` cmd +examples // examples of different config files internal logger // logic to write syncing information to stdout/files processor // Helper/Handler implementations for reconciler, storage, and syncer diff --git a/cmd/check.go b/cmd/check.go index eb45ad1d..2f44743a 100644 --- a/cmd/check.go +++ b/cmd/check.go @@ -49,15 +49,36 @@ const ( var ( checkCmd = &cobra.Command{ Use: "check", - Short: "Run a full check of the correctness of a Rosetta server", + Short: "Check the correctness of a Rosetta Node API Server", Long: `Check all server responses are properly constructed, that there are no duplicate blocks and transactions, that blocks can be processed from genesis to the current block (re-orgs handled automatically), and that computed balance changes are equal to balance changes reported by the node. -When re-running this command, it will start where it left off. If you want -to discard some number of blocks populate the --start flag with some block -index less than the last computed block index.`, +When re-running this command, it will start where it left off if you specify +some --data-dir. Otherwise, it will create a new temporary directory and start +again from the genesis block. If you want to discard some number of blocks +populate the --start flag with some block index. Starting from a given index +can be useful to debug a small range of blocks for issues but it is highly +recommended you sync from start to finish to ensure all correctness checks +are performed. + +By default, account balances are looked up at specific heights (instead of +only at the current block). If your node does not support this functionality +set --lookup-balance-by-block to false. This will make reconciliation much +less efficient but it will still work. + +To debug an INACTIVE account reconciliation error, set the +--interesting-accounts flag to the absolute path of a JSON file containing +accounts that will be actively checked for balance changes at each block. This +will return an error at the block where a balance change occurred with no +corresponding operations. + +If your blockchain has a genesis allocation of funds and you set +--lookup-balance-by-block to false, you must provide an +absolute path to a JSON file containing initial balances with the +--bootstrap-balances flag. You can look at the examples folder for an example +of what one of these files looks like.`, Run: runCheckCmd, } diff --git a/cmd/create_configuration.go b/cmd/create_configuration.go index 9e5cbdde..fd66cc62 100644 --- a/cmd/create_configuration.go +++ b/cmd/create_configuration.go @@ -28,16 +28,27 @@ import ( ) const ( + // fileMode 0600 indicates that the user/owner can read and write + // but can't execute. fileMode = 0600 ) var ( createConfigurationCmd = &cobra.Command{ Use: "create:configuration", - Short: "", - Long: ``, - Run: runCreateConfigurationCmd, - Args: cobra.ExactArgs(1), + Short: "Generate a static configuration file for the Asserter", + Long: `In production deployments, it is useful to initialize the response +Asserter (https://github.com/coinbase/rosetta-sdk-go/tree/master/asserter) using +a static configuration instead of intializing a configuration dynamically +from the node. This allows a client to error on new types/statuses that may +have been added in an update instead of silently erroring. + +To use this command, simply provide an absolute path as the argument for where +the configuration file should be saved (in JSON). Populate the optional +--server-url flag with the url of the server to generate the configuration +from.`, + Run: runCreateConfigurationCmd, + Args: cobra.ExactArgs(1), } ) diff --git a/cmd/view_account.go b/cmd/view_account.go index cb01da3e..994d413d 100644 --- a/cmd/view_account.go +++ b/cmd/view_account.go @@ -30,10 +30,17 @@ import ( var ( viewAccountCmd = &cobra.Command{ Use: "view:account", - Short: "", - Long: ``, - Run: runViewAccountCmd, - Args: cobra.MinimumNArgs(1), + Short: "View an account balance", + Long: `While debugging, it is often useful to inspect the state +of an account at a certain block. This command allows you to look up +any account by providing a JSON representation of a types.AccountIdentifier +(and optionally a height to perform the query). + +For example, you could run view:account '{"address":"interesting address"}' 1000 +to lookup the balance of an interesting address at block 1000. Allowing the +address to specified as JSON allows for querying by SubAccountIdentifier.`, + Run: runViewAccountCmd, + Args: cobra.MinimumNArgs(1), } ) @@ -55,10 +62,6 @@ func runViewAccountCmd(cmd *cobra.Command, args []string) { ) // Initialize the fetcher's asserter - // - // Behind the scenes this makes a call to get the - // network status and uses the response to inform - // the asserter what are valid responses. primaryNetwork, _, err := newFetcher.InitializeAsserter(ctx) if err != nil { log.Fatal(err) diff --git a/cmd/view_block.go b/cmd/view_block.go index 12633e8b..18bcfae8 100644 --- a/cmd/view_block.go +++ b/cmd/view_block.go @@ -28,10 +28,18 @@ import ( var ( viewBlockCmd = &cobra.Command{ Use: "view:block", - Short: "", - Long: ``, - Run: runViewBlockCmd, - Args: cobra.ExactArgs(1), + Short: "View a block", + Long: `While debugging a Node API implementation, it can be very +useful to inspect block contents. This command allows you to fetch any +block by index to inspect its contents. It uses the +fetcher (https://github.com/coinbase/rosetta-sdk-go/tree/master/fetcher) package +to automatically get all transactions in the block and assert the format +of the block is correct before printing. + +If this command errors, it is likely because the block you are trying to +fetch is formatted incorrectly.`, + Run: runViewBlockCmd, + Args: cobra.ExactArgs(1), } ) diff --git a/internal/reconciler/reconciler.go b/internal/reconciler/reconciler.go index e20ac44d..bffa17fb 100644 --- a/internal/reconciler/reconciler.go +++ b/internal/reconciler/reconciler.go @@ -572,7 +572,7 @@ func (r *Reconciler) reconcileInactiveAccounts( if err != nil { time.Sleep(inactiveReconciliationSleep) log.Printf( - "%s: unable to get current block for inactive reconciliation\n", + "%s: waiting to start inactive reconciliation until current block set\n", err.Error(), ) continue From 13c24ddd3bd9dc50bcab58e1f011ecd3901e3b12 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 7 May 2020 15:44:55 -0700 Subject: [PATCH 30/31] Cleanup dependencies --- Makefile | 18 ++- c.out | 374 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 5 +- go.sum | 80 ++++++++++-- 4 files changed, 457 insertions(+), 20 deletions(-) create mode 100644 c.out diff --git a/Makefile b/Makefile index ddb8b8fe..3603f825 100644 --- a/Makefile +++ b/Makefile @@ -2,16 +2,14 @@ check-license shorten-lines salus validate watch-blocks \ watch-transactions watch-balances watch-reconciliations \ view-block-benchmarks view-account-benchmarks -LICENCE_SCRIPT=addlicense -c "Coinbase, Inc." -l "apache" -v -GO_INSTALL=GO111MODULE=off go get +ADDLICENSE_CMD=go run github.com/google/addlicense +ADDLICENCE_SCRIPT=${ADDLICENSE_CMD} -c "Coinbase, Inc." -l "apache" -v +GOLINES_CMD=go run github.com/segmentio/golines +GOVERALLS_CMD=go run github.com/mattn/goveralls TEST_SCRIPT=go test -v ./internal/... deps: go get ./... - go get github.com/stretchr/testify - ${GO_INSTALL} github.com/google/addlicense - ${GO_INSTALL} github.com/segmentio/golines - ${GO_INSTALL} github.com/mattn/goveralls lint: golangci-lint run -v \ @@ -28,16 +26,16 @@ test: test-cover: ${TEST_SCRIPT} -coverprofile=c.out -covermode=count - goveralls -coverprofile=c.out -repotoken ${COVERALLS_TOKEN} + ${GOVERALLS_CMD} -coverprofile=c.out -repotoken ${COVERALLS_TOKEN} add-license: - ${LICENCE_SCRIPT} . + ${ADDLICENCE_SCRIPT} . check-license: - ${LICENCE_SCRIPT} -check . + ${ADDLICENCE_SCRIPT} -check . shorten-lines: - golines -w --shorten-comments internal cmd + ${GOLINES_CMD} -w --shorten-comments internal cmd salus: docker run --rm -t -v ${PWD}:/home/repo coinbase/salus diff --git a/c.out b/c.out new file mode 100644 index 00000000..4dd01683 --- /dev/null +++ b/c.out @@ -0,0 +1,374 @@ +mode: count +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:187.15,201.26 2 1 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:212.2,212.10 1 1 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:201.26,205.3 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:205.8,210.3 1 1 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:221.9,223.48 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:247.2,247.29 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:272.2,272.12 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:223.48,226.41 2 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:234.3,234.18 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:239.3,244.5 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:226.41,228.52 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:228.52,230.10 2 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:234.18,235.12 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:247.29,250.36 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:254.3,254.41 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:250.36,252.4 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:254.41,255.11 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:256.33,256.33 0 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:257.12,258.59 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:261.8,263.41 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:263.41,264.11 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:265.33,265.33 0 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:266.22,267.21 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:284.34,287.16 2 8 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:295.2,295.34 1 7 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:305.2,306.16 2 6 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:313.2,313.13 1 6 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:322.2,328.16 2 5 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:337.2,337.42 1 4 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:346.2,347.16 2 3 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:351.2,351.57 1 3 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:287.16,292.3 1 1 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:295.34,302.3 1 1 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:306.16,312.3 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:313.13,319.3 1 1 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:328.16,335.3 1 1 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:337.42,344.3 1 1 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:347.16,349.3 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:362.43,363.29 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:369.2,376.3 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:363.29,368.3 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:389.9,394.23 2 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:478.2,478.12 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:394.23,404.17 2 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:445.3,446.15 2 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:450.3,450.31 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:467.3,475.4 2 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:404.17,405.46 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:430.4,430.36 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:436.4,436.41 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:442.4,442.14 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:405.46,411.31 2 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:417.5,427.10 3 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:411.31,413.14 2 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:430.36,433.10 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:436.41,439.10 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:446.15,448.4 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:450.31,460.18 2 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:464.4,464.14 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:460.18,462.5 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:485.3,488.73 2 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:493.2,493.39 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:488.73,491.3 2 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:493.39,501.3 3 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:508.10,510.47 2 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:514.2,516.19 2 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:510.47,512.3 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:526.9,527.6 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:527.6,528.10 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:529.21,530.20 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:531.41,532.51 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:536.4,542.18 2 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:546.4,554.18 2 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:532.51,533.13 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:542.18,544.5 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:554.18,556.5 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:567.9,568.23 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:615.2,615.12 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:568.23,572.17 2 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:581.3,583.84 2 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:572.17,578.12 3 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:583.84,594.18 5 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:598.4,606.18 2 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:594.18,596.5 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:606.18,608.5 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:609.9,612.4 2 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:620.59,622.54 2 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:632.2,632.33 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:636.2,636.12 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:622.54,623.21 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:627.3,627.21 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:623.21,625.4 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:627.21,629.4 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:632.33,634.3 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:644.26,645.29 1 3 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:653.2,653.70 1 1 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:645.29,646.41 1 5 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:650.3,650.16 1 2 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:646.41,647.12 1 3 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:669.8,670.24 1 6 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:677.2,677.14 1 3 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:670.24,672.45 1 15 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:672.45,674.4 1 3 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:689.43,696.16 2 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:700.2,701.16 2 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:705.2,705.41 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:696.16,698.3 1 0 +github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:701.16,703.3 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:92.11,94.17 2 1 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:98.2,104.3 1 1 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:94.17,96.3 1 1 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:110.9,116.16 2 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:120.2,122.17 2 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:127.2,128.12 2 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:116.16,118.3 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:122.17,125.3 2 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:137.24,138.23 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:142.2,142.20 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:155.2,155.29 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:159.2,159.36 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:163.2,163.29 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:138.23,140.3 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:142.20,148.17 2 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:152.3,152.56 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:148.17,150.4 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:155.29,157.3 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:159.36,161.3 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:168.41,169.28 1 7 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:174.2,174.48 1 6 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:183.2,184.58 2 5 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:192.2,192.30 1 3 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:169.28,171.3 1 1 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:174.48,180.3 1 1 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:184.58,185.45 1 2 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:189.3,189.30 1 1 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:185.45,187.4 1 1 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:198.9,200.16 2 7 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:204.2,204.18 1 5 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:214.2,215.16 2 4 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:219.2,220.39 2 4 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:223.2,224.12 2 4 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:200.16,202.3 1 2 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:204.18,206.17 2 1 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:209.3,211.13 3 1 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:206.17,208.4 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:215.16,217.3 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:220.39,222.3 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:230.9,232.16 2 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:236.2,236.30 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:261.2,261.12 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:232.16,234.3 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:236.30,238.10 2 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:256.3,256.51 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:238.10,246.18 2 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:246.18,248.5 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:249.9,254.4 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:256.51,258.4 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:270.9,273.52 2 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:277.2,277.6 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:301.2,302.12 2 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:273.52,275.3 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:277.6,282.17 2 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:285.3,285.11 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:289.3,292.17 3 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:296.3,296.23 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:282.17,284.4 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:285.11,286.9 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:292.17,294.4 1 0 +github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:296.23,298.4 1 0 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:30.74,32.16 2 8 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:36.2,38.8 1 8 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:32.16,34.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:43.58,45.2 1 8 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:61.23,65.2 1 60 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:68.59,70.2 1 18 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:74.54,76.2 1 50 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:83.9,85.2 1 34 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:91.25,94.34 3 51 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:100.2,100.40 1 29 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:105.2,105.16 1 29 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:109.2,109.25 1 29 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:94.34,96.3 1 22 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:96.8,96.23 1 29 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:96.23,98.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:100.40,104.3 3 29 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:105.16,107.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:113.75,115.2 1 4 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:122.9,123.49 1 1 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:123.49,125.3 1 1 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:132.25,134.47 2 6 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:151.2,151.34 1 6 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:157.2,157.25 1 3 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:134.47,136.17 2 6 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:140.3,140.41 1 3 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:145.3,145.17 1 3 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:149.3,149.13 1 3 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:136.17,138.4 1 3 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:140.41,144.4 3 3 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:145.17,147.4 1 0 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:151.34,153.3 1 3 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:153.8,153.23 1 3 +github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:153.23,155.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:93.36,96.16 3 78 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:100.2,100.19 1 78 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:96.16,98.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:103.31,105.2 1 23 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:107.65,111.2 1 15 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:113.51,114.13 1 12 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:118.2,118.72 1 5 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:114.13,116.3 1 7 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:122.87,126.2 1 28 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:157.17,162.2 1 9 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:169.23,171.2 1 57 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:177.35,182.16 4 14 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:186.2,186.13 1 14 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:190.2,193.16 4 9 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:197.2,197.30 1 9 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:182.16,184.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:186.13,188.3 1 5 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:193.16,195.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:206.9,209.16 3 9 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:213.2,213.61 1 9 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:209.16,211.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:220.25,225.16 4 8 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:229.2,229.13 1 8 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:233.2,235.16 3 5 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:239.2,239.27 1 5 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:225.16,227.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:229.13,231.3 1 3 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:235.16,237.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:248.9,251.16 3 10 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:255.2,255.13 1 10 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:259.2,259.13 1 2 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:267.2,271.3 1 1 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:251.16,253.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:255.13,257.3 1 8 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:259.13,265.3 1 1 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:280.40,285.16 5 6 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:290.2,291.16 2 6 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:296.2,297.16 2 6 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:302.2,302.41 1 5 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:309.2,310.16 2 4 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:314.2,314.33 1 4 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:320.2,320.91 1 4 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:324.2,324.48 1 4 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:328.2,328.21 1 4 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:285.16,287.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:291.16,293.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:297.16,299.3 1 1 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:302.41,304.17 2 4 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:304.17,306.4 1 1 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:310.16,312.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:314.33,315.96 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:315.96,317.4 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:320.91,322.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:324.48,326.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:338.40,340.16 2 1 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:344.2,348.16 4 1 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:352.2,352.33 1 1 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:359.2,359.41 1 1 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:367.2,368.16 2 1 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:373.2,373.84 1 1 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:377.2,377.97 1 1 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:381.2,381.48 1 1 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:385.2,385.21 1 1 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:340.16,342.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:348.16,350.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:352.33,353.90 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:353.90,355.4 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:359.41,361.17 2 1 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:361.17,363.4 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:368.16,370.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:373.84,375.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:377.97,379.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:381.48,383.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:393.62,396.16 3 9 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:400.2,400.25 1 9 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:396.16,398.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:403.59,407.16 4 13 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:411.2,411.18 1 13 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:407.16,409.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:419.9,421.42 2 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:424.2,424.16 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:428.2,428.29 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:436.2,437.36 2 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:451.2,451.12 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:421.42,423.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:424.16,426.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:428.29,434.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:437.36,440.17 3 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:444.3,444.69 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:448.3,448.42 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:440.17,442.4 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:444.69,446.4 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:462.9,469.16 3 2 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:473.2,473.63 1 2 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:477.2,477.12 1 2 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:469.16,471.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:473.63,475.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:488.9,489.28 1 10 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:493.2,496.16 3 9 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:500.2,501.12 2 9 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:517.2,518.16 2 9 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:522.2,523.9 2 9 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:527.2,527.28 1 9 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:538.2,545.16 2 7 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:549.2,549.47 1 7 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:489.28,491.3 1 1 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:496.16,498.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:501.12,503.17 2 4 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:507.3,507.40 1 4 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:503.17,505.4 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:508.8,510.17 2 5 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:514.3,514.31 1 5 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:510.17,512.4 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:518.16,520.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:523.9,525.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:527.28,536.3 1 2 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:545.16,547.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:559.50,565.16 5 10 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:573.2,573.13 1 10 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:600.2,601.16 2 9 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:605.2,605.51 1 9 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:565.16,567.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:573.13,575.17 2 1 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:579.3,588.17 4 1 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:592.3,593.17 2 1 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:597.3,597.32 1 1 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:575.17,577.4 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:588.17,590.4 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:593.17,595.4 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:601.16,603.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:626.9,629.16 2 6 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:633.2,634.72 2 5 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:638.2,639.33 2 4 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:644.2,647.35 3 3 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:680.2,681.16 2 1 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:685.2,686.12 2 1 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:629.16,631.3 1 1 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:634.72,636.3 1 1 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:639.33,641.3 1 1 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:647.35,650.10 2 3 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:654.3,654.29 1 2 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:658.3,675.17 3 1 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:650.10,652.4 1 1 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:654.29,656.4 1 1 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:675.17,677.4 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:681.16,683.3 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:699.40,701.40 2 9 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:752.2,753.40 2 9 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:757.2,757.24 1 9 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:701.40,702.36 1 12 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:702.36,707.18 2 14 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:710.4,710.12 1 14 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:714.4,716.20 3 7 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:727.4,733.11 3 7 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:743.4,744.18 2 2 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:747.4,748.29 2 2 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:707.18,709.5 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:710.12,711.13 1 7 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:716.20,718.12 2 3 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:722.5,723.50 2 3 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:718.12,720.6 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:733.11,740.13 2 5 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:744.18,746.5 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:753.40,755.3 1 5 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:762.87,765.16 3 3 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:769.2,769.40 1 2 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:781.2,781.14 1 0 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:765.16,767.3 1 1 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:769.40,771.17 2 5 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:775.3,778.37 3 3 +github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:771.17,773.4 1 2 diff --git a/go.mod b/go.mod index 55ec8543..46b326b8 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,10 @@ go 1.13 require ( github.com/coinbase/rosetta-sdk-go v0.1.8 - github.com/davecgh/go-spew v1.1.1 github.com/dgraph-io/badger v1.6.0 - github.com/pkg/errors v0.8.1 + github.com/google/addlicense v0.0.0-20200422172452-68a83edd47bc // indirect + github.com/mattn/goveralls v0.0.5 // indirect + github.com/segmentio/golines v0.0.0-20200306054842-869934f8da7b // indirect github.com/spf13/cobra v0.0.5 github.com/stretchr/testify v1.5.1 golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a diff --git a/go.sum b/go.sum index 714c8fdd..afacd67c 100644 --- a/go.sum +++ b/go.sum @@ -1,23 +1,25 @@ github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9 h1:HD8gA2tkByhMAwYaFAX9w2l7vxvBQ5NMoxDrkhqhtn4= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d h1:UQZhZ2O0vMHr2cI+DC1Mbh0TJxzA3RcLoMsFw+aXw7E= +github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= -github.com/coinbase/rosetta-sdk-go v0.1.5 h1:fkYLDs8f7RuwKDJiaZv4qtVRntCOAcSm6VeWiHH9L8Y= -github.com/coinbase/rosetta-sdk-go v0.1.5/go.mod h1:lbmGsBpBiSZWP4WQqEW2CuTweGcU/ioQHwZ8CYo6yO8= -github.com/coinbase/rosetta-sdk-go v0.1.6 h1:l6vyt+Gad7TWIy2Tf7giXO8jniBcLPhhW6021QFgIq0= -github.com/coinbase/rosetta-sdk-go v0.1.6/go.mod h1:lbmGsBpBiSZWP4WQqEW2CuTweGcU/ioQHwZ8CYo6yO8= -github.com/coinbase/rosetta-sdk-go v0.1.7 h1:4NRMHWPSpmFNohtzQ/Nv9wsPLBEbgccNBbjwZTG1Owk= -github.com/coinbase/rosetta-sdk-go v0.1.7/go.mod h1:lbmGsBpBiSZWP4WQqEW2CuTweGcU/ioQHwZ8CYo6yO8= github.com/coinbase/rosetta-sdk-go v0.1.8 h1:YJf5Us7Pl4G8CRBZ5+LEUIV4k9ZI/18nFloPaHBySFk= github.com/coinbase/rosetta-sdk-go v0.1.8/go.mod h1:y1wXRc1wod4fEp3jhW+9D6MSFtnm/9X5yNxJv92Cv2E= -github.com/coinbase/rosetta-validator v0.1.2 h1:d1i/XuZu3tPEnc/HKLzixK0yXbMyg5UEzS2qba7tv5I= -github.com/coinbase/rosetta-validator v0.1.2/go.mod h1:SgJPFHi1Ikr+tC5MkUhkrnLQKLT0Qzf7g+s1rnZ3kGo= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/dave/dst v0.23.1 h1:2obX6c3RqALrEOp6u01qsqPvwp0t+RpOp9O4Bf9KhXs= +github.com/dave/dst v0.23.1/go.mod h1:LjPcLEauK4jC5hQ1fE/wr05O41zK91Pr4Qs22Ljq7gs= +github.com/dave/gopackages v0.0.0-20170318123100-46e7023ec56e/go.mod h1:i00+b/gKdIDIxuLDFob7ustLAVqhsZRk2qVZrArELGQ= +github.com/dave/jennifer v1.2.0/go.mod h1:fIb+770HOpJ2fmN9EPPKOqm1vMGhB+TwXKMZhrIygKg= +github.com/dave/kerr v0.0.0-20170318121727-bc25dd6abe8e/go.mod h1:qZqlPyPvfsDJt+3wHJ1EvSXDuVjFTK0j2p/ca+gtsb8= +github.com/dave/rebecca v0.9.1/go.mod h1:N6XYdMD/OKw3lkF3ywh8Z6wPGuwNFDNtWYEMFWEmXBA= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -28,23 +30,48 @@ github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUn github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/google/addlicense v0.0.0-20200422172452-68a83edd47bc h1:CHWlqgYPu3FMUOyAno2lVDyI9wmexZEuV6/nDvsvETc= +github.com/google/addlicense v0.0.0-20200422172452-68a83edd47bc/go.mod h1:EMjYTRimagHs1FwlIqKyX3wAM0u3rA+McvlIIWmSamA= +github.com/google/pprof v0.0.0-20181127221834-b4f47329b966/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.10/go.mod h1:qgIWMr58cqv1PHHyhnkY9lrL7etaEgOFcMEpPG5Rm84= +github.com/mattn/goveralls v0.0.5 h1:spfq8AyZ0cCk57Za6/juJ5btQxeE1FaEGMdfcI+XO48= +github.com/mattn/goveralls v0.0.5/go.mod h1:Xg2LHi51faXLyKXwsndxiW6uxEEQT9+3sjGzzwU4xy0= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/mapstructure v1.3.0 h1:iDwIio/3gk2QtLLEsqU5lInaMzos0hDTz8a6lazSFVw= github.com/mitchellh/mapstructure v1.3.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/ginkgo v1.10.2/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= +github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/segmentio/golines v0.0.0-20200306054842-869934f8da7b h1:Jk5Swz/AfwWl5yc/yquzUmNesPF2aTFuafpjikzczRg= +github.com/segmentio/golines v0.0.0-20200306054842-869934f8da7b/go.mod h1:K7zjgP8yJ/U8nb8nxaSykalAKSvbqr6TNbd9B7zzBFU= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= +github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s= @@ -53,25 +80,62 @@ github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb6 github.com/spf13/pflag v1.0.3 h1:zPAT6CGy6wXeQ7NtTnaTerfKOsV6V6F8agHXFiazDkg= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= +github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/x-cray/logrus-prefixed-formatter v0.5.2 h1:00txxvfBM9muc0jiLIEAkAcIMJzfthRT6usrui8uGmg= +github.com/x-cray/logrus-prefixed-formatter v0.5.2/go.mod h1:2duySbKsL6M18s5GU7VPsoEPHyzalCE06qoARUCeBBE= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +golang.org/x/arch v0.0.0-20180920145803-b19384d3c130/go.mod h1:cYlCBUl1MsqxdiKgmc4uh7TxZfWSFLOGSRR090WDxt8= +golang.org/x/crypto v0.0.0-20181127143415-eb0de9b17e85/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a h1:WXEvlFVvvGxCJLG6REjsT03iWnKLEWinaScsxF2Vm2o= golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180903190138-2b024373dcd9/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191008105621-543471e840be/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191024172528-b4ff53e7a1cb h1:ZxSglHghKPYD8WDeRUzRJrUJtDF0PxsTUSxyqr9/5BI= +golang.org/x/sys v0.0.0-20191024172528-b4ff53e7a1cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20181127232545-e782529d0ddd/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/tools v0.0.0-20191024220359-3d91e92cde03 h1:4gtJXHJ9ud0q8MNSDxJsRU/WH+afypbe4Vk4zq+8qow= +golang.org/x/tools v0.0.0-20191024220359-3d91e92cde03/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200113040837-eac381796e91 h1:OOkytthzFBKHY5EfEgLUabprb0LtJVkQtNxAQ02+UE4= +golang.org/x/tools v0.0.0-20200113040837-eac381796e91/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gopkg.in/alecthomas/kingpin.v2 v2.2.6 h1:jMFz6MfLP0/4fUyZle81rXUoxOBFi19VUFKVDOQfozc= +gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= +gopkg.in/src-d/go-billy.v4 v4.3.0/go.mod h1:tm33zBoOwxjYHZIE+OV8bxTWFMJLrconzFMd38aARFk= +gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= +gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= From 6e13cf23fe1675c14b6107db9c1f298e09d1b9ba Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Thu, 7 May 2020 16:16:05 -0700 Subject: [PATCH 31/31] nits --- Makefile | 5 + c.out | 374 --------------------------- cmd/check.go | 1 - internal/processor/syncer_handler.go | 12 +- internal/reconciler/reconciler.go | 2 +- 5 files changed, 10 insertions(+), 384 deletions(-) delete mode 100644 c.out diff --git a/Makefile b/Makefile index 3603f825..a02a2477 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,11 @@ check-license shorten-lines salus validate watch-blocks \ watch-transactions watch-balances watch-reconciliations \ view-block-benchmarks view-account-benchmarks + +# To run the the following packages as commands, +# it is necessary to use `go run `. Running `go get` does +# not install any binaries that could be used to run +# the commands directly. ADDLICENSE_CMD=go run github.com/google/addlicense ADDLICENCE_SCRIPT=${ADDLICENSE_CMD} -c "Coinbase, Inc." -l "apache" -v GOLINES_CMD=go run github.com/segmentio/golines diff --git a/c.out b/c.out deleted file mode 100644 index 4dd01683..00000000 --- a/c.out +++ /dev/null @@ -1,374 +0,0 @@ -mode: count -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:187.15,201.26 2 1 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:212.2,212.10 1 1 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:201.26,205.3 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:205.8,210.3 1 1 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:221.9,223.48 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:247.2,247.29 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:272.2,272.12 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:223.48,226.41 2 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:234.3,234.18 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:239.3,244.5 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:226.41,228.52 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:228.52,230.10 2 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:234.18,235.12 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:247.29,250.36 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:254.3,254.41 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:250.36,252.4 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:254.41,255.11 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:256.33,256.33 0 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:257.12,258.59 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:261.8,263.41 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:263.41,264.11 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:265.33,265.33 0 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:266.22,267.21 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:284.34,287.16 2 8 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:295.2,295.34 1 7 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:305.2,306.16 2 6 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:313.2,313.13 1 6 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:322.2,328.16 2 5 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:337.2,337.42 1 4 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:346.2,347.16 2 3 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:351.2,351.57 1 3 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:287.16,292.3 1 1 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:295.34,302.3 1 1 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:306.16,312.3 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:313.13,319.3 1 1 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:328.16,335.3 1 1 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:337.42,344.3 1 1 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:347.16,349.3 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:362.43,363.29 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:369.2,376.3 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:363.29,368.3 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:389.9,394.23 2 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:478.2,478.12 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:394.23,404.17 2 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:445.3,446.15 2 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:450.3,450.31 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:467.3,475.4 2 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:404.17,405.46 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:430.4,430.36 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:436.4,436.41 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:442.4,442.14 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:405.46,411.31 2 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:417.5,427.10 3 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:411.31,413.14 2 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:430.36,433.10 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:436.41,439.10 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:446.15,448.4 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:450.31,460.18 2 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:464.4,464.14 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:460.18,462.5 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:485.3,488.73 2 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:493.2,493.39 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:488.73,491.3 2 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:493.39,501.3 3 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:508.10,510.47 2 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:514.2,516.19 2 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:510.47,512.3 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:526.9,527.6 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:527.6,528.10 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:529.21,530.20 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:531.41,532.51 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:536.4,542.18 2 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:546.4,554.18 2 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:532.51,533.13 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:542.18,544.5 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:554.18,556.5 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:567.9,568.23 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:615.2,615.12 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:568.23,572.17 2 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:581.3,583.84 2 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:572.17,578.12 3 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:583.84,594.18 5 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:598.4,606.18 2 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:594.18,596.5 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:606.18,608.5 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:609.9,612.4 2 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:620.59,622.54 2 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:632.2,632.33 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:636.2,636.12 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:622.54,623.21 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:627.3,627.21 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:623.21,625.4 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:627.21,629.4 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:632.33,634.3 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:644.26,645.29 1 3 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:653.2,653.70 1 1 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:645.29,646.41 1 5 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:650.3,650.16 1 2 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:646.41,647.12 1 3 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:669.8,670.24 1 6 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:677.2,677.14 1 3 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:670.24,672.45 1 15 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:672.45,674.4 1 3 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:689.43,696.16 2 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:700.2,701.16 2 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:705.2,705.41 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:696.16,698.3 1 0 -github.com/coinbase/rosetta-cli/internal/reconciler/reconciler.go:701.16,703.3 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:92.11,94.17 2 1 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:98.2,104.3 1 1 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:94.17,96.3 1 1 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:110.9,116.16 2 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:120.2,122.17 2 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:127.2,128.12 2 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:116.16,118.3 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:122.17,125.3 2 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:137.24,138.23 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:142.2,142.20 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:155.2,155.29 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:159.2,159.36 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:163.2,163.29 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:138.23,140.3 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:142.20,148.17 2 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:152.3,152.56 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:148.17,150.4 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:155.29,157.3 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:159.36,161.3 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:168.41,169.28 1 7 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:174.2,174.48 1 6 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:183.2,184.58 2 5 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:192.2,192.30 1 3 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:169.28,171.3 1 1 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:174.48,180.3 1 1 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:184.58,185.45 1 2 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:189.3,189.30 1 1 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:185.45,187.4 1 1 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:198.9,200.16 2 7 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:204.2,204.18 1 5 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:214.2,215.16 2 4 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:219.2,220.39 2 4 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:223.2,224.12 2 4 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:200.16,202.3 1 2 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:204.18,206.17 2 1 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:209.3,211.13 3 1 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:206.17,208.4 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:215.16,217.3 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:220.39,222.3 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:230.9,232.16 2 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:236.2,236.30 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:261.2,261.12 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:232.16,234.3 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:236.30,238.10 2 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:256.3,256.51 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:238.10,246.18 2 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:246.18,248.5 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:249.9,254.4 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:256.51,258.4 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:270.9,273.52 2 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:277.2,277.6 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:301.2,302.12 2 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:273.52,275.3 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:277.6,282.17 2 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:285.3,285.11 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:289.3,292.17 3 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:296.3,296.23 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:282.17,284.4 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:285.11,286.9 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:292.17,294.4 1 0 -github.com/coinbase/rosetta-cli/internal/syncer/syncer.go:296.23,298.4 1 0 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:30.74,32.16 2 8 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:36.2,38.8 1 8 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:32.16,34.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:43.58,45.2 1 8 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:61.23,65.2 1 60 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:68.59,70.2 1 18 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:74.54,76.2 1 50 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:83.9,85.2 1 34 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:91.25,94.34 3 51 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:100.2,100.40 1 29 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:105.2,105.16 1 29 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:109.2,109.25 1 29 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:94.34,96.3 1 22 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:96.8,96.23 1 29 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:96.23,98.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:100.40,104.3 3 29 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:105.16,107.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:113.75,115.2 1 4 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:122.9,123.49 1 1 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:123.49,125.3 1 1 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:132.25,134.47 2 6 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:151.2,151.34 1 6 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:157.2,157.25 1 3 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:134.47,136.17 2 6 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:140.3,140.41 1 3 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:145.3,145.17 1 3 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:149.3,149.13 1 3 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:136.17,138.4 1 3 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:140.41,144.4 3 3 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:145.17,147.4 1 0 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:151.34,153.3 1 3 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:153.8,153.23 1 3 -github.com/coinbase/rosetta-cli/internal/storage/badger_storage.go:153.23,155.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:93.36,96.16 3 78 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:100.2,100.19 1 78 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:96.16,98.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:103.31,105.2 1 23 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:107.65,111.2 1 15 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:113.51,114.13 1 12 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:118.2,118.72 1 5 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:114.13,116.3 1 7 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:122.87,126.2 1 28 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:157.17,162.2 1 9 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:169.23,171.2 1 57 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:177.35,182.16 4 14 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:186.2,186.13 1 14 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:190.2,193.16 4 9 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:197.2,197.30 1 9 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:182.16,184.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:186.13,188.3 1 5 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:193.16,195.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:206.9,209.16 3 9 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:213.2,213.61 1 9 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:209.16,211.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:220.25,225.16 4 8 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:229.2,229.13 1 8 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:233.2,235.16 3 5 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:239.2,239.27 1 5 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:225.16,227.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:229.13,231.3 1 3 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:235.16,237.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:248.9,251.16 3 10 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:255.2,255.13 1 10 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:259.2,259.13 1 2 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:267.2,271.3 1 1 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:251.16,253.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:255.13,257.3 1 8 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:259.13,265.3 1 1 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:280.40,285.16 5 6 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:290.2,291.16 2 6 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:296.2,297.16 2 6 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:302.2,302.41 1 5 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:309.2,310.16 2 4 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:314.2,314.33 1 4 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:320.2,320.91 1 4 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:324.2,324.48 1 4 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:328.2,328.21 1 4 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:285.16,287.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:291.16,293.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:297.16,299.3 1 1 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:302.41,304.17 2 4 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:304.17,306.4 1 1 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:310.16,312.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:314.33,315.96 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:315.96,317.4 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:320.91,322.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:324.48,326.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:338.40,340.16 2 1 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:344.2,348.16 4 1 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:352.2,352.33 1 1 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:359.2,359.41 1 1 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:367.2,368.16 2 1 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:373.2,373.84 1 1 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:377.2,377.97 1 1 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:381.2,381.48 1 1 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:385.2,385.21 1 1 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:340.16,342.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:348.16,350.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:352.33,353.90 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:353.90,355.4 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:359.41,361.17 2 1 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:361.17,363.4 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:368.16,370.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:373.84,375.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:377.97,379.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:381.48,383.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:393.62,396.16 3 9 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:400.2,400.25 1 9 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:396.16,398.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:403.59,407.16 4 13 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:411.2,411.18 1 13 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:407.16,409.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:419.9,421.42 2 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:424.2,424.16 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:428.2,428.29 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:436.2,437.36 2 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:451.2,451.12 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:421.42,423.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:424.16,426.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:428.29,434.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:437.36,440.17 3 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:444.3,444.69 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:448.3,448.42 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:440.17,442.4 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:444.69,446.4 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:462.9,469.16 3 2 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:473.2,473.63 1 2 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:477.2,477.12 1 2 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:469.16,471.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:473.63,475.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:488.9,489.28 1 10 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:493.2,496.16 3 9 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:500.2,501.12 2 9 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:517.2,518.16 2 9 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:522.2,523.9 2 9 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:527.2,527.28 1 9 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:538.2,545.16 2 7 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:549.2,549.47 1 7 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:489.28,491.3 1 1 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:496.16,498.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:501.12,503.17 2 4 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:507.3,507.40 1 4 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:503.17,505.4 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:508.8,510.17 2 5 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:514.3,514.31 1 5 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:510.17,512.4 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:518.16,520.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:523.9,525.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:527.28,536.3 1 2 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:545.16,547.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:559.50,565.16 5 10 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:573.2,573.13 1 10 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:600.2,601.16 2 9 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:605.2,605.51 1 9 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:565.16,567.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:573.13,575.17 2 1 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:579.3,588.17 4 1 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:592.3,593.17 2 1 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:597.3,597.32 1 1 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:575.17,577.4 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:588.17,590.4 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:593.17,595.4 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:601.16,603.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:626.9,629.16 2 6 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:633.2,634.72 2 5 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:638.2,639.33 2 4 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:644.2,647.35 3 3 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:680.2,681.16 2 1 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:685.2,686.12 2 1 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:629.16,631.3 1 1 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:634.72,636.3 1 1 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:639.33,641.3 1 1 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:647.35,650.10 2 3 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:654.3,654.29 1 2 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:658.3,675.17 3 1 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:650.10,652.4 1 1 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:654.29,656.4 1 1 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:675.17,677.4 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:681.16,683.3 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:699.40,701.40 2 9 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:752.2,753.40 2 9 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:757.2,757.24 1 9 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:701.40,702.36 1 12 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:702.36,707.18 2 14 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:710.4,710.12 1 14 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:714.4,716.20 3 7 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:727.4,733.11 3 7 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:743.4,744.18 2 2 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:747.4,748.29 2 2 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:707.18,709.5 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:710.12,711.13 1 7 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:716.20,718.12 2 3 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:722.5,723.50 2 3 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:718.12,720.6 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:733.11,740.13 2 5 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:744.18,746.5 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:753.40,755.3 1 5 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:762.87,765.16 3 3 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:769.2,769.40 1 2 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:781.2,781.14 1 0 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:765.16,767.3 1 1 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:769.40,771.17 2 5 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:775.3,778.37 3 3 -github.com/coinbase/rosetta-cli/internal/storage/block_storage.go:771.17,773.4 1 2 diff --git a/cmd/check.go b/cmd/check.go index 2f44743a..8d029b79 100644 --- a/cmd/check.go +++ b/cmd/check.go @@ -380,7 +380,6 @@ func runCheckCmd(cmd *cobra.Command, args []string) { logger, r, fetcher, - exemptAccounts, ) g, ctx := errgroup.WithContext(ctx) diff --git a/internal/processor/syncer_handler.go b/internal/processor/syncer_handler.go index 93357aa6..d18fd771 100644 --- a/internal/processor/syncer_handler.go +++ b/internal/processor/syncer_handler.go @@ -32,8 +32,6 @@ type SyncerHandler struct { logger *logger.Logger reconciler *reconciler.Reconciler fetcher *fetcher.Fetcher - - exemptAccounts []*reconciler.AccountCurrency } // NewSyncerHandler returns a new SyncerHandler. @@ -42,14 +40,12 @@ func NewSyncerHandler( logger *logger.Logger, reconciler *reconciler.Reconciler, fetcher *fetcher.Fetcher, - exemptAccounts []*reconciler.AccountCurrency, ) *SyncerHandler { return &SyncerHandler{ - storage: storage, - logger: logger, - reconciler: reconciler, - fetcher: fetcher, - exemptAccounts: exemptAccounts, + storage: storage, + logger: logger, + reconciler: reconciler, + fetcher: fetcher, } } diff --git a/internal/reconciler/reconciler.go b/internal/reconciler/reconciler.go index bffa17fb..a46d3594 100644 --- a/internal/reconciler/reconciler.go +++ b/internal/reconciler/reconciler.go @@ -239,7 +239,7 @@ func (r *Reconciler) QueueChanges( balanceChanges = append(balanceChanges, &BalanceChange{ Account: account.Account, Currency: account.Currency, - Difference: "0", + Difference: zeroString, Block: block, }) }