From 10a48ae99d5c92023e1b7f67b8425216f96f9cf6 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 14 Jul 2020 14:32:51 -0700 Subject: [PATCH 1/9] Make inactiveFrequency configurable --- reconciler/configuration.go | 8 ++++++++ reconciler/reconciler.go | 11 ++++++----- 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/reconciler/configuration.go b/reconciler/configuration.go index bca875ef..9ac3ef46 100644 --- a/reconciler/configuration.go +++ b/reconciler/configuration.go @@ -85,3 +85,11 @@ func WithLookupBalanceByBlock(lookup bool) Option { r.lookupBalanceByBlock = lookup } } + +// WithInactiveFrequency is how many blocks the reconciler +// should wait between inactive reconciliations on each account. +func WithInactiveFrequency(blocks int64) Option { + return func(r *Reconciler) { + r.inactiveFrequency = blocks + } +} diff --git a/reconciler/reconciler.go b/reconciler/reconciler.go index 42ecc0a6..35ffc8c3 100644 --- a/reconciler/reconciler.go +++ b/reconciler/reconciler.go @@ -64,11 +64,10 @@ const ( // to sleep when there are no seen accounts to reconcile. inactiveReconciliationSleep = 5 * time.Second - // inactiveReconciliationRequiredDepth is the minimum + // defaultInactiveFrequency is the minimum // number of blocks the reconciler should wait between - // inactive reconciliations. - // TODO: make configurable - inactiveReconciliationRequiredDepth = 500 + // inactive reconciliations for each account. + defaultInactiveFrequency = 200 // defaultLookupBalanceByBlock is the default setting // for how to perform balance queries. It is preferable @@ -175,6 +174,7 @@ type Reconciler struct { lookupBalanceByBlock bool interestingAccounts []*AccountCurrency changeQueue chan *parser.BalanceChange + inactiveFrequency int64 // Reconciler concurrency is separated between // active and inactive concurrency to allow for @@ -219,6 +219,7 @@ func New( helper: helper, handler: handler, fetcher: fetcher, + inactiveFrequency: defaultInactiveFrequency, activeConcurrency: defaultReconcilerConcurrency, inactiveConcurrency: defaultReconcilerConcurrency, highWaterMark: -1, @@ -601,7 +602,7 @@ func (r *Reconciler) reconcileInactiveAccounts( r.inactiveQueueMutex.Lock() if len(r.inactiveQueue) > 0 && (r.inactiveQueue[0].LastCheck == nil || // block is set to nil when loaded from previous run - r.inactiveQueue[0].LastCheck.Index+inactiveReconciliationRequiredDepth < head.Index) { + r.inactiveQueue[0].LastCheck.Index+r.inactiveFrequency < head.Index) { randAcct := r.inactiveQueue[0] r.inactiveQueue = r.inactiveQueue[1:] r.inactiveQueueMutex.Unlock() From 614cc3e8cb23b09978aafd1adb99063f9bbb4c7a Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 14 Jul 2020 14:43:16 -0700 Subject: [PATCH 2/9] Add better logging when no accounts ready for reconcilation --- reconciler/reconciler.go | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/reconciler/reconciler.go b/reconciler/reconciler.go index 35ffc8c3..6a5f6dc2 100644 --- a/reconciler/reconciler.go +++ b/reconciler/reconciler.go @@ -62,7 +62,7 @@ const ( // inactiveReconciliationSleep is used as the time.Duration // to sleep when there are no seen accounts to reconcile. - inactiveReconciliationSleep = 5 * time.Second + inactiveReconciliationSleep = 1 * time.Second // defaultInactiveFrequency is the minimum // number of blocks the reconciler should wait between @@ -600,17 +600,18 @@ func (r *Reconciler) reconcileInactiveAccounts( } r.inactiveQueueMutex.Lock() + nextValidIndex := r.inactiveQueue[0].LastCheck.Index + r.inactiveFrequency if len(r.inactiveQueue) > 0 && (r.inactiveQueue[0].LastCheck == nil || // block is set to nil when loaded from previous run - r.inactiveQueue[0].LastCheck.Index+r.inactiveFrequency < head.Index) { - randAcct := r.inactiveQueue[0] + nextValidIndex <= head.Index) { + nextAcct := r.inactiveQueue[0] r.inactiveQueue = r.inactiveQueue[1:] r.inactiveQueueMutex.Unlock() block, amount, err := r.bestBalance( ctx, - randAcct.Entry.Account, - randAcct.Entry.Currency, + nextAcct.Entry.Account, + nextAcct.Entry.Currency, types.ConstructPartialBlockIdentifier(head), ) if err != nil { @@ -619,8 +620,8 @@ func (r *Reconciler) reconcileInactiveAccounts( err = r.accountReconciliation( ctx, - randAcct.Entry.Account, - randAcct.Entry.Currency, + nextAcct.Entry.Account, + nextAcct.Entry.Currency, amount, block, true, @@ -630,6 +631,11 @@ func (r *Reconciler) reconcileInactiveAccounts( } } else { r.inactiveQueueMutex.Unlock() + log.Printf( + "no accounts ready for inactive reconciliation (%d accounts in queue, will reconcile next account at index %d)\n", + len(r.inactiveQueue), + nextValidIndex, + ) time.Sleep(inactiveReconciliationSleep) } } From 58215f13e41b7b92534528e3247150b5582c30bc Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 14 Jul 2020 15:05:24 -0700 Subject: [PATCH 3/9] Use set to keep track of seen accounts not array --- reconciler/configuration.go | 3 ++- reconciler/reconciler.go | 25 +++++++++++-------------- reconciler/reconciler_test.go | 30 +++++++++++++++++++++--------- 3 files changed, 34 insertions(+), 24 deletions(-) diff --git a/reconciler/configuration.go b/reconciler/configuration.go index 9ac3ef46..612428c2 100644 --- a/reconciler/configuration.go +++ b/reconciler/configuration.go @@ -18,6 +18,7 @@ import ( "fmt" "github.com/coinbase/rosetta-sdk-go/parser" + "github.com/coinbase/rosetta-sdk-go/types" ) // Option is used to overwrite default values in @@ -59,7 +60,7 @@ func WithSeenAccounts(seen []*AccountCurrency) Option { r.inactiveQueue = append(r.inactiveQueue, &InactiveEntry{ Entry: acct, }) - r.seenAccounts = append(r.seenAccounts, acct) + r.seenAccounts[types.Hash(acct)] = struct{}{} } fmt.Printf( diff --git a/reconciler/reconciler.go b/reconciler/reconciler.go index 6a5f6dc2..dfaee92b 100644 --- a/reconciler/reconciler.go +++ b/reconciler/reconciler.go @@ -198,7 +198,7 @@ type Reconciler struct { // queue. If this is not done, it is possible a goroutine // could be processing an account (not in the queue) when // we do a lookup to determine if we should add to the queue. - seenAccounts []*AccountCurrency + seenAccounts map[string]struct{} inactiveQueue []*InactiveEntry // inactiveQueueMutex needed because we can't peek at the tip @@ -223,7 +223,7 @@ func New( activeConcurrency: defaultReconcilerConcurrency, inactiveConcurrency: defaultReconcilerConcurrency, highWaterMark: -1, - seenAccounts: []*AccountCurrency{}, + seenAccounts: map[string]struct{}{}, inactiveQueue: []*InactiveEntry{}, // When lookupBalanceByBlock is enabled, we check @@ -517,22 +517,24 @@ func (r *Reconciler) inactiveAccountQueue( accountCurrency *AccountCurrency, liveBlock *types.BlockIdentifier, ) error { + r.inactiveQueueMutex.Lock() + // Only enqueue the first time we see an account on an active reconciliation. shouldEnqueueInactive := false if !inactive && !ContainsAccountCurrency(r.seenAccounts, accountCurrency) { - r.seenAccounts = append(r.seenAccounts, accountCurrency) + r.seenAccounts[types.Hash(accountCurrency)] = struct{}{} shouldEnqueueInactive = true } if inactive || shouldEnqueueInactive { - r.inactiveQueueMutex.Lock() r.inactiveQueue = append(r.inactiveQueue, &InactiveEntry{ Entry: accountCurrency, LastCheck: liveBlock, }) - r.inactiveQueueMutex.Unlock() } + r.inactiveQueueMutex.Unlock() + return nil } @@ -682,18 +684,13 @@ func ExtractAmount( } // ContainsAccountCurrency returns a boolean indicating if a -// AccountCurrency slice already contains an Account and Currency combination. +// AccountCurrency set already contains an Account and Currency combination. func ContainsAccountCurrency( - arr []*AccountCurrency, + m map[string]struct{}, change *AccountCurrency, ) bool { - for _, a := range arr { - if types.Hash(a) == types.Hash(change) { - return true - } - } - - return false + _, exists := m[types.Hash(change)] + return exists } // GetCurrencyBalance fetches the balance of a *types.AccountIdentifier diff --git a/reconciler/reconciler_test.go b/reconciler/reconciler_test.go index fb41688b..530d629d 100644 --- a/reconciler/reconciler_test.go +++ b/reconciler/reconciler_test.go @@ -88,8 +88,8 @@ func TestNewReconciler(t *testing.T) { Entry: accountCurrency, }, } - r.seenAccounts = []*AccountCurrency{ - accountCurrency, + r.seenAccounts = map[string]struct{}{ + types.Hash(accountCurrency): struct{}{}, } return r @@ -113,7 +113,7 @@ func TestNewReconciler(t *testing.T) { t.Run(name, func(t *testing.T) { result := New(nil, nil, nil, nil, test.options...) assert.ElementsMatch(t, test.expected.inactiveQueue, result.inactiveQueue) - assert.ElementsMatch(t, test.expected.seenAccounts, result.seenAccounts) + assert.Equal(t, test.expected.seenAccounts, result.seenAccounts) assert.ElementsMatch(t, test.expected.interestingAccounts, result.interestingAccounts) assert.Equal(t, test.expected.inactiveConcurrency, result.inactiveConcurrency) assert.Equal(t, test.expected.activeConcurrency, result.activeConcurrency) @@ -132,7 +132,7 @@ func TestContainsAccountCurrency(t *testing.T) { Symbol: "Blah2", Decimals: 2, } - accts := []*AccountCurrency{ + acctSlice := []*AccountCurrency{ { Account: &types.AccountIdentifier{ Address: "test", @@ -162,6 +162,11 @@ func TestContainsAccountCurrency(t *testing.T) { }, } + accts := map[string]struct{}{} + for _, acct := range acctSlice { + accts[types.Hash(acct)] = struct{}{} + } + t.Run("Non-existent account", func(t *testing.T) { assert.False(t, ContainsAccountCurrency(accts, &AccountCurrency{ Account: &types.AccountIdentifier{ @@ -489,6 +494,13 @@ func TestCompareBalance(t *testing.T) { }) } +func assertContainsAllAccounts(t *testing.T, m map[string]struct{}, a []*AccountCurrency) { + for _, account := range a { + _, exists := m[types.Hash(account)] + assert.True(t, exists) + } +} + func TestInactiveAccountQueue(t *testing.T) { var ( handler = &MockReconcilerHandler{} @@ -528,7 +540,7 @@ func TestInactiveAccountQueue(t *testing.T) { block, ) assert.Nil(t, err) - assert.ElementsMatch(t, r.seenAccounts, []*AccountCurrency{accountCurrency}) + assertContainsAllAccounts(t, r.seenAccounts, []*AccountCurrency{accountCurrency}) assert.ElementsMatch(t, r.inactiveQueue, []*InactiveEntry{ { Entry: accountCurrency, @@ -544,7 +556,7 @@ func TestInactiveAccountQueue(t *testing.T) { block2, ) assert.Nil(t, err) - assert.ElementsMatch( + assertContainsAllAccounts( t, r.seenAccounts, []*AccountCurrency{accountCurrency, accountCurrency2}, @@ -570,7 +582,7 @@ func TestInactiveAccountQueue(t *testing.T) { block, ) assert.Nil(t, err) - assert.ElementsMatch( + assertContainsAllAccounts( t, r.seenAccounts, []*AccountCurrency{accountCurrency, accountCurrency2}, @@ -585,7 +597,7 @@ func TestInactiveAccountQueue(t *testing.T) { block, ) assert.Nil(t, err) - assert.ElementsMatch( + assertContainsAllAccounts( t, r.seenAccounts, []*AccountCurrency{accountCurrency, accountCurrency2}, @@ -605,7 +617,7 @@ func TestInactiveAccountQueue(t *testing.T) { block2, ) assert.Nil(t, err) - assert.ElementsMatch( + assertContainsAllAccounts( t, r.seenAccounts, []*AccountCurrency{accountCurrency, accountCurrency2}, From 0cc8cda5b6aad3e86f7551fd7dbb03b7ec11ea34 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 14 Jul 2020 15:24:12 -0700 Subject: [PATCH 4/9] Cleanup inactive queue touches --- reconciler/reconciler.go | 49 ++++++++++++++++++++++++++-------------- 1 file changed, 32 insertions(+), 17 deletions(-) diff --git a/reconciler/reconciler.go b/reconciler/reconciler.go index dfaee92b..ac33d6f6 100644 --- a/reconciler/reconciler.go +++ b/reconciler/reconciler.go @@ -270,23 +270,35 @@ func (r *Reconciler) QueueChanges( }) } - if !r.lookupBalanceByBlock { - // All changes will have the same block. Return - // if we are too far behind to start reconciling. - if block.Index < r.highWaterMark { - return nil + for _, change := range balanceChanges { + // Add all seen accounts to inactive reconciler queue. + // + // Note: accounts are only added if they have not been seen before. + err := r.inactiveAccountQueue(false, &AccountCurrency{ + Account: change.Account, + Currency: change.Currency, + }, block) + if err != nil { + return err } - for _, change := range balanceChanges { + if !r.lookupBalanceByBlock { + // All changes will have the same block. Continue + // if we are too far behind to start reconciling. + // + // Note: we don't return here so that we can ensure + // all seen accounts are added to the inactiveAccountQueue. + if block.Index < r.highWaterMark { + continue + } + select { case r.changeQueue <- change: default: log.Println("skipping active enqueue because backlog") } - } - } else { - // Block until all checked for a block or context is Done - for _, change := range balanceChanges { + } else { + // Block until all checked for a block or context is Done select { case r.changeQueue <- change: case <-ctx.Done(): @@ -487,18 +499,13 @@ func (r *Reconciler) accountReconciliation( liveAmount, liveBlock, ) - if err != nil { + if err != nil { // error only returned if we should exit on failure return err } return nil } - err = r.inactiveAccountQueue(inactive, accountCurrency, liveBlock) - if err != nil { - return err - } - return r.handler.ReconciliationSucceeded( ctx, reconciliationType, @@ -509,6 +516,7 @@ func (r *Reconciler) accountReconciliation( ) } + // We return here if we gave up trying to reconcile an account. return nil } @@ -596,7 +604,6 @@ func (r *Reconciler) reconcileInactiveAccounts( // When first start syncing, this loop may run before the genesis block is synced. // If this is the case, we should sleep and try again later instead of exiting. if err != nil { - log.Println("waiting to start intactive reconciliation until a block is synced...") time.Sleep(inactiveReconciliationSleep) continue } @@ -631,6 +638,14 @@ func (r *Reconciler) reconcileInactiveAccounts( if err != nil { return err } + + // Always re-enqueue accounts after they have been inactively + // reconciled. If we don't re-enqueue, we will never check + // these accounts again. + err = r.inactiveAccountQueue(true, nextAcct.Entry, block) + if err != nil { + return err + } } else { r.inactiveQueueMutex.Unlock() log.Printf( From b026dfeeaf91c5f844e33363e54e0d4f5b52f822 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 14 Jul 2020 15:44:23 -0700 Subject: [PATCH 5/9] Add debug logging configuration for reconciler --- reconciler/configuration.go | 8 ++++++++ reconciler/reconciler.go | 32 +++++++++++++++++++++----------- 2 files changed, 29 insertions(+), 11 deletions(-) diff --git a/reconciler/configuration.go b/reconciler/configuration.go index 612428c2..0b03c5b7 100644 --- a/reconciler/configuration.go +++ b/reconciler/configuration.go @@ -94,3 +94,11 @@ func WithInactiveFrequency(blocks int64) Option { r.inactiveFrequency = blocks } } + +// WithDebugLogging determines if verbose logs should +// be printed. +func WithDebugLogging(debug bool) Option { + return func(r *Reconciler) { + r.debugLogging = debug + } +} diff --git a/reconciler/reconciler.go b/reconciler/reconciler.go index ac33d6f6..6e5b9d03 100644 --- a/reconciler/reconciler.go +++ b/reconciler/reconciler.go @@ -175,6 +175,7 @@ type Reconciler struct { interestingAccounts []*AccountCurrency changeQueue chan *parser.BalanceChange inactiveFrequency int64 + debugLogging bool // Reconciler concurrency is separated between // active and inactive concurrency to allow for @@ -295,7 +296,9 @@ func (r *Reconciler) QueueChanges( select { case r.changeQueue <- change: default: - log.Println("skipping active enqueue because backlog") + if r.debugLogging { + log.Println("skipping active enqueue because backlog") + } } } else { // Block until all checked for a block or context is Done @@ -456,11 +459,13 @@ func (r *Reconciler) accountReconciliation( } // Don't wait to check if we are very far behind - log.Printf( - "Skipping reconciliation for %s: %d blocks behind\n", - types.PrettyPrintStruct(accountCurrency), - diff, - ) + if r.debugLogging { + log.Printf( + "Skipping reconciliation for %s: %d blocks behind\n", + types.PrettyPrintStruct(accountCurrency), + diff, + ) + } // Set a highWaterMark to not accept any new // reconciliation requests unless they happened @@ -604,6 +609,9 @@ func (r *Reconciler) reconcileInactiveAccounts( // When first start syncing, this loop may run before the genesis block is synced. // If this is the case, we should sleep and try again later instead of exiting. if err != nil { + if r.debugLogging { + log.Println("waiting to start intactive reconciliation until a block is synced...") + } time.Sleep(inactiveReconciliationSleep) continue } @@ -648,11 +656,13 @@ func (r *Reconciler) reconcileInactiveAccounts( } } else { r.inactiveQueueMutex.Unlock() - log.Printf( - "no accounts ready for inactive reconciliation (%d accounts in queue, will reconcile next account at index %d)\n", - len(r.inactiveQueue), - nextValidIndex, - ) + if r.debugLogging { + log.Printf( + "no accounts ready for inactive reconciliation (%d accounts in queue, will reconcile next account at index %d)\n", + len(r.inactiveQueue), + nextValidIndex, + ) + } time.Sleep(inactiveReconciliationSleep) } } From d21991f1ca14ef9e94e66cf923667d9ff7afa509 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 14 Jul 2020 15:54:03 -0700 Subject: [PATCH 6/9] Respect high water mark in inactive reconciler --- reconciler/reconciler.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/reconciler/reconciler.go b/reconciler/reconciler.go index 6e5b9d03..b6b8e5ec 100644 --- a/reconciler/reconciler.go +++ b/reconciler/reconciler.go @@ -616,6 +616,14 @@ func (r *Reconciler) reconcileInactiveAccounts( continue } + if head.Index < r.highWaterMark { + if r.debugLogging { + log.Println("waiting to continue intactive reconciliation until reaching high water mark...") + } + time.Sleep(inactiveReconciliationSleep) + continue + } + r.inactiveQueueMutex.Lock() nextValidIndex := r.inactiveQueue[0].LastCheck.Index + r.inactiveFrequency if len(r.inactiveQueue) > 0 && From 9cff6f8eb3dcf18f82797c81b54468f78f36985b Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 14 Jul 2020 16:04:03 -0700 Subject: [PATCH 7/9] Increase backlog threshold --- reconciler/reconciler.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/reconciler/reconciler.go b/reconciler/reconciler.go index b6b8e5ec..841d5926 100644 --- a/reconciler/reconciler.go +++ b/reconciler/reconciler.go @@ -43,7 +43,7 @@ const ( // that can be enqueued to reconcile before new // requests are dropped. // TODO: Make configurable - backlogThreshold = 1000 + backlogThreshold = 50000 // waitToCheckDiff is the syncing difference (live-head) // to retry instead of exiting. In other words, if the From 48272b8c36a3741ec488210928f0a8ed05dac6f7 Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 14 Jul 2020 16:13:28 -0700 Subject: [PATCH 8/9] nits --- reconciler/reconciler.go | 46 +++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 15 deletions(-) diff --git a/reconciler/reconciler.go b/reconciler/reconciler.go index 841d5926..b23d9849 100644 --- a/reconciler/reconciler.go +++ b/reconciler/reconciler.go @@ -593,6 +593,35 @@ func (r *Reconciler) reconcileActiveAccounts( } } +// shouldAttemptInactiveReconciliation returns a boolean indicating whether +// inactive reconciliation should be attempted based on syncing status. +func (r *Reconciler) shouldAttemptInactiveReconciliation( + ctx context.Context, +) (bool, *types.BlockIdentifier) { + head, err := r.helper.CurrentBlock(ctx) + // When first start syncing, this loop may run before the genesis block is synced. + // If this is the case, we should sleep and try again later instead of exiting. + if err != nil { + if r.debugLogging { + log.Println("waiting to start intactive reconciliation until a block is synced...") + } + + return false, nil + } + + if head.Index < r.highWaterMark { + if r.debugLogging { + log.Println( + "waiting to continue intactive reconciliation until reaching high water mark...", + ) + } + + return false, nil + } + + return true, head +} + // reconcileInactiveAccounts selects a random account // from all previously seen accounts and reconciles // the balance. This is useful for detecting balance @@ -605,21 +634,8 @@ func (r *Reconciler) reconcileInactiveAccounts( return ctx.Err() } - head, err := r.helper.CurrentBlock(ctx) - // When first start syncing, this loop may run before the genesis block is synced. - // If this is the case, we should sleep and try again later instead of exiting. - if err != nil { - if r.debugLogging { - log.Println("waiting to start intactive reconciliation until a block is synced...") - } - time.Sleep(inactiveReconciliationSleep) - continue - } - - if head.Index < r.highWaterMark { - if r.debugLogging { - log.Println("waiting to continue intactive reconciliation until reaching high water mark...") - } + shouldAttempt, head := r.shouldAttemptInactiveReconciliation(ctx) + if !shouldAttempt { time.Sleep(inactiveReconciliationSleep) continue } From 745b911433c9f29c4fd100e10797900688441e7f Mon Sep 17 00:00:00 2001 From: Patrick O'Grady Date: Tue, 14 Jul 2020 18:01:12 -0700 Subject: [PATCH 9/9] Improve logging when currency is missing --- reconciler/reconciler.go | 12 ++++++++++-- reconciler/reconciler_test.go | 9 ++++++++- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/reconciler/reconciler.go b/reconciler/reconciler.go index b23d9849..6b194439 100644 --- a/reconciler/reconciler.go +++ b/reconciler/reconciler.go @@ -729,7 +729,10 @@ func ExtractAmount( return b, nil } - return nil, fmt.Errorf("could not extract amount for %+v", currency) + return nil, fmt.Errorf( + "account balance response does could not contain currency %s", + types.PrettyPrintStruct(currency), + ) } // ContainsAccountCurrency returns a boolean indicating if a @@ -764,7 +767,12 @@ func GetCurrencyBalance( liveAmount, err := ExtractAmount(liveBalances, currency) if err != nil { - return nil, "", err + return nil, "", fmt.Errorf( + "%w: could not get %s currency balance for %s", + err, + types.PrettyPrintStruct(currency), + types.PrettyPrintStruct(account), + ) } return liveBlock, liveAmount.Value, nil diff --git a/reconciler/reconciler_test.go b/reconciler/reconciler_test.go index 530d629d..0d9773e5 100644 --- a/reconciler/reconciler_test.go +++ b/reconciler/reconciler_test.go @@ -273,7 +273,14 @@ func TestExtractAmount(t *testing.T) { t.Run("Non-existent currency", func(t *testing.T) { result, err := ExtractAmount(balances, badCurr) assert.Nil(t, result) - assert.EqualError(t, err, fmt.Errorf("could not extract amount for %+v", badCurr).Error()) + assert.EqualError( + t, + err, + fmt.Errorf( + "account balance response does could not contain currency %s", + types.PrettyPrintStruct(badCurr), + ).Error(), + ) }) t.Run("Simple account", func(t *testing.T) {