Configurable Client Errors

.changeset/lovely-jeans-confess.md

@@ -0,0 +1,5 @@
+---
+"chainlink": minor
+---
+
+Enable configurable client error regexes for error classification

core/chains/evm/client/errors.go

@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"regexp"
+	"sync"
 
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/core/types"
@@ -13,6 +14,7 @@ import (
 	"github.com/smartcontractkit/chainlink-common/pkg/logger"
 
 	commonclient "github.com/smartcontractkit/chainlink/v2/common/client"
+	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/config"
 	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/label"
 )
 
@@ -233,17 +235,62 @@ var zkEvm = ClientErrors{
 	OutOfCounters: regexp.MustCompile(`(?:: |^)not enough .* counters to continue the execution$`),
 }
 
-var clients = []ClientErrors{parity, geth, arbitrum, metis, substrate, avalanche, nethermind, harmony, besu, erigon, klaytn, celo, zkSync, zkEvm}
+var clientsLock = sync.Mutex{}
+var clients = map[string]ClientErrors{
+	"parity":     parity,
+	"geth":       geth,
+	"arbitrum":   arbitrum,
+	"metis":      metis,
+	"substrate":  substrate,
+	"avalanche":  avalanche,
+	"nethermind": nethermind,
+	"harmony":    harmony,
+	"besu":       besu,
+	"erigon":     erigon,
+	"klaytn":     klaytn,
+	"celo":       celo,
+	"zkSync":     zkSync,
+	"zkEvm":      zkEvm,
+}
+
+// SetClientErrorRegexes is called on startup to set the client errors from the config
+func SetClientErrorRegexes(errsRegex config.ClientErrors) {
+	clientsLock.Lock()
+	defer clientsLock.Unlock()
+	clients["tomlConfig"] = ClientErrors{
+		NonceTooLow:                       regexp.MustCompile(errsRegex.NonceTooLow()),
+		NonceTooHigh:                      regexp.MustCompile(errsRegex.NonceTooHigh()),
+		ReplacementTransactionUnderpriced: regexp.MustCompile(errsRegex.ReplacementTransactionUnderpriced()),
+		LimitReached:                      regexp.MustCompile(errsRegex.LimitReached()),
+		TransactionAlreadyInMempool:       regexp.MustCompile(errsRegex.TransactionAlreadyInMempool()),
+		TerminallyUnderpriced:             regexp.MustCompile(errsRegex.TerminallyUnderpriced()),
+		InsufficientEth:                   regexp.MustCompile(errsRegex.InsufficientEth()),
+		TxFeeExceedsCap:                   regexp.MustCompile(errsRegex.TxFeeExceedsCap()),
+		L2FeeTooLow:                       regexp.MustCompile(errsRegex.L2FeeTooLow()),
+		L2FeeTooHigh:                      regexp.MustCompile(errsRegex.L2FeeTooHigh()),
+		L2Full:                            regexp.MustCompile(errsRegex.L2Full()),
+		TransactionAlreadyMined:           regexp.MustCompile(errsRegex.TransactionAlreadyMined()),
+		Fatal:                             regexp.MustCompile(errsRegex.Fatal()),
+		ServiceUnavailable:                regexp.MustCompile(errsRegex.ServiceUnavailable()),
+	}
+}
 
 func (s *SendError) is(errorType int) bool {
 	if s == nil || s.err == nil {
 		return false
 	}
 	str := s.CauseStr()
+
+	clientsLock.Lock()
+	defer clientsLock.Unlock()
 	for _, client := range clients {
 		if _, ok := client[errorType]; !ok {
 			continue
 		}
+		if client[errorType].String() == "" {
+			// Skip empty regexes.
+			continue
+		}
 		if client[errorType].MatchString(str) {
 			return true
 		}
@@ -369,10 +416,17 @@ func isFatalSendError(err error) bool {
 		return false
 	}
 	str := pkgerrors.Cause(err).Error()
+
+	clientsLock.Lock()
+	defer clientsLock.Unlock()
 	for _, client := range clients {
 		if _, ok := client[Fatal]; !ok {
 			continue
 		}
+		if client[Fatal].String() == "" {
+			// Skip empty regexes.
+			continue
+		}
 		if client[Fatal].MatchString(str) {
 			return true
 		}

core/chains/evm/client/errors_test.go

@@ -19,15 +19,35 @@ type errorCase struct {
 	network string
 }
 
+func NewTestClientErrors() evmclient.TestClientErrors {
+	clientErrors := evmclient.TestClientErrors{}
+	clientErrors.SetNonceTooLow("client error nonce too low")
+	clientErrors.SetNonceTooHigh("client error nonce too high")
+	clientErrors.SetReplacementTransactionUnderpriced("client error replacement underpriced")
+	clientErrors.SetLimitReached("client error limit reached")
+	clientErrors.SetTransactionAlreadyInMempool("client error transaction already in mempool")
+	clientErrors.SetTerminallyUnderpriced("client error terminally underpriced")
+	clientErrors.SetInsufficientEth("client error insufficient eth")
+	clientErrors.SetTxFeeExceedsCap("client error tx fee exceeds cap")
+	clientErrors.SetL2FeeTooLow("client error l2 fee too low")
+	clientErrors.SetL2FeeTooHigh("client error l2 fee too high")
+	clientErrors.SetL2Full("client error l2 full")
+	clientErrors.SetTransactionAlreadyMined("client error transaction already mined")
+	clientErrors.SetFatal("client error fatal")
+	clientErrors.SetServiceUnavailable("client error service unavailable")
+	return clientErrors
+}
+
 func Test_Eth_Errors(t *testing.T) {
 	t.Parallel()
 
 	var err *evmclient.SendError
 	randomError := evmclient.NewSendErrorS("some old bollocks")
 
-	t.Run("IsNonceTooLowError", func(t *testing.T) {
-		assert.False(t, randomError.IsNonceTooLowError())
+	clientErrors := NewTestClientErrors()
+	evmclient.SetClientErrorRegexes(&clientErrors)
 
+	t.Run("IsNonceTooLowError", func(t *testing.T) {
 		tests := []errorCase{
 			{"nonce too low", true, "Geth"},
 			{"nonce too low: address 0x336394A3219e71D9d9bd18201d34E95C1Bb7122C, tx: 8089 state: 8090", true, "Arbitrum"},
@@ -41,6 +61,7 @@ func Test_Eth_Errors(t *testing.T) {
 			{"call failed: OldNonce", true, "Nethermind"},
 			{"call failed: OldNonce, Current nonce: 22, nonce of rejected tx: 17", true, "Nethermind"},
 			{"nonce too low. allowed nonce range: 427 - 447, actual: 426", true, "zkSync"},
+			{"client error nonce too low", true, "tomlConfig"},
 		}
 
 		for _, test := range tests {
@@ -54,14 +75,14 @@ func Test_Eth_Errors(t *testing.T) {
 	})
 
 	t.Run("IsNonceTooHigh", func(t *testing.T) {
-
 		tests := []errorCase{
 			{"call failed: NonceGap", true, "Nethermind"},
 			{"call failed: NonceGap, Future nonce. Expected nonce: 10", true, "Nethermind"},
 			{"nonce too high: address 0x336394A3219e71D9d9bd18201d34E95C1Bb7122C, tx: 8089 state: 8090", true, "Arbitrum"},
 			{"nonce too high", true, "Geth"},
 			{"nonce too high", true, "Erigon"},
 			{"nonce too high. allowed nonce range: 427 - 477, actual: 527", true, "zkSync"},
+			{"client error nonce too high", true, "tomlConfig"},
 		}
 
 		for _, test := range tests {
@@ -77,6 +98,7 @@ func Test_Eth_Errors(t *testing.T) {
 
 		tests := []errorCase{
 			{"transaction already finalized", true, "Harmony"},
+			{"client error transaction already mined", true, "tomlConfig"},
 		}
 
 		for _, test := range tests {
@@ -100,6 +122,7 @@ func Test_Eth_Errors(t *testing.T) {
 			{"Transaction gas price 100wei is too low. There is another transaction with same nonce in the queue with gas price 150wei. Try increasing the gas price or incrementing the nonce.", true, "Parity"},
 			{"There are too many transactions in the queue. Your transaction was dropped due to limit. Try increasing the fee.", false, "Parity"},
 			{"gas price too low", false, "Arbitrum"},
+			{"client error replacement underpriced", true, "tomlConfig"},
 		}
 
 		for _, test := range tests {
@@ -130,6 +153,7 @@ func Test_Eth_Errors(t *testing.T) {
 			{"known transaction. transaction with hash 0x6013…3053 is already in the system", true, "zkSync"},
 			// This seems to be an erroneous message from the zkSync client, we'll have to match it anyway
 			{"ErrorObject { code: ServerError(3), message: \\\"known transaction. transaction with hash 0xf016…ad63 is already in the system\\\", data: Some(RawValue(\\\"0x\\\")) }", true, "zkSync"},
+			{"client error transaction already in mempool", true, "tomlConfig"},
 		}
 		for _, test := range tests {
 			err = evmclient.NewSendErrorS(test.message)
@@ -159,6 +183,7 @@ func Test_Eth_Errors(t *testing.T) {
 			{"intrinsic gas too low", true, "Klaytn"},
 			{"max fee per gas less than block base fee", true, "zkSync"},
 			{"virtual machine entered unexpected state. please contact developers and provide transaction details that caused this error. Error description: The operator included transaction with an unacceptable gas price", true, "zkSync"},
+			{"client error terminally underpriced", true, "tomlConfig"},
 		}
 
 		for _, test := range tests {
@@ -204,6 +229,7 @@ func Test_Eth_Errors(t *testing.T) {
 			{"insufficient funds for gas * price + value + gatewayFee", true, "celo"},
 			{"insufficient balance for transfer", true, "zkSync"},
 			{"insufficient funds for gas + value. balance: 42719769622667482000, fee: 48098250000000, value: 42719769622667482000", true, "celo"},
+			{"client error insufficient eth", true, "tomlConfig"},
 		}
 		for _, test := range tests {
 			err = evmclient.NewSendErrorS(test.message)
@@ -217,6 +243,7 @@ func Test_Eth_Errors(t *testing.T) {
 		tests := []errorCase{
 			{"call failed: 503 Service Unavailable: <html>\r\n<head><title>503 Service Temporarily Unavailable</title></head>\r\n<body>\r\n<center><h1>503 Service Temporarily Unavailable</h1></center>\r\n</body>\r\n</html>\r\n", true, "Nethermind"},
 			{"call failed: 502 Bad Gateway: <html>\r\n<head><title>502 Bad Gateway</title></head>\r\n<body>\r\n<center><h1>502 Bad Gateway</h1></center>\r\n<hr><center>", true, "Arbitrum"},
+			{"client error service unavailable", true, "tomlConfig"},
 		}
 		for _, test := range tests {
 			err = evmclient.NewSendErrorS(test.message)
@@ -237,6 +264,7 @@ func Test_Eth_Errors(t *testing.T) {
 			{"max fee per gas higher than max priority fee per gas", true, "Klaytn"},
 			{"tx fee (1.10 of currency celo) exceeds the configured cap (1.00 celo)", true, "celo"},
 			{"max priority fee per gas higher than max fee per gas", true, "zkSync"},
+			{"client error tx fee exceeds cap", true, "tomlConfig"},
 		}
 		for _, test := range tests {
 			err = evmclient.NewSendErrorS(test.message)
@@ -299,6 +327,9 @@ func Test_Eth_Errors(t *testing.T) {
 func Test_Eth_Errors_Fatal(t *testing.T) {
 	t.Parallel()
 
+	clientErrors := NewTestClientErrors()
+	evmclient.SetClientErrorRegexes(&clientErrors)
+
 	tests := []errorCase{
 		{"some old bollocks", false, "none"},
 
@@ -363,6 +394,7 @@ func Test_Eth_Errors_Fatal(t *testing.T) {
 		{"Failed to serialize transaction: max fee per pubdata byte higher than 2^64-1", true, "zkSync"},
 		{"Failed to serialize transaction: max priority fee per gas higher than 2^64-1", true, "zkSync"},
 		{"Failed to serialize transaction: oversized data. max: 1000000; actual: 1000000", true, "zkSync"},
+		{"client error fatal", true, "tomlConfig"},
 	}
 
 	for _, test := range tests {

core/chains/evm/client/helpers_test.go

@@ -19,6 +19,73 @@ import (
 	evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types"
 )
 
+type TestClientErrors struct {
+	nonceTooLow                       string
+	nonceTooHigh                      string
+	replacementTransactionUnderpriced string
+	limitReached                      string
+	transactionAlreadyInMempool       string
+	terminallyUnderpriced             string
+	insufficientEth                   string
+	txFeeExceedsCap                   string
+	l2FeeTooLow                       string
+	l2FeeTooHigh                      string
+	l2Full                            string
+	transactionAlreadyMined           string
+	fatal                             string
+	serviceUnavailable                string
+}
+
+func (c *TestClientErrors) SetNonceTooLow(s string)  { c.nonceTooLow = s }
+func (c *TestClientErrors) SetNonceTooHigh(s string) { c.nonceTooHigh = s }
+
+func (c *TestClientErrors) SetReplacementTransactionUnderpriced(s string) {
+	c.replacementTransactionUnderpriced = s
+}
+
+func (c *TestClientErrors) SetLimitReached(s string) { c.limitReached = s }
+
+func (c *TestClientErrors) SetTransactionAlreadyInMempool(s string) {
+	c.transactionAlreadyInMempool = s
+}
+
+func (c *TestClientErrors) SetTerminallyUnderpriced(s string) { c.terminallyUnderpriced = s }
+func (c *TestClientErrors) SetInsufficientEth(s string)       { c.insufficientEth = s }
+func (c *TestClientErrors) SetTxFeeExceedsCap(s string)       { c.txFeeExceedsCap = s }
+func (c *TestClientErrors) SetL2FeeTooLow(s string)           { c.l2FeeTooLow = s }
+func (c *TestClientErrors) SetL2FeeTooHigh(s string)          { c.l2FeeTooHigh = s }
+func (c *TestClientErrors) SetL2Full(s string)                { c.l2Full = s }
+
+func (c *TestClientErrors) SetTransactionAlreadyMined(s string) {
+	c.transactionAlreadyMined = s
+}
+
+func (c *TestClientErrors) SetFatal(s string)              { c.fatal = s }
+func (c *TestClientErrors) SetServiceUnavailable(s string) { c.serviceUnavailable = s }
+
+func (c *TestClientErrors) NonceTooLow() string  { return c.nonceTooLow }
+func (c *TestClientErrors) NonceTooHigh() string { return c.nonceTooHigh }
+
+func (c *TestClientErrors) ReplacementTransactionUnderpriced() string {
+	return c.replacementTransactionUnderpriced
+}
+
+func (c *TestClientErrors) LimitReached() string { return c.limitReached }
+
+func (c *TestClientErrors) TransactionAlreadyInMempool() string {
+	return c.transactionAlreadyInMempool
+}
+
+func (c *TestClientErrors) TerminallyUnderpriced() string   { return c.terminallyUnderpriced }
+func (c *TestClientErrors) InsufficientEth() string         { return c.insufficientEth }
+func (c *TestClientErrors) TxFeeExceedsCap() string         { return c.txFeeExceedsCap }
+func (c *TestClientErrors) L2FeeTooLow() string             { return c.l2FeeTooLow }
+func (c *TestClientErrors) L2FeeTooHigh() string            { return c.l2FeeTooHigh }
+func (c *TestClientErrors) L2Full() string                  { return c.l2Full }
+func (c *TestClientErrors) TransactionAlreadyMined() string { return c.transactionAlreadyMined }
+func (c *TestClientErrors) Fatal() string                   { return c.fatal }
+func (c *TestClientErrors) ServiceUnavailable() string      { return c.serviceUnavailable }
+
 type TestNodePoolConfig struct {
 	NodePollFailureThreshold       uint32
 	NodePollInterval               time.Duration

core/chains/evm/config/chain_scoped.go

@@ -82,6 +82,10 @@ func (e *EVMConfig) Transactions() Transactions {
 	return &transactionsConfig{c: e.C.Transactions}
 }
 
+func (e *EVMConfig) ClientErrors() ClientErrors {
+	return &clientErrorsConfig{c: e.C.ClientErrors}
+}
+
 func (e *EVMConfig) HeadTracker() HeadTracker {
 	return &headTrackerConfig{c: e.C.HeadTracker}
 }

core/chains/evm/config/chain_scoped_client_errors.go

@@ -0,0 +1,52 @@
+package config
+
+import (
+	"github.com/smartcontractkit/chainlink/v2/core/chains/evm/config/toml"
+)
+
+var defaultRegex = ""
+
+func derefOrDefault(s *string) string {
+	if s == nil {
+		return defaultRegex
+	}
+
+	return *s
+}
+
+type clientErrorsConfig struct {
+	c toml.ClientErrors
+}
+
+func (c *clientErrorsConfig) NonceTooLow() string  { return derefOrDefault(c.c.NonceTooLow) }
+func (c *clientErrorsConfig) NonceTooHigh() string { return derefOrDefault(c.c.NonceTooHigh) }
+
+func (c *clientErrorsConfig) ReplacementTransactionUnderpriced() string {
+	return derefOrDefault(c.c.ReplacementTransactionUnderpriced)
+}
+
+func (c *clientErrorsConfig) LimitReached() string { return derefOrDefault(c.c.LimitReached) }
+
+func (c *clientErrorsConfig) TransactionAlreadyInMempool() string {
+	return derefOrDefault(c.c.TransactionAlreadyInMempool)
+}
+
+func (c *clientErrorsConfig) TerminallyUnderpriced() string {
+	return derefOrDefault(c.c.TerminallyUnderpriced)
+}
+
+func (c *clientErrorsConfig) InsufficientEth() string { return derefOrDefault(c.c.InsufficientEth) }
+func (c *clientErrorsConfig) TxFeeExceedsCap() string { return derefOrDefault(c.c.TxFeeExceedsCap) }
+func (c *clientErrorsConfig) L2FeeTooLow() string     { return derefOrDefault(c.c.L2FeeTooLow) }
+func (c *clientErrorsConfig) L2FeeTooHigh() string    { return derefOrDefault(c.c.L2FeeTooHigh) }
+func (c *clientErrorsConfig) L2Full() string          { return derefOrDefault(c.c.L2Full) }
+
+func (c *clientErrorsConfig) TransactionAlreadyMined() string {
+	return derefOrDefault(c.c.TransactionAlreadyMined)
+}
+
+func (c *clientErrorsConfig) Fatal() string { return derefOrDefault(c.c.Fatal) }
+
+func (c *clientErrorsConfig) ServiceUnavailable() string {
+	return derefOrDefault(c.c.ServiceUnavailable)
+}