From e89847ba346925dc7c2e3206582223686c498c9e Mon Sep 17 00:00:00 2001 From: Oliver Townsend Date: Tue, 17 Sep 2024 16:02:50 -0700 Subject: [PATCH 01/10] Bring KMS client and multiclient over to chainlink --- .../deployment/kms/evm_kmsclient.go | 186 ++++++++++++++++++ .../deployment/kms/evm_kmsclient_test.go | 27 +++ integration-tests/deployment/multiclient.go | 104 ++++++++++ 3 files changed, 317 insertions(+) create mode 100644 integration-tests/deployment/kms/evm_kmsclient.go create mode 100644 integration-tests/deployment/kms/evm_kmsclient_test.go create mode 100644 integration-tests/deployment/multiclient.go diff --git a/integration-tests/deployment/kms/evm_kmsclient.go b/integration-tests/deployment/kms/evm_kmsclient.go new file mode 100644 index 00000000000..9956bc2405b --- /dev/null +++ b/integration-tests/deployment/kms/evm_kmsclient.go @@ -0,0 +1,186 @@ +package kms + +import ( + "bytes" + "context" + "crypto/ecdsa" + "encoding/asn1" + "encoding/hex" + "fmt" + "math/big" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/service/kms" + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/crypto/secp256k1" +) + +var ( + secp256k1N = crypto.S256().Params().N + secp256k1HalfN = new(big.Int).Div(secp256k1N, big.NewInt(2)) +) + +// See https://docs.aws.amazon.com/kms/latest/APIReference/API_GetPublicKey.html#API_GetPublicKey_ResponseSyntax +// and https://datatracker.ietf.org/doc/html/rfc5280 for why we need to unpack the KMS public key. +type asn1SubjectPublicKeyInfo struct { + AlgorithmIdentifier asn1AlgorithmIdentifier + SubjectPublicKey asn1.BitString +} + +type asn1AlgorithmIdentifier struct { + Algorithm asn1.ObjectIdentifier + Parameters asn1.ObjectIdentifier +} + +// See https://aws.amazon.com/blogs/database/part2-use-aws-kms-to-securely-manage-ethereum-accounts/ for why we +// need to manually prep the signature for Ethereum. +type asn1ECDSASig struct { + R asn1.RawValue + S asn1.RawValue +} + +type KMSClient interface { + GetPublicKey(input *kms.GetPublicKeyInput) (*kms.GetPublicKeyOutput, error) + Sign(input *kms.SignInput) (*kms.SignOutput, error) +} + +type evmKMSClient struct { + Client KMSClient + KeyID string +} + +func NewEVMKMSClient(client KMSClient, keyID string) *evmKMSClient { + return &evmKMSClient{ + Client: client, + KeyID: keyID, + } +} + +func (c *evmKMSClient) GetKMSTransactOpts(ctx context.Context, chainID *big.Int) (*bind.TransactOpts, error) { + ecdsaPublicKey, err := c.GetECDSAPublicKey() + if err != nil { + return nil, err + } + + pubKeyBytes := secp256k1.S256().Marshal(ecdsaPublicKey.X, ecdsaPublicKey.Y) + keyAddr := crypto.PubkeyToAddress(*ecdsaPublicKey) + if chainID == nil { + return nil, fmt.Errorf("chainID is required") + } + signer := types.LatestSignerForChainID(chainID) + + signerFn := func(address common.Address, tx *types.Transaction) (*types.Transaction, error) { + if address != keyAddr { + return nil, bind.ErrNotAuthorized + } + + txHashBytes := signer.Hash(tx).Bytes() + + mType := kms.MessageTypeDigest + algo := kms.SigningAlgorithmSpecEcdsaSha256 + signOutput, err := c.Client.Sign( + &kms.SignInput{ + KeyId: &c.KeyID, + SigningAlgorithm: &algo, + MessageType: &mType, + Message: txHashBytes, + }) + if err != nil { + return nil, fmt.Errorf("failed to call kms.Sign() on transaction: %v", err) + } + + ethSig, err := kmsToEthSig(signOutput.Signature, pubKeyBytes, txHashBytes) + if err != nil { + return nil, fmt.Errorf("failed to convert KMS signature to Ethereum signature: %v", err) + } + + return tx.WithSignature(signer, ethSig) + } + + return &bind.TransactOpts{ + From: keyAddr, + Signer: signerFn, + Context: ctx, + }, nil +} + +// GetECDSAPublicKey retrieves the public key from KMS and converts it to its ECDSA representation. +func (c *evmKMSClient) GetECDSAPublicKey() (*ecdsa.PublicKey, error) { + getPubKeyOutput, err := c.Client.GetPublicKey(&kms.GetPublicKeyInput{ + KeyId: aws.String(c.KeyID), + }) + if err != nil { + return nil, fmt.Errorf("can not get public key from KMS for KeyId=%s: %s", c.KeyID, err) + } + + var asn1pubKeyInfo asn1SubjectPublicKeyInfo + _, err = asn1.Unmarshal(getPubKeyOutput.PublicKey, &asn1pubKeyInfo) + if err != nil { + return nil, fmt.Errorf("can not parse asn1 public key for KeyId=%s: %s", c.KeyID, err) + } + + pubKey, err := crypto.UnmarshalPubkey(asn1pubKeyInfo.SubjectPublicKey.Bytes) + if err != nil { + return nil, fmt.Errorf("can not unmarshal public key bytes: %s", err) + } + return pubKey, nil +} + +func kmsToEthSig(kmsSig, ecdsaPubKeyBytes, hash []byte) ([]byte, error) { + var asn1Sig asn1ECDSASig + _, err := asn1.Unmarshal(kmsSig, &asn1Sig) + if err != nil { + return nil, err + } + + rBytes := asn1Sig.R.Bytes + sBytes := asn1Sig.S.Bytes + + // Adjust S value from signature to match Eth standard. + // See: https://aws.amazon.com/blogs/database/part2-use-aws-kms-to-securely-manage-ethereum-accounts/ + // "After we extract r and s successfully, we have to test if the value of s is greater than secp256k1n/2 as + // specified in EIP-2 and flip it if required." + sBigInt := new(big.Int).SetBytes(sBytes) + if sBigInt.Cmp(secp256k1HalfN) > 0 { + sBytes = new(big.Int).Sub(secp256k1N, sBigInt).Bytes() + } + + return recoverEthSignature(ecdsaPubKeyBytes, hash, rBytes, sBytes) +} + +// See: https://aws.amazon.com/blogs/database/part2-use-aws-kms-to-securely-manage-ethereum-accounts/ +func recoverEthSignature(expectedPublicKeyBytes, txHash, r, s []byte) ([]byte, error) { + rsSig := append(padTo32Bytes(r), padTo32Bytes(s)...) + ethSig := append(rsSig, []byte{0}...) + + recoveredPublicKeyBytes, err := crypto.Ecrecover(txHash, ethSig) + if err != nil { + return nil, fmt.Errorf("failing to call Ecrecover: %v", err) + } + + if hex.EncodeToString(recoveredPublicKeyBytes) != hex.EncodeToString(expectedPublicKeyBytes) { + ethSig = append(rsSig, []byte{1}...) + recoveredPublicKeyBytes, err = crypto.Ecrecover(txHash, ethSig) + if err != nil { + return nil, fmt.Errorf("failing to call Ecrecover: %v", err) + } + + if hex.EncodeToString(recoveredPublicKeyBytes) != hex.EncodeToString(expectedPublicKeyBytes) { + return nil, fmt.Errorf("can not reconstruct public key from sig") + } + } + + return ethSig, nil +} + +func padTo32Bytes(buffer []byte) []byte { + buffer = bytes.TrimLeft(buffer, "\x00") + for len(buffer) < 32 { + zeroBuf := []byte{0} + buffer = append(zeroBuf, buffer...) + } + return buffer +} diff --git a/integration-tests/deployment/kms/evm_kmsclient_test.go b/integration-tests/deployment/kms/evm_kmsclient_test.go new file mode 100644 index 00000000000..cb909b7d018 --- /dev/null +++ b/integration-tests/deployment/kms/evm_kmsclient_test.go @@ -0,0 +1,27 @@ +package kms + +import ( + "encoding/hex" + "testing" + + "github.com/test-go/testify/require" +) + +func TestKMSToEthSigConversion(t *testing.T) { + kmsSigBytes, err := hex.DecodeString("304402206168865941bafcae3a8cf8b26edbb5693d62222b2e54d962c1aabbeaddf33b6802205edc7f597d2bf2d1eaa14fc514a6202bafcffe52b13ae3fec00674d92a874b73") + require.NoError(t, err) + ecdsaPublicKeyBytes, err := hex.DecodeString("04a735e9e3cb526f83be23b03f1f5ae7788a8654e3f0fcfb4f978290de07ebd47da30eeb72e904fdd4a81b46e320908ff4345e119148f89c1f04674c14a506e24b") + require.NoError(t, err) + txHashBytes, err := hex.DecodeString("a2f037301e90f58c084fe4bec2eef14b26e620d6b6cb46051037d03b29ab7d9a") + require.NoError(t, err) + expectedEthSignBytes, err := hex.DecodeString("6168865941bafcae3a8cf8b26edbb5693d62222b2e54d962c1aabbeaddf33b685edc7f597d2bf2d1eaa14fc514a6202bafcffe52b13ae3fec00674d92a874b7300") + require.NoError(t, err) + + actualEthSig, err := kmsToEthSig( + kmsSigBytes, + ecdsaPublicKeyBytes, + txHashBytes, + ) + require.NoError(t, err) + require.Equal(t, expectedEthSignBytes, actualEthSig) +} diff --git a/integration-tests/deployment/multiclient.go b/integration-tests/deployment/multiclient.go new file mode 100644 index 00000000000..2136c4439dd --- /dev/null +++ b/integration-tests/deployment/multiclient.go @@ -0,0 +1,104 @@ +package deployment + +import ( + "context" + "fmt" + "math/big" + "time" + + "github.com/avast/retry-go/v4" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/ethclient" +) + +const ( + RPC_RETRY_ATTEMPTS = 10 + RPC_RETRY_DELAY = 1000 * time.Millisecond +) + +// MultiClient should comply with the coreenv.OnchainClient interface +var _ OnchainClient = &MultiClient{} + +type MultiClient struct { + *ethclient.Client + backup []*ethclient.Client +} + +type RPC struct { + RPCName string `toml:"rpc_name"` + HTTPURL string `toml:"http_url"` + WSURL string `toml:"ws_url"` +} + +func NewMultiClient(rpcs []RPC) *MultiClient { + if len(rpcs) == 0 { + panic("No RPCs provided") + } + clients := make([]*ethclient.Client, 0, len(rpcs)) + for _, rpc := range rpcs { + client, err := ethclient.Dial(rpc.HTTPURL) + if err != nil { + panic(err) + } + clients = append(clients, client) + } + return &MultiClient{ + Client: clients[0], + backup: clients[1:], + } +} + +func (mc *MultiClient) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) { + var receipt *types.Receipt + err := mc.retryWithBackups(func(client *ethclient.Client) error { + var err error + receipt, err = client.TransactionReceipt(ctx, txHash) + return err + }) + return receipt, err +} + +func (mc *MultiClient) SendTransaction(ctx context.Context, tx *types.Transaction) error { + return mc.retryWithBackups(func(client *ethclient.Client) error { + return client.SendTransaction(ctx, tx) + }) +} + +func (mc *MultiClient) CodeAt(ctx context.Context, account common.Address, blockNumber *big.Int) ([]byte, error) { + var code []byte + err := mc.retryWithBackups(func(client *ethclient.Client) error { + var err error + code, err = client.CodeAt(ctx, account, blockNumber) + return err + }) + return code, err +} + +func (mc *MultiClient) NonceAt(ctx context.Context, account common.Address) (uint64, error) { + var count uint64 + err := mc.retryWithBackups(func(client *ethclient.Client) error { + var err error + count, err = client.NonceAt(ctx, account, nil) + return err + }) + return count, err +} + +func (mc *MultiClient) retryWithBackups(op func(*ethclient.Client) error) error { + var err error + for _, client := range append([]*ethclient.Client{mc.Client}, mc.backup...) { + err2 := retry.Do(func() error { + err = op(client) + if err != nil { + fmt.Printf(" [MultiClient RPC] Retrying with new client, error: %v\n", err) + return err + } + return nil + }, retry.Attempts(RPC_RETRY_ATTEMPTS), retry.Delay(RPC_RETRY_DELAY)) + if err2 == nil { + return nil + } + } + return err +} From 2381addf05a97194cf3f08286e5a8a1952227d8a Mon Sep 17 00:00:00 2001 From: Oliver Townsend Date: Wed, 18 Sep 2024 09:52:04 -0700 Subject: [PATCH 02/10] Pull seth client into multiclient --- .../deployment/{kms => }/evm_kmsclient.go | 26 +++- .../{kms => }/evm_kmsclient_test.go | 2 +- integration-tests/deployment/multiclient.go | 134 +++++++++++++++++- integration-tests/go.mod | 2 +- integration-tests/go.sum | 1 + 5 files changed, 157 insertions(+), 8 deletions(-) rename integration-tests/deployment/{kms => }/evm_kmsclient.go (87%) rename integration-tests/deployment/{kms => }/evm_kmsclient_test.go (98%) diff --git a/integration-tests/deployment/kms/evm_kmsclient.go b/integration-tests/deployment/evm_kmsclient.go similarity index 87% rename from integration-tests/deployment/kms/evm_kmsclient.go rename to integration-tests/deployment/evm_kmsclient.go index 9956bc2405b..b7fb1b6a4b9 100644 --- a/integration-tests/deployment/kms/evm_kmsclient.go +++ b/integration-tests/deployment/evm_kmsclient.go @@ -1,4 +1,4 @@ -package kms +package deployment import ( "bytes" @@ -9,6 +9,8 @@ import ( "fmt" "math/big" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/service/kms" "github.com/ethereum/go-ethereum/accounts/abi/bind" @@ -184,3 +186,25 @@ func padTo32Bytes(buffer []byte) []byte { } return buffer } + +type AwsSessionFn func(config Config) *session.Session + +var awsSessionFromEnvVarsFn = func(config Config) *session.Session { + return session.Must( + session.NewSession(&aws.Config{ + Region: aws.String(config.EnvConfig.KmsDeployerKeyRegion), + CredentialsChainVerboseErrors: aws.Bool(true), + })) +} + +var awsSessionFromProfileFn = func(config Config) *session.Session { + return session.Must( + session.NewSessionWithOptions(session.Options{ + SharedConfigState: session.SharedConfigEnable, + Profile: config.EnvConfig.AwsProfileName, + Config: aws.Config{ + Region: aws.String(config.EnvConfig.KmsDeployerKeyRegion), + CredentialsChainVerboseErrors: aws.Bool(true), + }, + })) +} diff --git a/integration-tests/deployment/kms/evm_kmsclient_test.go b/integration-tests/deployment/evm_kmsclient_test.go similarity index 98% rename from integration-tests/deployment/kms/evm_kmsclient_test.go rename to integration-tests/deployment/evm_kmsclient_test.go index cb909b7d018..8a889a56067 100644 --- a/integration-tests/deployment/kms/evm_kmsclient_test.go +++ b/integration-tests/deployment/evm_kmsclient_test.go @@ -1,4 +1,4 @@ -package kms +package deployment import ( "encoding/hex" diff --git a/integration-tests/deployment/multiclient.go b/integration-tests/deployment/multiclient.go index 2136c4439dd..b5cac349e2e 100644 --- a/integration-tests/deployment/multiclient.go +++ b/integration-tests/deployment/multiclient.go @@ -4,12 +4,17 @@ import ( "context" "fmt" "math/big" + "os" "time" "github.com/avast/retry-go/v4" + "github.com/aws/aws-sdk-go/service/kms" + "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethclient" + "github.com/pelletier/go-toml/v2" + "github.com/smartcontractkit/chainlink-testing-framework/seth" ) const ( @@ -17,12 +22,17 @@ const ( RPC_RETRY_DELAY = 1000 * time.Millisecond ) -// MultiClient should comply with the coreenv.OnchainClient interface +// MultiClient should comply with the OnchainClient interface var _ OnchainClient = &MultiClient{} type MultiClient struct { *ethclient.Client backup []*ethclient.Client + // we will use Seth only for gas estimations, confirming and tracing the transactions, but for sending transactions we will use pure ethclient + // so that MultiClient conforms to the OnchainClient interface + SethClient *seth.Client + EvmKMSClient *evmKMSClient + chainId uint64 } type RPC struct { @@ -31,11 +41,34 @@ type RPC struct { WSURL string `toml:"ws_url"` } -func NewMultiClient(rpcs []RPC) *MultiClient { +type Config struct { + EnvConfig EnvConfig `toml:"env_config"` +} + +type EnvConfig struct { + TestWalletKey string `toml:"test_wallet_key"` + KmsDeployerKeyId string `toml:"kms_deployer_key_id"` + KmsDeployerKeyRegion string `toml:"kms_deployer_key_region"` + AwsProfileName string `toml:"aws_profile_name"` + EvmNetworks []EvmNetwork `toml:"evm_networks"` + // Seth-related + GethWrappersDirs []string `toml:"geth_wrappers_dirs"` + SethConfigFile string `toml:"seth_config_file"` +} + +type EvmNetwork struct { + ChainID uint64 `toml:"chain_id"` + EtherscanAPIKey string `toml:"etherscan_api_key"` + EtherscanUrl string `toml:"etherscan_url"` + RPCs []RPC `toml:"rpcs"` +} + +func initRpcClients(rpcs []RPC) (*ethclient.Client, []*ethclient.Client) { if len(rpcs) == 0 { panic("No RPCs provided") } clients := make([]*ethclient.Client, 0, len(rpcs)) + for _, rpc := range rpcs { client, err := ethclient.Dial(rpc.HTTPURL) if err != nil { @@ -43,10 +76,101 @@ func NewMultiClient(rpcs []RPC) *MultiClient { } clients = append(clients, client) } - return &MultiClient{ - Client: clients[0], - backup: clients[1:], + return clients[0], clients[1:] +} + +func NewMultiClientWithSeth(rpcs []RPC, chainId uint64, config Config) *MultiClient { + mainClient, backupClients := initRpcClients(rpcs) + mc := &MultiClient{ + Client: mainClient, + backup: backupClients, + chainId: chainId, + } + + sethClient, err := buildSethClient(rpcs[0].HTTPURL, chainId, config) + if err != nil { + panic(err) + } + + mc.SethClient = sethClient + mc.EvmKMSClient = initialiseKMSClient(config) + + return mc +} + +func buildSethClient(rpc string, chainId uint64, config Config) (*seth.Client, error) { + var sethClient *seth.Client + var err error + + // if config path is provided use the TOML file to configure Seth to provide maximum flexibility + if config.EnvConfig.SethConfigFile != "" { + sethConfig, readErr := readSethConfigFromFile(config.EnvConfig.SethConfigFile) + if readErr != nil { + return nil, readErr + } + + sethClient, err = seth.NewClientBuilderWithConfig(sethConfig). + UseNetworkWithChainId(chainId). + WithRpcUrl(rpc). + WithPrivateKeys([]string{config.EnvConfig.TestWalletKey}). + Build() + } else { + // if full flexibility is not needed we create a client with reasonable defaults + // if you need to further tweak them, please refer to https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/seth/README.md + sethClient, err = seth.NewClientBuilder(). + WithRpcUrl(rpc). + WithPrivateKeys([]string{config.EnvConfig.TestWalletKey}). + WithProtections(true, true, seth.MustMakeDuration(1*time.Minute)). + WithGethWrappersFolders(config.EnvConfig.GethWrappersDirs). + // Fast priority will add a 20% buffer on top of what the node suggests + // we will use last 20 block to estimate block congestion and further bump gas price suggested by the node + WithGasPriceEstimations(true, 20, seth.Priority_Fast). + Build() + } + + return sethClient, err +} + +func readSethConfigFromFile(configPath string) (*seth.Config, error) { + d, err := os.ReadFile(configPath) + if err != nil { + return nil, err + } + + var sethConfig seth.Config + err = toml.Unmarshal(d, &sethConfig) + if err != nil { + return nil, err + } + + return &sethConfig, nil +} + +func initialiseKMSClient(config Config) *evmKMSClient { + if config.EnvConfig.KmsDeployerKeyId != "" && config.EnvConfig.KmsDeployerKeyRegion != "" { + var awsSessionFn AwsSessionFn + if config.EnvConfig.AwsProfileName != "" { + awsSessionFn = awsSessionFromProfileFn + } else { + awsSessionFn = awsSessionFromEnvVarsFn + } + return NewEVMKMSClient(kms.New(awsSessionFn(config)), config.EnvConfig.KmsDeployerKeyId) } + return nil +} + +func (mc *MultiClient) GetKMSKey() *bind.TransactOpts { + kmsTxOpts, err := mc.EvmKMSClient.GetKMSTransactOpts(context.Background(), big.NewInt(int64(mc.chainId))) + if err != nil { + panic(err) + } + // nonce needs to be `nil` so that RPC node sets it, otherwise Seth would set it to whatever it was, when we requested the key + return mc.SethClient.NewTXOpts(seth.WithNonce(nil), seth.WithFrom(kmsTxOpts.From), seth.WithSignerFn(kmsTxOpts.Signer)) +} + +func (mc *MultiClient) GetTestWalletKey() *bind.TransactOpts { + // nonce needs to be `nil` so that RPC node sets it, otherwise Seth would set it to whatever it was, when we requested the key + return mc.SethClient.NewTXOpts(seth.WithNonce(nil)) } func (mc *MultiClient) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) { diff --git a/integration-tests/go.mod b/integration-tests/go.mod index 714c415086c..fae9152e7d5 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -43,7 +43,7 @@ require ( github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.0 github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.5 github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0 - github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.1 + github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.4-0.20240912161944-13ade8436072 github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.0 github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 github.com/smartcontractkit/libocr v0.0.0-20240717100443-f6226e09bee7 diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 2494576bc4d..13ddaa47527 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1445,6 +1445,7 @@ github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0 h1:V github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0/go.mod h1:lyAu+oMXdNUzEDScj2DXB2IueY+SDXPPfyl/kb63tMM= github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.1 h1:2OxnPfvjC+zs0ZokSsRTRnJrEGJ4NVJwZgfroS1lPHs= github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.1/go.mod h1:afY3QmNgeR/VI1pRbGH8g3YXGy7C2RrFOwUzEFvL3L8= +github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.4-0.20240912161944-13ade8436072/go.mod h1:afY3QmNgeR/VI1pRbGH8g3YXGy7C2RrFOwUzEFvL3L8= github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.0 h1:gfhfTn7HkbUHNooSF3c9vzQyN8meWJVGt6G/pNUbpYk= github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.0/go.mod h1:tqajhpUJA/9OaMCLitghBXjAgqYO4i27St0F4TUO3+M= github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 h1:12ijqMM9tvYVEm+nR826WsrNi6zCKpwBhuApq127wHs= From 1425b3eb8dd36def64aa21246c2b9e8e7d7723f2 Mon Sep 17 00:00:00 2001 From: AnieeG Date: Wed, 18 Sep 2024 12:47:37 -0700 Subject: [PATCH 03/10] remove toml tags --- integration-tests/deployment/multiclient.go | 30 ++++++++++----------- integration-tests/go.mod | 2 +- integration-tests/go.sum | 3 +-- 3 files changed, 17 insertions(+), 18 deletions(-) diff --git a/integration-tests/deployment/multiclient.go b/integration-tests/deployment/multiclient.go index b5cac349e2e..4d3d90c87a0 100644 --- a/integration-tests/deployment/multiclient.go +++ b/integration-tests/deployment/multiclient.go @@ -36,31 +36,31 @@ type MultiClient struct { } type RPC struct { - RPCName string `toml:"rpc_name"` - HTTPURL string `toml:"http_url"` - WSURL string `toml:"ws_url"` + RPCName string + HTTPURL string + WSURL string } type Config struct { - EnvConfig EnvConfig `toml:"env_config"` + EnvConfig EnvConfig } type EnvConfig struct { - TestWalletKey string `toml:"test_wallet_key"` - KmsDeployerKeyId string `toml:"kms_deployer_key_id"` - KmsDeployerKeyRegion string `toml:"kms_deployer_key_region"` - AwsProfileName string `toml:"aws_profile_name"` - EvmNetworks []EvmNetwork `toml:"evm_networks"` + TestWalletKey string + KmsDeployerKeyId string + KmsDeployerKeyRegion string + AwsProfileName string + EvmNetworks []EvmNetwork // Seth-related - GethWrappersDirs []string `toml:"geth_wrappers_dirs"` - SethConfigFile string `toml:"seth_config_file"` + GethWrappersDirs []string + SethConfigFile string } type EvmNetwork struct { - ChainID uint64 `toml:"chain_id"` - EtherscanAPIKey string `toml:"etherscan_api_key"` - EtherscanUrl string `toml:"etherscan_url"` - RPCs []RPC `toml:"rpcs"` + ChainID uint64 + EtherscanAPIKey string + EtherscanUrl string + RPCs []RPC } func initRpcClients(rpcs []RPC) (*ethclient.Client, []*ethclient.Client) { diff --git a/integration-tests/go.mod b/integration-tests/go.mod index fae9152e7d5..f393d7b7b2e 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -11,6 +11,7 @@ require ( github.com/Khan/genqlient v0.7.0 github.com/Masterminds/semver/v3 v3.2.1 github.com/avast/retry-go/v4 v4.6.0 + github.com/aws/aws-sdk-go v1.45.25 github.com/barkimedes/go-deepcopy v0.0.0-20220514131651-17c30cfc62df github.com/chaos-mesh/chaos-mesh/api v0.0.0-20240821051457-da69c6d9617a github.com/cli/go-gh/v2 v2.0.0 @@ -107,7 +108,6 @@ require ( github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect github.com/avast/retry-go v3.0.0+incompatible // indirect github.com/awalterschulze/gographviz v2.0.3+incompatible // indirect - github.com/aws/aws-sdk-go v1.45.25 // indirect github.com/aws/aws-sdk-go-v2 v1.30.4 // indirect github.com/aws/aws-sdk-go-v2/config v1.27.28 // indirect github.com/aws/aws-sdk-go-v2/credentials v1.17.28 // indirect diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 13ddaa47527..9ee36ab7d41 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1443,8 +1443,7 @@ github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.5 h1:Owb1MQZn0 github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.5/go.mod h1:hS4yNF94C1lkS9gvtFXW8Km8K9NzGeR20aNfkqo5qbE= github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0 h1:VIxK8u0Jd0Q/VuhmsNm6Bls6Tb31H/sA3A/rbc5hnhg= github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0/go.mod h1:lyAu+oMXdNUzEDScj2DXB2IueY+SDXPPfyl/kb63tMM= -github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.1 h1:2OxnPfvjC+zs0ZokSsRTRnJrEGJ4NVJwZgfroS1lPHs= -github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.1/go.mod h1:afY3QmNgeR/VI1pRbGH8g3YXGy7C2RrFOwUzEFvL3L8= +github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.4-0.20240912161944-13ade8436072 h1:/wGR8PUytZBze4DPnhzF+V7IVl/EslDmDC2SsamXjqw= github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.4-0.20240912161944-13ade8436072/go.mod h1:afY3QmNgeR/VI1pRbGH8g3YXGy7C2RrFOwUzEFvL3L8= github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.0 h1:gfhfTn7HkbUHNooSF3c9vzQyN8meWJVGt6G/pNUbpYk= github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.0/go.mod h1:tqajhpUJA/9OaMCLitghBXjAgqYO4i27St0F4TUO3+M= From b5f1c48fe0b5115e5b472d8c5e546db4a8158975 Mon Sep 17 00:00:00 2001 From: connorwstein Date: Wed, 18 Sep 2024 16:42:31 -0400 Subject: [PATCH 04/10] Keep multiclient simple --- .../ccip/ccip_integration_tests/helpers.go | 2 - integration-tests/deployment/evm_kmsclient.go | 42 +++-- integration-tests/deployment/multiclient.go | 174 ++++-------------- .../deployment/multiclient_test.go | 29 +++ integration-tests/load/go.mod | 2 +- integration-tests/load/go.sum | 4 +- 6 files changed, 98 insertions(+), 155 deletions(-) create mode 100644 integration-tests/deployment/multiclient_test.go diff --git a/core/capabilities/ccip/ccip_integration_tests/helpers.go b/core/capabilities/ccip/ccip_integration_tests/helpers.go index 25baddfb48e..4670333e391 100644 --- a/core/capabilities/ccip/ccip_integration_tests/helpers.go +++ b/core/capabilities/ccip/ccip_integration_tests/helpers.go @@ -406,7 +406,6 @@ func (h *homeChain) AddNodes( p2pIDs [][32]byte, capabilityIDs [][32]byte, ) { - // Need to sort, otherwise _checkIsValidUniqueSubset onChain will fail sortP2PIDS(p2pIDs) var nodeParams []kcr.CapabilitiesRegistryNodeParams for _, p2pID := range p2pIDs { @@ -430,7 +429,6 @@ func AddChainConfig( p2pIDs [][32]byte, f uint8, ) ccip_config.CCIPConfigTypesChainConfigInfo { - // Need to sort, otherwise _checkIsValidUniqueSubset onChain will fail sortP2PIDS(p2pIDs) // First Add ChainConfig that includes all p2pIDs as readers encodedExtraChainConfig, err := chainconfig.EncodeChainConfig(chainconfig.ChainConfig{ diff --git a/integration-tests/deployment/evm_kmsclient.go b/integration-tests/deployment/evm_kmsclient.go index b7fb1b6a4b9..8e2de18d789 100644 --- a/integration-tests/deployment/evm_kmsclient.go +++ b/integration-tests/deployment/evm_kmsclient.go @@ -44,24 +44,44 @@ type asn1ECDSASig struct { S asn1.RawValue } +// TODO: Mockery gen then test with a regular eth key behind the interface. type KMSClient interface { GetPublicKey(input *kms.GetPublicKeyInput) (*kms.GetPublicKeyOutput, error) Sign(input *kms.SignInput) (*kms.SignOutput, error) } -type evmKMSClient struct { +type KMS struct { + KmsDeployerKeyId string + KmsDeployerKeyRegion string + AwsProfileName string +} + +func NewKMSClient(config KMS) KMSClient { + if config.KmsDeployerKeyId != "" && config.KmsDeployerKeyRegion != "" { + var awsSessionFn AwsSessionFn + if config.AwsProfileName != "" { + awsSessionFn = awsSessionFromProfileFn + } else { + awsSessionFn = awsSessionFromEnvVarsFn + } + return kms.New(awsSessionFn(config)) + } + return nil +} + +type EVMKMSClient struct { Client KMSClient KeyID string } -func NewEVMKMSClient(client KMSClient, keyID string) *evmKMSClient { - return &evmKMSClient{ +func NewEVMKMSClient(client KMSClient, keyID string) *EVMKMSClient { + return &EVMKMSClient{ Client: client, KeyID: keyID, } } -func (c *evmKMSClient) GetKMSTransactOpts(ctx context.Context, chainID *big.Int) (*bind.TransactOpts, error) { +func (c *EVMKMSClient) GetKMSTransactOpts(ctx context.Context, chainID *big.Int) (*bind.TransactOpts, error) { ecdsaPublicKey, err := c.GetECDSAPublicKey() if err != nil { return nil, err @@ -110,7 +130,7 @@ func (c *evmKMSClient) GetKMSTransactOpts(ctx context.Context, chainID *big.Int) } // GetECDSAPublicKey retrieves the public key from KMS and converts it to its ECDSA representation. -func (c *evmKMSClient) GetECDSAPublicKey() (*ecdsa.PublicKey, error) { +func (c *EVMKMSClient) GetECDSAPublicKey() (*ecdsa.PublicKey, error) { getPubKeyOutput, err := c.Client.GetPublicKey(&kms.GetPublicKeyInput{ KeyId: aws.String(c.KeyID), }) @@ -187,23 +207,23 @@ func padTo32Bytes(buffer []byte) []byte { return buffer } -type AwsSessionFn func(config Config) *session.Session +type AwsSessionFn func(config KMS) *session.Session -var awsSessionFromEnvVarsFn = func(config Config) *session.Session { +var awsSessionFromEnvVarsFn = func(config KMS) *session.Session { return session.Must( session.NewSession(&aws.Config{ - Region: aws.String(config.EnvConfig.KmsDeployerKeyRegion), + Region: aws.String(config.KmsDeployerKeyRegion), CredentialsChainVerboseErrors: aws.Bool(true), })) } -var awsSessionFromProfileFn = func(config Config) *session.Session { +var awsSessionFromProfileFn = func(config KMS) *session.Session { return session.Must( session.NewSessionWithOptions(session.Options{ SharedConfigState: session.SharedConfigEnable, - Profile: config.EnvConfig.AwsProfileName, + Profile: config.AwsProfileName, Config: aws.Config{ - Region: aws.String(config.EnvConfig.KmsDeployerKeyRegion), + Region: aws.String(config.KmsDeployerKeyRegion), CredentialsChainVerboseErrors: aws.Bool(true), }, })) diff --git a/integration-tests/deployment/multiclient.go b/integration-tests/deployment/multiclient.go index 4d3d90c87a0..927df43e9ae 100644 --- a/integration-tests/deployment/multiclient.go +++ b/integration-tests/deployment/multiclient.go @@ -4,173 +4,67 @@ import ( "context" "fmt" "math/big" - "os" "time" "github.com/avast/retry-go/v4" - "github.com/aws/aws-sdk-go/service/kms" - "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethclient" - "github.com/pelletier/go-toml/v2" - "github.com/smartcontractkit/chainlink-testing-framework/seth" + "github.com/pkg/errors" ) const ( - RPC_RETRY_ATTEMPTS = 10 - RPC_RETRY_DELAY = 1000 * time.Millisecond + RPC_DEFAULT_RETRY_ATTEMPTS = 10 + RPC_DEFAULT_RETRY_DELAY = 1000 * time.Millisecond ) -// MultiClient should comply with the OnchainClient interface -var _ OnchainClient = &MultiClient{} +type RetryConfig struct { + Attempts uint + Delay time.Duration +} -type MultiClient struct { - *ethclient.Client - backup []*ethclient.Client - // we will use Seth only for gas estimations, confirming and tracing the transactions, but for sending transactions we will use pure ethclient - // so that MultiClient conforms to the OnchainClient interface - SethClient *seth.Client - EvmKMSClient *evmKMSClient - chainId uint64 +func defaultRetryConfig() RetryConfig { + return RetryConfig{ + Attempts: RPC_DEFAULT_RETRY_ATTEMPTS, + Delay: RPC_DEFAULT_RETRY_DELAY, + } } type RPC struct { - RPCName string HTTPURL string - WSURL string + // TODO: ws support? } -type Config struct { - EnvConfig EnvConfig -} - -type EnvConfig struct { - TestWalletKey string - KmsDeployerKeyId string - KmsDeployerKeyRegion string - AwsProfileName string - EvmNetworks []EvmNetwork - // Seth-related - GethWrappersDirs []string - SethConfigFile string -} +// MultiClient should comply with the OnchainClient interface +var _ OnchainClient = &MultiClient{} -type EvmNetwork struct { - ChainID uint64 - EtherscanAPIKey string - EtherscanUrl string - RPCs []RPC +type MultiClient struct { + *ethclient.Client + Backups []*ethclient.Client + RetryConfig RetryConfig } -func initRpcClients(rpcs []RPC) (*ethclient.Client, []*ethclient.Client) { +func NewMultiClient(rpcs []RPC, opts ...func(client *MultiClient)) (*MultiClient, error) { if len(rpcs) == 0 { - panic("No RPCs provided") + return nil, fmt.Errorf("No RPCs provided, need at least one") } + var mc MultiClient clients := make([]*ethclient.Client, 0, len(rpcs)) - for _, rpc := range rpcs { client, err := ethclient.Dial(rpc.HTTPURL) if err != nil { - panic(err) + return nil, errors.Wrapf(err, "failed to dial %s", rpc.HTTPURL) } clients = append(clients, client) } - return clients[0], clients[1:] -} - -func NewMultiClientWithSeth(rpcs []RPC, chainId uint64, config Config) *MultiClient { - mainClient, backupClients := initRpcClients(rpcs) - mc := &MultiClient{ - Client: mainClient, - backup: backupClients, - chainId: chainId, - } - - sethClient, err := buildSethClient(rpcs[0].HTTPURL, chainId, config) - if err != nil { - panic(err) - } - - mc.SethClient = sethClient - mc.EvmKMSClient = initialiseKMSClient(config) - - return mc -} - -func buildSethClient(rpc string, chainId uint64, config Config) (*seth.Client, error) { - var sethClient *seth.Client - var err error - - // if config path is provided use the TOML file to configure Seth to provide maximum flexibility - if config.EnvConfig.SethConfigFile != "" { - sethConfig, readErr := readSethConfigFromFile(config.EnvConfig.SethConfigFile) - if readErr != nil { - return nil, readErr - } - - sethClient, err = seth.NewClientBuilderWithConfig(sethConfig). - UseNetworkWithChainId(chainId). - WithRpcUrl(rpc). - WithPrivateKeys([]string{config.EnvConfig.TestWalletKey}). - Build() - } else { - // if full flexibility is not needed we create a client with reasonable defaults - // if you need to further tweak them, please refer to https://github.com/smartcontractkit/chainlink-testing-framework/blob/main/seth/README.md - sethClient, err = seth.NewClientBuilder(). - WithRpcUrl(rpc). - WithPrivateKeys([]string{config.EnvConfig.TestWalletKey}). - WithProtections(true, true, seth.MustMakeDuration(1*time.Minute)). - WithGethWrappersFolders(config.EnvConfig.GethWrappersDirs). - // Fast priority will add a 20% buffer on top of what the node suggests - // we will use last 20 block to estimate block congestion and further bump gas price suggested by the node - WithGasPriceEstimations(true, 20, seth.Priority_Fast). - Build() - } - - return sethClient, err -} - -func readSethConfigFromFile(configPath string) (*seth.Config, error) { - d, err := os.ReadFile(configPath) - if err != nil { - return nil, err - } + mc.Client = clients[0] + mc.Backups = clients[1:] + mc.RetryConfig = defaultRetryConfig() - var sethConfig seth.Config - err = toml.Unmarshal(d, &sethConfig) - if err != nil { - return nil, err + for _, opt := range opts { + opt(&mc) } - - return &sethConfig, nil -} - -func initialiseKMSClient(config Config) *evmKMSClient { - if config.EnvConfig.KmsDeployerKeyId != "" && config.EnvConfig.KmsDeployerKeyRegion != "" { - var awsSessionFn AwsSessionFn - if config.EnvConfig.AwsProfileName != "" { - awsSessionFn = awsSessionFromProfileFn - } else { - awsSessionFn = awsSessionFromEnvVarsFn - } - return NewEVMKMSClient(kms.New(awsSessionFn(config)), config.EnvConfig.KmsDeployerKeyId) - } - return nil -} - -func (mc *MultiClient) GetKMSKey() *bind.TransactOpts { - kmsTxOpts, err := mc.EvmKMSClient.GetKMSTransactOpts(context.Background(), big.NewInt(int64(mc.chainId))) - if err != nil { - panic(err) - } - // nonce needs to be `nil` so that RPC node sets it, otherwise Seth would set it to whatever it was, when we requested the key - return mc.SethClient.NewTXOpts(seth.WithNonce(nil), seth.WithFrom(kmsTxOpts.From), seth.WithSignerFn(kmsTxOpts.Signer)) -} - -func (mc *MultiClient) GetTestWalletKey() *bind.TransactOpts { - // nonce needs to be `nil` so that RPC node sets it, otherwise Seth would set it to whatever it was, when we requested the key - return mc.SethClient.NewTXOpts(seth.WithNonce(nil)) + return &mc, nil } func (mc *MultiClient) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) { @@ -211,18 +105,20 @@ func (mc *MultiClient) NonceAt(ctx context.Context, account common.Address) (uin func (mc *MultiClient) retryWithBackups(op func(*ethclient.Client) error) error { var err error - for _, client := range append([]*ethclient.Client{mc.Client}, mc.backup...) { + for _, client := range append([]*ethclient.Client{mc.Client}, mc.Backups...) { err2 := retry.Do(func() error { err = op(client) if err != nil { - fmt.Printf(" [MultiClient RPC] Retrying with new client, error: %v\n", err) + // TODO: logger? + fmt.Printf("Error %v with client %v\n", err, client) return err } return nil - }, retry.Attempts(RPC_RETRY_ATTEMPTS), retry.Delay(RPC_RETRY_DELAY)) + }, retry.Attempts(mc.RetryConfig.Attempts), retry.Delay(mc.RetryConfig.Delay)) if err2 == nil { return nil } + fmt.Println("Client %v failed, trying next client", client) } - return err + return errors.Wrapf(err, "All backup clients %v failed", mc.Backups) } diff --git a/integration-tests/deployment/multiclient_test.go b/integration-tests/deployment/multiclient_test.go new file mode 100644 index 00000000000..40b0b8e6946 --- /dev/null +++ b/integration-tests/deployment/multiclient_test.go @@ -0,0 +1,29 @@ +package deployment + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMultiClient(t *testing.T) { + // Expect an error if no RPCs supplied. + _, err := NewMultiClient([]RPC{}) + require.Error(t, err) + + // Expect defaults to be set if not provided. + mc, err := NewMultiClient([]RPC{{HTTPURL: "http://localhost:8545"}}) + require.NoError(t, err) + assert.Equal(t, mc.RetryConfig.Attempts, uint(RPC_DEFAULT_RETRY_ATTEMPTS)) + assert.Equal(t, mc.RetryConfig.Delay, RPC_DEFAULT_RETRY_DELAY) + + // Expect second client to be set as backup. + mc, err = NewMultiClient([]RPC{ + {HTTPURL: "http://localhost:8545"}, + {HTTPURL: "http://localhost:8546"}, + }) + require.NoError(t, err) + require.Equal(t, len(mc.Backups), 1) + assert.Equal(t, mc.Backups[0], "http://localhost:8546") +} diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 673191e145b..adbc5d36f9a 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -17,7 +17,7 @@ require ( github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chainlink-common v0.2.2-0.20240916150342-36cb47701edf github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.5 - github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.1 + github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.4-0.20240912161944-13ade8436072 github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.0 github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c github.com/smartcontractkit/chainlink/v2 v2.9.0-beta0.0.20240216210048-da02459ddad8 diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index ee64962cb1c..708f9726936 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1417,8 +1417,8 @@ github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.5 h1:Owb1MQZn0 github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.5/go.mod h1:hS4yNF94C1lkS9gvtFXW8Km8K9NzGeR20aNfkqo5qbE= github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0 h1:VIxK8u0Jd0Q/VuhmsNm6Bls6Tb31H/sA3A/rbc5hnhg= github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0/go.mod h1:lyAu+oMXdNUzEDScj2DXB2IueY+SDXPPfyl/kb63tMM= -github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.1 h1:2OxnPfvjC+zs0ZokSsRTRnJrEGJ4NVJwZgfroS1lPHs= -github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.1/go.mod h1:afY3QmNgeR/VI1pRbGH8g3YXGy7C2RrFOwUzEFvL3L8= +github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.4-0.20240912161944-13ade8436072 h1:/wGR8PUytZBze4DPnhzF+V7IVl/EslDmDC2SsamXjqw= +github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.4-0.20240912161944-13ade8436072/go.mod h1:afY3QmNgeR/VI1pRbGH8g3YXGy7C2RrFOwUzEFvL3L8= github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.0 h1:gfhfTn7HkbUHNooSF3c9vzQyN8meWJVGt6G/pNUbpYk= github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.0/go.mod h1:tqajhpUJA/9OaMCLitghBXjAgqYO4i27St0F4TUO3+M= github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 h1:12ijqMM9tvYVEm+nR826WsrNi6zCKpwBhuApq127wHs= From 188e99356f5c2d16d3f046faa544e0eecc5bb882 Mon Sep 17 00:00:00 2001 From: connorwstein Date: Wed, 18 Sep 2024 16:46:47 -0400 Subject: [PATCH 05/10] Config validation for kms --- integration-tests/deployment/evm_kmsclient.go | 23 +++++++++++-------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/integration-tests/deployment/evm_kmsclient.go b/integration-tests/deployment/evm_kmsclient.go index 8e2de18d789..1f253f9a8b6 100644 --- a/integration-tests/deployment/evm_kmsclient.go +++ b/integration-tests/deployment/evm_kmsclient.go @@ -56,17 +56,20 @@ type KMS struct { AwsProfileName string } -func NewKMSClient(config KMS) KMSClient { - if config.KmsDeployerKeyId != "" && config.KmsDeployerKeyRegion != "" { - var awsSessionFn AwsSessionFn - if config.AwsProfileName != "" { - awsSessionFn = awsSessionFromProfileFn - } else { - awsSessionFn = awsSessionFromEnvVarsFn - } - return kms.New(awsSessionFn(config)) +func NewKMSClient(config KMS) (KMSClient, error) { + if config.KmsDeployerKeyId == "" { + return nil, fmt.Errorf("KMS key ID is required") + } + if config.KmsDeployerKeyRegion == "" { + return nil, fmt.Errorf("KMS key region is required") + } + var awsSessionFn AwsSessionFn + if config.AwsProfileName != "" { + awsSessionFn = awsSessionFromProfileFn + } else { + awsSessionFn = awsSessionFromEnvVarsFn } - return nil + return kms.New(awsSessionFn(config)), nil } type EVMKMSClient struct { From 7b1592b0eeeeb24bdc91bf90f2a505161ab06df4 Mon Sep 17 00:00:00 2001 From: connorwstein Date: Wed, 18 Sep 2024 16:52:22 -0400 Subject: [PATCH 06/10] Rename to ws, fix test --- integration-tests/deployment/multiclient.go | 10 +++++----- integration-tests/deployment/multiclient_test.go | 14 ++++++++++---- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/integration-tests/deployment/multiclient.go b/integration-tests/deployment/multiclient.go index 927df43e9ae..02a18f760df 100644 --- a/integration-tests/deployment/multiclient.go +++ b/integration-tests/deployment/multiclient.go @@ -31,8 +31,8 @@ func defaultRetryConfig() RetryConfig { } type RPC struct { - HTTPURL string - // TODO: ws support? + WSURL string + // TODO: http fallback needed for some networks? } // MultiClient should comply with the OnchainClient interface @@ -51,9 +51,9 @@ func NewMultiClient(rpcs []RPC, opts ...func(client *MultiClient)) (*MultiClient var mc MultiClient clients := make([]*ethclient.Client, 0, len(rpcs)) for _, rpc := range rpcs { - client, err := ethclient.Dial(rpc.HTTPURL) + client, err := ethclient.Dial(rpc.WSURL) if err != nil { - return nil, errors.Wrapf(err, "failed to dial %s", rpc.HTTPURL) + return nil, errors.Wrapf(err, "failed to dial %s", rpc.WSURL) } clients = append(clients, client) } @@ -118,7 +118,7 @@ func (mc *MultiClient) retryWithBackups(op func(*ethclient.Client) error) error if err2 == nil { return nil } - fmt.Println("Client %v failed, trying next client", client) + fmt.Printf("Client %v failed, trying next client\n", client) } return errors.Wrapf(err, "All backup clients %v failed", mc.Backups) } diff --git a/integration-tests/deployment/multiclient_test.go b/integration-tests/deployment/multiclient_test.go index 40b0b8e6946..b0ee6c2042f 100644 --- a/integration-tests/deployment/multiclient_test.go +++ b/integration-tests/deployment/multiclient_test.go @@ -1,6 +1,8 @@ package deployment import ( + "net/http" + "net/http/httptest" "testing" "github.com/stretchr/testify/assert" @@ -9,21 +11,25 @@ import ( func TestMultiClient(t *testing.T) { // Expect an error if no RPCs supplied. + s := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + writer.WriteHeader(http.StatusOK) + writer.Write([]byte(`{"jsonrpc":"2.0","id":1,"result":true}`)) + })) + defer s.Close() _, err := NewMultiClient([]RPC{}) require.Error(t, err) // Expect defaults to be set if not provided. - mc, err := NewMultiClient([]RPC{{HTTPURL: "http://localhost:8545"}}) + mc, err := NewMultiClient([]RPC{{WSURL: s.URL}}) require.NoError(t, err) assert.Equal(t, mc.RetryConfig.Attempts, uint(RPC_DEFAULT_RETRY_ATTEMPTS)) assert.Equal(t, mc.RetryConfig.Delay, RPC_DEFAULT_RETRY_DELAY) // Expect second client to be set as backup. mc, err = NewMultiClient([]RPC{ - {HTTPURL: "http://localhost:8545"}, - {HTTPURL: "http://localhost:8546"}, + {WSURL: s.URL}, + {WSURL: s.URL}, }) require.NoError(t, err) require.Equal(t, len(mc.Backups), 1) - assert.Equal(t, mc.Backups[0], "http://localhost:8546") } From 96e002db8445cbb044e3099bc175be385f68c218 Mon Sep 17 00:00:00 2001 From: connorwstein Date: Wed, 18 Sep 2024 16:58:20 -0400 Subject: [PATCH 07/10] Changeset --- .changeset/sour-ears-wink.md | 5 +++++ 1 file changed, 5 insertions(+) create mode 100644 .changeset/sour-ears-wink.md diff --git a/.changeset/sour-ears-wink.md b/.changeset/sour-ears-wink.md new file mode 100644 index 00000000000..d77fd548c51 --- /dev/null +++ b/.changeset/sour-ears-wink.md @@ -0,0 +1,5 @@ +--- +"chainlink": patch +--- + +#internal KMS client for deployment From c0adf8c44534d17c7a47688a54a644620acf8c2f Mon Sep 17 00:00:00 2001 From: connorwstein Date: Wed, 18 Sep 2024 17:46:18 -0400 Subject: [PATCH 08/10] Downgrade seth --- integration-tests/go.mod | 2 +- integration-tests/go.sum | 4 ++-- integration-tests/load/go.mod | 2 +- integration-tests/load/go.sum | 4 ++-- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/integration-tests/go.mod b/integration-tests/go.mod index f393d7b7b2e..f96def40fd5 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -44,7 +44,7 @@ require ( github.com/smartcontractkit/chainlink-testing-framework/havoc v1.50.0 github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.5 github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0 - github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.4-0.20240912161944-13ade8436072 + github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.1 github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.0 github.com/smartcontractkit/chainlink/v2 v2.0.0-00010101000000-000000000000 github.com/smartcontractkit/libocr v0.0.0-20240717100443-f6226e09bee7 diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 9ee36ab7d41..2494576bc4d 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1443,8 +1443,8 @@ github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.5 h1:Owb1MQZn0 github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.5/go.mod h1:hS4yNF94C1lkS9gvtFXW8Km8K9NzGeR20aNfkqo5qbE= github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0 h1:VIxK8u0Jd0Q/VuhmsNm6Bls6Tb31H/sA3A/rbc5hnhg= github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0/go.mod h1:lyAu+oMXdNUzEDScj2DXB2IueY+SDXPPfyl/kb63tMM= -github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.4-0.20240912161944-13ade8436072 h1:/wGR8PUytZBze4DPnhzF+V7IVl/EslDmDC2SsamXjqw= -github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.4-0.20240912161944-13ade8436072/go.mod h1:afY3QmNgeR/VI1pRbGH8g3YXGy7C2RrFOwUzEFvL3L8= +github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.1 h1:2OxnPfvjC+zs0ZokSsRTRnJrEGJ4NVJwZgfroS1lPHs= +github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.1/go.mod h1:afY3QmNgeR/VI1pRbGH8g3YXGy7C2RrFOwUzEFvL3L8= github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.0 h1:gfhfTn7HkbUHNooSF3c9vzQyN8meWJVGt6G/pNUbpYk= github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.0/go.mod h1:tqajhpUJA/9OaMCLitghBXjAgqYO4i27St0F4TUO3+M= github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 h1:12ijqMM9tvYVEm+nR826WsrNi6zCKpwBhuApq127wHs= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index adbc5d36f9a..673191e145b 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -17,7 +17,7 @@ require ( github.com/slack-go/slack v0.12.2 github.com/smartcontractkit/chainlink-common v0.2.2-0.20240916150342-36cb47701edf github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.5 - github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.4-0.20240912161944-13ade8436072 + github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.1 github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.0 github.com/smartcontractkit/chainlink/integration-tests v0.0.0-20240214231432-4ad5eb95178c github.com/smartcontractkit/chainlink/v2 v2.9.0-beta0.0.20240216210048-da02459ddad8 diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index 708f9726936..ee64962cb1c 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1417,8 +1417,8 @@ github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.5 h1:Owb1MQZn0 github.com/smartcontractkit/chainlink-testing-framework/lib v1.50.5/go.mod h1:hS4yNF94C1lkS9gvtFXW8Km8K9NzGeR20aNfkqo5qbE= github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0 h1:VIxK8u0Jd0Q/VuhmsNm6Bls6Tb31H/sA3A/rbc5hnhg= github.com/smartcontractkit/chainlink-testing-framework/lib/grafana v1.50.0/go.mod h1:lyAu+oMXdNUzEDScj2DXB2IueY+SDXPPfyl/kb63tMM= -github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.4-0.20240912161944-13ade8436072 h1:/wGR8PUytZBze4DPnhzF+V7IVl/EslDmDC2SsamXjqw= -github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.4-0.20240912161944-13ade8436072/go.mod h1:afY3QmNgeR/VI1pRbGH8g3YXGy7C2RrFOwUzEFvL3L8= +github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.1 h1:2OxnPfvjC+zs0ZokSsRTRnJrEGJ4NVJwZgfroS1lPHs= +github.com/smartcontractkit/chainlink-testing-framework/seth v1.50.1/go.mod h1:afY3QmNgeR/VI1pRbGH8g3YXGy7C2RrFOwUzEFvL3L8= github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.0 h1:gfhfTn7HkbUHNooSF3c9vzQyN8meWJVGt6G/pNUbpYk= github.com/smartcontractkit/chainlink-testing-framework/wasp v1.50.0/go.mod h1:tqajhpUJA/9OaMCLitghBXjAgqYO4i27St0F4TUO3+M= github.com/smartcontractkit/grpc-proxy v0.0.0-20240830132753-a7e17fec5ab7 h1:12ijqMM9tvYVEm+nR826WsrNi6zCKpwBhuApq127wHs= From 532483d99e9dde10afbb96a286069a5cfc2ce2f7 Mon Sep 17 00:00:00 2001 From: connorwstein Date: Thu, 19 Sep 2024 11:08:13 -0400 Subject: [PATCH 09/10] Lint --- integration-tests/deployment/evm_kmsclient.go | 14 +++++++------- integration-tests/deployment/multiclient.go | 2 +- integration-tests/deployment/multiclient_test.go | 3 ++- 3 files changed, 10 insertions(+), 9 deletions(-) diff --git a/integration-tests/deployment/evm_kmsclient.go b/integration-tests/deployment/evm_kmsclient.go index 1f253f9a8b6..07af77523c8 100644 --- a/integration-tests/deployment/evm_kmsclient.go +++ b/integration-tests/deployment/evm_kmsclient.go @@ -114,12 +114,12 @@ func (c *EVMKMSClient) GetKMSTransactOpts(ctx context.Context, chainID *big.Int) Message: txHashBytes, }) if err != nil { - return nil, fmt.Errorf("failed to call kms.Sign() on transaction: %v", err) + return nil, fmt.Errorf("failed to call kms.Sign() on transaction: %w", err) } ethSig, err := kmsToEthSig(signOutput.Signature, pubKeyBytes, txHashBytes) if err != nil { - return nil, fmt.Errorf("failed to convert KMS signature to Ethereum signature: %v", err) + return nil, fmt.Errorf("failed to convert KMS signature to Ethereum signature: %w", err) } return tx.WithSignature(signer, ethSig) @@ -138,18 +138,18 @@ func (c *EVMKMSClient) GetECDSAPublicKey() (*ecdsa.PublicKey, error) { KeyId: aws.String(c.KeyID), }) if err != nil { - return nil, fmt.Errorf("can not get public key from KMS for KeyId=%s: %s", c.KeyID, err) + return nil, fmt.Errorf("can not get public key from KMS for KeyId=%s: %w", c.KeyID, err) } var asn1pubKeyInfo asn1SubjectPublicKeyInfo _, err = asn1.Unmarshal(getPubKeyOutput.PublicKey, &asn1pubKeyInfo) if err != nil { - return nil, fmt.Errorf("can not parse asn1 public key for KeyId=%s: %s", c.KeyID, err) + return nil, fmt.Errorf("can not parse asn1 public key for KeyId=%s: %w", c.KeyID, err) } pubKey, err := crypto.UnmarshalPubkey(asn1pubKeyInfo.SubjectPublicKey.Bytes) if err != nil { - return nil, fmt.Errorf("can not unmarshal public key bytes: %s", err) + return nil, fmt.Errorf("can not unmarshal public key bytes: %w", err) } return pubKey, nil } @@ -183,14 +183,14 @@ func recoverEthSignature(expectedPublicKeyBytes, txHash, r, s []byte) ([]byte, e recoveredPublicKeyBytes, err := crypto.Ecrecover(txHash, ethSig) if err != nil { - return nil, fmt.Errorf("failing to call Ecrecover: %v", err) + return nil, fmt.Errorf("failing to call Ecrecover: %w", err) } if hex.EncodeToString(recoveredPublicKeyBytes) != hex.EncodeToString(expectedPublicKeyBytes) { ethSig = append(rsSig, []byte{1}...) recoveredPublicKeyBytes, err = crypto.Ecrecover(txHash, ethSig) if err != nil { - return nil, fmt.Errorf("failing to call Ecrecover: %v", err) + return nil, fmt.Errorf("failing to call Ecrecover: %w", err) } if hex.EncodeToString(recoveredPublicKeyBytes) != hex.EncodeToString(expectedPublicKeyBytes) { diff --git a/integration-tests/deployment/multiclient.go b/integration-tests/deployment/multiclient.go index 02a18f760df..c0db21714dc 100644 --- a/integration-tests/deployment/multiclient.go +++ b/integration-tests/deployment/multiclient.go @@ -110,7 +110,7 @@ func (mc *MultiClient) retryWithBackups(op func(*ethclient.Client) error) error err = op(client) if err != nil { // TODO: logger? - fmt.Printf("Error %v with client %v\n", err, client) + fmt.Printf("Error %w with client %v\n", err, client) return err } return nil diff --git a/integration-tests/deployment/multiclient_test.go b/integration-tests/deployment/multiclient_test.go index b0ee6c2042f..a3176691c0c 100644 --- a/integration-tests/deployment/multiclient_test.go +++ b/integration-tests/deployment/multiclient_test.go @@ -13,7 +13,8 @@ func TestMultiClient(t *testing.T) { // Expect an error if no RPCs supplied. s := httptest.NewServer(http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { writer.WriteHeader(http.StatusOK) - writer.Write([]byte(`{"jsonrpc":"2.0","id":1,"result":true}`)) + _, err := writer.Write([]byte(`{"jsonrpc":"2.0","id":1,"result":true}`)) + require.NoError(t, err) })) defer s.Close() _, err := NewMultiClient([]RPC{}) From f610a301feb1ebc1860de67c363a3a7426f365e9 Mon Sep 17 00:00:00 2001 From: connorwstein Date: Thu, 19 Sep 2024 11:52:50 -0400 Subject: [PATCH 10/10] More lint --- integration-tests/deployment/multiclient.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration-tests/deployment/multiclient.go b/integration-tests/deployment/multiclient.go index c0db21714dc..02a18f760df 100644 --- a/integration-tests/deployment/multiclient.go +++ b/integration-tests/deployment/multiclient.go @@ -110,7 +110,7 @@ func (mc *MultiClient) retryWithBackups(op func(*ethclient.Client) error) error err = op(client) if err != nil { // TODO: logger? - fmt.Printf("Error %w with client %v\n", err, client) + fmt.Printf("Error %v with client %v\n", err, client) return err } return nil