From 0676320169a27af44f1fea522853abf3522aeede Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 26 Nov 2019 12:21:11 +0200 Subject: [PATCH 001/128] params: begin v1.9.9 release cycle --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 3933faf68d..2ed8e37bd2 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 9 // Minor version component of the current release - VersionPatch = 8 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 9 // Minor version component of the current release + VersionPatch = 9 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From 23c8c741318cc9f7baba3d3555c1c9385494714e Mon Sep 17 00:00:00 2001 From: zaccoding Date: Tue, 26 Nov 2019 19:46:39 +0900 Subject: [PATCH 002/128] cmd: fix command help messages in modules (#20203) --- cmd/abigen/main.go | 15 +-------------- cmd/checkpoint-admin/main.go | 15 +-------------- cmd/clef/main.go | 1 + cmd/ethkey/main.go | 1 + cmd/evm/main.go | 1 + cmd/utils/flags.go | 11 +++++++++++ 6 files changed, 16 insertions(+), 28 deletions(-) diff --git a/cmd/abigen/main.go b/cmd/abigen/main.go index 659cf1b4c5..252d8ef800 100644 --- a/cmd/abigen/main.go +++ b/cmd/abigen/main.go @@ -32,19 +32,6 @@ import ( "gopkg.in/urfave/cli.v1" ) -const ( - commandHelperTemplate = `{{.Name}}{{if .Subcommands}} command{{end}}{{if .Flags}} [command options]{{end}} [arguments...] -{{if .Description}}{{.Description}} -{{end}}{{if .Subcommands}} -SUBCOMMANDS: - {{range .Subcommands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}} - {{end}}{{end}}{{if .Flags}} -OPTIONS: -{{range $.Flags}}{{"\t"}}{{.}} -{{end}} -{{end}}` -) - var ( // Git SHA1 commit hash of the release (set via linker flags) gitCommit = "" @@ -128,7 +115,7 @@ func init() { aliasFlag, } app.Action = utils.MigrateFlags(abigen) - cli.CommandHelpTemplate = commandHelperTemplate + cli.CommandHelpTemplate = utils.OriginCommandHelpTemplate } func abigen(c *cli.Context) error { diff --git a/cmd/checkpoint-admin/main.go b/cmd/checkpoint-admin/main.go index 26d751dd81..b4d8e0db5a 100644 --- a/cmd/checkpoint-admin/main.go +++ b/cmd/checkpoint-admin/main.go @@ -28,19 +28,6 @@ import ( "gopkg.in/urfave/cli.v1" ) -const ( - commandHelperTemplate = `{{.Name}}{{if .Subcommands}} command{{end}}{{if .Flags}} [command options]{{end}} [arguments...] -{{if .Description}}{{.Description}} -{{end}}{{if .Subcommands}} -SUBCOMMANDS: - {{range .Subcommands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}} - {{end}}{{end}}{{if .Flags}} -OPTIONS: -{{range $.Flags}}{{"\t"}}{{.}} -{{end}} -{{end}}` -) - var ( // Git SHA1 commit hash of the release (set via linker flags) gitCommit = "" @@ -61,7 +48,7 @@ func init() { oracleFlag, nodeURLFlag, } - cli.CommandHelpTemplate = commandHelperTemplate + cli.CommandHelpTemplate = utils.OriginCommandHelpTemplate } // Commonly used command line flags. diff --git a/cmd/clef/main.go b/cmd/clef/main.go index 5775a1049f..b2c8812ab2 100644 --- a/cmd/clef/main.go +++ b/cmd/clef/main.go @@ -223,6 +223,7 @@ func init() { } app.Action = signer app.Commands = []cli.Command{initCommand, attestCommand, setCredentialCommand, delCredentialCommand, gendocCommand} + cli.CommandHelpTemplate = utils.OriginCommandHelpTemplate } func main() { diff --git a/cmd/ethkey/main.go b/cmd/ethkey/main.go index 5b545d5f94..dbc4960588 100644 --- a/cmd/ethkey/main.go +++ b/cmd/ethkey/main.go @@ -43,6 +43,7 @@ func init() { commandSignMessage, commandVerifyMessage, } + cli.CommandHelpTemplate = utils.OriginCommandHelpTemplate } // Commonly used command line flags. diff --git a/cmd/evm/main.go b/cmd/evm/main.go index 5b8b950ab0..67447d58ae 100644 --- a/cmd/evm/main.go +++ b/cmd/evm/main.go @@ -152,6 +152,7 @@ func init() { runCommand, stateTestCommand, } + cli.CommandHelpTemplate = utils.OriginCommandHelpTemplate } func main() { diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 4780a81373..c6846d312b 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -77,6 +77,17 @@ SUBCOMMANDS: {{range $categorized.Flags}}{{"\t"}}{{.}} {{end}} {{end}}{{end}}` + + OriginCommandHelpTemplate = `{{.Name}}{{if .Subcommands}} command{{end}}{{if .Flags}} [command options]{{end}} [arguments...] +{{if .Description}}{{.Description}} +{{end}}{{if .Subcommands}} +SUBCOMMANDS: + {{range .Subcommands}}{{.Name}}{{with .ShortName}}, {{.}}{{end}}{{ "\t" }}{{.Usage}} + {{end}}{{end}}{{if .Flags}} +OPTIONS: +{{range $.Flags}}{{"\t"}}{{.}} +{{end}} +{{end}}` ) func init() { From c4844e9ee211f34d8023b0de8221d03a63817395 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Wed, 27 Nov 2019 09:49:41 +0100 Subject: [PATCH 003/128] les: fix staticcheck warnings (#20371) --- les/retrieve.go | 2 +- les/txrelay.go | 84 ++++++++++++++++++++++++------------------------- 2 files changed, 43 insertions(+), 43 deletions(-) diff --git a/les/retrieve.go b/les/retrieve.go index d17a02e1ae..c806117902 100644 --- a/les/retrieve.go +++ b/les/retrieve.go @@ -119,7 +119,7 @@ func (rm *retrieveManager) retrieve(ctx context.Context, reqID uint64, req *dist case <-ctx.Done(): sentReq.stop(ctx.Err()) case <-shutdown: - sentReq.stop(fmt.Errorf("Client is shutting down")) + sentReq.stop(fmt.Errorf("client is shutting down")) } return sentReq.getError() } diff --git a/les/txrelay.go b/les/txrelay.go index 49195161b7..d37a18faff 100644 --- a/les/txrelay.go +++ b/les/txrelay.go @@ -54,51 +54,51 @@ func newLesTxRelay(ps *peerSet, retriever *retrieveManager) *lesTxRelay { return r } -func (self *lesTxRelay) Stop() { - close(self.stop) +func (ltrx *lesTxRelay) Stop() { + close(ltrx.stop) } -func (self *lesTxRelay) registerPeer(p *peer) { - self.lock.Lock() - defer self.lock.Unlock() +func (ltrx *lesTxRelay) registerPeer(p *peer) { + ltrx.lock.Lock() + defer ltrx.lock.Unlock() - self.peerList = self.ps.AllPeers() + ltrx.peerList = ltrx.ps.AllPeers() } -func (self *lesTxRelay) unregisterPeer(p *peer) { - self.lock.Lock() - defer self.lock.Unlock() +func (ltrx *lesTxRelay) unregisterPeer(p *peer) { + ltrx.lock.Lock() + defer ltrx.lock.Unlock() - self.peerList = self.ps.AllPeers() + ltrx.peerList = ltrx.ps.AllPeers() } // send sends a list of transactions to at most a given number of peers at // once, never resending any particular transaction to the same peer twice -func (self *lesTxRelay) send(txs types.Transactions, count int) { +func (ltrx *lesTxRelay) send(txs types.Transactions, count int) { sendTo := make(map[*peer]types.Transactions) - self.peerStartPos++ // rotate the starting position of the peer list - if self.peerStartPos >= len(self.peerList) { - self.peerStartPos = 0 + ltrx.peerStartPos++ // rotate the starting position of the peer list + if ltrx.peerStartPos >= len(ltrx.peerList) { + ltrx.peerStartPos = 0 } for _, tx := range txs { hash := tx.Hash() - ltr, ok := self.txSent[hash] + ltr, ok := ltrx.txSent[hash] if !ok { ltr = <rInfo{ tx: tx, sentTo: make(map[*peer]struct{}), } - self.txSent[hash] = ltr - self.txPending[hash] = struct{}{} + ltrx.txSent[hash] = ltr + ltrx.txPending[hash] = struct{}{} } - if len(self.peerList) > 0 { + if len(ltrx.peerList) > 0 { cnt := count - pos := self.peerStartPos + pos := ltrx.peerStartPos for { - peer := self.peerList[pos] + peer := ltrx.peerList[pos] if _, ok := ltr.sentTo[peer]; !ok { sendTo[peer] = append(sendTo[peer], tx) ltr.sentTo[peer] = struct{}{} @@ -108,10 +108,10 @@ func (self *lesTxRelay) send(txs types.Transactions, count int) { break // sent it to the desired number of peers } pos++ - if pos == len(self.peerList) { + if pos == len(ltrx.peerList) { pos = 0 } - if pos == self.peerStartPos { + if pos == ltrx.peerStartPos { break // tried all available peers } } @@ -139,46 +139,46 @@ func (self *lesTxRelay) send(txs types.Transactions, count int) { return func() { peer.SendTxs(reqID, cost, enc) } }, } - go self.retriever.retrieve(context.Background(), reqID, rq, func(p distPeer, msg *Msg) error { return nil }, self.stop) + go ltrx.retriever.retrieve(context.Background(), reqID, rq, func(p distPeer, msg *Msg) error { return nil }, ltrx.stop) } } -func (self *lesTxRelay) Send(txs types.Transactions) { - self.lock.Lock() - defer self.lock.Unlock() +func (ltrx *lesTxRelay) Send(txs types.Transactions) { + ltrx.lock.Lock() + defer ltrx.lock.Unlock() - self.send(txs, 3) + ltrx.send(txs, 3) } -func (self *lesTxRelay) NewHead(head common.Hash, mined []common.Hash, rollback []common.Hash) { - self.lock.Lock() - defer self.lock.Unlock() +func (ltrx *lesTxRelay) NewHead(head common.Hash, mined []common.Hash, rollback []common.Hash) { + ltrx.lock.Lock() + defer ltrx.lock.Unlock() for _, hash := range mined { - delete(self.txPending, hash) + delete(ltrx.txPending, hash) } for _, hash := range rollback { - self.txPending[hash] = struct{}{} + ltrx.txPending[hash] = struct{}{} } - if len(self.txPending) > 0 { - txs := make(types.Transactions, len(self.txPending)) + if len(ltrx.txPending) > 0 { + txs := make(types.Transactions, len(ltrx.txPending)) i := 0 - for hash := range self.txPending { - txs[i] = self.txSent[hash].tx + for hash := range ltrx.txPending { + txs[i] = ltrx.txSent[hash].tx i++ } - self.send(txs, 1) + ltrx.send(txs, 1) } } -func (self *lesTxRelay) Discard(hashes []common.Hash) { - self.lock.Lock() - defer self.lock.Unlock() +func (ltrx *lesTxRelay) Discard(hashes []common.Hash) { + ltrx.lock.Lock() + defer ltrx.lock.Unlock() for _, hash := range hashes { - delete(self.txSent, hash) - delete(self.txPending, hash) + delete(ltrx.txSent, hash) + delete(ltrx.txPending, hash) } } From 7b189d6f1f7eedf46c6607901af291855b81112b Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Wed, 27 Nov 2019 09:50:30 +0100 Subject: [PATCH 004/128] core: fix staticcheck warnings (#20384) * core: fix staticcheck warnings * fix goimports --- core/rawdb/database.go | 11 +++++------ core/vm/contracts.go | 2 ++ core/vm/contracts_test.go | 26 +++++++++++++++++++++++++- core/vm/stack.go | 7 ------- 4 files changed, 32 insertions(+), 14 deletions(-) diff --git a/core/rawdb/database.go b/core/rawdb/database.go index 353b7dce62..838c084359 100644 --- a/core/rawdb/database.go +++ b/core/rawdb/database.go @@ -150,11 +150,10 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st } // Database contains only older data than the freezer, this happens if the // state was wiped and reinited from an existing freezer. - } else { - // Key-value store continues where the freezer left off, all is fine. We might - // have duplicate blocks (crash after freezer write but before kay-value store - // deletion, but that's fine). } + // Otherwise, key-value store continues where the freezer left off, all is fine. + // We might have duplicate blocks (crash after freezer write but before key-value + // store deletion, but that's fine). } else { // If the freezer is empty, ensure nothing was moved yet from the key-value // store, otherwise we'll end up missing data. We check block #1 to decide @@ -167,9 +166,9 @@ func NewDatabaseWithFreezer(db ethdb.KeyValueStore, freezer string, namespace st return nil, errors.New("ancient chain segments already extracted, please set --datadir.ancient to the correct path") } // Block #1 is still in the database, we're allowed to init a new feezer - } else { - // The head header is still the genesis, we're allowed to init a new feezer } + // Otherwise, the head header is still the genesis, we're allowed to init a new + // feezer. } } // Freezer is consistent with the key-value database, permit combining the two diff --git a/core/vm/contracts.go b/core/vm/contracts.go index 9b0ba09ed1..7d304613ce 100644 --- a/core/vm/contracts.go +++ b/core/vm/contracts.go @@ -28,6 +28,8 @@ import ( "github.com/ethereum/go-ethereum/crypto/blake2b" "github.com/ethereum/go-ethereum/crypto/bn256" "github.com/ethereum/go-ethereum/params" + + //lint:ignore SA1019 Needed for precompile "golang.org/x/crypto/ripemd160" ) diff --git a/core/vm/contracts_test.go b/core/vm/contracts_test.go index b4a0c07dca..be003a60c9 100644 --- a/core/vm/contracts_test.go +++ b/core/vm/contracts_test.go @@ -29,7 +29,6 @@ import ( // precompiledTest defines the input/output pairs for precompiled contract tests. type precompiledTest struct { input, expected string - gas uint64 name string noBenchmark bool // Benchmark primarily the worst-cases } @@ -418,6 +417,24 @@ func testPrecompiled(addr string, test precompiledTest, t *testing.T) { }) } +func testPrecompiledOOG(addr string, test precompiledTest, t *testing.T) { + p := PrecompiledContractsIstanbul[common.HexToAddress(addr)] + in := common.Hex2Bytes(test.input) + contract := NewContract(AccountRef(common.HexToAddress("1337")), + nil, new(big.Int), p.RequiredGas(in)-1) + t.Run(fmt.Sprintf("%s-Gas=%d", test.name, contract.Gas), func(t *testing.T) { + _, err := RunPrecompiledContract(p, in, contract) + if err.Error() != "out of gas" { + t.Errorf("Expected error [out of gas], got [%v]", err) + } + // Verify that the precompile did not touch the input buffer + exp := common.Hex2Bytes(test.input) + if !bytes.Equal(in, exp) { + t.Errorf("Precompiled %v modified input data", addr) + } + }) +} + func testPrecompiledFailure(addr string, test precompiledFailureTest, t *testing.T) { p := PrecompiledContractsIstanbul[common.HexToAddress(addr)] in := common.Hex2Bytes(test.input) @@ -541,6 +558,13 @@ func BenchmarkPrecompiledBn256Add(bench *testing.B) { } } +// Tests OOG +func TestPrecompiledModExpOOG(t *testing.T) { + for _, test := range modexpTests { + testPrecompiledOOG("05", test, t) + } +} + // Tests the sample inputs from the elliptic curve scalar multiplication EIP 213. func TestPrecompiledBn256ScalarMul(t *testing.T) { for _, test := range bn256ScalarMulTests { diff --git a/core/vm/stack.go b/core/vm/stack.go index 4c1b9e8037..c9c3d07f4b 100644 --- a/core/vm/stack.go +++ b/core/vm/stack.go @@ -74,13 +74,6 @@ func (st *Stack) Back(n int) *big.Int { return st.data[st.len()-n-1] } -func (st *Stack) require(n int) error { - if st.len() < n { - return fmt.Errorf("stack underflow (%d <=> %d)", len(st.data), n) - } - return nil -} - // Print dumps the content of the stack func (st *Stack) Print() { fmt.Println("### stack ###") From 717f8a4e8f768e2c3a34067f4a55f9c8e3267f3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 27 Nov 2019 14:41:47 +0200 Subject: [PATCH 005/128] core/rawdb: fix reinit regression caused by the hash check PR --- core/rawdb/freezer_reinit.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/rawdb/freezer_reinit.go b/core/rawdb/freezer_reinit.go index ea4dd33d1d..d6bf9ab1dd 100644 --- a/core/rawdb/freezer_reinit.go +++ b/core/rawdb/freezer_reinit.go @@ -55,10 +55,10 @@ func InitDatabaseFromFreezer(db ethdb.Database) error { if n >= frozen { return } - // Retrieve the block from the freezer (no need for the hash, we pull by - // number from the freezer). If successful, pre-cache the block hash and - // the individual transaction hashes for storing into the database. - block := ReadBlock(db, common.Hash{}, n) + // Retrieve the block from the freezer. If successful, pre-cache + // the block hash and the individual transaction hashes for storing + // into the database. + block := ReadBlock(db, ReadCanonicalHash(db, n), n) if block != nil { block.Hash() for _, tx := range block.Transactions() { From 7800ba978d4ba71422755ed3e30d12a4aef33381 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 27 Nov 2019 13:46:07 +0100 Subject: [PATCH 006/128] deps: update fastcache to 1.5.3 --- go.mod | 3 ++- go.sum | 6 ++++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e144d6477f..d4b4208336 100644 --- a/go.mod +++ b/go.mod @@ -7,10 +7,11 @@ require ( github.com/Azure/azure-storage-blob-go v0.7.0 github.com/Azure/go-autorest/autorest/adal v0.8.0 // indirect github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect - github.com/VictoriaMetrics/fastcache v1.5.2 + github.com/VictoriaMetrics/fastcache v1.5.3 github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847 github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6 github.com/cespare/cp v0.1.0 + github.com/cespare/xxhash/v2 v2.1.1 // indirect github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9 github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea diff --git a/go.sum b/go.sum index 5aefbd3784..515207bca3 100644 --- a/go.sum +++ b/go.sum @@ -25,8 +25,8 @@ github.com/OneOfOne/xxhash v1.2.5 h1:zl/OfRA6nftbBK9qTohYBJ5xvw6C/oNKizR7cZGl3cI github.com/OneOfOne/xxhash v1.2.5/go.mod h1:eZbhyaAYD41SGSSsnmcpxVoRiQ/MPUTjUdIIOT9Um7Q= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 h1:fLjPD/aNc3UIOA6tDi6QXUemppXK3P9BI7mr2hd6gx8= github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= -github.com/VictoriaMetrics/fastcache v1.5.2 h1:Erd8iIuBAL9kke8JzM4+WxkKuFkHh3ktwLanJvDgR44= -github.com/VictoriaMetrics/fastcache v1.5.2/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE= +github.com/VictoriaMetrics/fastcache v1.5.3 h1:2odJnXLbFZcoV9KYtQ+7TH1UOq3dn3AssMgieaezkR4= +github.com/VictoriaMetrics/fastcache v1.5.3/go.mod h1:+jv9Ckb+za/P1ZRg/sulP5Ni1v49daAVERr0H3CuscE= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah4HI848JfFxHt+iPb26b4zyfspmqY0/8= @@ -42,6 +42,8 @@ github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18 h1:pl4eWIqvFe/Kg3zkn7NxevNzILnZYWDCG7qbA1CJik0= github.com/cespare/xxhash/v2 v2.0.1-0.20190104013014-3767db7a7e18/go.mod h1:HD5P3vAIAh+Y2GAxg0PrPN1P8WkepXGpjbUPDHJqqKM= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= +github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9 h1:J82+/8rub3qSy0HxEnoYD8cs+HDlHWYrqYXe2Vqxluk= github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9/go.mod h1:1MxXX1Ux4x6mqPmjkUgTP1CdXIBXKX7T+Jk9Gxrmx+U= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= From a91b704b013aaabb3bfc796f5a3d8c940dfb7f05 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 28 Nov 2019 10:51:57 +0100 Subject: [PATCH 007/128] consensus/ethash: refactor remote sealer (#20335) The original idea behind this change was to remove a use of the deprecated CancelRequest method. Simply removing it would've been an option, but I couldn't resist and did a bit of a refactoring instead. All remote sealing code was contained in a single giant function. Remote sealing is now extracted into its own object, remoteSealer. --- consensus/ethash/algorithm_test.go | 2 +- consensus/ethash/api.go | 26 +-- consensus/ethash/ethash.go | 112 +++------ consensus/ethash/sealer.go | 351 +++++++++++++++++------------ consensus/ethash/sealer_test.go | 107 ++++----- 5 files changed, 294 insertions(+), 304 deletions(-) diff --git a/consensus/ethash/algorithm_test.go b/consensus/ethash/algorithm_test.go index cf8552f3ab..f4f65047be 100644 --- a/consensus/ethash/algorithm_test.go +++ b/consensus/ethash/algorithm_test.go @@ -729,7 +729,7 @@ func TestConcurrentDiskCacheGeneration(t *testing.T) { go func(idx int) { defer pend.Done() - ethash := New(Config{cachedir, 0, 1, "", 0, 0, ModeNormal}, nil, false) + ethash := New(Config{cachedir, 0, 1, "", 0, 0, ModeNormal, nil}, nil, false) defer ethash.Close() if err := ethash.VerifySeal(nil, block.Header()); err != nil { t.Errorf("proc %d: block verification failed: %v", idx, err) diff --git a/consensus/ethash/api.go b/consensus/ethash/api.go index 4d8eed4161..68b3a84b09 100644 --- a/consensus/ethash/api.go +++ b/consensus/ethash/api.go @@ -28,7 +28,7 @@ var errEthashStopped = errors.New("ethash stopped") // API exposes ethash related methods for the RPC interface. type API struct { - ethash *Ethash // Make sure the mode of ethash is normal. + ethash *Ethash } // GetWork returns a work package for external miner. @@ -39,7 +39,7 @@ type API struct { // result[2] - 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty // result[3] - hex encoded block number func (api *API) GetWork() ([4]string, error) { - if api.ethash.config.PowMode != ModeNormal && api.ethash.config.PowMode != ModeTest { + if api.ethash.remote == nil { return [4]string{}, errors.New("not supported") } @@ -47,13 +47,11 @@ func (api *API) GetWork() ([4]string, error) { workCh = make(chan [4]string, 1) errc = make(chan error, 1) ) - select { - case api.ethash.fetchWorkCh <- &sealWork{errc: errc, res: workCh}: - case <-api.ethash.exitCh: + case api.ethash.remote.fetchWorkCh <- &sealWork{errc: errc, res: workCh}: + case <-api.ethash.remote.exitCh: return [4]string{}, errEthashStopped } - select { case work := <-workCh: return work, nil @@ -66,23 +64,21 @@ func (api *API) GetWork() ([4]string, error) { // It returns an indication if the work was accepted. // Note either an invalid solution, a stale work a non-existent work will return false. func (api *API) SubmitWork(nonce types.BlockNonce, hash, digest common.Hash) bool { - if api.ethash.config.PowMode != ModeNormal && api.ethash.config.PowMode != ModeTest { + if api.ethash.remote == nil { return false } var errc = make(chan error, 1) - select { - case api.ethash.submitWorkCh <- &mineResult{ + case api.ethash.remote.submitWorkCh <- &mineResult{ nonce: nonce, mixDigest: digest, hash: hash, errc: errc, }: - case <-api.ethash.exitCh: + case <-api.ethash.remote.exitCh: return false } - err := <-errc return err == nil } @@ -94,21 +90,19 @@ func (api *API) SubmitWork(nonce types.BlockNonce, hash, digest common.Hash) boo // It accepts the miner hash rate and an identifier which must be unique // between nodes. func (api *API) SubmitHashRate(rate hexutil.Uint64, id common.Hash) bool { - if api.ethash.config.PowMode != ModeNormal && api.ethash.config.PowMode != ModeTest { + if api.ethash.remote == nil { return false } var done = make(chan struct{}, 1) - select { - case api.ethash.submitRateCh <- &hashrate{done: done, rate: uint64(rate), id: id}: - case <-api.ethash.exitCh: + case api.ethash.remote.submitRateCh <- &hashrate{done: done, rate: uint64(rate), id: id}: + case <-api.ethash.remote.exitCh: return false } // Block until hash rate submitted successfully. <-done - return true } diff --git a/consensus/ethash/ethash.go b/consensus/ethash/ethash.go index 78892e1da8..4bf78b0a1d 100644 --- a/consensus/ethash/ethash.go +++ b/consensus/ethash/ethash.go @@ -34,9 +34,7 @@ import ( "unsafe" mmap "github.com/edsrzf/mmap-go" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" - "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/rpc" @@ -50,7 +48,7 @@ var ( two256 = new(big.Int).Exp(big.NewInt(2), big.NewInt(256), big.NewInt(0)) // sharedEthash is a full instance that can be shared between multiple users. - sharedEthash = New(Config{"", 3, 0, "", 1, 0, ModeNormal}, nil, false) + sharedEthash = New(Config{"", 3, 0, "", 1, 0, ModeNormal, nil}, nil, false) // algorithmRevision is the data structure version used for file naming. algorithmRevision = 23 @@ -403,36 +401,8 @@ type Config struct { DatasetsInMem int DatasetsOnDisk int PowMode Mode -} - -// sealTask wraps a seal block with relative result channel for remote sealer thread. -type sealTask struct { - block *types.Block - results chan<- *types.Block -} - -// mineResult wraps the pow solution parameters for the specified block. -type mineResult struct { - nonce types.BlockNonce - mixDigest common.Hash - hash common.Hash - - errc chan error -} - -// hashrate wraps the hash rate submitted by the remote sealer. -type hashrate struct { - id common.Hash - ping time.Time - rate uint64 - done chan struct{} -} - -// sealWork wraps a seal work package for remote sealer. -type sealWork struct { - errc chan error - res chan [4]string + Log log.Logger `toml:"-"` } // Ethash is a consensus engine based on proof-of-work implementing the ethash @@ -448,52 +418,42 @@ type Ethash struct { threads int // Number of threads to mine on if mining update chan struct{} // Notification channel to update mining parameters hashrate metrics.Meter // Meter tracking the average hashrate - - // Remote sealer related fields - workCh chan *sealTask // Notification channel to push new work and relative result channel to remote sealer - fetchWorkCh chan *sealWork // Channel used for remote sealer to fetch mining work - submitWorkCh chan *mineResult // Channel used for remote sealer to submit their mining result - fetchRateCh chan chan uint64 // Channel used to gather submitted hash rate for local or remote sealer. - submitRateCh chan *hashrate // Channel used for remote sealer to submit their mining hashrate + remote *remoteSealer // The fields below are hooks for testing shared *Ethash // Shared PoW verifier to avoid cache regeneration fakeFail uint64 // Block number which fails PoW check even in fake mode fakeDelay time.Duration // Time delay to sleep for before returning from verify - lock sync.Mutex // Ensures thread safety for the in-memory caches and mining fields - closeOnce sync.Once // Ensures exit channel will not be closed twice. - exitCh chan chan error // Notification channel to exiting backend threads + lock sync.Mutex // Ensures thread safety for the in-memory caches and mining fields + closeOnce sync.Once // Ensures exit channel will not be closed twice. } // New creates a full sized ethash PoW scheme and starts a background thread for // remote mining, also optionally notifying a batch of remote services of new work // packages. func New(config Config, notify []string, noverify bool) *Ethash { + if config.Log == nil { + config.Log = log.Root() + } if config.CachesInMem <= 0 { - log.Warn("One ethash cache must always be in memory", "requested", config.CachesInMem) + config.Log.Warn("One ethash cache must always be in memory", "requested", config.CachesInMem) config.CachesInMem = 1 } if config.CacheDir != "" && config.CachesOnDisk > 0 { - log.Info("Disk storage enabled for ethash caches", "dir", config.CacheDir, "count", config.CachesOnDisk) + config.Log.Info("Disk storage enabled for ethash caches", "dir", config.CacheDir, "count", config.CachesOnDisk) } if config.DatasetDir != "" && config.DatasetsOnDisk > 0 { - log.Info("Disk storage enabled for ethash DAGs", "dir", config.DatasetDir, "count", config.DatasetsOnDisk) + config.Log.Info("Disk storage enabled for ethash DAGs", "dir", config.DatasetDir, "count", config.DatasetsOnDisk) } ethash := &Ethash{ - config: config, - caches: newlru("cache", config.CachesInMem, newCache), - datasets: newlru("dataset", config.DatasetsInMem, newDataset), - update: make(chan struct{}), - hashrate: metrics.NewMeterForced(), - workCh: make(chan *sealTask), - fetchWorkCh: make(chan *sealWork), - submitWorkCh: make(chan *mineResult), - fetchRateCh: make(chan chan uint64), - submitRateCh: make(chan *hashrate), - exitCh: make(chan chan error), - } - go ethash.remote(notify, noverify) + config: config, + caches: newlru("cache", config.CachesInMem, newCache), + datasets: newlru("dataset", config.DatasetsInMem, newDataset), + update: make(chan struct{}), + hashrate: metrics.NewMeterForced(), + } + ethash.remote = startRemoteSealer(ethash, notify, noverify) return ethash } @@ -501,19 +461,13 @@ func New(config Config, notify []string, noverify bool) *Ethash { // purposes. func NewTester(notify []string, noverify bool) *Ethash { ethash := &Ethash{ - config: Config{PowMode: ModeTest}, - caches: newlru("cache", 1, newCache), - datasets: newlru("dataset", 1, newDataset), - update: make(chan struct{}), - hashrate: metrics.NewMeterForced(), - workCh: make(chan *sealTask), - fetchWorkCh: make(chan *sealWork), - submitWorkCh: make(chan *mineResult), - fetchRateCh: make(chan chan uint64), - submitRateCh: make(chan *hashrate), - exitCh: make(chan chan error), - } - go ethash.remote(notify, noverify) + config: Config{PowMode: ModeTest, Log: log.Root()}, + caches: newlru("cache", 1, newCache), + datasets: newlru("dataset", 1, newDataset), + update: make(chan struct{}), + hashrate: metrics.NewMeterForced(), + } + ethash.remote = startRemoteSealer(ethash, notify, noverify) return ethash } @@ -524,6 +478,7 @@ func NewFaker() *Ethash { return &Ethash{ config: Config{ PowMode: ModeFake, + Log: log.Root(), }, } } @@ -535,6 +490,7 @@ func NewFakeFailer(fail uint64) *Ethash { return &Ethash{ config: Config{ PowMode: ModeFake, + Log: log.Root(), }, fakeFail: fail, } @@ -547,6 +503,7 @@ func NewFakeDelayer(delay time.Duration) *Ethash { return &Ethash{ config: Config{ PowMode: ModeFake, + Log: log.Root(), }, fakeDelay: delay, } @@ -558,6 +515,7 @@ func NewFullFaker() *Ethash { return &Ethash{ config: Config{ PowMode: ModeFullFake, + Log: log.Root(), }, } } @@ -573,13 +531,11 @@ func (ethash *Ethash) Close() error { var err error ethash.closeOnce.Do(func() { // Short circuit if the exit channel is not allocated. - if ethash.exitCh == nil { + if ethash.remote == nil { return } - errc := make(chan error) - ethash.exitCh <- errc - err = <-errc - close(ethash.exitCh) + close(ethash.remote.requestExit) + <-ethash.remote.exitCh }) return err } @@ -680,8 +636,8 @@ func (ethash *Ethash) Hashrate() float64 { var res = make(chan uint64, 1) select { - case ethash.fetchRateCh <- res: - case <-ethash.exitCh: + case ethash.remote.fetchRateCh <- res: + case <-ethash.remote.exitCh: // Return local hashrate only if ethash is stopped. return ethash.hashrate.Rate1() } diff --git a/consensus/ethash/sealer.go b/consensus/ethash/sealer.go index 43db1fcb7f..52c4ed46dc 100644 --- a/consensus/ethash/sealer.go +++ b/consensus/ethash/sealer.go @@ -18,6 +18,7 @@ package ethash import ( "bytes" + "context" crand "crypto/rand" "encoding/json" "errors" @@ -33,7 +34,6 @@ import ( "github.com/ethereum/go-ethereum/common/hexutil" "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/log" ) const ( @@ -56,7 +56,7 @@ func (ethash *Ethash) Seal(chain consensus.ChainReader, block *types.Block, resu select { case results <- block.WithSeal(header): default: - log.Warn("Sealing result is not read by miner", "mode", "fake", "sealhash", ethash.SealHash(block.Header())) + ethash.config.Log.Warn("Sealing result is not read by miner", "mode", "fake", "sealhash", ethash.SealHash(block.Header())) } return nil } @@ -85,8 +85,8 @@ func (ethash *Ethash) Seal(chain consensus.ChainReader, block *types.Block, resu threads = 0 // Allows disabling local mining without extra logic around local/remote } // Push new work to remote sealer - if ethash.workCh != nil { - ethash.workCh <- &sealTask{block: block, results: results} + if ethash.remote != nil { + ethash.remote.workCh <- &sealTask{block: block, results: results} } var ( pend sync.WaitGroup @@ -111,14 +111,14 @@ func (ethash *Ethash) Seal(chain consensus.ChainReader, block *types.Block, resu select { case results <- result: default: - log.Warn("Sealing result is not read by miner", "mode", "local", "sealhash", ethash.SealHash(block.Header())) + ethash.config.Log.Warn("Sealing result is not read by miner", "mode", "local", "sealhash", ethash.SealHash(block.Header())) } close(abort) case <-ethash.update: // Thread count was changed on user request, restart close(abort) if err := ethash.Seal(chain, block, results, stop); err != nil { - log.Error("Failed to restart sealing after update", "err", err) + ethash.config.Log.Error("Failed to restart sealing after update", "err", err) } } // Wait for all miners to terminate and return the block @@ -143,7 +143,7 @@ func (ethash *Ethash) mine(block *types.Block, id int, seed uint64, abort chan s attempts = int64(0) nonce = seed ) - logger := log.New("miner", id) + logger := ethash.config.Log.New("miner", id) logger.Trace("Started ethash search for new nonces", "seed", seed) search: for { @@ -186,160 +186,128 @@ search: runtime.KeepAlive(dataset) } -// remote is a standalone goroutine to handle remote mining related stuff. -func (ethash *Ethash) remote(notify []string, noverify bool) { - var ( - works = make(map[common.Hash]*types.Block) - rates = make(map[common.Hash]hashrate) +// This is the timeout for HTTP requests to notify external miners. +const remoteSealerTimeout = 1 * time.Second - results chan<- *types.Block - currentBlock *types.Block - currentWork [4]string +type remoteSealer struct { + works map[common.Hash]*types.Block + rates map[common.Hash]hashrate + currentBlock *types.Block + currentWork [4]string + notifyCtx context.Context + cancelNotify context.CancelFunc // cancels all notification requests + reqWG sync.WaitGroup // tracks notification request goroutines - notifyTransport = &http.Transport{} - notifyClient = &http.Client{ - Transport: notifyTransport, - Timeout: time.Second, - } - notifyReqs = make([]*http.Request, len(notify)) - ) - // notifyWork notifies all the specified mining endpoints of the availability of - // new work to be processed. - notifyWork := func() { - work := currentWork - blob, _ := json.Marshal(work) - - for i, url := range notify { - // Terminate any previously pending request and create the new work - if notifyReqs[i] != nil { - notifyTransport.CancelRequest(notifyReqs[i]) - } - notifyReqs[i], _ = http.NewRequest("POST", url, bytes.NewReader(blob)) - notifyReqs[i].Header.Set("Content-Type", "application/json") - - // Push the new work concurrently to all the remote nodes - go func(req *http.Request, url string) { - res, err := notifyClient.Do(req) - if err != nil { - log.Warn("Failed to notify remote miner", "err", err) - } else { - log.Trace("Notified remote miner", "miner", url, "hash", log.Lazy{Fn: func() common.Hash { return common.HexToHash(work[0]) }}, "target", work[2]) - res.Body.Close() - } - }(notifyReqs[i], url) - } - } - // makeWork creates a work package for external miner. - // - // The work package consists of 3 strings: - // result[0], 32 bytes hex encoded current block header pow-hash - // result[1], 32 bytes hex encoded seed hash used for DAG - // result[2], 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty - // result[3], hex encoded block number - makeWork := func(block *types.Block) { - hash := ethash.SealHash(block.Header()) - - currentWork[0] = hash.Hex() - currentWork[1] = common.BytesToHash(SeedHash(block.NumberU64())).Hex() - currentWork[2] = common.BytesToHash(new(big.Int).Div(two256, block.Difficulty()).Bytes()).Hex() - currentWork[3] = hexutil.EncodeBig(block.Number()) - - // Trace the seal work fetched by remote sealer. - currentBlock = block - works[hash] = block - } - // submitWork verifies the submitted pow solution, returning - // whether the solution was accepted or not (not can be both a bad pow as well as - // any other error, like no pending work or stale mining result). - submitWork := func(nonce types.BlockNonce, mixDigest common.Hash, sealhash common.Hash) bool { - if currentBlock == nil { - log.Error("Pending work without block", "sealhash", sealhash) - return false - } - // Make sure the work submitted is present - block := works[sealhash] - if block == nil { - log.Warn("Work submitted but none pending", "sealhash", sealhash, "curnumber", currentBlock.NumberU64()) - return false - } - // Verify the correctness of submitted result. - header := block.Header() - header.Nonce = nonce - header.MixDigest = mixDigest - - start := time.Now() - if !noverify { - if err := ethash.verifySeal(nil, header, true); err != nil { - log.Warn("Invalid proof-of-work submitted", "sealhash", sealhash, "elapsed", common.PrettyDuration(time.Since(start)), "err", err) - return false - } - } - // Make sure the result channel is assigned. - if results == nil { - log.Warn("Ethash result channel is empty, submitted mining result is rejected") - return false - } - log.Trace("Verified correct proof-of-work", "sealhash", sealhash, "elapsed", common.PrettyDuration(time.Since(start))) + ethash *Ethash + noverify bool + notifyURLs []string + results chan<- *types.Block + workCh chan *sealTask // Notification channel to push new work and relative result channel to remote sealer + fetchWorkCh chan *sealWork // Channel used for remote sealer to fetch mining work + submitWorkCh chan *mineResult // Channel used for remote sealer to submit their mining result + fetchRateCh chan chan uint64 // Channel used to gather submitted hash rate for local or remote sealer. + submitRateCh chan *hashrate // Channel used for remote sealer to submit their mining hashrate + requestExit chan struct{} + exitCh chan struct{} +} - // Solutions seems to be valid, return to the miner and notify acceptance. - solution := block.WithSeal(header) +// sealTask wraps a seal block with relative result channel for remote sealer thread. +type sealTask struct { + block *types.Block + results chan<- *types.Block +} - // The submitted solution is within the scope of acceptance. - if solution.NumberU64()+staleThreshold > currentBlock.NumberU64() { - select { - case results <- solution: - log.Debug("Work submitted is acceptable", "number", solution.NumberU64(), "sealhash", sealhash, "hash", solution.Hash()) - return true - default: - log.Warn("Sealing result is not read by miner", "mode", "remote", "sealhash", sealhash) - return false - } - } - // The submitted block is too old to accept, drop it. - log.Warn("Work submitted is too old", "number", solution.NumberU64(), "sealhash", sealhash, "hash", solution.Hash()) - return false +// mineResult wraps the pow solution parameters for the specified block. +type mineResult struct { + nonce types.BlockNonce + mixDigest common.Hash + hash common.Hash + + errc chan error +} + +// hashrate wraps the hash rate submitted by the remote sealer. +type hashrate struct { + id common.Hash + ping time.Time + rate uint64 + + done chan struct{} +} + +// sealWork wraps a seal work package for remote sealer. +type sealWork struct { + errc chan error + res chan [4]string +} + +func startRemoteSealer(ethash *Ethash, urls []string, noverify bool) *remoteSealer { + ctx, cancel := context.WithCancel(context.Background()) + s := &remoteSealer{ + ethash: ethash, + noverify: noverify, + notifyURLs: urls, + notifyCtx: ctx, + cancelNotify: cancel, + works: make(map[common.Hash]*types.Block), + rates: make(map[common.Hash]hashrate), + workCh: make(chan *sealTask), + fetchWorkCh: make(chan *sealWork), + submitWorkCh: make(chan *mineResult), + fetchRateCh: make(chan chan uint64), + submitRateCh: make(chan *hashrate), + requestExit: make(chan struct{}), + exitCh: make(chan struct{}), } + go s.loop() + return s +} + +func (s *remoteSealer) loop() { + defer func() { + s.ethash.config.Log.Trace("Ethash remote sealer is exiting") + s.cancelNotify() + s.reqWG.Wait() + close(s.exitCh) + }() ticker := time.NewTicker(5 * time.Second) defer ticker.Stop() for { select { - case work := <-ethash.workCh: + case work := <-s.workCh: // Update current work with new received block. // Note same work can be past twice, happens when changing CPU threads. - results = work.results + s.results = work.results + s.makeWork(work.block) + s.notifyWork() - makeWork(work.block) - - // Notify and requested URLs of the new work availability - notifyWork() - - case work := <-ethash.fetchWorkCh: + case work := <-s.fetchWorkCh: // Return current mining work to remote miner. - if currentBlock == nil { + if s.currentBlock == nil { work.errc <- errNoMiningWork } else { - work.res <- currentWork + work.res <- s.currentWork } - case result := <-ethash.submitWorkCh: + case result := <-s.submitWorkCh: // Verify submitted PoW solution based on maintained mining blocks. - if submitWork(result.nonce, result.mixDigest, result.hash) { + if s.submitWork(result.nonce, result.mixDigest, result.hash) { result.errc <- nil } else { result.errc <- errInvalidSealResult } - case result := <-ethash.submitRateCh: + case result := <-s.submitRateCh: // Trace remote sealer's hash rate by submitted value. - rates[result.id] = hashrate{rate: result.rate, ping: time.Now()} + s.rates[result.id] = hashrate{rate: result.rate, ping: time.Now()} close(result.done) - case req := <-ethash.fetchRateCh: + case req := <-s.fetchRateCh: // Gather all hash rate submitted by remote sealer. var total uint64 - for _, rate := range rates { + for _, rate := range s.rates { // this could overflow total += rate.rate } @@ -347,25 +315,126 @@ func (ethash *Ethash) remote(notify []string, noverify bool) { case <-ticker.C: // Clear stale submitted hash rate. - for id, rate := range rates { + for id, rate := range s.rates { if time.Since(rate.ping) > 10*time.Second { - delete(rates, id) + delete(s.rates, id) } } // Clear stale pending blocks - if currentBlock != nil { - for hash, block := range works { - if block.NumberU64()+staleThreshold <= currentBlock.NumberU64() { - delete(works, hash) + if s.currentBlock != nil { + for hash, block := range s.works { + if block.NumberU64()+staleThreshold <= s.currentBlock.NumberU64() { + delete(s.works, hash) } } } - case errc := <-ethash.exitCh: - // Exit remote loop if ethash is closed and return relevant error. - errc <- nil - log.Trace("Ethash remote sealer is exiting") + case <-s.requestExit: return } } } + +// makeWork creates a work package for external miner. +// +// The work package consists of 3 strings: +// result[0], 32 bytes hex encoded current block header pow-hash +// result[1], 32 bytes hex encoded seed hash used for DAG +// result[2], 32 bytes hex encoded boundary condition ("target"), 2^256/difficulty +// result[3], hex encoded block number +func (s *remoteSealer) makeWork(block *types.Block) { + hash := s.ethash.SealHash(block.Header()) + s.currentWork[0] = hash.Hex() + s.currentWork[1] = common.BytesToHash(SeedHash(block.NumberU64())).Hex() + s.currentWork[2] = common.BytesToHash(new(big.Int).Div(two256, block.Difficulty()).Bytes()).Hex() + s.currentWork[3] = hexutil.EncodeBig(block.Number()) + + // Trace the seal work fetched by remote sealer. + s.currentBlock = block + s.works[hash] = block +} + +// notifyWork notifies all the specified mining endpoints of the availability of +// new work to be processed. +func (s *remoteSealer) notifyWork() { + work := s.currentWork + blob, _ := json.Marshal(work) + s.reqWG.Add(len(s.notifyURLs)) + for _, url := range s.notifyURLs { + go s.sendNotification(s.notifyCtx, url, blob, work) + } +} + +func (s *remoteSealer) sendNotification(ctx context.Context, url string, json []byte, work [4]string) { + defer s.reqWG.Done() + + req, err := http.NewRequest("POST", url, bytes.NewReader(json)) + if err != nil { + s.ethash.config.Log.Warn("Can't create remote miner notification", "err", err) + return + } + ctx, cancel := context.WithTimeout(ctx, remoteSealerTimeout) + defer cancel() + req = req.WithContext(ctx) + req.Header.Set("Content-Type", "application/json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + s.ethash.config.Log.Warn("Failed to notify remote miner", "err", err) + } else { + s.ethash.config.Log.Trace("Notified remote miner", "miner", url, "hash", work[0], "target", work[2]) + resp.Body.Close() + } +} + +// submitWork verifies the submitted pow solution, returning +// whether the solution was accepted or not (not can be both a bad pow as well as +// any other error, like no pending work or stale mining result). +func (s *remoteSealer) submitWork(nonce types.BlockNonce, mixDigest common.Hash, sealhash common.Hash) bool { + if s.currentBlock == nil { + s.ethash.config.Log.Error("Pending work without block", "sealhash", sealhash) + return false + } + // Make sure the work submitted is present + block := s.works[sealhash] + if block == nil { + s.ethash.config.Log.Warn("Work submitted but none pending", "sealhash", sealhash, "curnumber", s.currentBlock.NumberU64()) + return false + } + // Verify the correctness of submitted result. + header := block.Header() + header.Nonce = nonce + header.MixDigest = mixDigest + + start := time.Now() + if !s.noverify { + if err := s.ethash.verifySeal(nil, header, true); err != nil { + s.ethash.config.Log.Warn("Invalid proof-of-work submitted", "sealhash", sealhash, "elapsed", common.PrettyDuration(time.Since(start)), "err", err) + return false + } + } + // Make sure the result channel is assigned. + if s.results == nil { + s.ethash.config.Log.Warn("Ethash result channel is empty, submitted mining result is rejected") + return false + } + s.ethash.config.Log.Trace("Verified correct proof-of-work", "sealhash", sealhash, "elapsed", common.PrettyDuration(time.Since(start))) + + // Solutions seems to be valid, return to the miner and notify acceptance. + solution := block.WithSeal(header) + + // The submitted solution is within the scope of acceptance. + if solution.NumberU64()+staleThreshold > s.currentBlock.NumberU64() { + select { + case s.results <- solution: + s.ethash.config.Log.Debug("Work submitted is acceptable", "number", solution.NumberU64(), "sealhash", sealhash, "hash", solution.Hash()) + return true + default: + s.ethash.config.Log.Warn("Sealing result is not read by miner", "mode", "remote", "sealhash", sealhash) + return false + } + } + // The submitted block is too old to accept, drop it. + s.ethash.config.Log.Warn("Work submitted is too old", "number", solution.NumberU64(), "sealhash", sealhash, "hash", solution.Hash()) + return false +} diff --git a/consensus/ethash/sealer_test.go b/consensus/ethash/sealer_test.go index 82f08d673c..7f83def072 100644 --- a/consensus/ethash/sealer_test.go +++ b/consensus/ethash/sealer_test.go @@ -20,59 +20,39 @@ import ( "encoding/json" "io/ioutil" "math/big" - "net" "net/http" + "net/http/httptest" "testing" "time" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/internal/testlog" + "github.com/ethereum/go-ethereum/log" ) // Tests whether remote HTTP servers are correctly notified of new work. func TestRemoteNotify(t *testing.T) { - // Start a simple webserver to capture notifications + // Start a simple web server to capture notifications. sink := make(chan [3]string) - - server := &http.Server{ - Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - blob, err := ioutil.ReadAll(req.Body) - if err != nil { - t.Fatalf("failed to read miner notification: %v", err) - } - var work [3]string - if err := json.Unmarshal(blob, &work); err != nil { - t.Fatalf("failed to unmarshal miner notification: %v", err) - } - sink <- work - }), - } - // Open a custom listener to extract its local address - listener, err := net.Listen("tcp", "localhost:0") - if err != nil { - t.Fatalf("failed to open notification server: %v", err) - } - defer listener.Close() - - go server.Serve(listener) - - // Wait for server to start listening - var tries int - for tries = 0; tries < 10; tries++ { - conn, _ := net.DialTimeout("tcp", listener.Addr().String(), 1*time.Second) - if conn != nil { - break + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + blob, err := ioutil.ReadAll(req.Body) + if err != nil { + t.Errorf("failed to read miner notification: %v", err) } - } - if tries == 10 { - t.Fatal("tcp listener not ready for more than 10 seconds") - } + var work [3]string + if err := json.Unmarshal(blob, &work); err != nil { + t.Errorf("failed to unmarshal miner notification: %v", err) + } + sink <- work + })) + defer server.Close() - // Create the custom ethash engine - ethash := NewTester([]string{"http://" + listener.Addr().String()}, false) + // Create the custom ethash engine. + ethash := NewTester([]string{server.URL}, false) defer ethash.Close() - // Stream a work task and ensure the notification bubbles out + // Stream a work task and ensure the notification bubbles out. header := &types.Header{Number: big.NewInt(1), Difficulty: big.NewInt(100)} block := types.NewBlockWithHeader(header) @@ -97,46 +77,37 @@ func TestRemoteNotify(t *testing.T) { // Tests that pushing work packages fast to the miner doesn't cause any data race // issues in the notifications. func TestRemoteMultiNotify(t *testing.T) { - // Start a simple webserver to capture notifications + // Start a simple web server to capture notifications. sink := make(chan [3]string, 64) + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { + blob, err := ioutil.ReadAll(req.Body) + if err != nil { + t.Errorf("failed to read miner notification: %v", err) + } + var work [3]string + if err := json.Unmarshal(blob, &work); err != nil { + t.Errorf("failed to unmarshal miner notification: %v", err) + } + sink <- work + })) + defer server.Close() - server := &http.Server{ - Handler: http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { - blob, err := ioutil.ReadAll(req.Body) - if err != nil { - t.Fatalf("failed to read miner notification: %v", err) - } - var work [3]string - if err := json.Unmarshal(blob, &work); err != nil { - t.Fatalf("failed to unmarshal miner notification: %v", err) - } - sink <- work - }), - } - // Open a custom listener to extract its local address - listener, err := net.Listen("tcp", "localhost:0") - if err != nil { - t.Fatalf("failed to open notification server: %v", err) - } - defer listener.Close() - - go server.Serve(listener) - - // Create the custom ethash engine - ethash := NewTester([]string{"http://" + listener.Addr().String()}, false) + // Create the custom ethash engine. + ethash := NewTester([]string{server.URL}, false) + ethash.config.Log = testlog.Logger(t, log.LvlWarn) defer ethash.Close() - // Stream a lot of work task and ensure all the notifications bubble out + // Stream a lot of work task and ensure all the notifications bubble out. for i := 0; i < cap(sink); i++ { header := &types.Header{Number: big.NewInt(int64(i)), Difficulty: big.NewInt(100)} block := types.NewBlockWithHeader(header) - ethash.Seal(nil, block, nil, nil) } + for i := 0; i < cap(sink); i++ { select { case <-sink: - case <-time.After(3 * time.Second): + case <-time.After(10 * time.Second): t.Fatalf("notification %d timed out", i) } } @@ -206,10 +177,10 @@ func TestStaleSubmission(t *testing.T) { select { case res := <-results: if res.Header().Nonce != fakeNonce { - t.Errorf("case %d block nonce mismatch, want %s, get %s", id+1, fakeNonce, res.Header().Nonce) + t.Errorf("case %d block nonce mismatch, want %x, get %x", id+1, fakeNonce, res.Header().Nonce) } if res.Header().MixDigest != fakeDigest { - t.Errorf("case %d block digest mismatch, want %s, get %s", id+1, fakeDigest, res.Header().MixDigest) + t.Errorf("case %d block digest mismatch, want %x, get %x", id+1, fakeDigest, res.Header().MixDigest) } if res.Header().Difficulty.Uint64() != c.headers[c.submitIndex].Difficulty.Uint64() { t.Errorf("case %d block difficulty mismatch, want %d, get %d", id+1, c.headers[c.submitIndex].Difficulty, res.Header().Difficulty) From 9a529d64d14f8ebf677e69cf1b5de836ca215049 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Thu, 28 Nov 2019 10:53:06 +0100 Subject: [PATCH 008/128] log: fix staticcheck warnings (#20388) --- log/handler_glog.go | 6 +++--- log/logger.go | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/log/handler_glog.go b/log/handler_glog.go index 83dae44bd5..9b1d4efaf4 100644 --- a/log/handler_glog.go +++ b/log/handler_glog.go @@ -207,7 +207,7 @@ func (h *GlogHandler) Log(r *Record) error { } // Check callsite cache for previously calculated log levels h.lock.RLock() - lvl, ok := h.siteCache[r.Call.PC()] + lvl, ok := h.siteCache[r.Call.Frame().PC] h.lock.RUnlock() // If we didn't cache the callsite yet, calculate it @@ -215,13 +215,13 @@ func (h *GlogHandler) Log(r *Record) error { h.lock.Lock() for _, rule := range h.patterns { if rule.pattern.MatchString(fmt.Sprintf("%+s", r.Call)) { - h.siteCache[r.Call.PC()], lvl, ok = rule.level, rule.level, true + h.siteCache[r.Call.Frame().PC], lvl, ok = rule.level, rule.level, true break } } // If no rule matched, remember to drop log the next time if !ok { - h.siteCache[r.Call.PC()] = 0 + h.siteCache[r.Call.Frame().PC] = 0 } h.lock.Unlock() } diff --git a/log/logger.go b/log/logger.go index ca3e0b0599..276d6969e2 100644 --- a/log/logger.go +++ b/log/logger.go @@ -83,7 +83,7 @@ func LvlFromString(lvlString string) (Lvl, error) { case "crit": return LvlCrit, nil default: - return LvlDebug, fmt.Errorf("Unknown level: %v", lvlString) + return LvlDebug, fmt.Errorf("unknown level: %v", lvlString) } } From 08611cfd75b503d8489e0640780637d49d2be7eb Mon Sep 17 00:00:00 2001 From: gary rong Date: Thu, 28 Nov 2019 18:47:35 +0800 Subject: [PATCH 009/128] trie: remove dead code (#20405) --- trie/database.go | 14 -------------- 1 file changed, 14 deletions(-) diff --git a/trie/database.go b/trie/database.go index d510a65885..931918efb5 100644 --- a/trie/database.go +++ b/trie/database.go @@ -17,7 +17,6 @@ package trie import ( - "encoding/binary" "errors" "fmt" "io" @@ -272,19 +271,6 @@ func expandNode(hash hashNode, n node) node { } } -// trienodeHasher is a struct to be used with BigCache, which uses a Hasher to -// determine which shard to place an entry into. It's not a cryptographic hash, -// just to provide a bit of anti-collision (default is FNV64a). -// -// Since trie keys are already hashes, we can just use the key directly to -// map shard id. -type trienodeHasher struct{} - -// Sum64 implements the bigcache.Hasher interface. -func (t trienodeHasher) Sum64(key string) uint64 { - return binary.BigEndian.Uint64([]byte(key)) -} - // NewDatabase creates a new trie database to store ephemeral trie content before // its written out to disk or garbage collected. No read cache is created, so all // data retrievals will hit the underlying disk database. From 1ff3d7c2d48d2625c5c96a6e785332ce5fdeea60 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Fri, 29 Nov 2019 11:38:34 +0100 Subject: [PATCH 010/128] cmd/faucet, cmd/geth: fix staticcheck warnings (#20374) --- cmd/faucet/faucet.go | 13 ++++++++++++- cmd/geth/retesteth.go | 10 +--------- 2 files changed, 13 insertions(+), 10 deletions(-) diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index 3700ff9c59..77938efabd 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -351,6 +351,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { if head == nil || balance == nil { // Report the faucet offline until initial stats are ready + //lint:ignore ST1005 This error is to be displayed in the browser if err = sendError(conn, errors.New("Faucet offline")); err != nil { log.Warn("Failed to send faucet error to client", "err", err) return @@ -392,6 +393,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { continue } if msg.Tier >= uint(*tiersFlag) { + //lint:ignore ST1005 This error is to be displayed in the browser if err = sendError(conn, errors.New("Invalid funding tier requested")); err != nil { log.Warn("Failed to send tier error to client", "err", err) return @@ -429,6 +431,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { } if !result.Success { log.Warn("Captcha verification failed", "err", string(result.Errors)) + //lint:ignore ST1005 it's funny and the robot won't mind if err = sendError(conn, errors.New("Beep-bop, you're a robot!")); err != nil { log.Warn("Failed to send captcha failure to client", "err", err) return @@ -450,6 +453,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { } continue case strings.HasPrefix(msg.URL, "https://plus.google.com/"): + //lint:ignore ST1005 Google is a company name and should be capitalized. if err = sendError(conn, errors.New("Google+ authentication discontinued as the service was sunset")); err != nil { log.Warn("Failed to send Google+ deprecation to client", "err", err) return @@ -462,6 +466,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { case *noauthFlag: username, avatar, address, err = authNoAuth(msg.URL) default: + //lint:ignore ST1005 This error is to be displayed in the browser err = errors.New("Something funky happened, please open an issue at https://github.com/ethereum/go-ethereum/issues") } if err != nil { @@ -520,7 +525,7 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { // Send an error if too frequent funding, othewise a success if !fund { - if err = sendError(conn, fmt.Errorf("%s left until next allowance", common.PrettyDuration(timeout.Sub(time.Now())))); err != nil { // nolint: gosimple + if err = sendError(conn, fmt.Errorf("%s left until next allowance", common.PrettyDuration(time.Until(timeout)))); err != nil { // nolint: gosimple log.Warn("Failed to send funding error to client", "err", err) return } @@ -682,6 +687,7 @@ func authTwitter(url string) (string, string, common.Address, error) { // Ensure the user specified a meaningful URL, no fancy nonsense parts := strings.Split(url, "/") if len(parts) < 4 || parts[len(parts)-2] != "status" { + //lint:ignore ST1005 This error is to be displayed in the browser return "", "", common.Address{}, errors.New("Invalid Twitter status URL") } // Twitter's API isn't really friendly with direct links. Still, we don't @@ -696,6 +702,7 @@ func authTwitter(url string) (string, string, common.Address, error) { // Resolve the username from the final redirect, no intermediate junk parts = strings.Split(res.Request.URL.String(), "/") if len(parts) < 4 || parts[len(parts)-2] != "status" { + //lint:ignore ST1005 This error is to be displayed in the browser return "", "", common.Address{}, errors.New("Invalid Twitter status URL") } username := parts[len(parts)-3] @@ -706,6 +713,7 @@ func authTwitter(url string) (string, string, common.Address, error) { } address := common.HexToAddress(string(regexp.MustCompile("0x[0-9a-fA-F]{40}").Find(body))) if address == (common.Address{}) { + //lint:ignore ST1005 This error is to be displayed in the browser return "", "", common.Address{}, errors.New("No Ethereum address found to fund") } var avatar string @@ -721,6 +729,7 @@ func authFacebook(url string) (string, string, common.Address, error) { // Ensure the user specified a meaningful URL, no fancy nonsense parts := strings.Split(url, "/") if len(parts) < 4 || parts[len(parts)-2] != "posts" { + //lint:ignore ST1005 This error is to be displayed in the browser return "", "", common.Address{}, errors.New("Invalid Facebook post URL") } username := parts[len(parts)-3] @@ -740,6 +749,7 @@ func authFacebook(url string) (string, string, common.Address, error) { } address := common.HexToAddress(string(regexp.MustCompile("0x[0-9a-fA-F]{40}").Find(body))) if address == (common.Address{}) { + //lint:ignore ST1005 This error is to be displayed in the browser return "", "", common.Address{}, errors.New("No Ethereum address found to fund") } var avatar string @@ -755,6 +765,7 @@ func authFacebook(url string) (string, string, common.Address, error) { func authNoAuth(url string) (string, string, common.Address, error) { address := common.HexToAddress(regexp.MustCompile("0x[0-9a-fA-F]{40}").FindString(url)) if address == (common.Address{}) { + //lint:ignore ST1005 This error is to be displayed in the browser return "", "", common.Address{}, errors.New("No Ethereum address found to fund") } return address.Hex() + "@noauth", "", address, nil diff --git a/cmd/geth/retesteth.go b/cmd/geth/retesteth.go index b6aa3706b2..2c3f8be78d 100644 --- a/cmd/geth/retesteth.go +++ b/cmd/geth/retesteth.go @@ -495,7 +495,6 @@ func (api *RetestethAPI) mineBlock() error { txCount := 0 var txs []*types.Transaction var receipts []*types.Receipt - var coalescedLogs []*types.Log var blockFull = gasPool.Gas() < params.TxGas for address := range api.txSenders { if blockFull { @@ -522,7 +521,6 @@ func (api *RetestethAPI) mineBlock() error { } txs = append(txs, tx) receipts = append(receipts, receipt) - coalescedLogs = append(coalescedLogs, receipt.Logs...) delete(m, nonce) if len(m) == 0 { // Last tx for the sender @@ -682,9 +680,6 @@ func (api *RetestethAPI) AccountRange(ctx context.Context, for i := 0; i < int(maxResults) && it.Next(); i++ { if preimage := accountTrie.GetKey(it.Key); preimage != nil { result.AddressMap[common.BytesToHash(it.Key)] = common.BytesToAddress(preimage) - //fmt.Printf("%x: %x\n", it.Key, preimage) - } else { - //fmt.Printf("could not find preimage for %x\n", it.Key) } } //fmt.Printf("Number of entries returned: %d\n", len(result.AddressMap)) @@ -808,9 +803,6 @@ func (api *RetestethAPI) StorageRangeAt(ctx context.Context, Key: string(ks), Value: string(vs), } - //fmt.Printf("Key: %s, Value: %s\n", ks, vs) - } else { - //fmt.Printf("Did not find preimage for %x\n", it.Key) } } if it.Next() { @@ -889,7 +881,7 @@ func retesteth(ctx *cli.Context) error { log.Info("HTTP endpoint closed", "url", httpEndpoint) }() - abortChan := make(chan os.Signal) + abortChan := make(chan os.Signal, 11) signal.Notify(abortChan, os.Interrupt) sig := <-abortChan From e0bf5f0ccbeed89de6fe7611045787e072b1ab32 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Fri, 29 Nov 2019 11:40:02 +0100 Subject: [PATCH 011/128] internal: fix staticcheck warnings (#20380) --- internal/cmdtest/test_cmd.go | 4 ++-- internal/ethapi/api.go | 8 ++++---- internal/jsre/deps/bindata.go | 29 +++++++++++++++++++++-------- 3 files changed, 27 insertions(+), 14 deletions(-) diff --git a/internal/cmdtest/test_cmd.go b/internal/cmdtest/test_cmd.go index f8b1b43c01..d4ccc58329 100644 --- a/internal/cmdtest/test_cmd.go +++ b/internal/cmdtest/test_cmd.go @@ -127,12 +127,12 @@ func (tt *TestCmd) matchExactOutput(want []byte) error { // Find the mismatch position. for i := 0; i < n; i++ { if want[i] != buf[i] { - return fmt.Errorf("Output mismatch at â—Š:\n---------------- (stdout text)\n%sâ—Š%s\n---------------- (expected text)\n%s", + return fmt.Errorf("output mismatch at â—Š:\n---------------- (stdout text)\n%sâ—Š%s\n---------------- (expected text)\n%s", buf[:i], buf[i:n], want) } } if n < len(want) { - return fmt.Errorf("Not enough output, got until â—Š:\n---------------- (stdout text)\n%s\n---------------- (expected text)\n%sâ—Š%s", + return fmt.Errorf("not enough output, got until â—Š:\n---------------- (stdout text)\n%s\n---------------- (expected text)\n%sâ—Š%s", buf, want[:n], want[n:]) } } diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 93042b7a78..2b299d3859 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -486,7 +486,7 @@ func (s *PrivateAccountAPI) InitializeWallet(ctx context.Context, url string) (s case *scwallet.Wallet: return mnemonic, wallet.Initialize(seed) default: - return "", fmt.Errorf("Specified wallet does not support initialization") + return "", fmt.Errorf("specified wallet does not support initialization") } } @@ -501,7 +501,7 @@ func (s *PrivateAccountAPI) Unpair(ctx context.Context, url string, pin string) case *scwallet.Wallet: return wallet.Unpair([]byte(pin)) default: - return fmt.Errorf("Specified wallet does not support pairing") + return fmt.Errorf("specified wallet does not support pairing") } } @@ -1389,7 +1389,7 @@ func (args *SendTxArgs) setDefaults(ctx context.Context, b Backend) error { args.Nonce = (*hexutil.Uint64)(&nonce) } if args.Data != nil && args.Input != nil && !bytes.Equal(*args.Data, *args.Input) { - return errors.New(`Both "data" and "input" are set and not equal. Please use "input" to pass transaction call data.`) + return errors.New(`both "data" and "input" are set and not equal. Please use "input" to pass transaction call data`) } if args.To == nil { // Contract creation @@ -1645,7 +1645,7 @@ func (s *PublicTransactionPoolAPI) Resend(ctx context.Context, sendArgs SendTxAr } } - return common.Hash{}, fmt.Errorf("Transaction %#x not found", matchTx.Hash()) + return common.Hash{}, fmt.Errorf("transaction %#x not found", matchTx.Hash()) } // PublicDebugAPI is the collection of Ethereum APIs exposed over the public diff --git a/internal/jsre/deps/bindata.go b/internal/jsre/deps/bindata.go index 7454c7cfcb..a6545b7140 100644 --- a/internal/jsre/deps/bindata.go +++ b/internal/jsre/deps/bindata.go @@ -1,8 +1,7 @@ -// Code generated by go-bindata. DO NOT EDIT. +// Package deps Code generated by go-bindata. (@generated) DO NOT EDIT. // sources: // bignumber.js // web3.js - package deps import ( @@ -20,7 +19,7 @@ import ( func bindataRead(data []byte, name string) ([]byte, error) { gz, err := gzip.NewReader(bytes.NewBuffer(data)) if err != nil { - return nil, fmt.Errorf("Read %q: %v", name, err) + return nil, fmt.Errorf("read %q: %v", name, err) } var buf bytes.Buffer @@ -28,7 +27,7 @@ func bindataRead(data []byte, name string) ([]byte, error) { clErr := gz.Close() if err != nil { - return nil, fmt.Errorf("Read %q: %v", name, err) + return nil, fmt.Errorf("read %q: %v", name, err) } if clErr != nil { return nil, err @@ -49,21 +48,32 @@ type bindataFileInfo struct { modTime time.Time } +// Name return file name func (fi bindataFileInfo) Name() string { return fi.name } + +// Size return file size func (fi bindataFileInfo) Size() int64 { return fi.size } + +// Mode return file mode func (fi bindataFileInfo) Mode() os.FileMode { return fi.mode } + +// ModTime return file modify time func (fi bindataFileInfo) ModTime() time.Time { return fi.modTime } + +// IsDir return file whether a directory func (fi bindataFileInfo) IsDir() bool { - return false + return fi.mode&os.ModeDir != 0 } + +// Sys return file is sys mode func (fi bindataFileInfo) Sys() interface{} { return nil } @@ -161,8 +171,7 @@ func AssetNames() []string { // _bindata is a table, holding each asset generator, mapped to its name. var _bindata = map[string]func() (*asset, error){ "bignumber.js": bignumberJs, - - "web3.js": web3Js, + "web3.js": web3Js, } // AssetDir returns the file names below a certain @@ -228,7 +237,11 @@ func RestoreAsset(dir, name string) error { if err != nil { return err } - return os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime()) + err = os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime()) + if err != nil { + return err + } + return nil } // RestoreAssets restores an asset under the given directory recursively From 54d332e1db268803223c2d8ea594b719a820f1a4 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Fri, 29 Nov 2019 11:42:53 +0100 Subject: [PATCH 012/128] accounts/scwallet: fix staticcheck warnings (#20370) --- accounts/scwallet/securechannel.go | 24 ++++++++++++------------ accounts/scwallet/wallet.go | 15 +++++++++------ 2 files changed, 21 insertions(+), 18 deletions(-) diff --git a/accounts/scwallet/securechannel.go b/accounts/scwallet/securechannel.go index fad876a019..9b70c69dcc 100644 --- a/accounts/scwallet/securechannel.go +++ b/accounts/scwallet/securechannel.go @@ -71,7 +71,7 @@ func NewSecureChannelSession(card *pcsc.Card, keyData []byte) (*SecureChannelSes cardPublic, ok := gen.Unmarshal(keyData) if !ok { - return nil, fmt.Errorf("Could not unmarshal public key from card") + return nil, fmt.Errorf("could not unmarshal public key from card") } secret, err := gen.GenerateSharedSecret(private, cardPublic) @@ -109,7 +109,7 @@ func (s *SecureChannelSession) Pair(pairingPassword []byte) error { cardChallenge := response.Data[32:64] if !bytes.Equal(expectedCryptogram, cardCryptogram) { - return fmt.Errorf("Invalid card cryptogram %v != %v", expectedCryptogram, cardCryptogram) + return fmt.Errorf("invalid card cryptogram %v != %v", expectedCryptogram, cardCryptogram) } md.Reset() @@ -132,7 +132,7 @@ func (s *SecureChannelSession) Pair(pairingPassword []byte) error { // Unpair disestablishes an existing pairing. func (s *SecureChannelSession) Unpair() error { if s.PairingKey == nil { - return fmt.Errorf("Cannot unpair: not paired") + return fmt.Errorf("cannot unpair: not paired") } _, err := s.transmitEncrypted(claSCWallet, insUnpair, s.PairingIndex, 0, []byte{}) @@ -148,7 +148,7 @@ func (s *SecureChannelSession) Unpair() error { // Open initializes the secure channel. func (s *SecureChannelSession) Open() error { if s.iv != nil { - return fmt.Errorf("Session already opened") + return fmt.Errorf("session already opened") } response, err := s.open() @@ -185,11 +185,11 @@ func (s *SecureChannelSession) mutuallyAuthenticate() error { return err } if response.Sw1 != 0x90 || response.Sw2 != 0x00 { - return fmt.Errorf("Got unexpected response from MUTUALLY_AUTHENTICATE: 0x%x%x", response.Sw1, response.Sw2) + return fmt.Errorf("got unexpected response from MUTUALLY_AUTHENTICATE: 0x%x%x", response.Sw1, response.Sw2) } if len(response.Data) != scSecretLength { - return fmt.Errorf("Response from MUTUALLY_AUTHENTICATE was %d bytes, expected %d", len(response.Data), scSecretLength) + return fmt.Errorf("response from MUTUALLY_AUTHENTICATE was %d bytes, expected %d", len(response.Data), scSecretLength) } return nil @@ -222,7 +222,7 @@ func (s *SecureChannelSession) pair(p1 uint8, data []byte) (*responseAPDU, error // transmitEncrypted sends an encrypted message, and decrypts and returns the response. func (s *SecureChannelSession) transmitEncrypted(cla, ins, p1, p2 byte, data []byte) (*responseAPDU, error) { if s.iv == nil { - return nil, fmt.Errorf("Channel not open") + return nil, fmt.Errorf("channel not open") } data, err := s.encryptAPDU(data) @@ -261,14 +261,14 @@ func (s *SecureChannelSession) transmitEncrypted(cla, ins, p1, p2 byte, data []b return nil, err } if !bytes.Equal(s.iv, rmac) { - return nil, fmt.Errorf("Invalid MAC in response") + return nil, fmt.Errorf("invalid MAC in response") } rapdu := &responseAPDU{} rapdu.deserialize(plainData) if rapdu.Sw1 != sw1Ok { - return nil, fmt.Errorf("Unexpected response status Cla=0x%x, Ins=0x%x, Sw=0x%x%x", cla, ins, rapdu.Sw1, rapdu.Sw2) + return nil, fmt.Errorf("unexpected response status Cla=0x%x, Ins=0x%x, Sw=0x%x%x", cla, ins, rapdu.Sw1, rapdu.Sw2) } return rapdu, nil @@ -277,7 +277,7 @@ func (s *SecureChannelSession) transmitEncrypted(cla, ins, p1, p2 byte, data []b // encryptAPDU is an internal method that serializes and encrypts an APDU. func (s *SecureChannelSession) encryptAPDU(data []byte) ([]byte, error) { if len(data) > maxPayloadSize { - return nil, fmt.Errorf("Payload of %d bytes exceeds maximum of %d", len(data), maxPayloadSize) + return nil, fmt.Errorf("payload of %d bytes exceeds maximum of %d", len(data), maxPayloadSize) } data = pad(data, 0x80) @@ -323,10 +323,10 @@ func unpad(data []byte, terminator byte) ([]byte, error) { case terminator: return data[:len(data)-i], nil default: - return nil, fmt.Errorf("Expected end of padding, got %d", data[len(data)-i]) + return nil, fmt.Errorf("expected end of padding, got %d", data[len(data)-i]) } } - return nil, fmt.Errorf("Expected end of padding, got 0") + return nil, fmt.Errorf("expected end of padding, got 0") } // updateIV is an internal method that updates the initialization vector after diff --git a/accounts/scwallet/wallet.go b/accounts/scwallet/wallet.go index 57b5977062..dd9266cb31 100644 --- a/accounts/scwallet/wallet.go +++ b/accounts/scwallet/wallet.go @@ -167,7 +167,7 @@ func transmit(card *pcsc.Card, command *commandAPDU) (*responseAPDU, error) { } if response.Sw1 != sw1Ok { - return nil, fmt.Errorf("Unexpected insecure response status Cla=0x%x, Ins=0x%x, Sw=0x%x%x", command.Cla, command.Ins, response.Sw1, response.Sw2) + return nil, fmt.Errorf("unexpected insecure response status Cla=0x%x, Ins=0x%x, Sw=0x%x%x", command.Cla, command.Ins, response.Sw1, response.Sw2) } return response, nil @@ -252,7 +252,7 @@ func (w *Wallet) release() error { // with the wallet. func (w *Wallet) pair(puk []byte) error { if w.session.paired() { - return fmt.Errorf("Wallet already paired") + return fmt.Errorf("wallet already paired") } pairing, err := w.session.pair(puk) if err != nil { @@ -773,12 +773,12 @@ func (w *Wallet) findAccountPath(account accounts.Account) (accounts.DerivationP // Look for the path in the URL if account.URL.Scheme != w.Hub.scheme { - return nil, fmt.Errorf("Scheme %s does not match wallet scheme %s", account.URL.Scheme, w.Hub.scheme) + return nil, fmt.Errorf("scheme %s does not match wallet scheme %s", account.URL.Scheme, w.Hub.scheme) } parts := strings.SplitN(account.URL.Path, "/", 2) if len(parts) != 2 { - return nil, fmt.Errorf("Invalid URL format: %s", account.URL) + return nil, fmt.Errorf("invalid URL format: %s", account.URL) } if parts[0] != fmt.Sprintf("%x", w.PublicKey[1:3]) { @@ -813,7 +813,7 @@ func (s *Session) pair(secret []byte) (smartcardPairing, error) { // unpair deletes an existing pairing. func (s *Session) unpair() error { if !s.verified { - return fmt.Errorf("Unpair requires that the PIN be verified") + return fmt.Errorf("unpair requires that the PIN be verified") } return s.Channel.Unpair() } @@ -850,7 +850,7 @@ func (s *Session) paired() bool { // authenticate uses an existing pairing to establish a secure channel. func (s *Session) authenticate(pairing smartcardPairing) error { if !bytes.Equal(s.Wallet.PublicKey, pairing.PublicKey) { - return fmt.Errorf("Cannot pair using another wallet's pairing; %x != %x", s.Wallet.PublicKey, pairing.PublicKey) + return fmt.Errorf("cannot pair using another wallet's pairing; %x != %x", s.Wallet.PublicKey, pairing.PublicKey) } s.Channel.PairingKey = pairing.PairingKey s.Channel.PairingIndex = pairing.PairingIndex @@ -879,6 +879,7 @@ func (s *Session) walletStatus() (*walletStatus, error) { } // derivationPath fetches the wallet's current derivation path from the card. +//lint:ignore U1000 needs to be added to the console interface func (s *Session) derivationPath() (accounts.DerivationPath, error) { response, err := s.Channel.transmitEncrypted(claSCWallet, insStatus, statusP1Path, 0, nil) if err != nil { @@ -993,12 +994,14 @@ func (s *Session) derive(path accounts.DerivationPath) (accounts.Account, error) } // keyExport contains information on an exported keypair. +//lint:ignore U1000 needs to be added to the console interface type keyExport struct { PublicKey []byte `asn1:"tag:0"` PrivateKey []byte `asn1:"tag:1,optional"` } // publicKey returns the public key for the current derivation path. +//lint:ignore U1000 needs to be added to the console interface func (s *Session) publicKey() ([]byte, error) { response, err := s.Channel.transmitEncrypted(claSCWallet, insExportKey, exportP1Any, exportP2Pubkey, nil) if err != nil { From d556d39a2cef8406de4c6deb5e558761c0b5af5d Mon Sep 17 00:00:00 2001 From: xinluyin <31590468+xinluyin@users.noreply.github.com> Date: Fri, 29 Nov 2019 18:46:12 +0800 Subject: [PATCH 013/128] internal/web3ext: add debug_accountRange (#20410) --- internal/web3ext/web3ext.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index dbffbd2a83..cd16fa79c6 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -227,6 +227,11 @@ const DebugJs = ` web3._extend({ property: 'debug', methods: [ + new web3._extend.Method({ + name: 'accountRange', + call: 'debug_accountRange', + params: 2 + }), new web3._extend.Method({ name: 'printBlock', call: 'debug_printBlock', From 5cc6e7a71e6d7677f4c075e5cc98e3d86a8cca08 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Fri, 29 Nov 2019 11:47:14 +0100 Subject: [PATCH 014/128] accounts/usbwallet: fix staticcheck warnings (#20372) --- accounts/usbwallet/ledger.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/accounts/usbwallet/ledger.go b/accounts/usbwallet/ledger.go index 17ca9223ff..64eae64f68 100644 --- a/accounts/usbwallet/ledger.go +++ b/accounts/usbwallet/ledger.go @@ -162,7 +162,8 @@ func (w *ledgerDriver) SignTx(path accounts.DerivationPath, tx *types.Transactio return common.Address{}, nil, accounts.ErrWalletClosed } // Ensure the wallet is capable of signing the given transaction - if chainID != nil && w.version[0] <= 1 && w.version[1] <= 0 && w.version[2] <= 2 { + if chainID != nil && w.version[0] <= 1 && w.version[2] <= 2 { + //lint:ignore ST1005 brand name displayed on the console return common.Address{}, nil, fmt.Errorf("Ledger v%d.%d.%d doesn't support signing this transaction, please update to v1.0.3 at least", w.version[0], w.version[1], w.version[2]) } // All infos gathered and metadata checks out, request signing From fc7e0fe6c75f5c0015935eb49bde677a53be52be Mon Sep 17 00:00:00 2001 From: gary rong Date: Fri, 29 Nov 2019 21:22:08 +0800 Subject: [PATCH 015/128] core, miner: remove PostChainEvents (#19396) This change: - removes the PostChainEvents method on core.BlockChain. - sorts 'removed log' events by block number. - fire the NewChainHead event if we inject a canonical block into the chain even if the entire insertion is not successful. - guarantees correct event ordering in all cases. --- core/blockchain.go | 179 ++++++++++++++++++++-------------------- core/blockchain_test.go | 107 ++++++++++++------------ miner/worker.go | 20 ++--- miner/worker_test.go | 12 ++- 4 files changed, 160 insertions(+), 158 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index 73b6ee65d7..ebab7a3ab5 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1260,16 +1260,16 @@ func (bc *BlockChain) writeKnownBlock(block *types.Block) error { } // WriteBlockWithState writes the block and all associated state to the database. -func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.Receipt, state *state.StateDB) (status WriteStatus, err error) { +func (bc *BlockChain) WriteBlockWithState(block *types.Block, receipts []*types.Receipt, logs []*types.Log, state *state.StateDB, emitHeadEvent bool) (status WriteStatus, err error) { bc.chainmu.Lock() defer bc.chainmu.Unlock() - return bc.writeBlockWithState(block, receipts, state) + return bc.writeBlockWithState(block, receipts, logs, state, emitHeadEvent) } // writeBlockWithState writes the block and all associated state to the database, // but is expects the chain mutex to be held. -func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.Receipt, state *state.StateDB) (status WriteStatus, err error) { +func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.Receipt, logs []*types.Log, state *state.StateDB, emitHeadEvent bool) (status WriteStatus, err error) { bc.wg.Add(1) defer bc.wg.Done() @@ -1394,6 +1394,23 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. bc.insert(block) } bc.futureBlocks.Remove(block.Hash()) + + if status == CanonStatTy { + bc.chainFeed.Send(ChainEvent{Block: block, Hash: block.Hash(), Logs: logs}) + if len(logs) > 0 { + bc.logsFeed.Send(logs) + } + // In theory we should fire a ChainHeadEvent when we inject + // a canonical block, but sometimes we can insert a batch of + // canonicial blocks. Avoid firing too much ChainHeadEvents, + // we will fire an accumulated ChainHeadEvent and disable fire + // event here. + if emitHeadEvent { + bc.chainHeadFeed.Send(ChainHeadEvent{Block: block}) + } + } else { + bc.chainSideFeed.Send(ChainSideEvent{Block: block}) + } return status, nil } @@ -1444,11 +1461,10 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { // Pre-checks passed, start the full block imports bc.wg.Add(1) bc.chainmu.Lock() - n, events, logs, err := bc.insertChain(chain, true) + n, err := bc.insertChain(chain, true) bc.chainmu.Unlock() bc.wg.Done() - bc.PostChainEvents(events, logs) return n, err } @@ -1460,23 +1476,24 @@ func (bc *BlockChain) InsertChain(chain types.Blocks) (int, error) { // racey behaviour. If a sidechain import is in progress, and the historic state // is imported, but then new canon-head is added before the actual sidechain // completes, then the historic state could be pruned again -func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, []interface{}, []*types.Log, error) { +func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, error) { // If the chain is terminating, don't even bother starting up if atomic.LoadInt32(&bc.procInterrupt) == 1 { - return 0, nil, nil, nil + return 0, nil } // Start a parallel signature recovery (signer will fluke on fork transition, minimal perf loss) senderCacher.recoverFromBlocks(types.MakeSigner(bc.chainConfig, chain[0].Number()), chain) - // A queued approach to delivering events. This is generally - // faster than direct delivery and requires much less mutex - // acquiring. var ( - stats = insertStats{startTime: mclock.Now()} - events = make([]interface{}, 0, len(chain)) - lastCanon *types.Block - coalescedLogs []*types.Log + stats = insertStats{startTime: mclock.Now()} + lastCanon *types.Block ) + // Fire a single chain head event if we've progressed the chain + defer func() { + if lastCanon != nil && bc.CurrentBlock().Hash() == lastCanon.Hash() { + bc.chainHeadFeed.Send(ChainHeadEvent{lastCanon}) + } + }() // Start the parallel header verifier headers := make([]*types.Header, len(chain)) seals := make([]bool, len(chain)) @@ -1526,7 +1543,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, [] for block != nil && err == ErrKnownBlock { log.Debug("Writing previously known block", "number", block.Number(), "hash", block.Hash()) if err := bc.writeKnownBlock(block); err != nil { - return it.index, nil, nil, err + return it.index, err } lastCanon = block @@ -1545,7 +1562,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, [] for block != nil && (it.index == 0 || err == consensus.ErrUnknownAncestor) { log.Debug("Future block, postponing import", "number", block.Number(), "hash", block.Hash()) if err := bc.addFutureBlock(block); err != nil { - return it.index, events, coalescedLogs, err + return it.index, err } block, err = it.next() } @@ -1553,14 +1570,14 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, [] stats.ignored += it.remaining() // If there are any still remaining, mark as ignored - return it.index, events, coalescedLogs, err + return it.index, err // Some other error occurred, abort case err != nil: bc.futureBlocks.Remove(block.Hash()) stats.ignored += len(it.chain) bc.reportBlock(block, nil, err) - return it.index, events, coalescedLogs, err + return it.index, err } // No validation errors for the first block (or chain prefix skipped) for ; block != nil && err == nil || err == ErrKnownBlock; block, err = it.next() { @@ -1572,7 +1589,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, [] // If the header is a banned one, straight out abort if BadHashes[block.Hash()] { bc.reportBlock(block, nil, ErrBlacklistedHash) - return it.index, events, coalescedLogs, ErrBlacklistedHash + return it.index, ErrBlacklistedHash } // If the block is known (in the middle of the chain), it's a special case for // Clique blocks where they can share state among each other, so importing an @@ -1589,15 +1606,13 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, [] "root", block.Root()) if err := bc.writeKnownBlock(block); err != nil { - return it.index, nil, nil, err + return it.index, err } stats.processed++ // We can assume that logs are empty here, since the only way for consecutive // Clique blocks to have the same state is if there are no transactions. - events = append(events, ChainEvent{block, block.Hash(), nil}) lastCanon = block - continue } // Retrieve the parent block and it's state to execute on top @@ -1609,7 +1624,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, [] } statedb, err := state.New(parent.Root, bc.stateCache) if err != nil { - return it.index, events, coalescedLogs, err + return it.index, err } // If we have a followup block, run that against the current state to pre-cache // transactions and probabilistically some of the account/storage trie nodes. @@ -1634,7 +1649,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, [] if err != nil { bc.reportBlock(block, receipts, err) atomic.StoreUint32(&followupInterrupt, 1) - return it.index, events, coalescedLogs, err + return it.index, err } // Update the metrics touched during block processing accountReadTimer.Update(statedb.AccountReads) // Account reads are complete, we can mark them @@ -1653,7 +1668,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, [] if err := bc.validator.ValidateState(block, statedb, receipts, usedGas); err != nil { bc.reportBlock(block, receipts, err) atomic.StoreUint32(&followupInterrupt, 1) - return it.index, events, coalescedLogs, err + return it.index, err } proctime := time.Since(start) @@ -1665,10 +1680,10 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, [] // Write the block to the chain and get the status. substart = time.Now() - status, err := bc.writeBlockWithState(block, receipts, statedb) + status, err := bc.writeBlockWithState(block, receipts, logs, statedb, false) if err != nil { atomic.StoreUint32(&followupInterrupt, 1) - return it.index, events, coalescedLogs, err + return it.index, err } atomic.StoreUint32(&followupInterrupt, 1) @@ -1686,8 +1701,6 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, [] "elapsed", common.PrettyDuration(time.Since(start)), "root", block.Root()) - coalescedLogs = append(coalescedLogs, logs...) - events = append(events, ChainEvent{block, block.Hash(), logs}) lastCanon = block // Only count canonical blocks for GC processing time @@ -1698,7 +1711,6 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, [] "diff", block.Difficulty(), "elapsed", common.PrettyDuration(time.Since(start)), "txs", len(block.Transactions()), "gas", block.GasUsed(), "uncles", len(block.Uncles()), "root", block.Root()) - events = append(events, ChainSideEvent{block}) default: // This in theory is impossible, but lets be nice to our future selves and leave @@ -1717,24 +1729,20 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, [] // Any blocks remaining here? The only ones we care about are the future ones if block != nil && err == consensus.ErrFutureBlock { if err := bc.addFutureBlock(block); err != nil { - return it.index, events, coalescedLogs, err + return it.index, err } block, err = it.next() for ; block != nil && err == consensus.ErrUnknownAncestor; block, err = it.next() { if err := bc.addFutureBlock(block); err != nil { - return it.index, events, coalescedLogs, err + return it.index, err } stats.queued++ } } stats.ignored += it.remaining() - // Append a single chain head event if we've progressed the chain - if lastCanon != nil && bc.CurrentBlock().Hash() == lastCanon.Hash() { - events = append(events, ChainHeadEvent{lastCanon}) - } - return it.index, events, coalescedLogs, err + return it.index, err } // insertSideChain is called when an import batch hits upon a pruned ancestor @@ -1743,7 +1751,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, [] // // The method writes all (header-and-body-valid) blocks to disk, then tries to // switch over to the new chain if the TD exceeded the current chain. -func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (int, []interface{}, []*types.Log, error) { +func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (int, error) { var ( externTd *big.Int current = bc.CurrentBlock() @@ -1779,7 +1787,7 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (i // If someone legitimately side-mines blocks, they would still be imported as usual. However, // we cannot risk writing unverified blocks to disk when they obviously target the pruning // mechanism. - return it.index, nil, nil, errors.New("sidechain ghost-state attack") + return it.index, errors.New("sidechain ghost-state attack") } } if externTd == nil { @@ -1790,7 +1798,7 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (i if !bc.HasBlock(block.Hash(), block.NumberU64()) { start := time.Now() if err := bc.writeBlockWithoutState(block, externTd); err != nil { - return it.index, nil, nil, err + return it.index, err } log.Debug("Injected sidechain block", "number", block.Number(), "hash", block.Hash(), "diff", block.Difficulty(), "elapsed", common.PrettyDuration(time.Since(start)), @@ -1807,7 +1815,7 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (i localTd := bc.GetTd(current.Hash(), current.NumberU64()) if localTd.Cmp(externTd) > 0 { log.Info("Sidechain written to disk", "start", it.first().NumberU64(), "end", it.previous().Number, "sidetd", externTd, "localtd", localTd) - return it.index, nil, nil, err + return it.index, err } // Gather all the sidechain hashes (full blocks may be memory heavy) var ( @@ -1822,7 +1830,7 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (i parent = bc.GetHeader(parent.ParentHash, parent.Number.Uint64()-1) } if parent == nil { - return it.index, nil, nil, errors.New("missing parent") + return it.index, errors.New("missing parent") } // Import all the pruned blocks to make the state available var ( @@ -1841,15 +1849,15 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (i // memory here. if len(blocks) >= 2048 || memory > 64*1024*1024 { log.Info("Importing heavy sidechain segment", "blocks", len(blocks), "start", blocks[0].NumberU64(), "end", block.NumberU64()) - if _, _, _, err := bc.insertChain(blocks, false); err != nil { - return 0, nil, nil, err + if _, err := bc.insertChain(blocks, false); err != nil { + return 0, err } blocks, memory = blocks[:0], 0 // If the chain is terminating, stop processing blocks if atomic.LoadInt32(&bc.procInterrupt) == 1 { log.Debug("Premature abort during blocks processing") - return 0, nil, nil, nil + return 0, nil } } } @@ -1857,7 +1865,7 @@ func (bc *BlockChain) insertSideChain(block *types.Block, it *insertIterator) (i log.Info("Importing sidechain segment", "start", blocks[0].NumberU64(), "end", blocks[len(blocks)-1].NumberU64()) return bc.insertChain(blocks, false) } - return 0, nil, nil, nil + return 0, nil } // reorg takes two blocks, an old chain and a new chain and will reconstruct the @@ -1872,11 +1880,11 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { deletedTxs types.Transactions addedTxs types.Transactions - deletedLogs []*types.Log - rebirthLogs []*types.Log + deletedLogs [][]*types.Log + rebirthLogs [][]*types.Log - // collectLogs collects the logs that were generated during the - // processing of the block that corresponds with the given hash. + // collectLogs collects the logs that were generated or removed during + // the processing of the block that corresponds with the given hash. // These logs are later announced as deleted or reborn collectLogs = func(hash common.Hash, removed bool) { number := bc.hc.GetBlockNumber(hash) @@ -1884,18 +1892,40 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { return } receipts := rawdb.ReadReceipts(bc.db, hash, *number, bc.chainConfig) + + var logs []*types.Log for _, receipt := range receipts { for _, log := range receipt.Logs { l := *log if removed { l.Removed = true - deletedLogs = append(deletedLogs, &l) } else { - rebirthLogs = append(rebirthLogs, &l) } + logs = append(logs, &l) + } + } + if len(logs) > 0 { + if removed { + deletedLogs = append(deletedLogs, logs) + } else { + rebirthLogs = append(rebirthLogs, logs) } } } + // mergeLogs returns a merged log slice with specified sort order. + mergeLogs = func(logs [][]*types.Log, reverse bool) []*types.Log { + var ret []*types.Log + if reverse { + for i := len(logs) - 1; i >= 0; i-- { + ret = append(ret, logs[i]...) + } + } else { + for i := 0; i < len(logs); i++ { + ret = append(ret, logs[i]...) + } + } + return ret + } ) // Reduce the longer chain to the same number as the shorter one if oldBlock.NumberU64() > newBlock.NumberU64() { @@ -1990,45 +2020,18 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { // this goroutine if there are no events to fire, but realistcally that only // ever happens if we're reorging empty blocks, which will only happen on idle // networks where performance is not an issue either way. - // - // TODO(karalabe): Can we get rid of the goroutine somehow to guarantee correct - // event ordering? - go func() { - if len(deletedLogs) > 0 { - bc.rmLogsFeed.Send(RemovedLogsEvent{deletedLogs}) - } - if len(rebirthLogs) > 0 { - bc.logsFeed.Send(rebirthLogs) - } - if len(oldChain) > 0 { - for _, block := range oldChain { - bc.chainSideFeed.Send(ChainSideEvent{Block: block}) - } - } - }() - return nil -} - -// PostChainEvents iterates over the events generated by a chain insertion and -// posts them into the event feed. -// TODO: Should not expose PostChainEvents. The chain events should be posted in WriteBlock. -func (bc *BlockChain) PostChainEvents(events []interface{}, logs []*types.Log) { - // post event logs for further processing - if logs != nil { - bc.logsFeed.Send(logs) + if len(deletedLogs) > 0 { + bc.rmLogsFeed.Send(RemovedLogsEvent{mergeLogs(deletedLogs, true)}) } - for _, event := range events { - switch ev := event.(type) { - case ChainEvent: - bc.chainFeed.Send(ev) - - case ChainHeadEvent: - bc.chainHeadFeed.Send(ev) - - case ChainSideEvent: - bc.chainSideFeed.Send(ev) + if len(rebirthLogs) > 0 { + bc.logsFeed.Send(mergeLogs(rebirthLogs, false)) + } + if len(oldChain) > 0 { + for i := len(oldChain) - 1; i >= 0; i-- { + bc.chainSideFeed.Send(ChainSideEvent{Block: oldChain[i]}) } } + return nil } func (bc *BlockChain) update() { diff --git a/core/blockchain_test.go b/core/blockchain_test.go index 3c2bba569b..de23ead210 100644 --- a/core/blockchain_test.go +++ b/core/blockchain_test.go @@ -22,6 +22,7 @@ import ( "math/big" "math/rand" "os" + "reflect" "sync" "testing" "time" @@ -960,16 +961,20 @@ func TestLogReorgs(t *testing.T) { } chain, _ = GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 3, func(i int, gen *BlockGen) {}) + done := make(chan struct{}) + go func() { + ev := <-rmLogsCh + if len(ev.Logs) == 0 { + t.Error("expected logs") + } + close(done) + }() if _, err := blockchain.InsertChain(chain); err != nil { t.Fatalf("failed to insert forked chain: %v", err) } - timeout := time.NewTimer(1 * time.Second) select { - case ev := <-rmLogsCh: - if len(ev.Logs) == 0 { - t.Error("expected logs") - } + case <-done: case <-timeout.C: t.Fatal("Timeout. There is no RemovedLogsEvent has been sent.") } @@ -982,39 +987,47 @@ func TestLogRebirth(t *testing.T) { db = rawdb.NewMemoryDatabase() // this code generates a log - code = common.Hex2Bytes("60606040525b7f24ec1d3ff24c2f6ff210738839dbc339cd45a5294d85c79361016243157aae7b60405180905060405180910390a15b600a8060416000396000f360606040526008565b00") - gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000)}}} - genesis = gspec.MustCommit(db) - signer = types.NewEIP155Signer(gspec.Config.ChainID) - newLogCh = make(chan bool) + code = common.Hex2Bytes("60606040525b7f24ec1d3ff24c2f6ff210738839dbc339cd45a5294d85c79361016243157aae7b60405180905060405180910390a15b600a8060416000396000f360606040526008565b00") + gspec = &Genesis{Config: params.TestChainConfig, Alloc: GenesisAlloc{addr1: {Balance: big.NewInt(10000000000000)}}} + genesis = gspec.MustCommit(db) + signer = types.NewEIP155Signer(gspec.Config.ChainID) + newLogCh = make(chan bool) + removeLogCh = make(chan bool) ) - // listenNewLog checks whether the received logs number is equal with expected. - listenNewLog := func(sink chan []*types.Log, expect int) { + // validateLogEvent checks whether the received logs number is equal with expected. + validateLogEvent := func(sink interface{}, result chan bool, expect int) { + chanval := reflect.ValueOf(sink) + chantyp := chanval.Type() + if chantyp.Kind() != reflect.Chan || chantyp.ChanDir()&reflect.RecvDir == 0 { + t.Fatalf("invalid channel, given type %v", chantyp) + } cnt := 0 + var recv []reflect.Value + timeout := time.After(1 * time.Second) + cases := []reflect.SelectCase{{Chan: chanval, Dir: reflect.SelectRecv}, {Chan: reflect.ValueOf(timeout), Dir: reflect.SelectRecv}} for { - select { - case logs := <-sink: - cnt += len(logs) - case <-time.NewTimer(5 * time.Second).C: - // new logs timeout - newLogCh <- false + chose, v, _ := reflect.Select(cases) + if chose == 1 { + // Not enough event received + result <- false return } + cnt += 1 + recv = append(recv, v) if cnt == expect { break - } else if cnt > expect { - // redundant logs received - newLogCh <- false - return } } - select { - case <-sink: - // redundant logs received - newLogCh <- false - case <-time.NewTimer(100 * time.Millisecond).C: - newLogCh <- true + done := time.After(50 * time.Millisecond) + cases = cases[:1] + cases = append(cases, reflect.SelectCase{Chan: reflect.ValueOf(done), Dir: reflect.SelectRecv}) + chose, _, _ := reflect.Select(cases) + // If chose equal 0, it means receiving redundant events. + if chose == 1 { + result <- true + } else { + result <- false } } @@ -1038,12 +1051,12 @@ func TestLogRebirth(t *testing.T) { }) // Spawn a goroutine to receive log events - go listenNewLog(logsCh, 1) + go validateLogEvent(logsCh, newLogCh, 1) if _, err := blockchain.InsertChain(chain); err != nil { t.Fatalf("failed to insert chain: %v", err) } if !<-newLogCh { - t.Fatalf("failed to receive new log event") + t.Fatal("failed to receive new log event") } // Generate long reorg chain @@ -1060,40 +1073,31 @@ func TestLogRebirth(t *testing.T) { }) // Spawn a goroutine to receive log events - go listenNewLog(logsCh, 1) + go validateLogEvent(logsCh, newLogCh, 1) + go validateLogEvent(rmLogsCh, removeLogCh, 1) if _, err := blockchain.InsertChain(forkChain); err != nil { t.Fatalf("failed to insert forked chain: %v", err) } if !<-newLogCh { - t.Fatalf("failed to receive new log event") + t.Fatal("failed to receive new log event") } - // Ensure removedLog events received - select { - case ev := <-rmLogsCh: - if len(ev.Logs) == 0 { - t.Error("expected logs") - } - case <-time.NewTimer(1 * time.Second).C: - t.Fatal("Timeout. There is no RemovedLogsEvent has been sent.") + if !<-removeLogCh { + t.Fatal("failed to receive removed log event") } newBlocks, _ := GenerateChain(params.TestChainConfig, chain[len(chain)-1], ethash.NewFaker(), db, 1, func(i int, gen *BlockGen) {}) - go listenNewLog(logsCh, 1) + go validateLogEvent(logsCh, newLogCh, 1) + go validateLogEvent(rmLogsCh, removeLogCh, 1) if _, err := blockchain.InsertChain(newBlocks); err != nil { t.Fatalf("failed to insert forked chain: %v", err) } - // Ensure removedLog events received - select { - case ev := <-rmLogsCh: - if len(ev.Logs) == 0 { - t.Error("expected logs") - } - case <-time.NewTimer(1 * time.Second).C: - t.Fatal("Timeout. There is no RemovedLogsEvent has been sent.") - } // Rebirth logs should omit a newLogEvent if !<-newLogCh { - t.Fatalf("failed to receive new log event") + t.Fatal("failed to receive new log event") + } + // Ensure removedLog events received + if !<-removeLogCh { + t.Fatal("failed to receive removed log event") } } @@ -1145,7 +1149,6 @@ func TestSideLogRebirth(t *testing.T) { logsCh := make(chan []*types.Log) blockchain.SubscribeLogsEvent(logsCh) - chain, _ := GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 2, func(i int, gen *BlockGen) { if i == 1 { // Higher block difficulty diff --git a/miner/worker.go b/miner/worker.go index 52f8919f0d..c95b32042f 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -590,7 +590,7 @@ func (w *worker) resultLoop() { logs = append(logs, receipt.Logs...) } // Commit block and state to database. - stat, err := w.chain.WriteBlockWithState(block, receipts, task.state) + _, err := w.chain.WriteBlockWithState(block, receipts, logs, task.state, true) if err != nil { log.Error("Failed writing block to chain", "err", err) continue @@ -601,16 +601,6 @@ func (w *worker) resultLoop() { // Broadcast the block and announce chain insertion event w.mux.Post(core.NewMinedBlockEvent{Block: block}) - var events []interface{} - switch stat { - case core.CanonStatTy: - events = append(events, core.ChainEvent{Block: block, Hash: block.Hash(), Logs: logs}) - events = append(events, core.ChainHeadEvent{Block: block}) - case core.SideStatTy: - events = append(events, core.ChainSideEvent{Block: block}) - } - w.chain.PostChainEvents(events, logs) - // Insert the block into the set of pending ones to resultLoop for confirmations w.unconfirmed.Insert(block.NumberU64(), block.Hash()) @@ -996,3 +986,11 @@ func (w *worker) commit(uncles []*types.Header, interval func(), update bool, st } return nil } + +// postSideBlock fires a side chain event, only use it for testing. +func (w *worker) postSideBlock(event core.ChainSideEvent) { + select { + case w.chainSideCh <- event: + case <-w.exitCh: + } +} diff --git a/miner/worker_test.go b/miner/worker_test.go index 95d5f64b1c..81fb8f1b94 100644 --- a/miner/worker_test.go +++ b/miner/worker_test.go @@ -149,9 +149,6 @@ func newTestWorkerBackend(t *testing.T, chainConfig *params.ChainConfig, engine func (b *testWorkerBackend) BlockChain() *core.BlockChain { return b.chain } func (b *testWorkerBackend) TxPool() *core.TxPool { return b.txPool } -func (b *testWorkerBackend) PostChainEvents(events []interface{}) { - b.chain.PostChainEvents(events, nil) -} func (b *testWorkerBackend) newRandomUncle() *types.Block { var parent *types.Block @@ -243,8 +240,8 @@ func testGenerateBlockAndImport(t *testing.T, isClique bool) { for i := 0; i < 5; i++ { b.txPool.AddLocal(b.newRandomTx(true)) b.txPool.AddLocal(b.newRandomTx(false)) - b.PostChainEvents([]interface{}{core.ChainSideEvent{Block: b.newRandomUncle()}}) - b.PostChainEvents([]interface{}{core.ChainSideEvent{Block: b.newRandomUncle()}}) + w.postSideBlock(core.ChainSideEvent{Block: b.newRandomUncle()}) + w.postSideBlock(core.ChainSideEvent{Block: b.newRandomUncle()}) select { case e := <-loopErr: t.Fatal(e) @@ -295,7 +292,7 @@ func testEmptyWork(t *testing.T, chainConfig *params.ChainConfig, engine consens } w.skipSealHook = func(task *task) bool { return true } w.fullTaskHook = func() { - // Aarch64 unit tests are running in a VM on travis, they must + // Arch64 unit tests are running in a VM on travis, they must // be given more time to execute. time.Sleep(time.Second) } @@ -351,7 +348,8 @@ func TestStreamUncleBlock(t *testing.T) { } } - b.PostChainEvents([]interface{}{core.ChainSideEvent{Block: b.uncleBlock}}) + w.postSideBlock(core.ChainSideEvent{Block: b.uncleBlock}) + select { case <-taskCh: case <-time.NewTimer(time.Second).C: From 7ce7c3967cf2c8ebf921fd90891a5105909e752c Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Mon, 2 Dec 2019 10:29:25 +0100 Subject: [PATCH 016/128] accounts/abi/bind: fix destructive packing of *big.Int (#20412) --- accounts/abi/pack.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/accounts/abi/pack.go b/accounts/abi/pack.go index 36c58265bd..dd1c9a5df8 100644 --- a/accounts/abi/pack.go +++ b/accounts/abi/pack.go @@ -73,7 +73,7 @@ func packNum(value reflect.Value) []byte { case reflect.Int, reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64: return U256(big.NewInt(value.Int())) case reflect.Ptr: - return U256(value.Interface().(*big.Int)) + return U256(new(big.Int).Set(value.Interface().(*big.Int))) default: panic("abi: fatal error") } From 13ccf6016e8b71a6a80ff7e78ae84aad6ed08326 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 2 Dec 2019 12:14:44 +0200 Subject: [PATCH 017/128] trie: track dirty cache metrics, track clean writes on commit --- trie/database.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/trie/database.go b/trie/database.go index 931918efb5..b0fd787444 100644 --- a/trie/database.go +++ b/trie/database.go @@ -38,6 +38,11 @@ var ( memcacheCleanReadMeter = metrics.NewRegisteredMeter("trie/memcache/clean/read", nil) memcacheCleanWriteMeter = metrics.NewRegisteredMeter("trie/memcache/clean/write", nil) + memcacheDirtyHitMeter = metrics.NewRegisteredMeter("trie/memcache/dirty/hit", nil) + memcacheDirtyMissMeter = metrics.NewRegisteredMeter("trie/memcache/dirty/miss", nil) + memcacheDirtyReadMeter = metrics.NewRegisteredMeter("trie/memcache/dirty/read", nil) + memcacheDirtyWriteMeter = metrics.NewRegisteredMeter("trie/memcache/dirty/write", nil) + memcacheFlushTimeTimer = metrics.NewRegisteredResettingTimer("trie/memcache/flush/time", nil) memcacheFlushNodesMeter = metrics.NewRegisteredMeter("trie/memcache/flush/nodes", nil) memcacheFlushSizeMeter = metrics.NewRegisteredMeter("trie/memcache/flush/size", nil) @@ -321,6 +326,8 @@ func (db *Database) insert(hash common.Hash, blob []byte, node node) { if _, ok := db.dirties[hash]; ok { return } + memcacheDirtyWriteMeter.Mark(int64(len(blob))) + // Create the cached entry for this node entry := &cachedNode{ node: simplifyNode(node), @@ -372,8 +379,12 @@ func (db *Database) node(hash common.Hash) node { db.lock.RUnlock() if dirty != nil { + memcacheDirtyHitMeter.Mark(1) + memcacheDirtyReadMeter.Mark(int64(dirty.size)) return dirty.obj(hash) } + memcacheDirtyMissMeter.Mark(1) + // Content unavailable in memory, attempt to retrieve from disk enc, err := db.diskdb.Get(hash[:]) if err != nil || enc == nil { @@ -408,8 +419,12 @@ func (db *Database) Node(hash common.Hash) ([]byte, error) { db.lock.RUnlock() if dirty != nil { + memcacheDirtyHitMeter.Mark(1) + memcacheDirtyReadMeter.Mark(int64(dirty.size)) return dirty.rlp(), nil } + memcacheDirtyMissMeter.Mark(1) + // Content unavailable in memory, attempt to retrieve from disk enc, err := db.diskdb.Get(hash[:]) if err == nil && enc != nil { @@ -812,6 +827,7 @@ func (c *cleaner) Put(key []byte, rlp []byte) error { // Move the flushed node into the clean cache to prevent insta-reloads if c.db.cleans != nil { c.db.cleans.Set(hash[:], rlp) + memcacheCleanWriteMeter.Mark(int64(len(rlp))) } return nil } From 6b6882f08b4b230fc6f740700d73b8cef049d6cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 5 Dec 2019 11:01:40 +0200 Subject: [PATCH 018/128] params: update CHTs for v1.9.9 release --- params/config.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/params/config.go b/params/config.go index 176247a30a..d622e77043 100644 --- a/params/config.go +++ b/params/config.go @@ -71,10 +71,10 @@ var ( // MainnetTrustedCheckpoint contains the light client trusted checkpoint for the main network. MainnetTrustedCheckpoint = &TrustedCheckpoint{ - SectionIndex: 270, - SectionHead: common.HexToHash("0xb67c33d838a60c282c2fb49b188fbbac1ef8565ffb4a1c4909b0a05885e72e40"), - CHTRoot: common.HexToHash("0x781daa4607782300da85d440df3813ba38a1262585231e35e9480726de81dbfc"), - BloomRoot: common.HexToHash("0xfd8951fa6d779cbc981df40dc31056ed1a549db529349d7dfae016f9d96cae72"), + SectionIndex: 275, + SectionHead: common.HexToHash("0x03159234a3699e31d27e5d83a55cbcf8ceb1f2d90855c219c55d79089b61abd4"), + CHTRoot: common.HexToHash("0xd0c1f3828a4dcb2ee76625fdbea85afeabfb61c04adf07439d2fc1cf00469f76"), + BloomRoot: common.HexToHash("0xab8ea2be8aa24703208fee3fc0afdbb536301013f412a7282b2692d6d68f92c5"), } // MainnetCheckpointOracle contains a set of configs for the main network oracle. @@ -109,10 +109,10 @@ var ( // TestnetTrustedCheckpoint contains the light client trusted checkpoint for the Ropsten test network. TestnetTrustedCheckpoint = &TrustedCheckpoint{ - SectionIndex: 204, - SectionHead: common.HexToHash("0xa39168b51c3205456f30ce6a91f3590a43295b15a1c8c2ab86bb8c06b8ad1808"), - CHTRoot: common.HexToHash("0x9a3654147b79882bfc4e16fbd3421512aa7e4dfadc6c511923980e0877bdf3b4"), - BloomRoot: common.HexToHash("0xe72b979522d94fa45c1331639316da234a9bb85062d64d72e13afe1d3f5c17d5"), + SectionIndex: 209, + SectionHead: common.HexToHash("0x8037eb6872b69397d434121424ed8d6ab74be32bf3cb3f12dc5d9657fc146860"), + CHTRoot: common.HexToHash("0xe64b7d6324e5cbdcbbc250adf4cf24a639a665aa83ccfd6a0b84a80faaaa0d41"), + BloomRoot: common.HexToHash("0x80fedbef680cd70d3dc4b50b14480fba82c74361a35e8dc7be9f11e03077c840"), } // TestnetCheckpointOracle contains a set of configs for the Ropsten test network oracle. @@ -150,10 +150,10 @@ var ( // RinkebyTrustedCheckpoint contains the light client trusted checkpoint for the Rinkeby test network. RinkebyTrustedCheckpoint = &TrustedCheckpoint{ - SectionIndex: 163, - SectionHead: common.HexToHash("0x36e5deaa46f258bece94b05d8e10f1ef68f422fb62ed47a2b6e616aa26e84997"), - CHTRoot: common.HexToHash("0x829b9feca1c2cdf5a4cf3efac554889e438ee4df8718c2ce3e02555a02d9e9e5"), - BloomRoot: common.HexToHash("0x58c01de24fdae7c082ebbe7665f189d0aa4d90ee10e72086bf56651c63269e54"), + SectionIndex: 168, + SectionHead: common.HexToHash("0x87301279595b16ac59360c839ef86b159e21fedbfcc8847d727ef446a14cf334"), + CHTRoot: common.HexToHash("0x00f522dd0705ff647cebdd36707d6779caaf77f5fe8f958aae85f36aa88e3f9c"), + BloomRoot: common.HexToHash("0xc908547a6b01c47c65a4581c68090e5602308d39e893f7c0ae3e16c52ce2abf2"), } // RinkebyCheckpointOracle contains a set of configs for the Rinkeby test network oracle. @@ -189,10 +189,10 @@ var ( // GoerliTrustedCheckpoint contains the light client trusted checkpoint for the Görli test network. GoerliTrustedCheckpoint = &TrustedCheckpoint{ - SectionIndex: 47, - SectionHead: common.HexToHash("0x00c5b54c6c9a73660501fd9273ccdb4c5bbdbe5d7b8b650e28f881ec9d2337f6"), - CHTRoot: common.HexToHash("0xef35caa155fd659f57167e7d507de2f8132cbb31f771526481211d8a977d704c"), - BloomRoot: common.HexToHash("0xbda330402f66008d52e7adc748da28535b1212a7912a21244acd2ba77ff0ff06"), + SectionIndex: 52, + SectionHead: common.HexToHash("0x64c3bbc896578cbf782e343db48e334177e87fb8b16106b75e1dcebf59ca59dc"), + CHTRoot: common.HexToHash("0x5d092e644f3815de40b8c4196698d3e34a9097cf3066a499c96e83e3927d8b8d"), + BloomRoot: common.HexToHash("0xb2ceb966b499dd9e6e5bf6adbf35440a0e15cbccc0f527f89a1c522a9f36250a"), } // GoerliCheckpointOracle contains a set of configs for the Goerli test network oracle. From c9dce0bfd769808f75aedbe119caa5f67be26b76 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Thu, 5 Dec 2019 22:16:35 +0100 Subject: [PATCH 019/128] p2p/enode: remove data race in sliceIter (#20421) --- p2p/enode/iter.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/p2p/enode/iter.go b/p2p/enode/iter.go index 112b76d06a..664964f534 100644 --- a/p2p/enode/iter.go +++ b/p2p/enode/iter.go @@ -88,6 +88,8 @@ func (it *sliceIter) Next() bool { } func (it *sliceIter) Node() *Node { + it.mu.Lock() + defer it.mu.Unlock() if len(it.nodes) == 0 { return nil } From bc01593afbb2f0bdbf7f21adc790f161b2a8b41d Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 6 Dec 2019 10:36:40 +0100 Subject: [PATCH 020/128] consensus/ethash, params: eip-2384: bump difficulty bomb (#20347) * consensus/ethash, params: implement eip-2384: bump difficulty bomb * params: EIP 2384 compat checks * consensus, params: add Muir Glacier block number (mainnet,ropsten) + official name * core/forkid: forkid tests for muir glacier * params/config: address review concerns * params, core/forkid: review nitpicks * cmd/geth,eth,les: add override option for muir glacier * params: nit fix --- cmd/geth/config.go | 3 +++ cmd/geth/main.go | 1 + cmd/utils/flags.go | 4 ++++ consensus/ethash/consensus.go | 7 +++++++ core/forkid/forkid_test.go | 16 ++++++++++------ core/genesis.go | 7 +++++-- eth/backend.go | 2 +- eth/config.go | 3 +++ les/client.go | 3 ++- params/config.go | 21 +++++++++++++++++---- 10 files changed, 53 insertions(+), 14 deletions(-) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index 3fe8bd7668..b450dd9c8b 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -150,6 +150,9 @@ func makeFullNode(ctx *cli.Context) *node.Node { if ctx.GlobalIsSet(utils.OverrideIstanbulFlag.Name) { cfg.Eth.OverrideIstanbul = new(big.Int).SetUint64(ctx.GlobalUint64(utils.OverrideIstanbulFlag.Name)) } + if ctx.GlobalIsSet(utils.OverrideMuirGlacierFlag.Name) { + cfg.Eth.OverrideMuirGlacier = new(big.Int).SetUint64(ctx.GlobalUint64(utils.OverrideMuirGlacierFlag.Name)) + } utils.RegisterEthService(stack, &cfg.Eth) // Whisper must be explicitly enabled by specifying at least 1 whisper flag or in dev mode diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 7cfa3e8adf..80ab0a44d8 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -70,6 +70,7 @@ var ( utils.NoUSBFlag, utils.SmartCardDaemonPathFlag, utils.OverrideIstanbulFlag, + utils.OverrideMuirGlacierFlag, utils.EthashCacheDirFlag, utils.EthashCachesInMemoryFlag, utils.EthashCachesOnDiskFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index c6846d312b..d019c1fdcc 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -237,6 +237,10 @@ var ( Name: "override.istanbul", Usage: "Manually specify Istanbul fork-block, overriding the bundled setting", } + OverrideMuirGlacierFlag = cli.Uint64Flag{ + Name: "override.muirglacier", + Usage: "Manually specify Muir Glacier fork-block, overriding the bundled setting", + } // Light server and client settings LightLegacyServFlag = cli.IntFlag{ // Deprecated in favor of light.serve, remove in 2021 Name: "lightserv", diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 3cff2d9fe5..270c5358dd 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -44,6 +44,11 @@ var ( maxUncles = 2 // Maximum number of uncles allowed in a single block allowedFutureBlockTime = 15 * time.Second // Max time from current time allowed for blocks, before they're considered future blocks + // calcDifficultyEip2384 is the difficulty adjustment algorithm as specified by EIP 2384. + // It offsets the bomb 4M blocks from Constantinople, so in total 9M blocks. + // Specification EIP-2384: https://eips.ethereum.org/EIPS/eip-2384 + calcDifficultyEip2384 = makeDifficultyCalculator(big.NewInt(9000000)) + // calcDifficultyConstantinople is the difficulty adjustment algorithm for Constantinople. // It returns the difficulty that a new block should have when created at time given the // parent block's time and difficulty. The calculation uses the Byzantium rules, but with @@ -311,6 +316,8 @@ func (ethash *Ethash) CalcDifficulty(chain consensus.ChainReader, time uint64, p func CalcDifficulty(config *params.ChainConfig, time uint64, parent *types.Header) *big.Int { next := new(big.Int).Add(parent.Number, big1) switch { + case config.IsMuirGlacier(next): + return calcDifficultyEip2384(time, parent) case config.IsConstantinople(next): return calcDifficultyConstantinople(time, parent) case config.IsByzantium(next): diff --git a/core/forkid/forkid_test.go b/core/forkid/forkid_test.go index ee201ae9ae..f3364c3d69 100644 --- a/core/forkid/forkid_test.go +++ b/core/forkid/forkid_test.go @@ -57,8 +57,10 @@ func TestCreation(t *testing.T) { {7279999, ID{Hash: checksumToBytes(0xa00bc324), Next: 7280000}}, // Last Byzantium block {7280000, ID{Hash: checksumToBytes(0x668db0af), Next: 9069000}}, // First and last Constantinople, first Petersburg block {9068999, ID{Hash: checksumToBytes(0x668db0af), Next: 9069000}}, // Last Petersburg block - {9069000, ID{Hash: checksumToBytes(0x879d6e30), Next: 0}}, // Today Istanbul block - {10000000, ID{Hash: checksumToBytes(0x879d6e30), Next: 0}}, // Future Istanbul block + {9069000, ID{Hash: checksumToBytes(0x879d6e30), Next: 9200000}}, // First Istanbul and first Muir Glacier block + {9199999, ID{Hash: checksumToBytes(0x879d6e30), Next: 9200000}}, // Last Istanbul and first Muir Glacier block + {9200000, ID{Hash: checksumToBytes(0xe029e991), Next: 0}}, // First Muir Glacier block + {10000000, ID{Hash: checksumToBytes(0xe029e991), Next: 0}}, // Future Muir Glacier block }, }, // Ropsten test cases @@ -76,8 +78,10 @@ func TestCreation(t *testing.T) { {4939393, ID{Hash: checksumToBytes(0x97b544f3), Next: 4939394}}, // Last Constantinople block {4939394, ID{Hash: checksumToBytes(0xd6e2149b), Next: 6485846}}, // First Petersburg block {6485845, ID{Hash: checksumToBytes(0xd6e2149b), Next: 6485846}}, // Last Petersburg block - {6485846, ID{Hash: checksumToBytes(0x4bc66396), Next: 0}}, // First Istanbul block - {7500000, ID{Hash: checksumToBytes(0x4bc66396), Next: 0}}, // Future Istanbul block + {6485846, ID{Hash: checksumToBytes(0x4bc66396), Next: 7117117}}, // First Istanbul block + {7117116, ID{Hash: checksumToBytes(0x4bc66396), Next: 7117117}}, // Last Istanbul block + {7117117, ID{Hash: checksumToBytes(0x6727ef90), Next: 0}}, // First Muir Glacier block + {7500000, ID{Hash: checksumToBytes(0x6727ef90), Next: 0}}, // Future }, }, // Rinkeby test cases @@ -181,11 +185,11 @@ func TestValidation(t *testing.T) { // Local is mainnet Petersburg, remote is Rinkeby Petersburg. {7987396, ID{Hash: checksumToBytes(0xafec6b27), Next: 0}, ErrLocalIncompatibleOrStale}, - // Local is mainnet Istanbul, far in the future. Remote announces Gopherium (non existing fork) + // Local is mainnet Muir Glacier, far in the future. Remote announces Gopherium (non existing fork) // at some future block 88888888, for itself, but past block for local. Local is incompatible. // // This case detects non-upgraded nodes with majority hash power (typical Ropsten mess). - {88888888, ID{Hash: checksumToBytes(0x879d6e30), Next: 88888888}, ErrLocalIncompatibleOrStale}, + {88888888, ID{Hash: checksumToBytes(0xe029e991), Next: 88888888}, ErrLocalIncompatibleOrStale}, // Local is mainnet Byzantium. Remote is also in Byzantium, but announces Gopherium (non existing // fork) at block 7279999, before Petersburg. Local is incompatible. diff --git a/core/genesis.go b/core/genesis.go index df0c967980..e15cffda6b 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -152,10 +152,10 @@ func (e *GenesisMismatchError) Error() string { // // The returned chain configuration is never nil. func SetupGenesisBlock(db ethdb.Database, genesis *Genesis) (*params.ChainConfig, common.Hash, error) { - return SetupGenesisBlockWithOverride(db, genesis, nil) + return SetupGenesisBlockWithOverride(db, genesis, nil, nil) } -func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, overrideIstanbul *big.Int) (*params.ChainConfig, common.Hash, error) { +func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, overrideIstanbul, overrideMuirGlacier *big.Int) (*params.ChainConfig, common.Hash, error) { if genesis != nil && genesis.Config == nil { return params.AllEthashProtocolChanges, common.Hash{}, errGenesisNoConfig } @@ -207,6 +207,9 @@ func SetupGenesisBlockWithOverride(db ethdb.Database, genesis *Genesis, override if overrideIstanbul != nil { newcfg.IstanbulBlock = overrideIstanbul } + if overrideMuirGlacier != nil { + newcfg.MuirGlacierBlock = overrideMuirGlacier + } if err := newcfg.CheckConfigForkOrder(); err != nil { return newcfg, common.Hash{}, err } diff --git a/eth/backend.go b/eth/backend.go index 83e05e96a8..adde609de0 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -135,7 +135,7 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { if err != nil { return nil, err } - chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideIstanbul) + chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideIstanbul, config.OverrideMuirGlacier) if _, ok := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !ok { return nil, genesisErr } diff --git a/eth/config.go b/eth/config.go index 5094a533bf..8240391114 100644 --- a/eth/config.go +++ b/eth/config.go @@ -157,4 +157,7 @@ type Config struct { // Istanbul block override (TODO: remove after the fork) OverrideIstanbul *big.Int + + // MuirGlacier block override (TODO: remove after the fork) + OverrideMuirGlacier *big.Int } diff --git a/les/client.go b/les/client.go index 1ad44e16d7..c460f4c09d 100644 --- a/les/client.go +++ b/les/client.go @@ -72,7 +72,8 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { if err != nil { return nil, err } - chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, config.OverrideIstanbul) + chainConfig, genesisHash, genesisErr := core.SetupGenesisBlockWithOverride(chainDb, config.Genesis, + config.OverrideIstanbul, config.OverrideMuirGlacier) if _, isCompat := genesisErr.(*params.ConfigCompatError); genesisErr != nil && !isCompat { return nil, genesisErr } diff --git a/params/config.go b/params/config.go index d622e77043..dac56b1bd7 100644 --- a/params/config.go +++ b/params/config.go @@ -66,6 +66,7 @@ var ( ConstantinopleBlock: big.NewInt(7280000), PetersburgBlock: big.NewInt(7280000), IstanbulBlock: big.NewInt(9069000), + MuirGlacierBlock: big.NewInt(9200000), Ethash: new(EthashConfig), } @@ -104,6 +105,7 @@ var ( ConstantinopleBlock: big.NewInt(4230000), PetersburgBlock: big.NewInt(4939394), IstanbulBlock: big.NewInt(6485846), + MuirGlacierBlock: big.NewInt(7117117), Ethash: new(EthashConfig), } @@ -213,16 +215,16 @@ var ( // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil} + AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil} // AllCliqueProtocolChanges contains every protocol change (EIPs) introduced // and accepted by the Ethereum core developers into the Clique consensus. // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}} + AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}} - TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil} + TestChainConfig = &ChainConfig{big.NewInt(1), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil} TestRules = TestChainConfig.Rules(new(big.Int)) ) @@ -292,6 +294,7 @@ type ChainConfig struct { ConstantinopleBlock *big.Int `json:"constantinopleBlock,omitempty"` // Constantinople switch block (nil = no fork, 0 = already activated) PetersburgBlock *big.Int `json:"petersburgBlock,omitempty"` // Petersburg switch block (nil = same as Constantinople) IstanbulBlock *big.Int `json:"istanbulBlock,omitempty"` // Istanbul switch block (nil = no fork, 0 = already on istanbul) + MuirGlacierBlock *big.Int `json:"muirGlacierBlock,omitempty"` // Eip-2384 (bomb delay) switch block (nil = no fork, 0 = already activated) EWASMBlock *big.Int `json:"ewasmBlock,omitempty"` // EWASM switch block (nil = no fork, 0 = already activated) // Various consensus engines @@ -329,7 +332,7 @@ func (c *ChainConfig) String() string { default: engine = "unknown" } - return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v Petersburg: %v Istanbul: %v Engine: %v}", + return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v Constantinople: %v Petersburg: %v Istanbul: %v, Muir Glacier: %v, Engine: %v}", c.ChainID, c.HomesteadBlock, c.DAOForkBlock, @@ -341,6 +344,7 @@ func (c *ChainConfig) String() string { c.ConstantinopleBlock, c.PetersburgBlock, c.IstanbulBlock, + c.MuirGlacierBlock, engine, ) } @@ -380,6 +384,11 @@ func (c *ChainConfig) IsConstantinople(num *big.Int) bool { return isForked(c.ConstantinopleBlock, num) } +// IsMuirGlacier returns whether num is either equal to the Muir Glacier (EIP-2384) fork block or greater. +func (c *ChainConfig) IsMuirGlacier(num *big.Int) bool { + return isForked(c.MuirGlacierBlock, num) +} + // IsPetersburg returns whether num is either // - equal to or greater than the PetersburgBlock fork block, // - OR is nil, and Constantinople is active @@ -432,6 +441,7 @@ func (c *ChainConfig) CheckConfigForkOrder() error { {"constantinopleBlock", c.ConstantinopleBlock}, {"petersburgBlock", c.PetersburgBlock}, {"istanbulBlock", c.IstanbulBlock}, + {"muirGlacierBlock", c.MuirGlacierBlock}, } { if lastFork.name != "" { // Next one must be higher number @@ -485,6 +495,9 @@ func (c *ChainConfig) checkCompatible(newcfg *ChainConfig, head *big.Int) *Confi if isForkIncompatible(c.IstanbulBlock, newcfg.IstanbulBlock, head) { return newCompatError("Istanbul fork block", c.IstanbulBlock, newcfg.IstanbulBlock) } + if isForkIncompatible(c.MuirGlacierBlock, newcfg.MuirGlacierBlock, head) { + return newCompatError("Muir Glacier fork block", c.MuirGlacierBlock, newcfg.MuirGlacierBlock) + } if isForkIncompatible(c.EWASMBlock, newcfg.EWASMBlock, head) { return newCompatError("ewasm fork block", c.EWASMBlock, newcfg.EWASMBlock) } From 017449971e1e9e220efcd97d3313a0e27f47003b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 6 Dec 2019 11:51:37 +0200 Subject: [PATCH 021/128] params: release Geth v1.9.9 --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 2ed8e37bd2..aaeac6a581 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 9 // Minor version component of the current release - VersionPatch = 9 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 9 // Minor version component of the current release + VersionPatch = 9 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From 370cb95b7ff5990879a4f47fcc818be7d6366357 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Fri, 6 Dec 2019 11:53:25 +0200 Subject: [PATCH 022/128] params: begin v1.9.10 release cycle --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index aaeac6a581..d37ea9d995 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 9 // Minor version component of the current release - VersionPatch = 9 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 9 // Minor version component of the current release + VersionPatch = 10 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From 4b40b5377b5069141cbb87026da96b079f393fff Mon Sep 17 00:00:00 2001 From: Charing Date: Tue, 10 Dec 2019 16:26:07 +0800 Subject: [PATCH 023/128] miner: add dependency for stress tests (#20436) 1.to build stress tests Depends-On: 6269e5574c024bb82617b33f673550231b3a3b37 --- miner/stress_clique.go | 3 ++- miner/stress_ethash.go | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/miner/stress_clique.go b/miner/stress_clique.go index 7f5db2e520..2f8a28b68f 100644 --- a/miner/stress_clique.go +++ b/miner/stress_clique.go @@ -37,6 +37,7 @@ import ( "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" @@ -199,7 +200,7 @@ func makeSealer(genesis *core.Genesis) (*node.Node, error) { DatabaseHandles: 256, TxPool: core.DefaultTxPoolConfig, GPO: eth.DefaultConfig.GPO, - Miner: Config{ + Miner: miner.Config{ GasFloor: genesis.GasLimit * 9 / 10, GasCeil: genesis.GasLimit * 11 / 10, GasPrice: big.NewInt(1), diff --git a/miner/stress_ethash.go b/miner/stress_ethash.go index 7d4a7d24f7..988a15c488 100644 --- a/miner/stress_ethash.go +++ b/miner/stress_ethash.go @@ -38,6 +38,7 @@ import ( "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/eth/downloader" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/enode" @@ -179,7 +180,7 @@ func makeMiner(genesis *core.Genesis) (*node.Node, error) { TxPool: core.DefaultTxPoolConfig, GPO: eth.DefaultConfig.GPO, Ethash: eth.DefaultConfig.Ethash, - Miner: Config{ + Miner: miner.Config{ GasFloor: genesis.GasLimit * 9 / 10, GasCeil: genesis.GasLimit * 11 / 10, GasPrice: big.NewInt(1), From cecc7230c054dd1a4fc2783ad558bfa6e92062fe Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 10 Dec 2019 10:57:37 +0100 Subject: [PATCH 024/128] tests/fuzzers: fuzzbuzz fuzzers for keystore, rlp, trie, whisper (#19910) * fuzzers: fuzzers for keystore, rlp, trie, whisper (cred to @guidovranken) * fuzzers: move fuzzers to testdata * testdata/fuzzers: documentation * testdata/fuzzers: corpus for rlp * tests/fuzzers: fixup --- fuzzbuzz.yaml | 36 ++++ tests/fuzzers/README.txt | 45 +++++ ...0176eaf52ed014ec5c91cf4afa070dd3fd469077-1 | 1 + tests/fuzzers/keystore/keystore-fuzzer.go | 37 ++++ tests/fuzzers/rlp/corpus/block_with_uncle.rlp | Bin 0 -> 1120 bytes tests/fuzzers/rlp/corpus/r.bin | 1 + tests/fuzzers/rlp/corpus/transaction.rlp | 2 + tests/fuzzers/rlp/rlp_fuzzer.go | 127 ++++++++++++ tests/fuzzers/trie/corpus/data | 1 + tests/fuzzers/trie/trie-fuzzer.go | 189 ++++++++++++++++++ .../009c5adfa4fd685caef58e1ce932fa7fb209730a | Bin 0 -> 61 bytes tests/fuzzers/whisperv6/whisper-fuzzer.go | 90 +++++++++ 12 files changed, 529 insertions(+) create mode 100644 fuzzbuzz.yaml create mode 100644 tests/fuzzers/README.txt create mode 100644 tests/fuzzers/keystore/corpus/0176eaf52ed014ec5c91cf4afa070dd3fd469077-1 create mode 100644 tests/fuzzers/keystore/keystore-fuzzer.go create mode 100644 tests/fuzzers/rlp/corpus/block_with_uncle.rlp create mode 100644 tests/fuzzers/rlp/corpus/r.bin create mode 100644 tests/fuzzers/rlp/corpus/transaction.rlp create mode 100644 tests/fuzzers/rlp/rlp_fuzzer.go create mode 100644 tests/fuzzers/trie/corpus/data create mode 100644 tests/fuzzers/trie/trie-fuzzer.go create mode 100644 tests/fuzzers/whisperv6/corpus/009c5adfa4fd685caef58e1ce932fa7fb209730a create mode 100644 tests/fuzzers/whisperv6/whisper-fuzzer.go diff --git a/fuzzbuzz.yaml b/fuzzbuzz.yaml new file mode 100644 index 0000000000..72c3835ace --- /dev/null +++ b/fuzzbuzz.yaml @@ -0,0 +1,36 @@ +# bmt keystore rlp trie whisperv6 + +base: ubuntu:16.04 +targets: + - name: rlp + language: go + version: "1.13" + corpus: ./fuzzers/rlp/corpus + harness: + function: Fuzz + package: github.com/ethereum/go-ethereum/tests/fuzzers/rlp + checkout: github.com/ethereum/go-ethereum/ + - name: keystore + language: go + version: "1.13" + corpus: ./fuzzers/keystore/corpus + harness: + function: Fuzz + package: github.com/ethereum/go-ethereum/tests/fuzzers/keystore + checkout: github.com/ethereum/go-ethereum/ + - name: trie + language: go + version: "1.13" + corpus: ./fuzzers/trie/corpus + harness: + function: Fuzz + package: github.com/ethereum/go-ethereum/tests/fuzzers/trie + checkout: github.com/ethereum/go-ethereum/ + - name: whisperv6 + language: go + version: "1.13" + corpus: ./fuzzers/whisperv6/corpus + harness: + function: Fuzz + package: github.com/ethereum/go-ethereum/tests/fuzzers/whisperv6 + checkout: github.com/ethereum/go-ethereum/ diff --git a/tests/fuzzers/README.txt b/tests/fuzzers/README.txt new file mode 100644 index 0000000000..fd8c4ec57f --- /dev/null +++ b/tests/fuzzers/README.txt @@ -0,0 +1,45 @@ +## Fuzzers + +To run a fuzzer locally, you need [go-fuzz](https://github.com/dvyukov/go-fuzz) installed. + +First build a fuzzing-binary out of the selected package: + +``` +(cd ./rlp && CGO_ENABLED=0 go-fuzz-build .) +``` +That command should generate a `rlp-fuzz.zip` in the `rlp/` directory. If you are already in that directory, you can do + +``` +[user@work rlp]$ go-fuzz +2019/11/26 13:36:54 workers: 6, corpus: 3 (3s ago), crashers: 0, restarts: 1/0, execs: 0 (0/sec), cover: 0, uptime: 3s +2019/11/26 13:36:57 workers: 6, corpus: 3 (6s ago), crashers: 0, restarts: 1/0, execs: 0 (0/sec), cover: 1054, uptime: 6s +2019/11/26 13:37:00 workers: 6, corpus: 3 (9s ago), crashers: 0, restarts: 1/8358, execs: 25074 (2786/sec), cover: 1054, uptime: 9s +2019/11/26 13:37:03 workers: 6, corpus: 3 (12s ago), crashers: 0, restarts: 1/8497, execs: 50986 (4249/sec), cover: 1054, uptime: 12s +2019/11/26 13:37:06 workers: 6, corpus: 3 (15s ago), crashers: 0, restarts: 1/9330, execs: 74640 (4976/sec), cover: 1054, uptime: 15s +2019/11/26 13:37:09 workers: 6, corpus: 3 (18s ago), crashers: 0, restarts: 1/9948, execs: 99482 (5527/sec), cover: 1054, uptime: 18s +2019/11/26 13:37:12 workers: 6, corpus: 3 (21s ago), crashers: 0, restarts: 1/9428, execs: 122568 (5836/sec), cover: 1054, uptime: 21s +2019/11/26 13:37:15 workers: 6, corpus: 3 (24s ago), crashers: 0, restarts: 1/9676, execs: 145152 (6048/sec), cover: 1054, uptime: 24s +2019/11/26 13:37:18 workers: 6, corpus: 3 (27s ago), crashers: 0, restarts: 1/9855, execs: 167538 (6205/sec), cover: 1054, uptime: 27s +2019/11/26 13:37:21 workers: 6, corpus: 3 (30s ago), crashers: 0, restarts: 1/9645, execs: 192901 (6430/sec), cover: 1054, uptime: 30s +2019/11/26 13:37:24 workers: 6, corpus: 3 (33s ago), crashers: 0, restarts: 1/9967, execs: 219294 (6645/sec), cover: 1054, uptime: 33s + +``` +Otherwise: +``` +go-fuzz -bin ./rlp/rlp-fuzz.zip +``` + +### Notes + +Once a 'crasher' is found, the fuzzer tries to avoid reporting the same vector twice, so stores the fault in the `suppressions` folder. Thus, if you +e.g. make changes to fix a bug, you should _remove_ all data from the `suppressions`-folder, to verify that the issue is indeed resolved. + +Also, if you have only one and the same exit-point for multiple different types of test, the suppression can make the fuzzer hide differnent types of errors. So make +sure that each type of failure is unique (for an example, see the rlp fuzzer, where a counter `i` is used to differentiate between failures: + +```golang + if !bytes.Equal(input, output) { + panic(fmt.Sprintf("case %d: encode-decode is not equal, \ninput : %x\noutput: %x", i, input, output)) + } +``` + diff --git a/tests/fuzzers/keystore/corpus/0176eaf52ed014ec5c91cf4afa070dd3fd469077-1 b/tests/fuzzers/keystore/corpus/0176eaf52ed014ec5c91cf4afa070dd3fd469077-1 new file mode 100644 index 0000000000..1c0ecf5250 --- /dev/null +++ b/tests/fuzzers/keystore/corpus/0176eaf52ed014ec5c91cf4afa070dd3fd469077-1 @@ -0,0 +1 @@ +ns��,�� \ No newline at end of file diff --git a/tests/fuzzers/keystore/keystore-fuzzer.go b/tests/fuzzers/keystore/keystore-fuzzer.go new file mode 100644 index 0000000000..704f29dc48 --- /dev/null +++ b/tests/fuzzers/keystore/keystore-fuzzer.go @@ -0,0 +1,37 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package keystore + +import ( + "os" + + "github.com/ethereum/go-ethereum/accounts/keystore" +) + +func Fuzz(input []byte) int { + ks := keystore.NewKeyStore("/tmp/ks", keystore.LightScryptN, keystore.LightScryptP) + + a, err := ks.NewAccount(string(input)) + if err != nil { + panic(err) + } + if err := ks.Unlock(a, string(input)); err != nil { + panic(err) + } + os.Remove(a.URL.Path) + return 0 +} diff --git a/tests/fuzzers/rlp/corpus/block_with_uncle.rlp b/tests/fuzzers/rlp/corpus/block_with_uncle.rlp new file mode 100644 index 0000000000000000000000000000000000000000..1b49fe6a095f6086ba3b2a22980818adb535c18f GIT binary patch literal 1120 zcmey#68n?!=Yq7}+ib@e9?7`~NX2fJjiyVr+U(iMH>U2nKQ*W-BmB-4lRK+dE|_lmWY(#>TW%QlUAFykmm$S- z6L;&2dG1PHC*JheElMa{z?!Z6dzS~>v~BTEm(K22^jN4lk@?8i+8>t6R-exMEV{Il zkzu5RW+sLMEY14wcQplZw8XfVZEsjWC`dYB3VtO0NML4cW;v}tg)^>t-LhrJj~u%H zdV1Zj!)oh8b=$cbq!%pT{^{3bruc);R!JOO#eP`JHN~>jPcmn_zkaRIdAkX+?CBzT!UCj;%*;$U2*-e{<(!TeAEou1H-La+RLaRU-Uhq0?W7 zE0f))?70_Q^5^N>J$+hj(+>t4rq|`|=Uxz_7ZtdC?$Y*&Xa9w3pI?0c<%xv-^=oUV zTP$34F6{keqXl6vLf(k~H7y~eg RIWU2eSOX}5Z6zUr0RUb(3%vjU literal 0 HcmV?d00001 diff --git a/tests/fuzzers/rlp/corpus/r.bin b/tests/fuzzers/rlp/corpus/r.bin new file mode 100644 index 0000000000..cb98a76a8a --- /dev/null +++ b/tests/fuzzers/rlp/corpus/r.bin @@ -0,0 +1 @@ +ˀ���������� \ No newline at end of file diff --git a/tests/fuzzers/rlp/corpus/transaction.rlp b/tests/fuzzers/rlp/corpus/transaction.rlp new file mode 100644 index 0000000000..80eea1aec6 --- /dev/null +++ b/tests/fuzzers/rlp/corpus/transaction.rlp @@ -0,0 +1,2 @@ +ďż˝Nďż˝������ +���a����P?-'ďż˝{�ЋDďż˝Yďż˝��fďż˝j\ďż˝E��~읕��F?1(ďż˝ij6ďż˝@ďż˝v ďż˝L��ڑ� \ No newline at end of file diff --git a/tests/fuzzers/rlp/rlp_fuzzer.go b/tests/fuzzers/rlp/rlp_fuzzer.go new file mode 100644 index 0000000000..534540476c --- /dev/null +++ b/tests/fuzzers/rlp/rlp_fuzzer.go @@ -0,0 +1,127 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package rlp + +import ( + "bytes" + "fmt" + + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/rlp" +) + +func decodeEncode(input []byte, val interface{}, i int) { + if err := rlp.DecodeBytes(input, val); err == nil { + output, err := rlp.EncodeToBytes(val) + if err != nil { + panic(err) + } + if !bytes.Equal(input, output) { + panic(fmt.Sprintf("case %d: encode-decode is not equal, \ninput : %x\noutput: %x", i, input, output)) + } + } +} + +func Fuzz(input []byte) int { + var i int + { + if len(input) > 0 { + rlp.Split(input) + } + } + { + if len(input) > 0 { + if elems, _, err := rlp.SplitList(input); err == nil { + rlp.CountValues(elems) + } + } + } + + { + rlp.NewStream(bytes.NewReader(input), 0).Decode(new(interface{})) + } + + { + decodeEncode(input, new(interface{}), i) + i++ + } + { + var v struct { + Int uint + String string + Bytes []byte + } + decodeEncode(input, &v, i) + i++ + } + + { + type Types struct { + Bool bool + Raw rlp.RawValue + Slice []*Types + Iface []interface{} + } + var v Types + decodeEncode(input, &v, i) + i++ + } + { + type AllTypes struct { + Int uint + String string + Bytes []byte + Bool bool + Raw rlp.RawValue + Slice []*AllTypes + Array [3]*AllTypes + Iface []interface{} + } + var v AllTypes + decodeEncode(input, &v, i) + i++ + } + { + decodeEncode(input, [10]byte{}, i) + i++ + } + { + var v struct { + Byte [10]byte + Rool [10]bool + } + decodeEncode(input, &v, i) + i++ + } + { + var h types.Header + decodeEncode(input, &h, i) + i++ + var b types.Block + decodeEncode(input, &b, i) + i++ + var t types.Transaction + decodeEncode(input, &t, i) + i++ + var txs types.Transactions + decodeEncode(input, &txs, i) + i++ + var rs types.Receipts + decodeEncode(input, &rs, i) + } + return 0 +} diff --git a/tests/fuzzers/trie/corpus/data b/tests/fuzzers/trie/corpus/data new file mode 100644 index 0000000000..c4a4839cb8 --- /dev/null +++ b/tests/fuzzers/trie/corpus/data @@ -0,0 +1 @@ +asdlfkjasf23oiejfasdfadkfqlkjfasdlkfjalwk4jfalsdkfjawlefkjsadlfkjasldkfjwalefkjasdlfkjM \ No newline at end of file diff --git a/tests/fuzzers/trie/trie-fuzzer.go b/tests/fuzzers/trie/trie-fuzzer.go new file mode 100644 index 0000000000..9818838053 --- /dev/null +++ b/tests/fuzzers/trie/trie-fuzzer.go @@ -0,0 +1,189 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "bytes" + "encoding/binary" + "fmt" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/ethdb/memorydb" + "github.com/ethereum/go-ethereum/trie" +) + +// randTest performs random trie operations. +// Instances of this test are created by Generate. +type randTest []randTestStep + +type randTestStep struct { + op int + key []byte // for opUpdate, opDelete, opGet + value []byte // for opUpdate + err error // for debugging +} + +type proofDb struct{} + +func (proofDb) Put(key []byte, value []byte) error { + return nil +} + +func (proofDb) Delete(key []byte) error { + return nil +} + +const ( + opUpdate = iota + opDelete + opGet + opCommit + opHash + opReset + opItercheckhash + opProve + opMax // boundary value, not an actual op +) + +type dataSource struct { + input []byte + reader *bytes.Reader +} + +func newDataSource(input []byte) *dataSource { + return &dataSource{ + input, bytes.NewReader(input), + } +} +func (ds *dataSource) ReadByte() byte { + if b, err := ds.reader.ReadByte(); err != nil { + return 0 + } else { + return b + } +} +func (ds *dataSource) Read(buf []byte) (int, error) { + return ds.reader.Read(buf) +} +func (ds *dataSource) Ended() bool { + return ds.reader.Len() == 0 +} + +func Generate(input []byte) randTest { + + var allKeys [][]byte + r := newDataSource(input) + genKey := func() []byte { + + if len(allKeys) < 2 || r.ReadByte() < 0x0f { + // new key + key := make([]byte, r.ReadByte()%50) + r.Read(key) + allKeys = append(allKeys, key) + return key + } + // use existing key + return allKeys[int(r.ReadByte())%len(allKeys)] + } + + var steps randTest + + for i := 0; !r.Ended(); i++ { + + step := randTestStep{op: int(r.ReadByte()) % opMax} + switch step.op { + case opUpdate: + step.key = genKey() + step.value = make([]byte, 8) + binary.BigEndian.PutUint64(step.value, uint64(i)) + case opGet, opDelete, opProve: + step.key = genKey() + } + steps = append(steps, step) + if len(steps) > 500 { + break + } + } + + return steps +} + +func Fuzz(input []byte) int { + program := Generate(input) + if len(program) == 0 { + return -1 + } + if err := runRandTest(program); err != nil { + panic(err) + } + return 0 +} + +func runRandTest(rt randTest) error { + + triedb := trie.NewDatabase(memorydb.New()) + + tr, _ := trie.New(common.Hash{}, triedb) + values := make(map[string]string) // tracks content of the trie + + for i, step := range rt { + switch step.op { + case opUpdate: + tr.Update(step.key, step.value) + values[string(step.key)] = string(step.value) + case opDelete: + tr.Delete(step.key) + delete(values, string(step.key)) + case opGet: + v := tr.Get(step.key) + want := values[string(step.key)] + if string(v) != want { + rt[i].err = fmt.Errorf("mismatch for key 0x%x, got 0x%x want 0x%x", step.key, v, want) + } + case opCommit: + _, rt[i].err = tr.Commit(nil) + case opHash: + tr.Hash() + case opReset: + hash, err := tr.Commit(nil) + if err != nil { + return err + } + newtr, err := trie.New(hash, triedb) + if err != nil { + return err + } + tr = newtr + case opItercheckhash: + checktr, _ := trie.New(common.Hash{}, triedb) + it := trie.NewIterator(tr.NodeIterator(nil)) + for it.Next() { + checktr.Update(it.Key, it.Value) + } + if tr.Hash() != checktr.Hash() { + return fmt.Errorf("hash mismatch in opItercheckhash") + } + case opProve: + rt[i].err = tr.Prove(step.key, 0, proofDb{}) + } + // Abort the test on error. + if rt[i].err != nil { + return rt[i].err + } + } + return nil +} diff --git a/tests/fuzzers/whisperv6/corpus/009c5adfa4fd685caef58e1ce932fa7fb209730a b/tests/fuzzers/whisperv6/corpus/009c5adfa4fd685caef58e1ce932fa7fb209730a new file mode 100644 index 0000000000000000000000000000000000000000..af2f0826730441cc5abb122074afd090fad2eb95 GIT binary patch literal 61 XcmaEK>c|9A!Hu2QZfJ0QyTbqgS<53t literal 0 HcmV?d00001 diff --git a/tests/fuzzers/whisperv6/whisper-fuzzer.go b/tests/fuzzers/whisperv6/whisper-fuzzer.go new file mode 100644 index 0000000000..379e4224fd --- /dev/null +++ b/tests/fuzzers/whisperv6/whisper-fuzzer.go @@ -0,0 +1,90 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package whisperv6 + +import ( + "bytes" + + "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/rlp" + "github.com/ethereum/go-ethereum/whisper/whisperv6" +) + +type MessageParams struct { + Topic whisperv6.TopicType + WorkTime uint32 + TTL uint32 + KeySym []byte + Payload []byte +} + +//export fuzzer_entry +func Fuzz(input []byte) int { + + var paramsDecoded MessageParams + err := rlp.DecodeBytes(input, ¶msDecoded) + if err != nil { + return 0 + } + var params whisperv6.MessageParams + params.KeySym = make([]byte, 32) + if len(paramsDecoded.KeySym) <= 32 { + copy(params.KeySym, paramsDecoded.KeySym) + } + if input[0] == 255 { + params.PoW = 0.01 + params.WorkTime = 1 + } else { + params.PoW = 0 + params.WorkTime = 0 + } + params.TTL = paramsDecoded.TTL + params.Payload = paramsDecoded.Payload + text := make([]byte, 0, 512) + text = append(text, params.Payload...) + params.Topic = paramsDecoded.Topic + params.Src, err = crypto.GenerateKey() + if err != nil { + return 0 + } + msg, err := whisperv6.NewSentMessage(¶ms) + if err != nil { + panic(err) + //return + } + env, err := msg.Wrap(¶ms) + if err != nil { + panic(err) + } + decrypted, err := env.OpenSymmetric(params.KeySym) + if err != nil { + panic(err) + } + if !decrypted.ValidateAndParse() { + panic("ValidateAndParse failed") + } + if !bytes.Equal(text, decrypted.Payload) { + panic("text != decrypted.Payload") + } + if len(decrypted.Signature) != 65 { + panic("Unexpected signature length") + } + if !whisperv6.IsPubKeyEqual(decrypted.Src, ¶ms.Src.PublicKey) { + panic("Unexpected public key") + } + return 0 +} From f383eaa102d11490a85c0a238f904e5d06b95178 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 10 Dec 2019 11:50:16 +0100 Subject: [PATCH 025/128] core: removed old invalid comment --- core/genesis.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/genesis.go b/core/genesis.go index e15cffda6b..92e654da83 100644 --- a/core/genesis.go +++ b/core/genesis.go @@ -383,8 +383,7 @@ func DefaultGoerliGenesisBlock() *Genesis { } } -// DeveloperGenesisBlock returns the 'geth --dev' genesis block. Note, this must -// be seeded with the +// DeveloperGenesisBlock returns the 'geth --dev' genesis block. func DeveloperGenesisBlock(period uint64, faucet common.Address) *Genesis { // Override the default period to the user requested one config := *params.AllCliqueProtocolChanges From d90d1db609c8d77baa422d49bd371207c06b4711 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 10 Dec 2019 12:39:14 +0100 Subject: [PATCH 026/128] eth/filters: remove use of event.TypeMux for pending logs (#20312) --- accounts/abi/bind/backends/simulated.go | 22 ++- core/events.go | 5 - eth/api_backend.go | 4 + eth/filters/api.go | 5 +- eth/filters/bench_test.go | 7 +- eth/filters/filter.go | 2 +- eth/filters/filter_system.go | 167 ++++++++++---------- eth/filters/filter_system_test.go | 198 ++++++++++++------------ eth/filters/filter_test.go | 33 ++-- internal/ethapi/backend.go | 2 +- les/api_backend.go | 11 +- miner/miner.go | 6 + miner/worker.go | 5 +- 13 files changed, 230 insertions(+), 237 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index e30572e0a1..e625a42868 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -76,7 +76,7 @@ func NewSimulatedBackendWithDatabase(database ethdb.Database, alloc core.Genesis database: database, blockchain: blockchain, config: genesis.Config, - events: filters.NewEventSystem(new(event.TypeMux), &filterBackend{database, blockchain}, false), + events: filters.NewEventSystem(&filterBackend{database, blockchain}, false), } backend.rollback() return backend @@ -502,22 +502,34 @@ func (fb *filterBackend) GetLogs(ctx context.Context, hash common.Hash) ([][]*ty } func (fb *filterBackend) SubscribeNewTxsEvent(ch chan<- core.NewTxsEvent) event.Subscription { - return event.NewSubscription(func(quit <-chan struct{}) error { - <-quit - return nil - }) + return nullSubscription() } + func (fb *filterBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { return fb.bc.SubscribeChainEvent(ch) } + func (fb *filterBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { return fb.bc.SubscribeRemovedLogsEvent(ch) } + func (fb *filterBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription { return fb.bc.SubscribeLogsEvent(ch) } +func (fb *filterBackend) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { + return nullSubscription() +} + func (fb *filterBackend) BloomStatus() (uint64, uint64) { return 4096, 0 } + func (fb *filterBackend) ServiceFilter(ctx context.Context, ms *bloombits.MatcherSession) { panic("not supported") } + +func nullSubscription() event.Subscription { + return event.NewSubscription(func(quit <-chan struct{}) error { + <-quit + return nil + }) +} diff --git a/core/events.go b/core/events.go index 710bdb5894..ac935a137f 100644 --- a/core/events.go +++ b/core/events.go @@ -24,11 +24,6 @@ import ( // NewTxsEvent is posted when a batch of transactions enter the transaction pool. type NewTxsEvent struct{ Txs []*types.Transaction } -// PendingLogsEvent is posted pre mining and notifies of pending logs. -type PendingLogsEvent struct { - Logs []*types.Log -} - // NewMinedBlockEvent is posted when a block has been imported. type NewMinedBlockEvent struct{ Block *types.Block } diff --git a/eth/api_backend.go b/eth/api_backend.go index 4b74ccff51..0ad2d57fd2 100644 --- a/eth/api_backend.go +++ b/eth/api_backend.go @@ -202,6 +202,10 @@ func (b *EthAPIBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEven return b.eth.BlockChain().SubscribeRemovedLogsEvent(ch) } +func (b *EthAPIBackend) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { + return b.eth.miner.SubscribePendingLogs(ch) +} + func (b *EthAPIBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { return b.eth.BlockChain().SubscribeChainEvent(ch) } diff --git a/eth/filters/api.go b/eth/filters/api.go index 5ed80a8875..46c9f44bb0 100644 --- a/eth/filters/api.go +++ b/eth/filters/api.go @@ -65,9 +65,8 @@ type PublicFilterAPI struct { func NewPublicFilterAPI(backend Backend, lightMode bool) *PublicFilterAPI { api := &PublicFilterAPI{ backend: backend, - mux: backend.EventMux(), chainDb: backend.ChainDb(), - events: NewEventSystem(backend.EventMux(), backend, lightMode), + events: NewEventSystem(backend, lightMode), filters: make(map[rpc.ID]*filter), } go api.timeoutLoop() @@ -428,7 +427,7 @@ func (api *PublicFilterAPI) GetFilterChanges(id rpc.ID) (interface{}, error) { hashes := f.hashes f.hashes = nil return returnHashes(hashes), nil - case LogsSubscription: + case LogsSubscription, MinedAndPendingLogsSubscription: logs := f.logs f.logs = nil return returnLogs(logs), nil diff --git a/eth/filters/bench_test.go b/eth/filters/bench_test.go index fc7e6f5273..584ea23d02 100644 --- a/eth/filters/bench_test.go +++ b/eth/filters/bench_test.go @@ -28,7 +28,6 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/ethdb" - "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/node" ) @@ -122,14 +121,13 @@ func benchmarkBloomBits(b *testing.B, sectionSize uint64) { b.Log("Running filter benchmarks...") start = time.Now() - mux := new(event.TypeMux) var backend *testBackend for i := 0; i < benchFilterCnt; i++ { if i%20 == 0 { db.Close() db, _ = rawdb.NewLevelDBDatabase(benchDataDir, 128, 1024, "") - backend = &testBackend{mux, db, cnt, new(event.Feed), new(event.Feed), new(event.Feed), new(event.Feed)} + backend = &testBackend{db: db, sections: cnt} } var addr common.Address addr[0] = byte(i) @@ -173,8 +171,7 @@ func BenchmarkNoBloomBits(b *testing.B) { b.Log("Running filter benchmarks...") start := time.Now() - mux := new(event.TypeMux) - backend := &testBackend{mux, db, 0, new(event.Feed), new(event.Feed), new(event.Feed), new(event.Feed)} + backend := &testBackend{db: db} filter := NewRangeFilter(backend, 0, int64(*headNum), []common.Address{{}}, nil) filter.Logs(context.Background()) d := time.Since(start) diff --git a/eth/filters/filter.go b/eth/filters/filter.go index 071613ad7a..2184092071 100644 --- a/eth/filters/filter.go +++ b/eth/filters/filter.go @@ -32,7 +32,6 @@ import ( type Backend interface { ChainDb() ethdb.Database - EventMux() *event.TypeMux HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) HeaderByHash(ctx context.Context, blockHash common.Hash) (*types.Header, error) GetReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) @@ -42,6 +41,7 @@ type Backend interface { SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription + SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription BloomStatus() (uint64, uint64) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) diff --git a/eth/filters/filter_system.go b/eth/filters/filter_system.go index 70139c1a96..a105ec51c3 100644 --- a/eth/filters/filter_system.go +++ b/eth/filters/filter_system.go @@ -20,7 +20,6 @@ package filters import ( "context" - "errors" "fmt" "sync" "time" @@ -58,7 +57,6 @@ const ( ) const ( - // txChanSize is the size of channel listening to NewTxsEvent. // The number is referenced from the size of tx pool. txChanSize = 4096 @@ -70,10 +68,6 @@ const ( chainEvChanSize = 10 ) -var ( - ErrInvalidSubscriptionID = errors.New("invalid id") -) - type subscription struct { id rpc.ID typ Type @@ -89,25 +83,25 @@ type subscription struct { // EventSystem creates subscriptions, processes events and broadcasts them to the // subscription which match the subscription criteria. type EventSystem struct { - mux *event.TypeMux backend Backend lightMode bool lastHead *types.Header // Subscriptions - txsSub event.Subscription // Subscription for new transaction event - logsSub event.Subscription // Subscription for new log event - rmLogsSub event.Subscription // Subscription for removed log event - chainSub event.Subscription // Subscription for new chain event - pendingLogSub *event.TypeMuxSubscription // Subscription for pending log event + txsSub event.Subscription // Subscription for new transaction event + logsSub event.Subscription // Subscription for new log event + rmLogsSub event.Subscription // Subscription for removed log event + pendingLogsSub event.Subscription // Subscription for pending log event + chainSub event.Subscription // Subscription for new chain event // Channels - install chan *subscription // install filter for event notification - uninstall chan *subscription // remove filter for event notification - txsCh chan core.NewTxsEvent // Channel to receive new transactions event - logsCh chan []*types.Log // Channel to receive new log event - rmLogsCh chan core.RemovedLogsEvent // Channel to receive removed log event - chainCh chan core.ChainEvent // Channel to receive new chain event + install chan *subscription // install filter for event notification + uninstall chan *subscription // remove filter for event notification + txsCh chan core.NewTxsEvent // Channel to receive new transactions event + logsCh chan []*types.Log // Channel to receive new log event + pendingLogsCh chan []*types.Log // Channel to receive new log event + rmLogsCh chan core.RemovedLogsEvent // Channel to receive removed log event + chainCh chan core.ChainEvent // Channel to receive new chain event } // NewEventSystem creates a new manager that listens for event on the given mux, @@ -116,17 +110,17 @@ type EventSystem struct { // // The returned manager has a loop that needs to be stopped with the Stop function // or by stopping the given mux. -func NewEventSystem(mux *event.TypeMux, backend Backend, lightMode bool) *EventSystem { +func NewEventSystem(backend Backend, lightMode bool) *EventSystem { m := &EventSystem{ - mux: mux, - backend: backend, - lightMode: lightMode, - install: make(chan *subscription), - uninstall: make(chan *subscription), - txsCh: make(chan core.NewTxsEvent, txChanSize), - logsCh: make(chan []*types.Log, logsChanSize), - rmLogsCh: make(chan core.RemovedLogsEvent, rmLogsChanSize), - chainCh: make(chan core.ChainEvent, chainEvChanSize), + backend: backend, + lightMode: lightMode, + install: make(chan *subscription), + uninstall: make(chan *subscription), + txsCh: make(chan core.NewTxsEvent, txChanSize), + logsCh: make(chan []*types.Log, logsChanSize), + rmLogsCh: make(chan core.RemovedLogsEvent, rmLogsChanSize), + pendingLogsCh: make(chan []*types.Log, logsChanSize), + chainCh: make(chan core.ChainEvent, chainEvChanSize), } // Subscribe events @@ -134,12 +128,10 @@ func NewEventSystem(mux *event.TypeMux, backend Backend, lightMode bool) *EventS m.logsSub = m.backend.SubscribeLogsEvent(m.logsCh) m.rmLogsSub = m.backend.SubscribeRemovedLogsEvent(m.rmLogsCh) m.chainSub = m.backend.SubscribeChainEvent(m.chainCh) - // TODO(rjl493456442): use feed to subscribe pending log event - m.pendingLogSub = m.mux.Subscribe(core.PendingLogsEvent{}) + m.pendingLogsSub = m.backend.SubscribePendingLogsEvent(m.pendingLogsCh) // Make sure none of the subscriptions are empty - if m.txsSub == nil || m.logsSub == nil || m.rmLogsSub == nil || m.chainSub == nil || - m.pendingLogSub.Closed() { + if m.txsSub == nil || m.logsSub == nil || m.rmLogsSub == nil || m.chainSub == nil || m.pendingLogsSub == nil { log.Crit("Subscribe for event system failed") } @@ -316,58 +308,61 @@ func (es *EventSystem) SubscribePendingTxs(hashes chan []common.Hash) *Subscript type filterIndex map[Type]map[rpc.ID]*subscription -// broadcast event to filters that match criteria. -func (es *EventSystem) broadcast(filters filterIndex, ev interface{}) { - if ev == nil { +func (es *EventSystem) handleLogs(filters filterIndex, ev []*types.Log) { + if len(ev) == 0 { return } + for _, f := range filters[LogsSubscription] { + matchedLogs := filterLogs(ev, f.logsCrit.FromBlock, f.logsCrit.ToBlock, f.logsCrit.Addresses, f.logsCrit.Topics) + if len(matchedLogs) > 0 { + f.logs <- matchedLogs + } + } +} - switch e := ev.(type) { - case []*types.Log: - if len(e) > 0 { - for _, f := range filters[LogsSubscription] { - if matchedLogs := filterLogs(e, f.logsCrit.FromBlock, f.logsCrit.ToBlock, f.logsCrit.Addresses, f.logsCrit.Topics); len(matchedLogs) > 0 { - f.logs <- matchedLogs - } - } +func (es *EventSystem) handlePendingLogs(filters filterIndex, ev []*types.Log) { + if len(ev) == 0 { + return + } + for _, f := range filters[PendingLogsSubscription] { + matchedLogs := filterLogs(ev, nil, f.logsCrit.ToBlock, f.logsCrit.Addresses, f.logsCrit.Topics) + if len(matchedLogs) > 0 { + f.logs <- matchedLogs } - case core.RemovedLogsEvent: - for _, f := range filters[LogsSubscription] { - if matchedLogs := filterLogs(e.Logs, f.logsCrit.FromBlock, f.logsCrit.ToBlock, f.logsCrit.Addresses, f.logsCrit.Topics); len(matchedLogs) > 0 { - f.logs <- matchedLogs - } + } +} + +func (es *EventSystem) handleRemovedLogs(filters filterIndex, ev core.RemovedLogsEvent) { + for _, f := range filters[LogsSubscription] { + matchedLogs := filterLogs(ev.Logs, f.logsCrit.FromBlock, f.logsCrit.ToBlock, f.logsCrit.Addresses, f.logsCrit.Topics) + if len(matchedLogs) > 0 { + f.logs <- matchedLogs } - case *event.TypeMuxEvent: - if muxe, ok := e.Data.(core.PendingLogsEvent); ok { - for _, f := range filters[PendingLogsSubscription] { - if e.Time.After(f.created) { - if matchedLogs := filterLogs(muxe.Logs, nil, f.logsCrit.ToBlock, f.logsCrit.Addresses, f.logsCrit.Topics); len(matchedLogs) > 0 { - f.logs <- matchedLogs - } + } +} + +func (es *EventSystem) handleTxsEvent(filters filterIndex, ev core.NewTxsEvent) { + hashes := make([]common.Hash, 0, len(ev.Txs)) + for _, tx := range ev.Txs { + hashes = append(hashes, tx.Hash()) + } + for _, f := range filters[PendingTransactionsSubscription] { + f.hashes <- hashes + } +} + +func (es *EventSystem) handleChainEvent(filters filterIndex, ev core.ChainEvent) { + for _, f := range filters[BlocksSubscription] { + f.headers <- ev.Block.Header() + } + if es.lightMode && len(filters[LogsSubscription]) > 0 { + es.lightFilterNewHead(ev.Block.Header(), func(header *types.Header, remove bool) { + for _, f := range filters[LogsSubscription] { + if matchedLogs := es.lightFilterLogs(header, f.logsCrit.Addresses, f.logsCrit.Topics, remove); len(matchedLogs) > 0 { + f.logs <- matchedLogs } } - } - case core.NewTxsEvent: - hashes := make([]common.Hash, 0, len(e.Txs)) - for _, tx := range e.Txs { - hashes = append(hashes, tx.Hash()) - } - for _, f := range filters[PendingTransactionsSubscription] { - f.hashes <- hashes - } - case core.ChainEvent: - for _, f := range filters[BlocksSubscription] { - f.headers <- e.Block.Header() - } - if es.lightMode && len(filters[LogsSubscription]) > 0 { - es.lightFilterNewHead(e.Block.Header(), func(header *types.Header, remove bool) { - for _, f := range filters[LogsSubscription] { - if matchedLogs := es.lightFilterLogs(header, f.logsCrit.Addresses, f.logsCrit.Topics, remove); len(matchedLogs) > 0 { - f.logs <- matchedLogs - } - } - }) - } + }) } } @@ -448,10 +443,10 @@ func (es *EventSystem) lightFilterLogs(header *types.Header, addresses []common. func (es *EventSystem) eventLoop() { // Ensure all subscriptions get cleaned up defer func() { - es.pendingLogSub.Unsubscribe() es.txsSub.Unsubscribe() es.logsSub.Unsubscribe() es.rmLogsSub.Unsubscribe() + es.pendingLogsSub.Unsubscribe() es.chainSub.Unsubscribe() }() @@ -462,20 +457,16 @@ func (es *EventSystem) eventLoop() { for { select { - // Handle subscribed events case ev := <-es.txsCh: - es.broadcast(index, ev) + es.handleTxsEvent(index, ev) case ev := <-es.logsCh: - es.broadcast(index, ev) + es.handleLogs(index, ev) case ev := <-es.rmLogsCh: - es.broadcast(index, ev) + es.handleRemovedLogs(index, ev) + case ev := <-es.pendingLogsCh: + es.handlePendingLogs(index, ev) case ev := <-es.chainCh: - es.broadcast(index, ev) - case ev, active := <-es.pendingLogSub.Chan(): - if !active { // system stopped - return - } - es.broadcast(index, ev) + es.handleChainEvent(index, ev) case f := <-es.install: if f.typ == MinedAndPendingLogsSubscription { diff --git a/eth/filters/filter_system_test.go b/eth/filters/filter_system_test.go index 93cb43123f..c8d1d43abb 100644 --- a/eth/filters/filter_system_test.go +++ b/eth/filters/filter_system_test.go @@ -39,23 +39,20 @@ import ( ) type testBackend struct { - mux *event.TypeMux - db ethdb.Database - sections uint64 - txFeed *event.Feed - rmLogsFeed *event.Feed - logsFeed *event.Feed - chainFeed *event.Feed + mux *event.TypeMux + db ethdb.Database + sections uint64 + txFeed event.Feed + logsFeed event.Feed + rmLogsFeed event.Feed + pendingLogsFeed event.Feed + chainFeed event.Feed } func (b *testBackend) ChainDb() ethdb.Database { return b.db } -func (b *testBackend) EventMux() *event.TypeMux { - return b.mux -} - func (b *testBackend) HeaderByNumber(ctx context.Context, blockNr rpc.BlockNumber) (*types.Header, error) { var ( hash common.Hash @@ -116,6 +113,10 @@ func (b *testBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscript return b.logsFeed.Subscribe(ch) } +func (b *testBackend) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { + return b.pendingLogsFeed.Subscribe(ch) +} + func (b *testBackend) SubscribeChainEvent(ch chan<- core.ChainEvent) event.Subscription { return b.chainFeed.Subscribe(ch) } @@ -160,13 +161,8 @@ func TestBlockSubscription(t *testing.T) { t.Parallel() var ( - mux = new(event.TypeMux) db = rawdb.NewMemoryDatabase() - txFeed = new(event.Feed) - rmLogsFeed = new(event.Feed) - logsFeed = new(event.Feed) - chainFeed = new(event.Feed) - backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} + backend = &testBackend{db: db} api = NewPublicFilterAPI(backend, false) genesis = new(core.Genesis).MustCommit(db) chain, _ = core.GenerateChain(params.TestChainConfig, genesis, ethash.NewFaker(), db, 10, func(i int, gen *core.BlockGen) {}) @@ -205,7 +201,7 @@ func TestBlockSubscription(t *testing.T) { time.Sleep(1 * time.Second) for _, e := range chainEvents { - chainFeed.Send(e) + backend.chainFeed.Send(e) } <-sub0.Err() @@ -217,14 +213,9 @@ func TestPendingTxFilter(t *testing.T) { t.Parallel() var ( - mux = new(event.TypeMux) - db = rawdb.NewMemoryDatabase() - txFeed = new(event.Feed) - rmLogsFeed = new(event.Feed) - logsFeed = new(event.Feed) - chainFeed = new(event.Feed) - backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} - api = NewPublicFilterAPI(backend, false) + db = rawdb.NewMemoryDatabase() + backend = &testBackend{db: db} + api = NewPublicFilterAPI(backend, false) transactions = []*types.Transaction{ types.NewTransaction(0, common.HexToAddress("0xb794f5ea0ba39494ce83a213fffba74279579268"), new(big.Int), 0, new(big.Int), nil), @@ -240,7 +231,7 @@ func TestPendingTxFilter(t *testing.T) { fid0 := api.NewPendingTransactionFilter() time.Sleep(1 * time.Second) - txFeed.Send(core.NewTxsEvent{Txs: transactions}) + backend.txFeed.Send(core.NewTxsEvent{Txs: transactions}) timeout := time.Now().Add(1 * time.Second) for { @@ -277,14 +268,9 @@ func TestPendingTxFilter(t *testing.T) { // If not it must return an error. func TestLogFilterCreation(t *testing.T) { var ( - mux = new(event.TypeMux) - db = rawdb.NewMemoryDatabase() - txFeed = new(event.Feed) - rmLogsFeed = new(event.Feed) - logsFeed = new(event.Feed) - chainFeed = new(event.Feed) - backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} - api = NewPublicFilterAPI(backend, false) + db = rawdb.NewMemoryDatabase() + backend = &testBackend{db: db} + api = NewPublicFilterAPI(backend, false) testCases = []struct { crit FilterCriteria @@ -326,14 +312,9 @@ func TestInvalidLogFilterCreation(t *testing.T) { t.Parallel() var ( - mux = new(event.TypeMux) - db = rawdb.NewMemoryDatabase() - txFeed = new(event.Feed) - rmLogsFeed = new(event.Feed) - logsFeed = new(event.Feed) - chainFeed = new(event.Feed) - backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} - api = NewPublicFilterAPI(backend, false) + db = rawdb.NewMemoryDatabase() + backend = &testBackend{db: db} + api = NewPublicFilterAPI(backend, false) ) // different situations where log filter creation should fail. @@ -353,15 +334,10 @@ func TestInvalidLogFilterCreation(t *testing.T) { func TestInvalidGetLogsRequest(t *testing.T) { var ( - mux = new(event.TypeMux) - db = rawdb.NewMemoryDatabase() - txFeed = new(event.Feed) - rmLogsFeed = new(event.Feed) - logsFeed = new(event.Feed) - chainFeed = new(event.Feed) - backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} - api = NewPublicFilterAPI(backend, false) - blockHash = common.HexToHash("0x1111111111111111111111111111111111111111111111111111111111111111") + db = rawdb.NewMemoryDatabase() + backend = &testBackend{db: db} + api = NewPublicFilterAPI(backend, false) + blockHash = common.HexToHash("0x1111111111111111111111111111111111111111111111111111111111111111") ) // Reason: Cannot specify both BlockHash and FromBlock/ToBlock) @@ -383,14 +359,9 @@ func TestLogFilter(t *testing.T) { t.Parallel() var ( - mux = new(event.TypeMux) - db = rawdb.NewMemoryDatabase() - txFeed = new(event.Feed) - rmLogsFeed = new(event.Feed) - logsFeed = new(event.Feed) - chainFeed = new(event.Feed) - backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} - api = NewPublicFilterAPI(backend, false) + db = rawdb.NewMemoryDatabase() + backend = &testBackend{db: db} + api = NewPublicFilterAPI(backend, false) firstAddr = common.HexToAddress("0x1111111111111111111111111111111111111111") secondAddr = common.HexToAddress("0x2222222222222222222222222222222222222222") @@ -400,7 +371,7 @@ func TestLogFilter(t *testing.T) { secondTopic = common.HexToHash("0x2222222222222222222222222222222222222222222222222222222222222222") notUsedTopic = common.HexToHash("0x9999999999999999999999999999999999999999999999999999999999999999") - // posted twice, once as vm.Logs and once as core.PendingLogsEvent + // posted twice, once as regular logs and once as pending logs. allLogs = []*types.Log{ {Address: firstAddr}, {Address: firstAddr, Topics: []common.Hash{firstTopic}, BlockNumber: 1}, @@ -453,11 +424,11 @@ func TestLogFilter(t *testing.T) { // raise events time.Sleep(1 * time.Second) - if nsend := logsFeed.Send(allLogs); nsend == 0 { - t.Fatal("Shoud have at least one subscription") + if nsend := backend.logsFeed.Send(allLogs); nsend == 0 { + t.Fatal("Logs event not delivered") } - if err := mux.Post(core.PendingLogsEvent{Logs: allLogs}); err != nil { - t.Fatal(err) + if nsend := backend.pendingLogsFeed.Send(allLogs); nsend == 0 { + t.Fatal("Pending logs event not delivered") } for i, tt := range testCases { @@ -502,14 +473,9 @@ func TestPendingLogsSubscription(t *testing.T) { t.Parallel() var ( - mux = new(event.TypeMux) - db = rawdb.NewMemoryDatabase() - txFeed = new(event.Feed) - rmLogsFeed = new(event.Feed) - logsFeed = new(event.Feed) - chainFeed = new(event.Feed) - backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} - api = NewPublicFilterAPI(backend, false) + db = rawdb.NewMemoryDatabase() + backend = &testBackend{db: db} + api = NewPublicFilterAPI(backend, false) firstAddr = common.HexToAddress("0x1111111111111111111111111111111111111111") secondAddr = common.HexToAddress("0x2222222222222222222222222222222222222222") @@ -521,26 +487,18 @@ func TestPendingLogsSubscription(t *testing.T) { fourthTopic = common.HexToHash("0x4444444444444444444444444444444444444444444444444444444444444444") notUsedTopic = common.HexToHash("0x9999999999999999999999999999999999999999999999999999999999999999") - allLogs = []core.PendingLogsEvent{ - {Logs: []*types.Log{{Address: firstAddr, Topics: []common.Hash{}, BlockNumber: 0}}}, - {Logs: []*types.Log{{Address: firstAddr, Topics: []common.Hash{firstTopic}, BlockNumber: 1}}}, - {Logs: []*types.Log{{Address: secondAddr, Topics: []common.Hash{firstTopic}, BlockNumber: 2}}}, - {Logs: []*types.Log{{Address: thirdAddress, Topics: []common.Hash{secondTopic}, BlockNumber: 3}}}, - {Logs: []*types.Log{{Address: thirdAddress, Topics: []common.Hash{secondTopic}, BlockNumber: 4}}}, - {Logs: []*types.Log{ + allLogs = [][]*types.Log{ + {{Address: firstAddr, Topics: []common.Hash{}, BlockNumber: 0}}, + {{Address: firstAddr, Topics: []common.Hash{firstTopic}, BlockNumber: 1}}, + {{Address: secondAddr, Topics: []common.Hash{firstTopic}, BlockNumber: 2}}, + {{Address: thirdAddress, Topics: []common.Hash{secondTopic}, BlockNumber: 3}}, + {{Address: thirdAddress, Topics: []common.Hash{secondTopic}, BlockNumber: 4}}, + { {Address: thirdAddress, Topics: []common.Hash{firstTopic}, BlockNumber: 5}, {Address: thirdAddress, Topics: []common.Hash{thirdTopic}, BlockNumber: 5}, {Address: thirdAddress, Topics: []common.Hash{fourthTopic}, BlockNumber: 5}, {Address: firstAddr, Topics: []common.Hash{firstTopic}, BlockNumber: 5}, - }}, - } - - convertLogs = func(pl []core.PendingLogsEvent) []*types.Log { - var logs []*types.Log - for _, l := range pl { - logs = append(logs, l.Logs...) - } - return logs + }, } testCases = []struct { @@ -550,21 +508,52 @@ func TestPendingLogsSubscription(t *testing.T) { sub *Subscription }{ // match all - {ethereum.FilterQuery{}, convertLogs(allLogs), nil, nil}, + { + ethereum.FilterQuery{}, flattenLogs(allLogs), + nil, nil, + }, // match none due to no matching addresses - {ethereum.FilterQuery{Addresses: []common.Address{{}, notUsedAddress}, Topics: [][]common.Hash{nil}}, []*types.Log{}, nil, nil}, + { + ethereum.FilterQuery{Addresses: []common.Address{{}, notUsedAddress}, Topics: [][]common.Hash{nil}}, + nil, + nil, nil, + }, // match logs based on addresses, ignore topics - {ethereum.FilterQuery{Addresses: []common.Address{firstAddr}}, append(convertLogs(allLogs[:2]), allLogs[5].Logs[3]), nil, nil}, + { + ethereum.FilterQuery{Addresses: []common.Address{firstAddr}}, + append(flattenLogs(allLogs[:2]), allLogs[5][3]), + nil, nil, + }, // match none due to no matching topics (match with address) - {ethereum.FilterQuery{Addresses: []common.Address{secondAddr}, Topics: [][]common.Hash{{notUsedTopic}}}, []*types.Log{}, nil, nil}, + { + ethereum.FilterQuery{Addresses: []common.Address{secondAddr}, Topics: [][]common.Hash{{notUsedTopic}}}, + nil, nil, nil, + }, // match logs based on addresses and topics - {ethereum.FilterQuery{Addresses: []common.Address{thirdAddress}, Topics: [][]common.Hash{{firstTopic, secondTopic}}}, append(convertLogs(allLogs[3:5]), allLogs[5].Logs[0]), nil, nil}, + { + ethereum.FilterQuery{Addresses: []common.Address{thirdAddress}, Topics: [][]common.Hash{{firstTopic, secondTopic}}}, + append(flattenLogs(allLogs[3:5]), allLogs[5][0]), + nil, nil, + }, // match logs based on multiple addresses and "or" topics - {ethereum.FilterQuery{Addresses: []common.Address{secondAddr, thirdAddress}, Topics: [][]common.Hash{{firstTopic, secondTopic}}}, append(convertLogs(allLogs[2:5]), allLogs[5].Logs[0]), nil, nil}, + { + ethereum.FilterQuery{Addresses: []common.Address{secondAddr, thirdAddress}, Topics: [][]common.Hash{{firstTopic, secondTopic}}}, + append(flattenLogs(allLogs[2:5]), allLogs[5][0]), + nil, + nil, + }, // block numbers are ignored for filters created with New***Filter, these return all logs that match the given criteria when the state changes - {ethereum.FilterQuery{Addresses: []common.Address{firstAddr}, FromBlock: big.NewInt(2), ToBlock: big.NewInt(3)}, append(convertLogs(allLogs[:2]), allLogs[5].Logs[3]), nil, nil}, + { + ethereum.FilterQuery{Addresses: []common.Address{firstAddr}, FromBlock: big.NewInt(2), ToBlock: big.NewInt(3)}, + append(flattenLogs(allLogs[:2]), allLogs[5][3]), + nil, nil, + }, // multiple pending logs, should match only 2 topics from the logs in block 5 - {ethereum.FilterQuery{Addresses: []common.Address{thirdAddress}, Topics: [][]common.Hash{{firstTopic, fourthTopic}}}, []*types.Log{allLogs[5].Logs[0], allLogs[5].Logs[2]}, nil, nil}, + { + ethereum.FilterQuery{Addresses: []common.Address{thirdAddress}, Topics: [][]common.Hash{{firstTopic, fourthTopic}}}, + []*types.Log{allLogs[5][0], allLogs[5][2]}, + nil, nil, + }, } ) @@ -607,10 +596,15 @@ func TestPendingLogsSubscription(t *testing.T) { // raise events time.Sleep(1 * time.Second) - // allLogs are type of core.PendingLogsEvent - for _, l := range allLogs { - if err := mux.Post(l); err != nil { - t.Fatal(err) - } + for _, ev := range allLogs { + backend.pendingLogsFeed.Send(ev) + } +} + +func flattenLogs(pl [][]*types.Log) []*types.Log { + var logs []*types.Log + for _, l := range pl { + logs = append(logs, l...) } + return logs } diff --git a/eth/filters/filter_test.go b/eth/filters/filter_test.go index eafa19cdfa..f45720d5a9 100644 --- a/eth/filters/filter_test.go +++ b/eth/filters/filter_test.go @@ -29,7 +29,6 @@ import ( "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/params" ) @@ -50,18 +49,13 @@ func BenchmarkFilters(b *testing.B) { defer os.RemoveAll(dir) var ( - db, _ = rawdb.NewLevelDBDatabase(dir, 0, 0, "") - mux = new(event.TypeMux) - txFeed = new(event.Feed) - rmLogsFeed = new(event.Feed) - logsFeed = new(event.Feed) - chainFeed = new(event.Feed) - backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} - key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - addr1 = crypto.PubkeyToAddress(key1.PublicKey) - addr2 = common.BytesToAddress([]byte("jeff")) - addr3 = common.BytesToAddress([]byte("ethereum")) - addr4 = common.BytesToAddress([]byte("random addresses please")) + db, _ = rawdb.NewLevelDBDatabase(dir, 0, 0, "") + backend = &testBackend{db: db} + key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + addr1 = crypto.PubkeyToAddress(key1.PublicKey) + addr2 = common.BytesToAddress([]byte("jeff")) + addr3 = common.BytesToAddress([]byte("ethereum")) + addr4 = common.BytesToAddress([]byte("random addresses please")) ) defer db.Close() @@ -109,15 +103,10 @@ func TestFilters(t *testing.T) { defer os.RemoveAll(dir) var ( - db, _ = rawdb.NewLevelDBDatabase(dir, 0, 0, "") - mux = new(event.TypeMux) - txFeed = new(event.Feed) - rmLogsFeed = new(event.Feed) - logsFeed = new(event.Feed) - chainFeed = new(event.Feed) - backend = &testBackend{mux, db, 0, txFeed, rmLogsFeed, logsFeed, chainFeed} - key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") - addr = crypto.PubkeyToAddress(key1.PublicKey) + db, _ = rawdb.NewLevelDBDatabase(dir, 0, 0, "") + backend = &testBackend{db: db} + key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + addr = crypto.PubkeyToAddress(key1.PublicKey) hash1 = common.BytesToHash([]byte("topic1")) hash2 = common.BytesToHash([]byte("topic2")) diff --git a/internal/ethapi/backend.go b/internal/ethapi/backend.go index 73b6c89cea..245091df35 100644 --- a/internal/ethapi/backend.go +++ b/internal/ethapi/backend.go @@ -43,7 +43,6 @@ type Backend interface { ProtocolVersion() int SuggestPrice(ctx context.Context) (*big.Int, error) ChainDb() ethdb.Database - EventMux() *event.TypeMux AccountManager() *accounts.Manager ExtRPCEnabled() bool RPCGasCap() *big.Int // global gas cap for eth_call over rpc: DoS protection @@ -80,6 +79,7 @@ type Backend interface { GetLogs(ctx context.Context, blockHash common.Hash) ([][]*types.Log, error) ServiceFilter(ctx context.Context, session *bloombits.MatcherSession) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscription + SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription ChainConfig() *params.ChainConfig diff --git a/les/api_backend.go b/les/api_backend.go index e01e1be98b..5627c34d6a 100644 --- a/les/api_backend.go +++ b/les/api_backend.go @@ -225,6 +225,13 @@ func (b *LesApiBackend) SubscribeLogsEvent(ch chan<- []*types.Log) event.Subscri return b.eth.blockchain.SubscribeLogsEvent(ch) } +func (b *LesApiBackend) SubscribePendingLogsEvent(ch chan<- []*types.Log) event.Subscription { + return event.NewSubscription(func(quit <-chan struct{}) error { + <-quit + return nil + }) +} + func (b *LesApiBackend) SubscribeRemovedLogsEvent(ch chan<- core.RemovedLogsEvent) event.Subscription { return b.eth.blockchain.SubscribeRemovedLogsEvent(ch) } @@ -245,10 +252,6 @@ func (b *LesApiBackend) ChainDb() ethdb.Database { return b.eth.chainDb } -func (b *LesApiBackend) EventMux() *event.TypeMux { - return b.eth.eventMux -} - func (b *LesApiBackend) AccountManager() *accounts.Manager { return b.eth.accountManager } diff --git a/miner/miner.go b/miner/miner.go index bc9af0f060..b968b3a92a 100644 --- a/miner/miner.go +++ b/miner/miner.go @@ -182,3 +182,9 @@ func (miner *Miner) SetEtherbase(addr common.Address) { miner.coinbase = addr miner.worker.setEtherbase(addr) } + +// SubscribePendingLogs starts delivering logs from pending transactions +// to the given channel. +func (self *Miner) SubscribePendingLogs(ch chan<- []*types.Log) event.Subscription { + return self.worker.pendingLogsFeed.Subscribe(ch) +} diff --git a/miner/worker.go b/miner/worker.go index c95b32042f..d3cd10ed26 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -128,6 +128,9 @@ type worker struct { eth Backend chain *core.BlockChain + // Feeds + pendingLogsFeed event.Feed + // Subscriptions mux *event.TypeMux txsCh chan core.NewTxsEvent @@ -809,7 +812,7 @@ func (w *worker) commitTransactions(txs *types.TransactionsByPriceAndNonce, coin cpy[i] = new(types.Log) *cpy[i] = *l } - go w.mux.Post(core.PendingLogsEvent{Logs: cpy}) + w.pendingLogsFeed.Send(cpy) } // Notify resubmit loop to decrease resubmitting interval if current interval is larger // than the user-specified one. From 191364c3507a87c72a49a68e7d4bc79773d61f47 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 12 Dec 2019 10:15:36 +0100 Subject: [PATCH 027/128] p2p/dnsdisc: add enode.Iterator API (#20437) * p2p/dnsdisc: add support for enode.Iterator This changes the dnsdisc.Client API to support the enode.Iterator interface. * p2p/dnsdisc: rate-limit DNS requests * p2p/dnsdisc: preserve linked trees across root updates This improves the way links are handled when the link root changes. Previously, sync would simply remove all links from the current tree and garbage-collect all unreachable trees before syncing the new list of links. This behavior isn't great in certain cases: Consider a structure where trees A, B, and C reference each other and D links to A. If D's link root changed, the sync code would first remove trees A, B and C, only to re-sync them later when the link to A was found again. The fix for this problem is to track the current set of links in each clientTree and removing old links only AFTER all links are synced. * p2p/dnsdisc: deflake iterator test * cmd/devp2p: adapt dnsClient to new p2p/dnsdisc API * p2p/dnsdisc: tiny comment fix --- cmd/devp2p/dnscmd.go | 3 +- go.mod | 1 + p2p/dnsdisc/client.go | 248 +++++++++++++++++++++++-------------- p2p/dnsdisc/client_test.go | 129 ++++++++++++------- p2p/dnsdisc/sync.go | 129 +++++++++---------- p2p/dnsdisc/sync_test.go | 83 +++++++++++++ p2p/dnsdisc/tree.go | 17 ++- p2p/dnsdisc/tree_test.go | 2 +- 8 files changed, 393 insertions(+), 219 deletions(-) create mode 100644 p2p/dnsdisc/sync_test.go diff --git a/cmd/devp2p/dnscmd.go b/cmd/devp2p/dnscmd.go index eb15764b04..f245104053 100644 --- a/cmd/devp2p/dnscmd.go +++ b/cmd/devp2p/dnscmd.go @@ -214,8 +214,7 @@ func dnsClient(ctx *cli.Context) *dnsdisc.Client { if commandHasFlag(ctx, dnsTimeoutFlag) { cfg.Timeout = ctx.Duration(dnsTimeoutFlag.Name) } - c, _ := dnsdisc.NewClient(cfg) // cannot fail because no URLs given - return c + return dnsdisc.NewClient(cfg) } // There are two file formats for DNS node trees on disk: diff --git a/go.mod b/go.mod index d4b4208336..a280949e94 100644 --- a/go.mod +++ b/go.mod @@ -60,6 +60,7 @@ require ( golang.org/x/sync v0.0.0-20181108010431-42b317875d0f golang.org/x/sys v0.0.0-20190712062909-fae7ac547cb7 golang.org/x/text v0.3.2 + golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190213234257-ec84240a7772 gopkg.in/sourcemap.v1 v1.0.5 // indirect diff --git a/p2p/dnsdisc/client.go b/p2p/dnsdisc/client.go index 677c0aa922..a29f82cd81 100644 --- a/p2p/dnsdisc/client.go +++ b/p2p/dnsdisc/client.go @@ -23,6 +23,7 @@ import ( "math/rand" "net" "strings" + "sync" "time" "github.com/ethereum/go-ethereum/common/mclock" @@ -31,15 +32,13 @@ import ( "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" lru "github.com/hashicorp/golang-lru" + "golang.org/x/time/rate" ) // Client discovers nodes by querying DNS servers. type Client struct { - cfg Config - clock mclock.Clock - linkCache linkCache - trees map[string]*clientTree - + cfg Config + clock mclock.Clock entries *lru.Cache } @@ -48,6 +47,7 @@ type Config struct { Timeout time.Duration // timeout used for DNS lookups (default 5s) RecheckInterval time.Duration // time between tree root update checks (default 30min) CacheLimit int // maximum number of cached records (default 1000) + RateLimit float64 // maximum DNS requests / second (default 3) ValidSchemes enr.IdentityScheme // acceptable ENR identity schemes (default enode.ValidSchemes) Resolver Resolver // the DNS resolver to use (defaults to system DNS) Logger log.Logger // destination of client log messages (defaults to root logger) @@ -60,9 +60,10 @@ type Resolver interface { func (cfg Config) withDefaults() Config { const ( - defaultTimeout = 5 * time.Second - defaultRecheck = 30 * time.Minute - defaultCache = 1000 + defaultTimeout = 5 * time.Second + defaultRecheck = 30 * time.Minute + defaultRateLimit = 3 + defaultCache = 1000 ) if cfg.Timeout == 0 { cfg.Timeout = defaultTimeout @@ -73,6 +74,9 @@ func (cfg Config) withDefaults() Config { if cfg.CacheLimit == 0 { cfg.CacheLimit = defaultCache } + if cfg.RateLimit == 0 { + cfg.RateLimit = defaultRateLimit + } if cfg.ValidSchemes == nil { cfg.ValidSchemes = enode.ValidSchemes } @@ -86,32 +90,24 @@ func (cfg Config) withDefaults() Config { } // NewClient creates a client. -func NewClient(cfg Config, urls ...string) (*Client, error) { - c := &Client{ - cfg: cfg.withDefaults(), - clock: mclock.System{}, - trees: make(map[string]*clientTree), - } - var err error - if c.entries, err = lru.New(c.cfg.CacheLimit); err != nil { - return nil, err - } - for _, url := range urls { - if err := c.AddTree(url); err != nil { - return nil, err - } +func NewClient(cfg Config) *Client { + cfg = cfg.withDefaults() + cache, err := lru.New(cfg.CacheLimit) + if err != nil { + panic(err) } - return c, nil + rlimit := rate.NewLimiter(rate.Limit(cfg.RateLimit), 10) + cfg.Resolver = &rateLimitResolver{cfg.Resolver, rlimit} + return &Client{cfg: cfg, entries: cache, clock: mclock.System{}} } -// SyncTree downloads the entire node tree at the given URL. This doesn't add the tree for -// later use, but any previously-synced entries are reused. +// SyncTree downloads the entire node tree at the given URL. func (c *Client) SyncTree(url string) (*Tree, error) { le, err := parseLink(url) if err != nil { return nil, fmt.Errorf("invalid enrtree URL: %v", err) } - ct := newClientTree(c, le) + ct := newClientTree(c, new(linkCache), le) t := &Tree{entries: make(map[string]entry)} if err := ct.syncAll(t.entries); err != nil { return nil, err @@ -120,75 +116,16 @@ func (c *Client) SyncTree(url string) (*Tree, error) { return t, nil } -// AddTree adds a enrtree:// URL to crawl. -func (c *Client) AddTree(url string) error { - le, err := parseLink(url) - if err != nil { - return fmt.Errorf("invalid enrtree URL: %v", err) - } - ct, err := c.ensureTree(le) - if err != nil { - return err - } - c.linkCache.add(ct) - return nil -} - -func (c *Client) ensureTree(le *linkEntry) (*clientTree, error) { - if tree, ok := c.trees[le.domain]; ok { - if !tree.matchPubkey(le.pubkey) { - return nil, fmt.Errorf("conflicting public keys for domain %q", le.domain) - } - return tree, nil - } - ct := newClientTree(c, le) - c.trees[le.domain] = ct - return ct, nil -} - -// RandomNode retrieves the next random node. -func (c *Client) RandomNode(ctx context.Context) *enode.Node { - for { - ct := c.randomTree() - if ct == nil { - return nil - } - n, err := ct.syncRandom(ctx) - if err != nil { - if err == ctx.Err() { - return nil // context canceled. - } - c.cfg.Logger.Debug("Error in DNS random node sync", "tree", ct.loc.domain, "err", err) - continue - } - if n != nil { - return n - } - } -} - -// randomTree returns a random tree. -func (c *Client) randomTree() *clientTree { - if !c.linkCache.valid() { - c.gcTrees() - } - limit := rand.Intn(len(c.trees)) - for _, ct := range c.trees { - if limit == 0 { - return ct +// NewIterator creates an iterator that visits all nodes at the +// given tree URLs. +func (c *Client) NewIterator(urls ...string) (enode.Iterator, error) { + it := c.newRandomIterator() + for _, url := range urls { + if err := it.addTree(url); err != nil { + return nil, err } - limit-- - } - return nil -} - -// gcTrees rebuilds the 'trees' map. -func (c *Client) gcTrees() { - trees := make(map[string]*clientTree) - for t := range c.linkCache.all() { - trees[t.loc.domain] = t } - c.trees = trees + return it, nil } // resolveRoot retrieves a root entry via DNS. @@ -258,3 +195,128 @@ func (c *Client) doResolveEntry(ctx context.Context, domain, hash string) (entry } return nil, nameError{name, errNoEntry} } + +// rateLimitResolver applies a rate limit to a Resolver. +type rateLimitResolver struct { + r Resolver + limiter *rate.Limiter +} + +func (r *rateLimitResolver) LookupTXT(ctx context.Context, domain string) ([]string, error) { + if err := r.limiter.Wait(ctx); err != nil { + return nil, err + } + return r.r.LookupTXT(ctx, domain) +} + +// randomIterator traverses a set of trees and returns nodes found in them. +type randomIterator struct { + cur *enode.Node + ctx context.Context + cancelFn context.CancelFunc + c *Client + + mu sync.Mutex + trees map[string]*clientTree // all trees + lc linkCache // tracks tree dependencies +} + +func (c *Client) newRandomIterator() *randomIterator { + ctx, cancel := context.WithCancel(context.Background()) + return &randomIterator{ + c: c, + ctx: ctx, + cancelFn: cancel, + trees: make(map[string]*clientTree), + } +} + +// Node returns the current node. +func (it *randomIterator) Node() *enode.Node { + return it.cur +} + +// Close closes the iterator. +func (it *randomIterator) Close() { + it.mu.Lock() + defer it.mu.Unlock() + + it.cancelFn() + it.trees = nil +} + +// Next moves the iterator to the next node. +func (it *randomIterator) Next() bool { + it.cur = it.nextNode() + return it.cur != nil +} + +// addTree adds a enrtree:// URL to the iterator. +func (it *randomIterator) addTree(url string) error { + le, err := parseLink(url) + if err != nil { + return fmt.Errorf("invalid enrtree URL: %v", err) + } + it.lc.addLink("", le.str) + return nil +} + +// nextNode syncs random tree entries until it finds a node. +func (it *randomIterator) nextNode() *enode.Node { + for { + ct := it.nextTree() + if ct == nil { + return nil + } + n, err := ct.syncRandom(it.ctx) + if err != nil { + if err == it.ctx.Err() { + return nil // context canceled. + } + it.c.cfg.Logger.Debug("Error in DNS random node sync", "tree", ct.loc.domain, "err", err) + continue + } + if n != nil { + return n + } + } +} + +// nextTree returns a random tree. +func (it *randomIterator) nextTree() *clientTree { + it.mu.Lock() + defer it.mu.Unlock() + + if it.lc.changed { + it.rebuildTrees() + it.lc.changed = false + } + if len(it.trees) == 0 { + return nil + } + limit := rand.Intn(len(it.trees)) + for _, ct := range it.trees { + if limit == 0 { + return ct + } + limit-- + } + return nil +} + +// rebuildTrees rebuilds the 'trees' map. +func (it *randomIterator) rebuildTrees() { + // Delete removed trees. + for loc := range it.trees { + if !it.lc.isReferenced(loc) { + delete(it.trees, loc) + } + } + // Add new trees. + for loc := range it.lc.backrefs { + if it.trees[loc] == nil { + link, _ := parseLink(linkPrefix + loc) + it.trees[loc] = newClientTree(it.c, &it.lc, link) + } + } +} diff --git a/p2p/dnsdisc/client_test.go b/p2p/dnsdisc/client_test.go index d8e3ecee39..1a1d5ade95 100644 --- a/p2p/dnsdisc/client_test.go +++ b/p2p/dnsdisc/client_test.go @@ -54,7 +54,7 @@ func TestClientSyncTree(t *testing.T) { wantSeq = uint(1) ) - c, _ := NewClient(Config{Resolver: r, Logger: testlog.Logger(t, log.LvlTrace)}) + c := NewClient(Config{Resolver: r, Logger: testlog.Logger(t, log.LvlTrace)}) stree, err := c.SyncTree("enrtree://AKPYQIUQIL7PSIACI32J7FGZW56E5FKHEFCCOFHILBIMW3M6LWXS2@n") if err != nil { t.Fatal("sync error:", err) @@ -68,9 +68,6 @@ func TestClientSyncTree(t *testing.T) { if stree.Seq() != wantSeq { t.Errorf("synced tree has wrong seq: %d", stree.Seq()) } - if len(c.trees) > 0 { - t.Errorf("tree from SyncTree added to client") - } } // In this test, syncing the tree fails because it contains an invalid ENR entry. @@ -91,7 +88,7 @@ func TestClientSyncTreeBadNode(t *testing.T) { "C7HRFPF3BLGF3YR4DY5KX3SMBE.n": "enrtree://AM5FCQLWIZX2QFPNJAP7VUERCCRNGRHWZG3YYHIUV7BVDQ5FDPRT2@morenodes.example.org", "INDMVBZEEQ4ESVYAKGIYU74EAA.n": "enr:-----", } - c, _ := NewClient(Config{Resolver: r, Logger: testlog.Logger(t, log.LvlTrace)}) + c := NewClient(Config{Resolver: r, Logger: testlog.Logger(t, log.LvlTrace)}) _, err := c.SyncTree("enrtree://AKPYQIUQIL7PSIACI32J7FGZW56E5FKHEFCCOFHILBIMW3M6LWXS2@n") wantErr := nameError{name: "INDMVBZEEQ4ESVYAKGIYU74EAA.n", err: entryError{typ: "enr", err: errInvalidENR}} if err != wantErr { @@ -99,57 +96,89 @@ func TestClientSyncTreeBadNode(t *testing.T) { } } -// This test checks that RandomNode hits all entries. -func TestClientRandomNode(t *testing.T) { +// This test checks that randomIterator finds all entries. +func TestIterator(t *testing.T) { nodes := testNodes(nodesSeed1, 30) tree, url := makeTestTree("n", nodes, nil) r := mapResolver(tree.ToTXT("n")) - c, _ := NewClient(Config{Resolver: r, Logger: testlog.Logger(t, log.LvlTrace)}) - if err := c.AddTree(url); err != nil { + c := NewClient(Config{ + Resolver: r, + Logger: testlog.Logger(t, log.LvlTrace), + RateLimit: 500, + }) + it, err := c.NewIterator(url) + if err != nil { t.Fatal(err) } - checkRandomNode(t, c, nodes) + checkIterator(t, it, nodes) } -// This test checks that RandomNode traverses linked trees as well as explicitly added trees. -func TestClientRandomNodeLinks(t *testing.T) { +// This test checks if closing randomIterator races. +func TestIteratorClose(t *testing.T) { + nodes := testNodes(nodesSeed1, 500) + tree1, url1 := makeTestTree("t1", nodes, nil) + c := NewClient(Config{Resolver: newMapResolver(tree1.ToTXT("t1"))}) + it, err := c.NewIterator(url1) + if err != nil { + t.Fatal(err) + } + + done := make(chan struct{}) + go func() { + for it.Next() { + _ = it.Node() + } + close(done) + }() + + time.Sleep(50 * time.Millisecond) + it.Close() + <-done +} + +// This test checks that randomIterator traverses linked trees as well as explicitly added trees. +func TestIteratorLinks(t *testing.T) { nodes := testNodes(nodesSeed1, 40) tree1, url1 := makeTestTree("t1", nodes[:10], nil) tree2, url2 := makeTestTree("t2", nodes[10:], []string{url1}) - cfg := Config{ - Resolver: newMapResolver(tree1.ToTXT("t1"), tree2.ToTXT("t2")), - Logger: testlog.Logger(t, log.LvlTrace), - } - c, _ := NewClient(cfg) - if err := c.AddTree(url2); err != nil { + c := NewClient(Config{ + Resolver: newMapResolver(tree1.ToTXT("t1"), tree2.ToTXT("t2")), + Logger: testlog.Logger(t, log.LvlTrace), + RateLimit: 500, + }) + it, err := c.NewIterator(url2) + if err != nil { t.Fatal(err) } - checkRandomNode(t, c, nodes) + checkIterator(t, it, nodes) } -// This test verifies that RandomNode re-checks the root of the tree to catch +// This test verifies that randomIterator re-checks the root of the tree to catch // updates to nodes. -func TestClientRandomNodeUpdates(t *testing.T) { +func TestIteratorNodeUpdates(t *testing.T) { var ( clock = new(mclock.Simulated) nodes = testNodes(nodesSeed1, 30) resolver = newMapResolver() - cfg = Config{ + c = NewClient(Config{ Resolver: resolver, Logger: testlog.Logger(t, log.LvlTrace), RecheckInterval: 20 * time.Minute, - } - c, _ = NewClient(cfg) + RateLimit: 500, + }) ) c.clock = clock tree1, url := makeTestTree("n", nodes[:25], nil) + it, err := c.NewIterator(url) + if err != nil { + t.Fatal(err) + } - // Sync the original tree. + // sync the original tree. resolver.add(tree1.ToTXT("n")) - c.AddTree(url) - checkRandomNode(t, c, nodes[:25]) + checkIterator(t, it, nodes[:25]) // Update some nodes and ensure RandomNode returns the new nodes as well. keys := testKeys(nodesSeed1, len(nodes)) @@ -162,25 +191,25 @@ func TestClientRandomNodeUpdates(t *testing.T) { nodes[i] = n2 } tree2, _ := makeTestTree("n", nodes, nil) - clock.Run(cfg.RecheckInterval + 1*time.Second) + clock.Run(c.cfg.RecheckInterval + 1*time.Second) resolver.clear() resolver.add(tree2.ToTXT("n")) - checkRandomNode(t, c, nodes) + checkIterator(t, it, nodes) } -// This test verifies that RandomNode re-checks the root of the tree to catch +// This test verifies that randomIterator re-checks the root of the tree to catch // updates to links. -func TestClientRandomNodeLinkUpdates(t *testing.T) { +func TestIteratorLinkUpdates(t *testing.T) { var ( clock = new(mclock.Simulated) nodes = testNodes(nodesSeed1, 30) resolver = newMapResolver() - cfg = Config{ + c = NewClient(Config{ Resolver: resolver, Logger: testlog.Logger(t, log.LvlTrace), RecheckInterval: 20 * time.Minute, - } - c, _ = NewClient(cfg) + RateLimit: 500, + }) ) c.clock = clock tree3, url3 := makeTestTree("t3", nodes[20:30], nil) @@ -190,49 +219,53 @@ func TestClientRandomNodeLinkUpdates(t *testing.T) { resolver.add(tree2.ToTXT("t2")) resolver.add(tree3.ToTXT("t3")) + it, err := c.NewIterator(url1) + if err != nil { + t.Fatal(err) + } + // Sync tree1 using RandomNode. - c.AddTree(url1) - checkRandomNode(t, c, nodes[:20]) + checkIterator(t, it, nodes[:20]) // Add link to tree3, remove link to tree2. tree1, _ = makeTestTree("t1", nodes[:10], []string{url3}) resolver.add(tree1.ToTXT("t1")) - clock.Run(cfg.RecheckInterval + 1*time.Second) + clock.Run(c.cfg.RecheckInterval + 1*time.Second) t.Log("tree1 updated") var wantNodes []*enode.Node wantNodes = append(wantNodes, tree1.Nodes()...) wantNodes = append(wantNodes, tree3.Nodes()...) - checkRandomNode(t, c, wantNodes) + checkIterator(t, it, wantNodes) // Check that linked trees are GCed when they're no longer referenced. - if len(c.trees) != 2 { - t.Errorf("client knows %d trees, want 2", len(c.trees)) + knownTrees := it.(*randomIterator).trees + if len(knownTrees) != 2 { + t.Errorf("client knows %d trees, want 2", len(knownTrees)) } } -func checkRandomNode(t *testing.T, c *Client, wantNodes []*enode.Node) { +func checkIterator(t *testing.T, it enode.Iterator, wantNodes []*enode.Node) { t.Helper() var ( want = make(map[enode.ID]*enode.Node) - maxCalls = len(wantNodes) * 2 + maxCalls = len(wantNodes) * 3 calls = 0 - ctx = context.Background() ) for _, n := range wantNodes { want[n.ID()] = n } for ; len(want) > 0 && calls < maxCalls; calls++ { - n := c.RandomNode(ctx) - if n == nil { - t.Fatalf("RandomNode returned nil (call %d)", calls) + if !it.Next() { + t.Fatalf("Next returned false (call %d)", calls) } + n := it.Node() delete(want, n.ID()) } - t.Logf("checkRandomNode called RandomNode %d times to find %d nodes", calls, len(wantNodes)) + t.Logf("checkIterator called Next %d times to find %d nodes", calls, len(wantNodes)) for _, n := range want { - t.Errorf("RandomNode didn't discover node %v", n.ID()) + t.Errorf("iterator didn't discover node %v", n.ID()) } } diff --git a/p2p/dnsdisc/sync.go b/p2p/dnsdisc/sync.go index 533dacc653..53423527a5 100644 --- a/p2p/dnsdisc/sync.go +++ b/p2p/dnsdisc/sync.go @@ -18,7 +18,6 @@ package dnsdisc import ( "context" - "crypto/ecdsa" "math/rand" "time" @@ -28,27 +27,21 @@ import ( // clientTree is a full tree being synced. type clientTree struct { - c *Client - loc *linkEntry - root *rootEntry + c *Client + loc *linkEntry // link to this tree + lastRootCheck mclock.AbsTime // last revalidation of root + root *rootEntry enrs *subtreeSync links *subtreeSync - linkCache linkCache -} - -func newClientTree(c *Client, loc *linkEntry) *clientTree { - ct := &clientTree{c: c, loc: loc} - ct.linkCache.self = ct - return ct -} -func (ct *clientTree) matchPubkey(key *ecdsa.PublicKey) bool { - return keysEqual(ct.loc.pubkey, key) + lc *linkCache // tracks all links between all trees + curLinks map[string]struct{} // links contained in this tree + linkGCRoot string // root on which last link GC has run } -func keysEqual(k1, k2 *ecdsa.PublicKey) bool { - return k1.Curve == k2.Curve && k1.X.Cmp(k2.X) == 0 && k1.Y.Cmp(k2.Y) == 0 +func newClientTree(c *Client, lc *linkCache, loc *linkEntry) *clientTree { + return &clientTree{c: c, lc: lc, loc: loc} } // syncAll retrieves all entries of the tree. @@ -78,6 +71,7 @@ func (ct *clientTree) syncRandom(ctx context.Context) (*enode.Node, error) { err := ct.syncNextLink(ctx) return nil, err } + ct.gcLinks() // Sync next random entry in ENR tree. Once every node has been visited, we simply // start over. This is fine because entries are cached. @@ -87,6 +81,16 @@ func (ct *clientTree) syncRandom(ctx context.Context) (*enode.Node, error) { return ct.syncNextRandomENR(ctx) } +// gcLinks removes outdated links from the global link cache. GC runs once +// when the link sync finishes. +func (ct *clientTree) gcLinks() { + if !ct.links.done() || ct.root.lroot == ct.linkGCRoot { + return + } + ct.lc.resetLinks(ct.loc.str, ct.curLinks) + ct.linkGCRoot = ct.root.lroot +} + func (ct *clientTree) syncNextLink(ctx context.Context) error { hash := ct.links.missing[0] e, err := ct.links.resolveNext(ctx, hash) @@ -95,12 +99,9 @@ func (ct *clientTree) syncNextLink(ctx context.Context) error { } ct.links.missing = ct.links.missing[1:] - if le, ok := e.(*linkEntry); ok { - lt, err := ct.c.ensureTree(le) - if err != nil { - return err - } - ct.linkCache.add(lt) + if dest, ok := e.(*linkEntry); ok { + ct.lc.addLink(ct.loc.str, dest.str) + ct.curLinks[dest.str] = struct{}{} } return nil } @@ -150,7 +151,7 @@ func (ct *clientTree) updateRoot() error { // Invalidate subtrees if changed. if ct.links == nil || root.lroot != ct.links.root { ct.links = newSubtreeSync(ct.c, ct.loc, root.lroot, true) - ct.linkCache.reset() + ct.curLinks = make(map[string]struct{}) } if ct.enrs == nil || root.eroot != ct.enrs.root { ct.enrs = newSubtreeSync(ct.c, ct.loc, root.eroot, false) @@ -215,63 +216,51 @@ func (ts *subtreeSync) resolveNext(ctx context.Context, hash string) (entry, err return e, nil } -// linkCache tracks the links of a tree. +// linkCache tracks links between trees. type linkCache struct { - self *clientTree - directM map[*clientTree]struct{} // direct links - allM map[*clientTree]struct{} // direct & transitive links + backrefs map[string]map[string]struct{} + changed bool } -// reset clears the cache. -func (lc *linkCache) reset() { - lc.directM = nil - lc.allM = nil +func (lc *linkCache) isReferenced(r string) bool { + return len(lc.backrefs[r]) != 0 } -// add adds a direct link to the cache. -func (lc *linkCache) add(ct *clientTree) { - if lc.directM == nil { - lc.directM = make(map[*clientTree]struct{}) - } - if _, ok := lc.directM[ct]; !ok { - lc.invalidate() +func (lc *linkCache) addLink(from, to string) { + if _, ok := lc.backrefs[to][from]; ok { + return } - lc.directM[ct] = struct{}{} -} - -// invalidate resets the cache of transitive links. -func (lc *linkCache) invalidate() { - lc.allM = nil -} -// valid returns true when the cache of transitive links is up-to-date. -func (lc *linkCache) valid() bool { - // Re-check validity of child caches to catch updates. - for ct := range lc.allM { - if ct != lc.self && !ct.linkCache.valid() { - lc.allM = nil - break - } + if lc.backrefs == nil { + lc.backrefs = make(map[string]map[string]struct{}) + } + if _, ok := lc.backrefs[to]; !ok { + lc.backrefs[to] = make(map[string]struct{}) } - return lc.allM != nil + lc.backrefs[to][from] = struct{}{} + lc.changed = true } -// all returns all trees reachable through the cache. -func (lc *linkCache) all() map[*clientTree]struct{} { - if lc.valid() { - return lc.allM - } - // Remake lc.allM it by taking the union of all() across children. - m := make(map[*clientTree]struct{}) - if lc.self != nil { - m[lc.self] = struct{}{} - } - for ct := range lc.directM { - m[ct] = struct{}{} - for lt := range ct.linkCache.all() { - m[lt] = struct{}{} +// resetLinks clears all links of the given tree. +func (lc *linkCache) resetLinks(from string, keep map[string]struct{}) { + stk := []string{from} + for len(stk) > 0 { + item := stk[len(stk)-1] + stk = stk[:len(stk)-1] + + for r, refs := range lc.backrefs { + if _, ok := keep[r]; ok { + continue + } + if _, ok := refs[item]; !ok { + continue + } + lc.changed = true + delete(refs, item) + if len(refs) == 0 { + delete(lc.backrefs, r) + stk = append(stk, r) + } } } - lc.allM = m - return m } diff --git a/p2p/dnsdisc/sync_test.go b/p2p/dnsdisc/sync_test.go new file mode 100644 index 0000000000..32af3656ef --- /dev/null +++ b/p2p/dnsdisc/sync_test.go @@ -0,0 +1,83 @@ +// Copyright 2018 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package dnsdisc + +import ( + "math/rand" + "strconv" + "testing" +) + +func TestLinkCache(t *testing.T) { + var lc linkCache + + // Check adding links. + lc.addLink("1", "2") + if !lc.changed { + t.Error("changed flag not set") + } + lc.changed = false + lc.addLink("1", "2") + if lc.changed { + t.Error("changed flag set after adding link that's already present") + } + lc.addLink("2", "3") + lc.addLink("3", "1") + lc.addLink("2", "4") + lc.changed = false + + if !lc.isReferenced("3") { + t.Error("3 not referenced") + } + if lc.isReferenced("6") { + t.Error("6 is referenced") + } + + lc.resetLinks("1", nil) + if !lc.changed { + t.Error("changed flag not set") + } + if len(lc.backrefs) != 0 { + t.Logf("%+v", lc) + t.Error("reference maps should be empty") + } +} + +func TestLinkCacheRandom(t *testing.T) { + tags := make([]string, 1000) + for i := range tags { + tags[i] = strconv.Itoa(i) + } + + // Create random links. + var lc linkCache + var remove []string + for i := 0; i < 100; i++ { + a, b := tags[rand.Intn(len(tags))], tags[rand.Intn(len(tags))] + lc.addLink(a, b) + remove = append(remove, a) + } + + // Remove all the links. + for _, s := range remove { + lc.resetLinks(s, nil) + } + if len(lc.backrefs) != 0 { + t.Logf("%+v", lc) + t.Error("reference maps should be empty") + } +} diff --git a/p2p/dnsdisc/tree.go b/p2p/dnsdisc/tree.go index eba2ff9c0c..82a935ca41 100644 --- a/p2p/dnsdisc/tree.go +++ b/p2p/dnsdisc/tree.go @@ -48,7 +48,7 @@ func (t *Tree) Sign(key *ecdsa.PrivateKey, domain string) (url string, err error } root.sig = sig t.root = &root - link := &linkEntry{domain, &key.PublicKey} + link := newLinkEntry(domain, &key.PublicKey) return link.String(), nil } @@ -209,6 +209,7 @@ type ( node *enode.Node } linkEntry struct { + str string domain string pubkey *ecdsa.PublicKey } @@ -246,7 +247,8 @@ func (e *rootEntry) sigHash() []byte { func (e *rootEntry) verifySignature(pubkey *ecdsa.PublicKey) bool { sig := e.sig[:crypto.RecoveryIDOffset] // remove recovery id - return crypto.VerifySignature(crypto.FromECDSAPub(pubkey), e.sigHash(), sig) + enckey := crypto.FromECDSAPub(pubkey) + return crypto.VerifySignature(enckey, e.sigHash(), sig) } func (e *branchEntry) String() string { @@ -258,8 +260,13 @@ func (e *enrEntry) String() string { } func (e *linkEntry) String() string { - pubkey := b32format.EncodeToString(crypto.CompressPubkey(e.pubkey)) - return fmt.Sprintf("%s%s@%s", linkPrefix, pubkey, e.domain) + return linkPrefix + e.str +} + +func newLinkEntry(domain string, pubkey *ecdsa.PublicKey) *linkEntry { + key := b32format.EncodeToString(crypto.CompressPubkey(pubkey)) + str := key + "@" + domain + return &linkEntry{str, domain, pubkey} } // Entry Parsing @@ -319,7 +326,7 @@ func parseLink(e string) (*linkEntry, error) { if err != nil { return nil, entryError{"link", errBadPubkey} } - return &linkEntry{domain, key}, nil + return &linkEntry{e, domain, key}, nil } func parseBranch(e string) (entry, error) { diff --git a/p2p/dnsdisc/tree_test.go b/p2p/dnsdisc/tree_test.go index b6d0a84336..4048c35d63 100644 --- a/p2p/dnsdisc/tree_test.go +++ b/p2p/dnsdisc/tree_test.go @@ -91,7 +91,7 @@ func TestParseEntry(t *testing.T) { // Links { input: "enrtree://AKPYQIUQIL7PSIACI32J7FGZW56E5FKHEFCCOFHILBIMW3M6LWXS2@nodes.example.org", - e: &linkEntry{"nodes.example.org", &testkey.PublicKey}, + e: &linkEntry{"AKPYQIUQIL7PSIACI32J7FGZW56E5FKHEFCCOFHILBIMW3M6LWXS2@nodes.example.org", "nodes.example.org", &testkey.PublicKey}, }, { input: "enrtree://nodes.example.org", From f51cf573b5c602914d22f5a31c58378c5aa17ffd Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 12 Dec 2019 22:25:12 +0100 Subject: [PATCH 028/128] cmd/devp2p: implement AWS Route53 enrtree deployer (#20446) --- cmd/devp2p/dns_route53.go | 260 ++++++++++++++++++++++++++++++++++++++ cmd/devp2p/dnscmd.go | 23 +++- go.mod | 1 + go.sum | 3 + 4 files changed, 286 insertions(+), 1 deletion(-) create mode 100644 cmd/devp2p/dns_route53.go diff --git a/cmd/devp2p/dns_route53.go b/cmd/devp2p/dns_route53.go new file mode 100644 index 0000000000..1e9b39b0ea --- /dev/null +++ b/cmd/devp2p/dns_route53.go @@ -0,0 +1,260 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "errors" + "fmt" + "strconv" + "strings" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/aws/aws-sdk-go/service/route53" + "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/dnsdisc" + "gopkg.in/urfave/cli.v1" +) + +var ( + route53AccessKeyFlag = cli.StringFlag{ + Name: "access-key-id", + Usage: "AWS Access Key ID", + EnvVar: "AWS_ACCESS_KEY_ID", + } + route53AccessSecretFlag = cli.StringFlag{ + Name: "access-key-secret", + Usage: "AWS Access Key Secret", + EnvVar: "AWS_SECRET_ACCESS_KEY", + } + route53ZoneIDFlag = cli.StringFlag{ + Name: "zone-id", + Usage: "Route53 Zone ID", + } +) + +type route53Client struct { + api *route53.Route53 + zoneID string +} + +// newRoute53Client sets up a Route53 API client from command line flags. +func newRoute53Client(ctx *cli.Context) *route53Client { + akey := ctx.String(route53AccessKeyFlag.Name) + asec := ctx.String(route53AccessSecretFlag.Name) + if akey == "" || asec == "" { + exit(fmt.Errorf("need Route53 Access Key ID and secret proceed")) + } + config := &aws.Config{Credentials: credentials.NewStaticCredentials(akey, asec, "")} + session, err := session.NewSession(config) + if err != nil { + exit(fmt.Errorf("can't create AWS session: %v", err)) + } + return &route53Client{ + api: route53.New(session), + zoneID: ctx.String(route53ZoneIDFlag.Name), + } +} + +// deploy uploads the given tree to Route53. +func (c *route53Client) deploy(name string, t *dnsdisc.Tree) error { + if err := c.checkZone(name); err != nil { + return err + } + + // Compute DNS changes. + records := t.ToTXT(name) + changes, err := c.computeChanges(name, records) + if err != nil { + return err + } + if len(changes) == 0 { + log.Info("No DNS changes needed") + return nil + } + + // Submit change request. + log.Info(fmt.Sprintf("Submitting %d changes to Route53", len(changes))) + batch := new(route53.ChangeBatch) + batch.SetChanges(changes) + batch.SetComment(fmt.Sprintf("enrtree update of %s at seq %d", name, t.Seq())) + req := &route53.ChangeResourceRecordSetsInput{HostedZoneId: &c.zoneID, ChangeBatch: batch} + resp, err := c.api.ChangeResourceRecordSets(req) + if err != nil { + return err + } + + // Wait for the change to be applied. + log.Info(fmt.Sprintf("Waiting for change request %s", *resp.ChangeInfo.Id)) + wreq := &route53.GetChangeInput{Id: resp.ChangeInfo.Id} + return c.api.WaitUntilResourceRecordSetsChanged(wreq) +} + +// checkZone verifies zone information for the given domain. +func (c *route53Client) checkZone(name string) (err error) { + if c.zoneID == "" { + c.zoneID, err = c.findZoneID(name) + } + return err +} + +// findZoneID searches for the Zone ID containing the given domain. +func (c *route53Client) findZoneID(name string) (string, error) { + log.Info(fmt.Sprintf("Finding Route53 Zone ID for %s", name)) + var req route53.ListHostedZonesByNameInput + for { + resp, err := c.api.ListHostedZonesByName(&req) + if err != nil { + return "", err + } + for _, zone := range resp.HostedZones { + if isSubdomain(name, *zone.Name) { + return *zone.Id, nil + } + } + if !*resp.IsTruncated { + break + } + req.DNSName = resp.NextDNSName + req.HostedZoneId = resp.NextHostedZoneId + } + return "", errors.New("can't find zone ID for " + name) +} + +// computeChanges creates DNS changes for the given record. +func (c *route53Client) computeChanges(name string, records map[string]string) ([]*route53.Change, error) { + // Convert all names to lowercase. + lrecords := make(map[string]string, len(records)) + for name, r := range records { + lrecords[strings.ToLower(name)] = r + } + records = lrecords + + // Get existing records. + existing, err := c.collectRecords(name) + if err != nil { + return nil, err + } + log.Info(fmt.Sprintf("Found %d TXT records", len(existing))) + + var changes []*route53.Change + for path, val := range records { + ttl := 1 + if path != name { + ttl = 2147483647 + } + + prevRecords, exists := existing[path] + prevValue := combineTXT(prevRecords) + if !exists { + // Entry is unknown, push a new one + log.Info(fmt.Sprintf("Creating %s = %q", path, val)) + changes = append(changes, newTXTChange("CREATE", path, ttl, splitTXT(val))) + } else if prevValue != val { + // Entry already exists, only change its content. + log.Info(fmt.Sprintf("Updating %s from %q to %q", path, prevValue, val)) + changes = append(changes, newTXTChange("UPSERT", path, ttl, splitTXT(val))) + } else { + log.Info(fmt.Sprintf("Skipping %s = %q", path, val)) + } + } + + // Iterate over the old records and delete anything stale. + for path, values := range existing { + if _, ok := records[path]; ok { + continue + } + // Stale entry, nuke it. + log.Info(fmt.Sprintf("Deleting %s = %q", path, combineTXT(values))) + changes = append(changes, newTXTChange("DELETE", path, 1, values)) + } + return changes, nil +} + +// collectRecords collects all TXT records below the given name. +func (c *route53Client) collectRecords(name string) (map[string][]string, error) { + log.Info(fmt.Sprintf("Retrieving existing TXT records on %s (%s)", name, c.zoneID)) + var req route53.ListResourceRecordSetsInput + req.SetHostedZoneId(c.zoneID) + existing := make(map[string][]string) + err := c.api.ListResourceRecordSetsPages(&req, func(resp *route53.ListResourceRecordSetsOutput, last bool) bool { + for _, set := range resp.ResourceRecordSets { + if !isSubdomain(*set.Name, name) || *set.Type != "TXT" { + continue + } + name := strings.TrimSuffix(*set.Name, ".") + for _, rec := range set.ResourceRecords { + existing[name] = append(existing[name], *rec.Value) + } + } + return true + }) + return existing, err +} + +// newTXTChange creates a change to a TXT record. +func newTXTChange(action, name string, ttl int, values []string) *route53.Change { + var c route53.Change + var r route53.ResourceRecordSet + var rrs []*route53.ResourceRecord + for _, val := range values { + rr := new(route53.ResourceRecord) + rr.SetValue(val) + rrs = append(rrs, rr) + } + r.SetType("TXT") + r.SetName(name) + r.SetTTL(int64(ttl)) + r.SetResourceRecords(rrs) + c.SetAction(action) + c.SetResourceRecordSet(&r) + return &c +} + +// isSubdomain returns true if name is a subdomain of domain. +func isSubdomain(name, domain string) bool { + domain = strings.TrimSuffix(domain, ".") + name = strings.TrimSuffix(name, ".") + return strings.HasSuffix("."+name, "."+domain) +} + +// combineTXT concatenates the given quoted strings into a single unquoted string. +func combineTXT(values []string) string { + result := "" + for _, v := range values { + if v[0] == '"' { + v = v[1 : len(v)-1] + } + result += v + } + return result +} + +// splitTXT splits value into a list of quoted 255-character strings. +func splitTXT(value string) []string { + var result []string + for len(value) > 0 { + rlen := len(value) + if rlen > 253 { + rlen = 253 + } + result = append(result, strconv.Quote(value[:rlen])) + value = value[rlen:] + } + return result +} diff --git a/cmd/devp2p/dnscmd.go b/cmd/devp2p/dnscmd.go index f245104053..287d6e6c76 100644 --- a/cmd/devp2p/dnscmd.go +++ b/cmd/devp2p/dnscmd.go @@ -42,6 +42,7 @@ var ( dnsSignCommand, dnsTXTCommand, dnsCloudflareCommand, + dnsRoute53Command, }, } dnsSyncCommand = cli.Command{ @@ -66,11 +67,18 @@ var ( } dnsCloudflareCommand = cli.Command{ Name: "to-cloudflare", - Usage: "Deploy DNS TXT records to cloudflare", + Usage: "Deploy DNS TXT records to CloudFlare", ArgsUsage: "", Action: dnsToCloudflare, Flags: []cli.Flag{cloudflareTokenFlag, cloudflareZoneIDFlag}, } + dnsRoute53Command = cli.Command{ + Name: "to-route53", + Usage: "Deploy DNS TXT records to Amazon Route53", + ArgsUsage: "", + Action: dnsToRoute53, + Flags: []cli.Flag{route53AccessKeyFlag, route53AccessSecretFlag, route53ZoneIDFlag}, + } ) var ( @@ -194,6 +202,19 @@ func dnsToCloudflare(ctx *cli.Context) error { return client.deploy(domain, t) } +// dnsToRoute53 peforms dnsRoute53Command. +func dnsToRoute53(ctx *cli.Context) error { + if ctx.NArg() < 1 { + return fmt.Errorf("need tree definition directory as argument") + } + domain, t, err := loadTreeDefinitionForExport(ctx.Args().Get(0)) + if err != nil { + return err + } + client := newRoute53Client(ctx) + return client.deploy(domain, t) +} + // loadSigningKey loads a private key in Ethereum keystore format. func loadSigningKey(keyfile string) *ecdsa.PrivateKey { keyjson, err := ioutil.ReadFile(keyfile) diff --git a/go.mod b/go.mod index a280949e94..223086f8c5 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6 // indirect github.com/VictoriaMetrics/fastcache v1.5.3 github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847 + github.com/aws/aws-sdk-go v1.25.48 github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6 github.com/cespare/cp v0.1.0 github.com/cespare/xxhash/v2 v2.1.1 // indirect diff --git a/go.sum b/go.sum index 515207bca3..edbb5ea2e0 100644 --- a/go.sum +++ b/go.sum @@ -33,6 +33,8 @@ github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156 h1:eMwmnE/GDgah github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847 h1:rtI0fD4oG/8eVokGVPYJEW1F88p1ZNgXiEIs9thEE4A= github.com/aristanetworks/goarista v0.0.0-20170210015632-ea17b1a17847/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= +github.com/aws/aws-sdk-go v1.25.48 h1:J82DYDGZHOKHdhx6hD24Tm30c2C3GchYGfN0mf9iKUk= +github.com/aws/aws-sdk-go v1.25.48/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6 h1:Eey/GGQ/E5Xp1P2Lyx1qj007hLZfbi0+CoVeJruGCtI= github.com/btcsuite/btcd v0.0.0-20171128150713-2e60448ffcc6/go.mod h1:Dmm/EzmjnCiweXmzRIAiUWCInVmPgjkzgv5k4tVyXiQ= @@ -99,6 +101,7 @@ github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883 h1:FSeK4fZCo github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21 h1:F/iKcka0K2LgnKy/fgSBf235AETtm1n1TvBzqu40LE0= github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/karalabe/usb v0.0.0-20190919080040-51dc0efba356 h1:I/yrLt2WilKxlQKCM52clh5rGzTKpVctGT1lH4Dc8Jw= From 275cd4988dbef4b81e856a6c6ae8cb12242e3976 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Mon, 16 Dec 2019 13:37:15 +0100 Subject: [PATCH 029/128] cmd/abigen: Sanitize vyper's combined json names (#20419) * cmd/abigen: Sanitize vyper's combined json names * Review feedback: handle full paths --- cmd/abigen/main.go | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/cmd/abigen/main.go b/cmd/abigen/main.go index 252d8ef800..ed4a3b8870 100644 --- a/cmd/abigen/main.go +++ b/cmd/abigen/main.go @@ -21,9 +21,11 @@ import ( "fmt" "io/ioutil" "os" + "path/filepath" "regexp" "strings" + "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/accounts/abi/bind" "github.com/ethereum/go-ethereum/cmd/utils" "github.com/ethereum/go-ethereum/common/compiler" @@ -193,10 +195,22 @@ func abigen(c *cli.Context) error { utils.Fatalf("Failed to build Solidity contract: %v", err) } case c.GlobalIsSet(vyFlag.Name): - contracts, err = compiler.CompileVyper(c.GlobalString(vyperFlag.Name), c.GlobalString(vyFlag.Name)) + output, err := compiler.CompileVyper(c.GlobalString(vyperFlag.Name), c.GlobalString(vyFlag.Name)) if err != nil { utils.Fatalf("Failed to build Vyper contract: %v", err) } + contracts = make(map[string]*compiler.Contract) + for n, contract := range output { + name := n + // Sanitize the combined json names to match the + // format expected by solidity. + if !strings.Contains(n, ":") { + // Remove extra path components + name = abi.ToCamelCase(strings.TrimSuffix(filepath.Base(name), ".vy")) + } + contracts[name] = contract + } + case c.GlobalIsSet(jsonFlag.Name): jsonOutput, err := ioutil.ReadFile(c.GlobalString(jsonFlag.Name)) if err != nil { From c4b7fdd27ed7ad44d4cc8ea953bae78efac536da Mon Sep 17 00:00:00 2001 From: Ryan Schneider Date: Tue, 17 Dec 2019 03:10:15 -0800 Subject: [PATCH 030/128] eth, internal/web3ext: add optional first and last arguments to the `admin_exportChain` RPC. (#20107) --- eth/api.go | 18 +++++++++++++++--- internal/web3ext/web3ext.go | 4 ++-- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/eth/api.go b/eth/api.go index f8c51c09bd..a874582e19 100644 --- a/eth/api.go +++ b/eth/api.go @@ -166,8 +166,16 @@ func NewPrivateAdminAPI(eth *Ethereum) *PrivateAdminAPI { return &PrivateAdminAPI{eth: eth} } -// ExportChain exports the current blockchain into a local file. -func (api *PrivateAdminAPI) ExportChain(file string) (bool, error) { +// ExportChain exports the current blockchain into a local file, +// or a range of blocks if first and last are non-nil +func (api *PrivateAdminAPI) ExportChain(file string, first *uint64, last *uint64) (bool, error) { + if first == nil && last != nil { + return false, errors.New("last cannot be specified without first") + } + if first != nil && last == nil { + head := api.eth.BlockChain().CurrentHeader().Number.Uint64() + last = &head + } if _, err := os.Stat(file); err == nil { // File already exists. Allowing overwrite could be a DoS vecotor, // since the 'file' may point to arbitrary paths on the drive @@ -187,7 +195,11 @@ func (api *PrivateAdminAPI) ExportChain(file string) (bool, error) { } // Export the blockchain - if err := api.eth.BlockChain().Export(writer); err != nil { + if first != nil { + if err := api.eth.BlockChain().ExportN(writer, *first, *last); err != nil { + return false, err + } + } else if err := api.eth.BlockChain().Export(writer); err != nil { return false, err } return true, nil diff --git a/internal/web3ext/web3ext.go b/internal/web3ext/web3ext.go index cd16fa79c6..bc105ef37c 100644 --- a/internal/web3ext/web3ext.go +++ b/internal/web3ext/web3ext.go @@ -172,8 +172,8 @@ web3._extend({ new web3._extend.Method({ name: 'exportChain', call: 'admin_exportChain', - params: 1, - inputFormatter: [null] + params: 3, + inputFormatter: [null, null, null] }), new web3._extend.Method({ name: 'importChain', From 49cf000df7a5fad11323ed8e1186710c4034260d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Pawe=C5=82=20Bylica?= Date: Wed, 18 Dec 2019 09:43:18 +0100 Subject: [PATCH 031/128] cmd/evm: Add --bench flag for benchmarking (#20330) The --bench flag uses the testing.B to execute the EVM bytecode many times and get the average exeuction time out of it. --- cmd/evm/main.go | 5 +++++ cmd/evm/runner.go | 54 +++++++++++++++++++++++++++++++++++++---------- 2 files changed, 48 insertions(+), 11 deletions(-) diff --git a/cmd/evm/main.go b/cmd/evm/main.go index 67447d58ae..72cb1ab852 100644 --- a/cmd/evm/main.go +++ b/cmd/evm/main.go @@ -87,6 +87,10 @@ var ( Name: "verbosity", Usage: "sets the verbosity level", } + BenchFlag = cli.BoolFlag{ + Name: "bench", + Usage: "benchmark the execution", + } CreateFlag = cli.BoolFlag{ Name: "create", Usage: "indicates the action should be create rather than call", @@ -124,6 +128,7 @@ var ( func init() { app.Flags = []cli.Flag{ + BenchFlag, CreateFlag, DebugFlag, VerbosityFlag, diff --git a/cmd/evm/runner.go b/cmd/evm/runner.go index cecbf36063..da301ff5ee 100644 --- a/cmd/evm/runner.go +++ b/cmd/evm/runner.go @@ -25,6 +25,7 @@ import ( "os" goruntime "runtime" "runtime/pprof" + "testing" "time" "github.com/ethereum/go-ethereum/cmd/evm/internal/compiler" @@ -69,6 +70,33 @@ func readGenesis(genesisPath string) *core.Genesis { return genesis } +func timedExec(bench bool, execFunc func() ([]byte, uint64, error)) ([]byte, uint64, time.Duration, error) { + var ( + output []byte + gasLeft uint64 + execTime time.Duration + err error + ) + + if bench { + result := testing.Benchmark(func(b *testing.B) { + for i := 0; i < b.N; i++ { + output, gasLeft, err = execFunc() + } + }) + + // Get the average execution time from the benchmarking result. + // There are other useful stats here that could be reported. + execTime = time.Duration(result.NsPerOp()) + } else { + startTime := time.Now() + output, gasLeft, err = execFunc() + execTime = time.Since(startTime) + } + + return output, gasLeft, execTime, err +} + func runCmd(ctx *cli.Context) error { glogger := log.NewGlogHandler(log.StreamHandler(os.Stderr, log.TerminalFormat(false))) glogger.Verbosity(log.Lvl(ctx.GlobalInt(VerbosityFlag.Name))) @@ -116,11 +144,7 @@ func runCmd(ctx *cli.Context) error { receiver = common.HexToAddress(ctx.GlobalString(ReceiverFlag.Name)) } - var ( - code []byte - ret []byte - err error - ) + var code []byte codeFileFlag := ctx.GlobalString(CodeFileFlag.Name) codeFlag := ctx.GlobalString(CodeFlag.Name) @@ -203,10 +227,10 @@ func runCmd(ctx *cli.Context) error { } else { runtimeConfig.ChainConfig = params.AllEthashProtocolChanges } - tstart := time.Now() - var leftOverGas uint64 + var hexInput []byte if inputFileFlag := ctx.GlobalString(InputFileFlag.Name); inputFileFlag != "" { + var err error if hexInput, err = ioutil.ReadFile(inputFileFlag); err != nil { fmt.Printf("could not load input from file: %v\n", err) os.Exit(1) @@ -215,16 +239,24 @@ func runCmd(ctx *cli.Context) error { hexInput = []byte(ctx.GlobalString(InputFlag.Name)) } input := common.FromHex(string(bytes.TrimSpace(hexInput))) + + var execFunc func() ([]byte, uint64, error) if ctx.GlobalBool(CreateFlag.Name) { input = append(code, input...) - ret, _, leftOverGas, err = runtime.Create(input, &runtimeConfig) + execFunc = func() ([]byte, uint64, error) { + output, _, gasLeft, err := runtime.Create(input, &runtimeConfig) + return output, gasLeft, err + } } else { if len(code) > 0 { statedb.SetCode(receiver, code) } - ret, leftOverGas, err = runtime.Call(receiver, input, &runtimeConfig) + execFunc = func() ([]byte, uint64, error) { + return runtime.Call(receiver, input, &runtimeConfig) + } } - execTime := time.Since(tstart) + + output, leftOverGas, execTime, err := timedExec(ctx.GlobalBool(BenchFlag.Name), execFunc) if ctx.GlobalBool(DumpFlag.Name) { statedb.Commit(true) @@ -267,7 +299,7 @@ Gas used: %d `, execTime, mem.HeapObjects, mem.Alloc, mem.TotalAlloc, mem.NumGC, initialGas-leftOverGas) } if tracer == nil { - fmt.Printf("0x%x\n", ret) + fmt.Printf("0x%x\n", output) if err != nil { fmt.Printf(" error: %v\n", err) } From 6ae9dc15cc51310f33a011dcafab5739a6ceea93 Mon Sep 17 00:00:00 2001 From: Jeff Wentworth Date: Wed, 18 Dec 2019 19:16:07 +0900 Subject: [PATCH 032/128] [#20266] Fix bugs decoding integers and fixed bytes in indexed event fields (#20269) * fix parseTopics() and add tests * remove printf * add ParseTopicsIntoMap() tests * fix FixedBytesTy * fix int and fixed bytes * golint topics_test.go --- accounts/abi/bind/topics.go | 16 ++++- accounts/abi/bind/topics_test.go | 112 ++++++++++++++++++++++++++----- accounts/abi/unpack.go | 22 +++--- 3 files changed, 122 insertions(+), 28 deletions(-) diff --git a/accounts/abi/bind/topics.go b/accounts/abi/bind/topics.go index e27fa54842..c908c92582 100644 --- a/accounts/abi/bind/topics.go +++ b/accounts/abi/bind/topics.go @@ -178,6 +178,13 @@ func parseTopics(out interface{}, fields abi.Arguments, topics []common.Hash) er case reflectBigInt: num := new(big.Int).SetBytes(topics[0][:]) + if arg.Type.T == abi.IntTy { + if num.Cmp(abi.MaxInt256) > 0 { + num.Add(abi.MaxUint256, big.NewInt(0).Neg(num)) + num.Add(num, big.NewInt(1)) + num.Neg(num) + } + } field.Set(reflect.ValueOf(num)) default: @@ -212,8 +219,7 @@ func parseTopicsIntoMap(out map[string]interface{}, fields abi.Arguments, topics case abi.BoolTy: out[arg.Name] = topics[0][common.HashLength-1] == 1 case abi.IntTy, abi.UintTy: - num := new(big.Int).SetBytes(topics[0][:]) - out[arg.Name] = num + out[arg.Name] = abi.ReadInteger(arg.Type.T, arg.Type.Kind, topics[0].Bytes()) case abi.AddressTy: var addr common.Address copy(addr[:], topics[0][common.HashLength-common.AddressLength:]) @@ -221,7 +227,11 @@ func parseTopicsIntoMap(out map[string]interface{}, fields abi.Arguments, topics case abi.HashTy: out[arg.Name] = topics[0] case abi.FixedBytesTy: - out[arg.Name] = topics[0][:] + array, err := abi.ReadFixedBytes(arg.Type, topics[0].Bytes()) + if err != nil { + return err + } + out[arg.Name] = array case abi.StringTy, abi.BytesTy, abi.SliceTy, abi.ArrayTy: // Array types (including strings and bytes) have their keccak256 hashes stored in the topic- not a hash // whose bytes can be decoded to the actual value- so the best we can do is retrieve that hash diff --git a/accounts/abi/bind/topics_test.go b/accounts/abi/bind/topics_test.go index f18e2d1bd2..c62f5bab32 100644 --- a/accounts/abi/bind/topics_test.go +++ b/accounts/abi/bind/topics_test.go @@ -17,6 +17,7 @@ package bind import ( + "math/big" "reflect" "testing" @@ -55,27 +56,44 @@ func TestMakeTopics(t *testing.T) { } } -func TestParseTopics(t *testing.T) { - type bytesStruct struct { - StaticBytes [5]byte - } +type args struct { + createObj func() interface{} + resultObj func() interface{} + resultMap func() map[string]interface{} + fields abi.Arguments + topics []common.Hash +} + +type bytesStruct struct { + StaticBytes [5]byte +} +type int8Struct struct { + Int8Value int8 +} +type int256Struct struct { + Int256Value *big.Int +} + +type topicTest struct { + name string + args args + wantErr bool +} + +func setupTopicsTests() []topicTest { bytesType, _ := abi.NewType("bytes5", "", nil) - type args struct { - createObj func() interface{} - resultObj func() interface{} - fields abi.Arguments - topics []common.Hash - } - tests := []struct { - name string - args args - wantErr bool - }{ + int8Type, _ := abi.NewType("int8", "", nil) + int256Type, _ := abi.NewType("int256", "", nil) + + tests := []topicTest{ { name: "support fixed byte types, right padded to 32 bytes", args: args{ createObj: func() interface{} { return &bytesStruct{} }, resultObj: func() interface{} { return &bytesStruct{StaticBytes: [5]byte{1, 2, 3, 4, 5}} }, + resultMap: func() map[string]interface{} { + return map[string]interface{}{"staticBytes": [5]byte{1, 2, 3, 4, 5}} + }, fields: abi.Arguments{abi.Argument{ Name: "staticBytes", Type: bytesType, @@ -87,7 +105,54 @@ func TestParseTopics(t *testing.T) { }, wantErr: false, }, + { + name: "int8 with negative value", + args: args{ + createObj: func() interface{} { return &int8Struct{} }, + resultObj: func() interface{} { return &int8Struct{Int8Value: -1} }, + resultMap: func() map[string]interface{} { + return map[string]interface{}{"int8Value": int8(-1)} + }, + fields: abi.Arguments{abi.Argument{ + Name: "int8Value", + Type: int8Type, + Indexed: true, + }}, + topics: []common.Hash{ + {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, + }, + }, + wantErr: false, + }, + { + name: "int256 with negative value", + args: args{ + createObj: func() interface{} { return &int256Struct{} }, + resultObj: func() interface{} { return &int256Struct{Int256Value: big.NewInt(-1)} }, + resultMap: func() map[string]interface{} { + return map[string]interface{}{"int256Value": big.NewInt(-1)} + }, + fields: abi.Arguments{abi.Argument{ + Name: "int256Value", + Type: int256Type, + Indexed: true, + }}, + topics: []common.Hash{ + {255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, + 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255, 255}, + }, + }, + wantErr: false, + }, } + + return tests +} + +func TestParseTopics(t *testing.T) { + tests := setupTopicsTests() + for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { createObj := tt.args.createObj() @@ -101,3 +166,20 @@ func TestParseTopics(t *testing.T) { }) } } + +func TestParseTopicsIntoMap(t *testing.T) { + tests := setupTopicsTests() + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + outMap := make(map[string]interface{}) + if err := parseTopicsIntoMap(outMap, tt.args.fields, tt.args.topics); (err != nil) != tt.wantErr { + t.Errorf("parseTopicsIntoMap() error = %v, wantErr %v", err, tt.wantErr) + } + resultMap := tt.args.resultMap() + if !reflect.DeepEqual(outMap, resultMap) { + t.Errorf("parseTopicsIntoMap() = %v, want %v", outMap, resultMap) + } + }) + } +} diff --git a/accounts/abi/unpack.go b/accounts/abi/unpack.go index b2e61d06c4..2a5db3b315 100644 --- a/accounts/abi/unpack.go +++ b/accounts/abi/unpack.go @@ -26,16 +26,18 @@ import ( ) var ( - maxUint256 = big.NewInt(0).Add( + // MaxUint256 is the maximum value that can be represented by a uint256 + MaxUint256 = big.NewInt(0).Add( big.NewInt(0).Exp(big.NewInt(2), big.NewInt(256), nil), big.NewInt(-1)) - maxInt256 = big.NewInt(0).Add( + // MaxInt256 is the maximum value that can be represented by a int256 + MaxInt256 = big.NewInt(0).Add( big.NewInt(0).Exp(big.NewInt(2), big.NewInt(255), nil), big.NewInt(-1)) ) -// reads the integer based on its kind -func readInteger(typ byte, kind reflect.Kind, b []byte) interface{} { +// ReadInteger reads the integer based on its kind and returns the appropriate value +func ReadInteger(typ byte, kind reflect.Kind, b []byte) interface{} { switch kind { case reflect.Uint8: return b[len(b)-1] @@ -62,8 +64,8 @@ func readInteger(typ byte, kind reflect.Kind, b []byte) interface{} { return ret } - if ret.Cmp(maxInt256) > 0 { - ret.Add(maxUint256, big.NewInt(0).Neg(ret)) + if ret.Cmp(MaxInt256) > 0 { + ret.Add(MaxUint256, big.NewInt(0).Neg(ret)) ret.Add(ret, big.NewInt(1)) ret.Neg(ret) } @@ -102,8 +104,8 @@ func readFunctionType(t Type, word []byte) (funcTy [24]byte, err error) { return } -// through reflection, creates a fixed array to be read from -func readFixedBytes(t Type, word []byte) (interface{}, error) { +// ReadFixedBytes uses reflection to create a fixed array to be read from +func ReadFixedBytes(t Type, word []byte) (interface{}, error) { if t.T != FixedBytesTy { return nil, fmt.Errorf("abi: invalid type in call to make fixed byte array") } @@ -230,7 +232,7 @@ func toGoType(index int, t Type, output []byte) (interface{}, error) { case StringTy: // variable arrays are written at the end of the return bytes return string(output[begin : begin+length]), nil case IntTy, UintTy: - return readInteger(t.T, t.Kind, returnOutput), nil + return ReadInteger(t.T, t.Kind, returnOutput), nil case BoolTy: return readBool(returnOutput) case AddressTy: @@ -240,7 +242,7 @@ func toGoType(index int, t Type, output []byte) (interface{}, error) { case BytesTy: return output[begin : begin+length], nil case FixedBytesTy: - return readFixedBytes(t, returnOutput) + return ReadFixedBytes(t, returnOutput) case FunctionTy: return readFunctionType(t, returnOutput) default: From 93b1171316d8f6e0252973eeaabf4cf8fbd9414d Mon Sep 17 00:00:00 2001 From: Ilan Gitter <8359193+gitteri@users.noreply.github.com> Date: Fri, 20 Dec 2019 05:33:32 -0500 Subject: [PATCH 033/128] accounts/abi/backends/simulated: add more API methods (#5) (#20208) * Add more functionality to the sim (#5) * backends: implement more of ethclient in sim * backends: add BlockByNumber to simulated backend * backends: make simulated progress function agree with syncprogress interface for client * backends: add more tests * backends: add more comments * backends: fix sim for index in tx and add tests * backends: add lock back to estimategas * backends: goimports * backends: go ci lint * Add more functionality to the sim (#5) * backends: implement more of ethclient in sim * backends: add BlockByNumber to simulated backend * backends: make simulated progress function agree with syncprogress interface for client * backends: add more tests * backends: add more comments * backends: fix sim for index in tx and add tests * backends: add lock back to estimategas * backends: goimports * backends: go ci lint * assert errs --- accounts/abi/bind/backends/simulated.go | 149 +++- accounts/abi/bind/backends/simulated_test.go | 767 ++++++++++++++++++- 2 files changed, 910 insertions(+), 6 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index e625a42868..0b2555c994 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -46,12 +46,17 @@ import ( var _ bind.ContractBackend = (*SimulatedBackend)(nil) var ( - errBlockNumberUnsupported = errors.New("simulatedBackend cannot access blocks other than the latest block") - errGasEstimationFailed = errors.New("gas required exceeds allowance or always failing transaction") + errBlockNumberUnsupported = errors.New("simulatedBackend cannot access blocks other than the latest block") + errBlockDoesNotExist = errors.New("block does not exist in blockchain") + errTransactionDoesNotExist = errors.New("transaction does not exist") + errGasEstimationFailed = errors.New("gas required exceeds allowance or always failing transaction") ) // SimulatedBackend implements bind.ContractBackend, simulating a blockchain in // the background. Its main purpose is to allow easily testing contract bindings. +// Simulated backend implements the following interfaces: +// ChainReader, ChainStateReader, ContractBackend, ContractCaller, ContractFilterer, ContractTransactor, +// DeployBackend, GasEstimator, GasPricer, LogFilterer, PendingContractCaller, TransactionReader, and TransactionSender type SimulatedBackend struct { database ethdb.Database // In memory database to store our testing data blockchain *core.BlockChain // Ethereum blockchain to handle the consensus @@ -173,6 +178,9 @@ func (b *SimulatedBackend) StorageAt(ctx context.Context, contract common.Addres // TransactionReceipt returns the receipt of a transaction. func (b *SimulatedBackend) TransactionReceipt(ctx context.Context, txHash common.Hash) (*types.Receipt, error) { + b.mu.Lock() + defer b.mu.Unlock() + receipt, _, _, _ := rawdb.ReadReceipt(b.database, txHash, b.config) return receipt, nil } @@ -196,6 +204,115 @@ func (b *SimulatedBackend) TransactionByHash(ctx context.Context, txHash common. return nil, false, ethereum.NotFound } +// BlockByHash retrieves a block based on the block hash +func (b *SimulatedBackend) BlockByHash(ctx context.Context, hash common.Hash) (*types.Block, error) { + b.mu.Lock() + defer b.mu.Unlock() + + if hash == b.pendingBlock.Hash() { + return b.pendingBlock, nil + } + + block := b.blockchain.GetBlockByHash(hash) + if block != nil { + return block, nil + } + + return nil, errBlockDoesNotExist +} + +// BlockByNumber retrieves a block from the database by number, caching it +// (associated with its hash) if found. +func (b *SimulatedBackend) BlockByNumber(ctx context.Context, number *big.Int) (*types.Block, error) { + b.mu.Lock() + defer b.mu.Unlock() + + if number == nil || number.Cmp(b.pendingBlock.Number()) == 0 { + return b.blockchain.CurrentBlock(), nil + } + + block := b.blockchain.GetBlockByNumber(uint64(number.Int64())) + if block == nil { + return nil, errBlockDoesNotExist + } + + return block, nil +} + +// HeaderByHash returns a block header from the current canonical chain. +func (b *SimulatedBackend) HeaderByHash(ctx context.Context, hash common.Hash) (*types.Header, error) { + b.mu.Lock() + defer b.mu.Unlock() + + if hash == b.pendingBlock.Hash() { + return b.pendingBlock.Header(), nil + } + + header := b.blockchain.GetHeaderByHash(hash) + if header == nil { + return nil, errBlockDoesNotExist + } + + return header, nil +} + +// HeaderByNumber returns a block header from the current canonical chain. If number is +// nil, the latest known header is returned. +func (b *SimulatedBackend) HeaderByNumber(ctx context.Context, block *big.Int) (*types.Header, error) { + b.mu.Lock() + defer b.mu.Unlock() + + if block == nil || block.Cmp(b.pendingBlock.Number()) == 0 { + return b.blockchain.CurrentHeader(), nil + } + + return b.blockchain.GetHeaderByNumber(uint64(block.Int64())), nil +} + +// TransactionCount returns the number of transactions in a given block +func (b *SimulatedBackend) TransactionCount(ctx context.Context, blockHash common.Hash) (uint, error) { + b.mu.Lock() + defer b.mu.Unlock() + + if blockHash == b.pendingBlock.Hash() { + return uint(b.pendingBlock.Transactions().Len()), nil + } + + block := b.blockchain.GetBlockByHash(blockHash) + if block == nil { + return uint(0), errBlockDoesNotExist + } + + return uint(block.Transactions().Len()), nil +} + +// TransactionInBlock returns the transaction for a specific block at a specific index +func (b *SimulatedBackend) TransactionInBlock(ctx context.Context, blockHash common.Hash, index uint) (*types.Transaction, error) { + b.mu.Lock() + defer b.mu.Unlock() + + if blockHash == b.pendingBlock.Hash() { + transactions := b.pendingBlock.Transactions() + if uint(len(transactions)) < index+1 { + return nil, errTransactionDoesNotExist + } + + return transactions[index], nil + } + + block := b.blockchain.GetBlockByHash(blockHash) + if block == nil { + return nil, errBlockDoesNotExist + } + + transactions := block.Transactions() + if uint(len(transactions)) < index+1 { + return nil, errTransactionDoesNotExist + } + + return transactions[index], nil +} + // PendingCodeAt returns the code associated with an account in the pending state. func (b *SimulatedBackend) PendingCodeAt(ctx context.Context, contract common.Address) ([]byte, error) { b.mu.Lock() @@ -419,10 +536,38 @@ func (b *SimulatedBackend) SubscribeFilterLogs(ctx context.Context, query ethere }), nil } +// SubscribeNewHead returns an event subscription for a new header +func (b *SimulatedBackend) SubscribeNewHead(ctx context.Context, ch chan<- *types.Header) (ethereum.Subscription, error) { + // subscribe to a new head + sink := make(chan *types.Header) + sub := b.events.SubscribeNewHeads(sink) + + return event.NewSubscription(func(quit <-chan struct{}) error { + defer sub.Unsubscribe() + for { + select { + case head := <-sink: + select { + case ch <- head: + case err := <-sub.Err(): + return err + case <-quit: + return nil + } + case err := <-sub.Err(): + return err + case <-quit: + return nil + } + } + }), nil +} + // AdjustTime adds a time shift to the simulated clock. func (b *SimulatedBackend) AdjustTime(adjustment time.Duration) error { b.mu.Lock() defer b.mu.Unlock() + blocks, _ := core.GenerateChain(b.config, b.blockchain.CurrentBlock(), ethash.NewFaker(), b.database, 1, func(number int, block *core.BlockGen) { for _, tx := range b.pendingBlock.Transactions() { block.AddTx(tx) diff --git a/accounts/abi/bind/backends/simulated_test.go b/accounts/abi/bind/backends/simulated_test.go index 785b25d10f..455d89c1e3 100644 --- a/accounts/abi/bind/backends/simulated_test.go +++ b/accounts/abi/bind/backends/simulated_test.go @@ -14,20 +14,24 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package backends_test +package backends import ( + "bytes" "context" "math/big" + "strings" "testing" + "time" - ethereum "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/crypto" + "github.com/ethereum/go-ethereum/params" ) func TestSimulatedBackend(t *testing.T) { @@ -37,7 +41,7 @@ func TestSimulatedBackend(t *testing.T) { genAlloc := make(core.GenesisAlloc) genAlloc[auth.From] = core.GenesisAccount{Balance: big.NewInt(9223372036854775807)} - sim := backends.NewSimulatedBackend(genAlloc, gasLimit) + sim := NewSimulatedBackend(genAlloc, gasLimit) defer sim.Close() // should return an error if the tx is not found @@ -79,5 +83,760 @@ func TestSimulatedBackend(t *testing.T) { if isPending { t.Fatal("transaction should not have pending status") } +} + +var testKey, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") + +// the following is based on this contract: +// contract T { +// event received(address sender, uint amount, bytes memo); +// event receivedAddr(address sender); +// +// function receive(bytes calldata memo) external payable returns (string memory res) { +// emit received(msg.sender, msg.value, memo); +// emit receivedAddr(msg.sender); +// return "hello world"; +// } +// } +const abiJSON = `[ { "constant": false, "inputs": [ { "name": "memo", "type": "bytes" } ], "name": "receive", "outputs": [ { "name": "res", "type": "string" } ], "payable": true, "stateMutability": "payable", "type": "function" }, { "anonymous": false, "inputs": [ { "indexed": false, "name": "sender", "type": "address" }, { "indexed": false, "name": "amount", "type": "uint256" }, { "indexed": false, "name": "memo", "type": "bytes" } ], "name": "received", "type": "event" }, { "anonymous": false, "inputs": [ { "indexed": false, "name": "sender", "type": "address" } ], "name": "receivedAddr", "type": "event" } ]` +const abiBin = `0x608060405234801561001057600080fd5b506102a0806100206000396000f3fe60806040526004361061003b576000357c010000000000000000000000000000000000000000000000000000000090048063a69b6ed014610040575b600080fd5b6100b76004803603602081101561005657600080fd5b810190808035906020019064010000000081111561007357600080fd5b82018360208201111561008557600080fd5b803590602001918460018302840111640100000000831117156100a757600080fd5b9091929391929390505050610132565b6040518080602001828103825283818151815260200191508051906020019080838360005b838110156100f75780820151818401526020810190506100dc565b50505050905090810190601f1680156101245780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b60607f75fd880d39c1daf53b6547ab6cb59451fc6452d27caa90e5b6649dd8293b9eed33348585604051808573ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff168152602001848152602001806020018281038252848482818152602001925080828437600081840152601f19601f8201169050808301925050509550505050505060405180910390a17f46923992397eac56cf13058aced2a1871933622717e27b24eabc13bf9dd329c833604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390a16040805190810160405280600b81526020017f68656c6c6f20776f726c6400000000000000000000000000000000000000000081525090509291505056fea165627a7a72305820ff0c57dad254cfeda48c9cfb47f1353a558bccb4d1bc31da1dae69315772d29e0029` +const deployedCode = `60806040526004361061003b576000357c010000000000000000000000000000000000000000000000000000000090048063a69b6ed014610040575b600080fd5b6100b76004803603602081101561005657600080fd5b810190808035906020019064010000000081111561007357600080fd5b82018360208201111561008557600080fd5b803590602001918460018302840111640100000000831117156100a757600080fd5b9091929391929390505050610132565b6040518080602001828103825283818151815260200191508051906020019080838360005b838110156100f75780820151818401526020810190506100dc565b50505050905090810190601f1680156101245780820380516001836020036101000a031916815260200191505b509250505060405180910390f35b60607f75fd880d39c1daf53b6547ab6cb59451fc6452d27caa90e5b6649dd8293b9eed33348585604051808573ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff168152602001848152602001806020018281038252848482818152602001925080828437600081840152601f19601f8201169050808301925050509550505050505060405180910390a17f46923992397eac56cf13058aced2a1871933622717e27b24eabc13bf9dd329c833604051808273ffffffffffffffffffffffffffffffffffffffff1673ffffffffffffffffffffffffffffffffffffffff16815260200191505060405180910390a16040805190810160405280600b81526020017f68656c6c6f20776f726c6400000000000000000000000000000000000000000081525090509291505056fea165627a7a72305820ff0c57dad254cfeda48c9cfb47f1353a558bccb4d1bc31da1dae69315772d29e0029` + +// expected return value contains "hello world" +var expectedReturn = []byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 32, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 11, 104, 101, 108, 108, 111, 32, 119, 111, 114, 108, 100, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0} + +func TestNewSimulatedBackend(t *testing.T) { + testAddr := crypto.PubkeyToAddress(testKey.PublicKey) + expectedBal := big.NewInt(10000000000) + sim := NewSimulatedBackend( + core.GenesisAlloc{ + testAddr: {Balance: expectedBal}, + }, 10000000, + ) + defer sim.Close() + + if sim.config != params.AllEthashProtocolChanges { + t.Errorf("expected sim config to equal params.AllEthashProtocolChanges, got %v", sim.config) + } + + if sim.blockchain.Config() != params.AllEthashProtocolChanges { + t.Errorf("expected sim blockchain config to equal params.AllEthashProtocolChanges, got %v", sim.config) + } + + statedb, _ := sim.blockchain.State() + bal := statedb.GetBalance(testAddr) + if bal.Cmp(expectedBal) != 0 { + t.Errorf("expected balance for test address not received. expected: %v actual: %v", expectedBal, bal) + } +} + +func TestSimulatedBackend_AdjustTime(t *testing.T) { + sim := NewSimulatedBackend( + core.GenesisAlloc{}, 10000000, + ) + defer sim.Close() + + prevTime := sim.pendingBlock.Time() + err := sim.AdjustTime(time.Second) + if err != nil { + t.Error(err) + } + newTime := sim.pendingBlock.Time() + + if newTime-prevTime != uint64(time.Second.Seconds()) { + t.Errorf("adjusted time not equal to a second. prev: %v, new: %v", prevTime, newTime) + } +} + +func TestSimulatedBackend_BalanceAt(t *testing.T) { + testAddr := crypto.PubkeyToAddress(testKey.PublicKey) + expectedBal := big.NewInt(10000000000) + sim := NewSimulatedBackend( + core.GenesisAlloc{ + testAddr: {Balance: expectedBal}, + }, 10000000, + ) + defer sim.Close() + bgCtx := context.Background() + + bal, err := sim.BalanceAt(bgCtx, testAddr, nil) + if err != nil { + t.Error(err) + } + + if bal.Cmp(expectedBal) != 0 { + t.Errorf("expected balance for test address not received. expected: %v actual: %v", expectedBal, bal) + } +} + +func TestSimulatedBackend_BlockByHash(t *testing.T) { + sim := NewSimulatedBackend( + core.GenesisAlloc{}, 10000000, + ) + defer sim.Close() + bgCtx := context.Background() + + block, err := sim.BlockByNumber(bgCtx, nil) + if err != nil { + t.Errorf("could not get recent block: %v", err) + } + blockByHash, err := sim.BlockByHash(bgCtx, block.Hash()) + if err != nil { + t.Errorf("could not get recent block: %v", err) + } + + if block.Hash() != blockByHash.Hash() { + t.Errorf("did not get expected block") + } +} + +func TestSimulatedBackend_BlockByNumber(t *testing.T) { + sim := NewSimulatedBackend( + core.GenesisAlloc{}, 10000000, + ) + defer sim.Close() + bgCtx := context.Background() + + block, err := sim.BlockByNumber(bgCtx, nil) + if err != nil { + t.Errorf("could not get recent block: %v", err) + } + if block.NumberU64() != 0 { + t.Errorf("did not get most recent block, instead got block number %v", block.NumberU64()) + } + + // create one block + sim.Commit() + + block, err = sim.BlockByNumber(bgCtx, nil) + if err != nil { + t.Errorf("could not get recent block: %v", err) + } + if block.NumberU64() != 1 { + t.Errorf("did not get most recent block, instead got block number %v", block.NumberU64()) + } + blockByNumber, err := sim.BlockByNumber(bgCtx, big.NewInt(1)) + if err != nil { + t.Errorf("could not get block by number: %v", err) + } + if blockByNumber.Hash() != block.Hash() { + t.Errorf("did not get the same block with height of 1 as before") + } +} + +func TestSimulatedBackend_NonceAt(t *testing.T) { + testAddr := crypto.PubkeyToAddress(testKey.PublicKey) + + sim := NewSimulatedBackend( + core.GenesisAlloc{ + testAddr: {Balance: big.NewInt(10000000000)}, + }, 10000000, + ) + defer sim.Close() + bgCtx := context.Background() + + nonce, err := sim.NonceAt(bgCtx, testAddr, big.NewInt(0)) + if err != nil { + t.Errorf("could not get nonce for test addr: %v", err) + } + + if nonce != uint64(0) { + t.Errorf("received incorrect nonce. expected 0, got %v", nonce) + } + + // create a signed transaction to send + tx := types.NewTransaction(nonce, testAddr, big.NewInt(1000), params.TxGas, big.NewInt(1), nil) + signedTx, err := types.SignTx(tx, types.HomesteadSigner{}, testKey) + if err != nil { + t.Errorf("could not sign tx: %v", err) + } + + // send tx to simulated backend + err = sim.SendTransaction(bgCtx, signedTx) + if err != nil { + t.Errorf("could not add tx to pending block: %v", err) + } + sim.Commit() + + newNonce, err := sim.NonceAt(bgCtx, testAddr, big.NewInt(1)) + if err != nil { + t.Errorf("could not get nonce for test addr: %v", err) + } + + if newNonce != nonce+uint64(1) { + t.Errorf("received incorrect nonce. expected 1, got %v", nonce) + } +} + +func TestSimulatedBackend_SendTransaction(t *testing.T) { + testAddr := crypto.PubkeyToAddress(testKey.PublicKey) + + sim := NewSimulatedBackend( + core.GenesisAlloc{ + testAddr: {Balance: big.NewInt(10000000000)}, + }, 10000000, + ) + defer sim.Close() + bgCtx := context.Background() + + // create a signed transaction to send + tx := types.NewTransaction(uint64(0), testAddr, big.NewInt(1000), params.TxGas, big.NewInt(1), nil) + signedTx, err := types.SignTx(tx, types.HomesteadSigner{}, testKey) + if err != nil { + t.Errorf("could not sign tx: %v", err) + } + + // send tx to simulated backend + err = sim.SendTransaction(bgCtx, signedTx) + if err != nil { + t.Errorf("could not add tx to pending block: %v", err) + } + sim.Commit() + + block, err := sim.BlockByNumber(bgCtx, big.NewInt(1)) + if err != nil { + t.Errorf("could not get block at height 1: %v", err) + } + + if signedTx.Hash() != block.Transactions()[0].Hash() { + t.Errorf("did not commit sent transaction. expected hash %v got hash %v", block.Transactions()[0].Hash(), signedTx.Hash()) + } +} + +func TestSimulatedBackend_TransactionByHash(t *testing.T) { + testAddr := crypto.PubkeyToAddress(testKey.PublicKey) + + sim := NewSimulatedBackend( + core.GenesisAlloc{ + testAddr: {Balance: big.NewInt(10000000000)}, + }, 10000000, + ) + defer sim.Close() + bgCtx := context.Background() + + // create a signed transaction to send + tx := types.NewTransaction(uint64(0), testAddr, big.NewInt(1000), params.TxGas, big.NewInt(1), nil) + signedTx, err := types.SignTx(tx, types.HomesteadSigner{}, testKey) + if err != nil { + t.Errorf("could not sign tx: %v", err) + } + + // send tx to simulated backend + err = sim.SendTransaction(bgCtx, signedTx) + if err != nil { + t.Errorf("could not add tx to pending block: %v", err) + } + + // ensure tx is committed pending + receivedTx, pending, err := sim.TransactionByHash(bgCtx, signedTx.Hash()) + if err != nil { + t.Errorf("could not get transaction by hash %v: %v", signedTx.Hash(), err) + } + if !pending { + t.Errorf("expected transaction to be in pending state") + } + if receivedTx.Hash() != signedTx.Hash() { + t.Errorf("did not received committed transaction. expected hash %v got hash %v", signedTx.Hash(), receivedTx.Hash()) + } + + sim.Commit() + + // ensure tx is not and committed pending + receivedTx, pending, err = sim.TransactionByHash(bgCtx, signedTx.Hash()) + if err != nil { + t.Errorf("could not get transaction by hash %v: %v", signedTx.Hash(), err) + } + if pending { + t.Errorf("expected transaction to not be in pending state") + } + if receivedTx.Hash() != signedTx.Hash() { + t.Errorf("did not received committed transaction. expected hash %v got hash %v", signedTx.Hash(), receivedTx.Hash()) + } +} + +func TestSimulatedBackend_EstimateGas(t *testing.T) { + sim := NewSimulatedBackend( + core.GenesisAlloc{}, 10000000, + ) + defer sim.Close() + bgCtx := context.Background() + testAddr := crypto.PubkeyToAddress(testKey.PublicKey) + + gas, err := sim.EstimateGas(bgCtx, ethereum.CallMsg{ + From: testAddr, + To: &testAddr, + Value: big.NewInt(1000), + Data: []byte{}, + }) + if err != nil { + t.Errorf("could not estimate gas: %v", err) + } + + if gas != params.TxGas { + t.Errorf("expected 21000 gas cost for a transaction got %v", gas) + } +} + +func TestSimulatedBackend_HeaderByHash(t *testing.T) { + testAddr := crypto.PubkeyToAddress(testKey.PublicKey) + + sim := NewSimulatedBackend( + core.GenesisAlloc{ + testAddr: {Balance: big.NewInt(10000000000)}, + }, 10000000, + ) + defer sim.Close() + bgCtx := context.Background() + + header, err := sim.HeaderByNumber(bgCtx, nil) + if err != nil { + t.Errorf("could not get recent block: %v", err) + } + headerByHash, err := sim.HeaderByHash(bgCtx, header.Hash()) + if err != nil { + t.Errorf("could not get recent block: %v", err) + } + + if header.Hash() != headerByHash.Hash() { + t.Errorf("did not get expected block") + } +} + +func TestSimulatedBackend_HeaderByNumber(t *testing.T) { + testAddr := crypto.PubkeyToAddress(testKey.PublicKey) + + sim := NewSimulatedBackend( + core.GenesisAlloc{ + testAddr: {Balance: big.NewInt(10000000000)}, + }, 10000000, + ) + defer sim.Close() + bgCtx := context.Background() + + latestBlockHeader, err := sim.HeaderByNumber(bgCtx, nil) + if err != nil { + t.Errorf("could not get header for tip of chain: %v", err) + } + if latestBlockHeader == nil { + t.Errorf("received a nil block header") + } + if latestBlockHeader.Number.Uint64() != uint64(0) { + t.Errorf("expected block header number 0, instead got %v", latestBlockHeader.Number.Uint64()) + } + + sim.Commit() + + latestBlockHeader, err = sim.HeaderByNumber(bgCtx, nil) + if err != nil { + t.Errorf("could not get header for blockheight of 1: %v", err) + } + + blockHeader, err := sim.HeaderByNumber(bgCtx, big.NewInt(1)) + if err != nil { + t.Errorf("could not get header for blockheight of 1: %v", err) + } + + if blockHeader.Hash() != latestBlockHeader.Hash() { + t.Errorf("block header and latest block header are not the same") + } + if blockHeader.Number.Int64() != int64(1) { + t.Errorf("did not get blockheader for block 1. instead got block %v", blockHeader.Number.Int64()) + } + + block, err := sim.BlockByNumber(bgCtx, big.NewInt(1)) + if err != nil { + t.Errorf("could not get block for blockheight of 1: %v", err) + } + + if block.Hash() != blockHeader.Hash() { + t.Errorf("block hash and block header hash do not match. expected %v, got %v", block.Hash(), blockHeader.Hash()) + } +} + +func TestSimulatedBackend_TransactionCount(t *testing.T) { + testAddr := crypto.PubkeyToAddress(testKey.PublicKey) + + sim := NewSimulatedBackend( + core.GenesisAlloc{ + testAddr: {Balance: big.NewInt(10000000000)}, + }, 10000000, + ) + defer sim.Close() + bgCtx := context.Background() + currentBlock, err := sim.BlockByNumber(bgCtx, nil) + if err != nil || currentBlock == nil { + t.Error("could not get current block") + } + + count, err := sim.TransactionCount(bgCtx, currentBlock.Hash()) + if err != nil { + t.Error("could not get current block's transaction count") + } + + if count != 0 { + t.Errorf("expected transaction count of %v does not match actual count of %v", 0, count) + } + + // create a signed transaction to send + tx := types.NewTransaction(uint64(0), testAddr, big.NewInt(1000), params.TxGas, big.NewInt(1), nil) + signedTx, err := types.SignTx(tx, types.HomesteadSigner{}, testKey) + if err != nil { + t.Errorf("could not sign tx: %v", err) + } + + // send tx to simulated backend + err = sim.SendTransaction(bgCtx, signedTx) + if err != nil { + t.Errorf("could not add tx to pending block: %v", err) + } + + sim.Commit() + + lastBlock, err := sim.BlockByNumber(bgCtx, nil) + if err != nil { + t.Errorf("could not get header for tip of chain: %v", err) + } + + count, err = sim.TransactionCount(bgCtx, lastBlock.Hash()) + if err != nil { + t.Error("could not get current block's transaction count") + } + + if count != 1 { + t.Errorf("expected transaction count of %v does not match actual count of %v", 1, count) + } +} + +func TestSimulatedBackend_TransactionInBlock(t *testing.T) { + testAddr := crypto.PubkeyToAddress(testKey.PublicKey) + + sim := NewSimulatedBackend( + core.GenesisAlloc{ + testAddr: {Balance: big.NewInt(10000000000)}, + }, 10000000, + ) + defer sim.Close() + bgCtx := context.Background() + + transaction, err := sim.TransactionInBlock(bgCtx, sim.pendingBlock.Hash(), uint(0)) + if err == nil && err != errTransactionDoesNotExist { + t.Errorf("expected a transaction does not exist error to be received but received %v", err) + } + if transaction != nil { + t.Errorf("expected transaction to be nil but received %v", transaction) + } + + // expect pending nonce to be 0 since account has not been used + pendingNonce, err := sim.PendingNonceAt(bgCtx, testAddr) + if err != nil { + t.Errorf("did not get the pending nonce: %v", err) + } + + if pendingNonce != uint64(0) { + t.Errorf("expected pending nonce of 0 got %v", pendingNonce) + } + + // create a signed transaction to send + tx := types.NewTransaction(uint64(0), testAddr, big.NewInt(1000), params.TxGas, big.NewInt(1), nil) + signedTx, err := types.SignTx(tx, types.HomesteadSigner{}, testKey) + if err != nil { + t.Errorf("could not sign tx: %v", err) + } + + // send tx to simulated backend + err = sim.SendTransaction(bgCtx, signedTx) + if err != nil { + t.Errorf("could not add tx to pending block: %v", err) + } + + sim.Commit() + + lastBlock, err := sim.BlockByNumber(bgCtx, nil) + if err != nil { + t.Errorf("could not get header for tip of chain: %v", err) + } + + transaction, err = sim.TransactionInBlock(bgCtx, lastBlock.Hash(), uint(1)) + if err == nil && err != errTransactionDoesNotExist { + t.Errorf("expected a transaction does not exist error to be received but received %v", err) + } + if transaction != nil { + t.Errorf("expected transaction to be nil but received %v", transaction) + } + + transaction, err = sim.TransactionInBlock(bgCtx, lastBlock.Hash(), uint(0)) + if err != nil { + t.Errorf("could not get transaction in the lastest block with hash %v: %v", lastBlock.Hash().String(), err) + } + + if signedTx.Hash().String() != transaction.Hash().String() { + t.Errorf("received transaction that did not match the sent transaction. expected hash %v, got hash %v", signedTx.Hash().String(), transaction.Hash().String()) + } +} + +func TestSimulatedBackend_PendingNonceAt(t *testing.T) { + testAddr := crypto.PubkeyToAddress(testKey.PublicKey) + + sim := NewSimulatedBackend( + core.GenesisAlloc{ + testAddr: {Balance: big.NewInt(10000000000)}, + }, 10000000, + ) + defer sim.Close() + bgCtx := context.Background() + + // expect pending nonce to be 0 since account has not been used + pendingNonce, err := sim.PendingNonceAt(bgCtx, testAddr) + if err != nil { + t.Errorf("did not get the pending nonce: %v", err) + } + + if pendingNonce != uint64(0) { + t.Errorf("expected pending nonce of 0 got %v", pendingNonce) + } + + // create a signed transaction to send + tx := types.NewTransaction(uint64(0), testAddr, big.NewInt(1000), params.TxGas, big.NewInt(1), nil) + signedTx, err := types.SignTx(tx, types.HomesteadSigner{}, testKey) + if err != nil { + t.Errorf("could not sign tx: %v", err) + } + + // send tx to simulated backend + err = sim.SendTransaction(bgCtx, signedTx) + if err != nil { + t.Errorf("could not add tx to pending block: %v", err) + } + + // expect pending nonce to be 1 since account has submitted one transaction + pendingNonce, err = sim.PendingNonceAt(bgCtx, testAddr) + if err != nil { + t.Errorf("did not get the pending nonce: %v", err) + } + + if pendingNonce != uint64(1) { + t.Errorf("expected pending nonce of 1 got %v", pendingNonce) + } + + // make a new transaction with a nonce of 1 + tx = types.NewTransaction(uint64(1), testAddr, big.NewInt(1000), params.TxGas, big.NewInt(1), nil) + signedTx, err = types.SignTx(tx, types.HomesteadSigner{}, testKey) + if err != nil { + t.Errorf("could not sign tx: %v", err) + } + err = sim.SendTransaction(bgCtx, signedTx) + if err != nil { + t.Errorf("could not send tx: %v", err) + } + + // expect pending nonce to be 2 since account now has two transactions + pendingNonce, err = sim.PendingNonceAt(bgCtx, testAddr) + if err != nil { + t.Errorf("did not get the pending nonce: %v", err) + } + + if pendingNonce != uint64(2) { + t.Errorf("expected pending nonce of 2 got %v", pendingNonce) + } +} + +func TestSimulatedBackend_TransactionReceipt(t *testing.T) { + testAddr := crypto.PubkeyToAddress(testKey.PublicKey) + + sim := NewSimulatedBackend( + core.GenesisAlloc{ + testAddr: {Balance: big.NewInt(10000000000)}, + }, 10000000, + ) + defer sim.Close() + bgCtx := context.Background() + + // create a signed transaction to send + tx := types.NewTransaction(uint64(0), testAddr, big.NewInt(1000), params.TxGas, big.NewInt(1), nil) + signedTx, err := types.SignTx(tx, types.HomesteadSigner{}, testKey) + if err != nil { + t.Errorf("could not sign tx: %v", err) + } + + // send tx to simulated backend + err = sim.SendTransaction(bgCtx, signedTx) + if err != nil { + t.Errorf("could not add tx to pending block: %v", err) + } + sim.Commit() + + receipt, err := sim.TransactionReceipt(bgCtx, signedTx.Hash()) + if err != nil { + t.Errorf("could not get transaction receipt: %v", err) + } + + if receipt.ContractAddress != testAddr && receipt.TxHash != signedTx.Hash() { + t.Errorf("received receipt is not correct: %v", receipt) + } +} + +func TestSimulatedBackend_SuggestGasPrice(t *testing.T) { + sim := NewSimulatedBackend( + core.GenesisAlloc{}, + 10000000, + ) + defer sim.Close() + bgCtx := context.Background() + gasPrice, err := sim.SuggestGasPrice(bgCtx) + if err != nil { + t.Errorf("could not get gas price: %v", err) + } + if gasPrice.Uint64() != uint64(1) { + t.Errorf("gas price was not expected value of 1. actual: %v", gasPrice.Uint64()) + } +} + +func TestSimulatedBackend_PendingCodeAt(t *testing.T) { + testAddr := crypto.PubkeyToAddress(testKey.PublicKey) + sim := NewSimulatedBackend( + core.GenesisAlloc{ + testAddr: {Balance: big.NewInt(10000000000)}, + }, + 10000000, + ) + defer sim.Close() + bgCtx := context.Background() + code, err := sim.CodeAt(bgCtx, testAddr, nil) + if err != nil { + t.Errorf("could not get code at test addr: %v", err) + } + if len(code) != 0 { + t.Errorf("got code for account that does not have contract code") + } + + parsed, err := abi.JSON(strings.NewReader(abiJSON)) + if err != nil { + t.Errorf("could not get code at test addr: %v", err) + } + auth := bind.NewKeyedTransactor(testKey) + contractAddr, tx, contract, err := bind.DeployContract(auth, parsed, common.FromHex(abiBin), sim) + if err != nil { + t.Errorf("could not deploy contract: %v tx: %v contract: %v", err, tx, contract) + } + + code, err = sim.PendingCodeAt(bgCtx, contractAddr) + if err != nil { + t.Errorf("could not get code at test addr: %v", err) + } + if len(code) == 0 { + t.Errorf("did not get code for account that has contract code") + } + // ensure code received equals code deployed + if !bytes.Equal(code, common.FromHex(deployedCode)) { + t.Errorf("code received did not match expected deployed code:\n expected %v\n actual %v", common.FromHex(deployedCode), code) + } +} + +func TestSimulatedBackend_CodeAt(t *testing.T) { + testAddr := crypto.PubkeyToAddress(testKey.PublicKey) + sim := NewSimulatedBackend( + core.GenesisAlloc{ + testAddr: {Balance: big.NewInt(10000000000)}, + }, + 10000000, + ) + defer sim.Close() + bgCtx := context.Background() + code, err := sim.CodeAt(bgCtx, testAddr, nil) + if err != nil { + t.Errorf("could not get code at test addr: %v", err) + } + if len(code) != 0 { + t.Errorf("got code for account that does not have contract code") + } + + parsed, err := abi.JSON(strings.NewReader(abiJSON)) + if err != nil { + t.Errorf("could not get code at test addr: %v", err) + } + auth := bind.NewKeyedTransactor(testKey) + contractAddr, tx, contract, err := bind.DeployContract(auth, parsed, common.FromHex(abiBin), sim) + if err != nil { + t.Errorf("could not deploy contract: %v tx: %v contract: %v", err, tx, contract) + } + + sim.Commit() + code, err = sim.CodeAt(bgCtx, contractAddr, nil) + if err != nil { + t.Errorf("could not get code at test addr: %v", err) + } + if len(code) == 0 { + t.Errorf("did not get code for account that has contract code") + } + // ensure code received equals code deployed + if !bytes.Equal(code, common.FromHex(deployedCode)) { + t.Errorf("code received did not match expected deployed code:\n expected %v\n actual %v", common.FromHex(deployedCode), code) + } +} + +// When receive("X") is called with sender 0x00... and value 1, it produces this tx receipt: +// receipt{status=1 cgas=23949 bloom=00000000004000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000800000000000000000000000000000000000040200000000000000000000000000000000001000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000080000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 logs=[log: b6818c8064f645cd82d99b59a1a267d6d61117ef [75fd880d39c1daf53b6547ab6cb59451fc6452d27caa90e5b6649dd8293b9eed] 000000000000000000000000376c47978271565f56deb45495afa69e59c16ab200000000000000000000000000000000000000000000000000000000000000010000000000000000000000000000000000000000000000000000000000000060000000000000000000000000000000000000000000000000000000000000000158 9ae378b6d4409eada347a5dc0c180f186cb62dc68fcc0f043425eb917335aa28 0 95d429d309bb9d753954195fe2d69bd140b4ae731b9b5b605c34323de162cf00 0]} +func TestSimulatedBackend_PendingAndCallContract(t *testing.T) { + testAddr := crypto.PubkeyToAddress(testKey.PublicKey) + sim := NewSimulatedBackend( + core.GenesisAlloc{ + testAddr: {Balance: big.NewInt(10000000000)}, + }, + 10000000, + ) + defer sim.Close() + bgCtx := context.Background() + + parsed, err := abi.JSON(strings.NewReader(abiJSON)) + if err != nil { + t.Errorf("could not get code at test addr: %v", err) + } + contractAuth := bind.NewKeyedTransactor(testKey) + addr, _, _, err := bind.DeployContract(contractAuth, parsed, common.FromHex(abiBin), sim) + if err != nil { + t.Errorf("could not deploy contract: %v", err) + } + + input, err := parsed.Pack("receive", []byte("X")) + if err != nil { + t.Errorf("could pack receive function on contract: %v", err) + } + + // make sure you can call the contract in pending state + res, err := sim.PendingCallContract(bgCtx, ethereum.CallMsg{ + From: testAddr, + To: &addr, + Data: input, + }) + if err != nil { + t.Errorf("could not call receive method on contract: %v", err) + } + if len(res) == 0 { + t.Errorf("result of contract call was empty: %v", res) + } + + // while comparing against the byte array is more exact, also compare against the human readable string for readability + if !bytes.Equal(res, expectedReturn) || !strings.Contains(string(res), "hello world") { + t.Errorf("response from calling contract was expected to be 'hello world' instead received %v", string(res)) + } + + sim.Commit() + + // make sure you can call the contract + res, err = sim.CallContract(bgCtx, ethereum.CallMsg{ + From: testAddr, + To: &addr, + Data: input, + }, nil) + if err != nil { + t.Errorf("could not call receive method on contract: %v", err) + } + if len(res) == 0 { + t.Errorf("result of contract call was empty: %v", res) + } + + if !bytes.Equal(res, expectedReturn) || !strings.Contains(string(res), "hello world") { + t.Errorf("response from calling contract was expected to be 'hello world' instead received %v", string(res)) + } } From a67fe48b43bfaa750d264bab54c1ea1a15c2a8f7 Mon Sep 17 00:00:00 2001 From: Gerald Nash Date: Fri, 20 Dec 2019 07:19:01 -0500 Subject: [PATCH 034/128] Change file extension of the ./tests/fuzzers README (#20474) --- tests/fuzzers/{README.txt => README.md} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename tests/fuzzers/{README.txt => README.md} (100%) diff --git a/tests/fuzzers/README.txt b/tests/fuzzers/README.md similarity index 100% rename from tests/fuzzers/README.txt rename to tests/fuzzers/README.md From 3bb6815fc1ade61e85b7039cf2e08a0871bb47e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Felf=C3=B6ldi=20Zsolt?= Date: Wed, 25 Dec 2019 02:06:00 +0100 Subject: [PATCH 035/128] les: do not disconnect another server (#20453) --- les/peer.go | 16 +++++++++++----- les/server_handler.go | 13 +++++++++---- 2 files changed, 20 insertions(+), 9 deletions(-) diff --git a/les/peer.go b/les/peer.go index ab5b30a657..801147df6b 100644 --- a/les/peer.go +++ b/les/peer.go @@ -113,7 +113,7 @@ type peer struct { fcParams flowcontrol.ServerParams fcCosts requestCostTable - trusted bool + trusted, server bool onlyAnnounce bool chainSince, chainRecent uint64 stateSince, stateRecent uint64 @@ -675,11 +675,16 @@ func (p *peer) Handshake(td *big.Int, head common.Hash, headNum uint64, genesis } if server != nil { - if recv.get("announceType", &p.announceType) != nil { - // set default announceType on server side - p.announceType = announceTypeSimple + p.server = recv.get("flowControl/MRR", nil) == nil + if p.server { + p.announceType = announceTypeNone // connected to another server, send no messages + } else { + if recv.get("announceType", &p.announceType) != nil { + // set default announceType on server side + p.announceType = announceTypeSimple + } + p.fcClient = flowcontrol.NewClientNode(server.fcManager, server.defParams) } - p.fcClient = flowcontrol.NewClientNode(server.fcManager, server.defParams) } else { if recv.get("serveChainSince", &p.chainSince) != nil { p.onlyAnnounce = true @@ -726,6 +731,7 @@ func (p *peer) Handshake(td *big.Int, head common.Hash, headNum uint64, genesis } } } + p.server = true } p.headInfo = &announceData{Td: rTd, Hash: rHash, Number: rNum} return nil diff --git a/les/server_handler.go b/les/server_handler.go index 16249ef1ba..4b505c2bc4 100644 --- a/les/server_handler.go +++ b/les/server_handler.go @@ -108,10 +108,6 @@ func (h *serverHandler) runPeer(version uint, p *p2p.Peer, rw p2p.MsgReadWriter) } func (h *serverHandler) handle(p *peer) error { - // Reject light clients if server is not synced. - if !h.synced() { - return p2p.DiscRequested - } p.Log().Debug("Light Ethereum peer connected", "name", p.Name()) // Execute the LES handshake @@ -125,6 +121,15 @@ func (h *serverHandler) handle(p *peer) error { p.Log().Debug("Light Ethereum handshake failed", "err", err) return err } + if p.server { + // connected to another server, no messages expected, just wait for disconnection + _, err := p.rw.ReadMsg() + return err + } + // Reject light clients if server is not synced. + if !h.synced() { + return p2p.DiscRequested + } defer p.fcClient.Disconnect() // Disconnect the inbound peer if it's rejected by clientPool From b7cf41e4b38812fafab43bb353b7637c4a75cc5f Mon Sep 17 00:00:00 2001 From: Sylvain Laurent Date: Mon, 6 Jan 2020 12:03:26 +0100 Subject: [PATCH 036/128] accounts/abi: fix method constant flag for solidity 6.0 (#20482) --- accounts/abi/abi.go | 16 +++++----- accounts/abi/bind/bind_test.go | 55 ++++++++++++++++++++++++++++++++++ 2 files changed, 64 insertions(+), 7 deletions(-) diff --git a/accounts/abi/abi.go b/accounts/abi/abi.go index 603e956b9d..fdb4c48b39 100644 --- a/accounts/abi/abi.go +++ b/accounts/abi/abi.go @@ -108,12 +108,13 @@ func (abi ABI) UnpackIntoMap(v map[string]interface{}, name string, data []byte) // UnmarshalJSON implements json.Unmarshaler interface func (abi *ABI) UnmarshalJSON(data []byte) error { var fields []struct { - Type string - Name string - Constant bool - Anonymous bool - Inputs []Argument - Outputs []Argument + Type string + Name string + Constant bool + StateMutability string + Anonymous bool + Inputs []Argument + Outputs []Argument } if err := json.Unmarshal(data, &fields); err != nil { return err @@ -134,10 +135,11 @@ func (abi *ABI) UnmarshalJSON(data []byte) error { name = fmt.Sprintf("%s%d", field.Name, idx) _, ok = abi.Methods[name] } + isConst := field.Constant || field.StateMutability == "pure" || field.StateMutability == "view" abi.Methods[name] = Method{ Name: name, RawName: field.Name, - Const: field.Constant, + Const: isConst, Inputs: field.Inputs, Outputs: field.Outputs, } diff --git a/accounts/abi/bind/bind_test.go b/accounts/abi/bind/bind_test.go index 7594af775e..fa1a7b1ca7 100644 --- a/accounts/abi/bind/bind_test.go +++ b/accounts/abi/bind/bind_test.go @@ -1530,6 +1530,61 @@ var bindTests = []struct { nil, []string{"ContractOne", "ContractTwo", "ExternalLib"}, }, + // Test the existence of the free retrieval calls + { + `PureAndView`, + `pragma solidity >=0.6.0; + contract PureAndView { + function PureFunc() public pure returns (uint) { + return 42; + } + function ViewFunc() public view returns (uint) { + return block.number; + } + } + `, + []string{`608060405234801561001057600080fd5b5060b68061001f6000396000f3fe6080604052348015600f57600080fd5b506004361060325760003560e01c806376b5686a146037578063bb38c66c146053575b600080fd5b603d606f565b6040518082815260200191505060405180910390f35b60596077565b6040518082815260200191505060405180910390f35b600043905090565b6000602a90509056fea2646970667358221220d158c2ab7fdfce366a7998ec79ab84edd43b9815630bbaede2c760ea77f29f7f64736f6c63430006000033`}, + []string{`[{"inputs": [],"name": "PureFunc","outputs": [{"internalType": "uint256","name": "","type": "uint256"}],"stateMutability": "pure","type": "function"},{"inputs": [],"name": "ViewFunc","outputs": [{"internalType": "uint256","name": "","type": "uint256"}],"stateMutability": "view","type": "function"}]`}, + ` + "math/big" + + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/accounts/abi/bind/backends" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/crypto" + `, + ` + // Generate a new random account and a funded simulator + key, _ := crypto.GenerateKey() + auth := bind.NewKeyedTransactor(key) + + sim := backends.NewSimulatedBackend(core.GenesisAlloc{auth.From: {Balance: big.NewInt(10000000000)}}, 10000000) + defer sim.Close() + + // Deploy a tester contract and execute a structured call on it + _, _, pav, err := DeployPureAndView(auth, sim) + if err != nil { + t.Fatalf("Failed to deploy PureAndView contract: %v", err) + } + sim.Commit() + + // This test the existence of the free retreiver call for view and pure functions + if num, err := pav.PureFunc(nil); err != nil { + t.Fatalf("Failed to call anonymous field retriever: %v", err) + } else if num.Cmp(big.NewInt(42)) != 0 { + t.Fatalf("Retrieved value mismatch: have %v, want %v", num, 42) + } + if num, err := pav.ViewFunc(nil); err != nil { + t.Fatalf("Failed to call anonymous field retriever: %v", err) + } else if num.Cmp(big.NewInt(1)) != 0 { + t.Fatalf("Retrieved value mismatch: have %v, want %v", num, 1) + } + `, + nil, + nil, + nil, + nil, + }, } // Tests that packages generated by the binder can be successfully compiled and From 2eeb8dd27188f6500ae28576aa258ff0b4e4d3c0 Mon Sep 17 00:00:00 2001 From: Chris Pacia Date: Mon, 6 Jan 2020 06:06:29 -0500 Subject: [PATCH 037/128] rpc: add DialWebsocketWithDialer (#20471) This commit intents to replicate the DialHTTPWithClient function which allows creating a RPC Client using a custom dialer but for websockets. We introduce a new DialWebsocketWithDialer function which allows the caller to instantiate a new websocket client using a custom dialer. --- rpc/websocket.go | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/rpc/websocket.go b/rpc/websocket.go index e34241e47f..b7ec56c6a6 100644 --- a/rpc/websocket.go +++ b/rpc/websocket.go @@ -124,21 +124,13 @@ func (e wsHandshakeError) Error() string { return s } -// DialWebsocket creates a new RPC client that communicates with a JSON-RPC server -// that is listening on the given endpoint. -// -// The context is used for the initial connection establishment. It does not -// affect subsequent interactions with the client. -func DialWebsocket(ctx context.Context, endpoint, origin string) (*Client, error) { +// DialWebsocketWithDialer creates a new RPC client that communicates with a JSON-RPC server +// that is listening on the given endpoint using the provided dialer. +func DialWebsocketWithDialer(ctx context.Context, endpoint, origin string, dialer websocket.Dialer) (*Client, error) { endpoint, header, err := wsClientHeaders(endpoint, origin) if err != nil { return nil, err } - dialer := websocket.Dialer{ - ReadBufferSize: wsReadBuffer, - WriteBufferSize: wsWriteBuffer, - WriteBufferPool: wsBufferPool, - } return newClient(ctx, func(ctx context.Context) (ServerCodec, error) { conn, resp, err := dialer.DialContext(ctx, endpoint, header) if err != nil { @@ -152,6 +144,20 @@ func DialWebsocket(ctx context.Context, endpoint, origin string) (*Client, error }) } +// DialWebsocket creates a new RPC client that communicates with a JSON-RPC server +// that is listening on the given endpoint. +// +// The context is used for the initial connection establishment. It does not +// affect subsequent interactions with the client. +func DialWebsocket(ctx context.Context, endpoint, origin string) (*Client, error) { + dialer := websocket.Dialer{ + ReadBufferSize: wsReadBuffer, + WriteBufferSize: wsWriteBuffer, + WriteBufferPool: wsBufferPool, + } + return DialWebsocketWithDialer(ctx, endpoint, origin, dialer) +} + func wsClientHeaders(endpoint, origin string) (string, http.Header, error) { endpointURL, err := url.Parse(endpoint) if err != nil { From 433937fb42a1cb43fda247313797874e92d3c9fe Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Mon, 6 Jan 2020 12:07:21 +0100 Subject: [PATCH 038/128] cmd/geth: fix forked exe leak in console tests (#20480) --- cmd/geth/consolecmd_test.go | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/cmd/geth/consolecmd_test.go b/cmd/geth/consolecmd_test.go index 33c83b7ede..45d4daff07 100644 --- a/cmd/geth/consolecmd_test.go +++ b/cmd/geth/consolecmd_test.go @@ -87,11 +87,14 @@ func TestIPCAttachWelcome(t *testing.T) { "--port", "0", "--maxpeers", "0", "--nodiscover", "--nat", "none", "--etherbase", coinbase, "--shh", "--ipcpath", ipc) + defer func() { + geth.Interrupt() + geth.ExpectExit() + }() + waitForEndpoint(t, ipc, 3*time.Second) testAttachWelcome(t, geth, "ipc:"+ipc, ipcAPIs) - geth.Interrupt() - geth.ExpectExit() } func TestHTTPAttachWelcome(t *testing.T) { @@ -100,13 +103,14 @@ func TestHTTPAttachWelcome(t *testing.T) { geth := runGeth(t, "--port", "0", "--maxpeers", "0", "--nodiscover", "--nat", "none", "--etherbase", coinbase, "--rpc", "--rpcport", port) + defer func() { + geth.Interrupt() + geth.ExpectExit() + }() endpoint := "http://127.0.0.1:" + port waitForEndpoint(t, endpoint, 3*time.Second) testAttachWelcome(t, geth, endpoint, httpAPIs) - - geth.Interrupt() - geth.ExpectExit() } func TestWSAttachWelcome(t *testing.T) { @@ -116,13 +120,14 @@ func TestWSAttachWelcome(t *testing.T) { geth := runGeth(t, "--port", "0", "--maxpeers", "0", "--nodiscover", "--nat", "none", "--etherbase", coinbase, "--ws", "--wsport", port) + defer func() { + geth.Interrupt() + geth.ExpectExit() + }() endpoint := "ws://127.0.0.1:" + port waitForEndpoint(t, endpoint, 3*time.Second) testAttachWelcome(t, geth, endpoint, httpAPIs) - - geth.Interrupt() - geth.ExpectExit() } func testAttachWelcome(t *testing.T, geth *testgeth, endpoint, apis string) { From 7a509b47321a28e36f437170a3cc5c283ee7909d Mon Sep 17 00:00:00 2001 From: Prince Sinha Date: Mon, 6 Jan 2020 16:55:38 +0530 Subject: [PATCH 039/128] internal/ethapi: fix encoding of uncle headers and pending blocks (#20460) Fixes #19024 Fixes #19332 --- internal/ethapi/api.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 2b299d3859..37433f335a 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -642,7 +642,7 @@ func (s *PublicBlockChainAPI) GetBlockByNumber(ctx context.Context, number rpc.B response, err := s.rpcMarshalBlock(block, true, fullTx) if err == nil && number == rpc.PendingBlockNumber { // Pending blocks need to nil out a few fields - for _, field := range []string{"hash", "nonce", "miner"} { + for _, field := range []string{"hash", "nonce", "miner", "number"} { response[field] = nil } } @@ -1088,7 +1088,9 @@ func (s *PublicBlockChainAPI) rpcMarshalBlock(b *types.Block, inclTx bool, fullT if err != nil { return nil, err } - fields["totalDifficulty"] = (*hexutil.Big)(s.b.GetTd(b.Hash())) + if inclTx { + fields["totalDifficulty"] = (*hexutil.Big)(s.b.GetTd(b.Hash())) + } return fields, err } From 35f95aef6fecca68eb18035bc81ad7a4c712908f Mon Sep 17 00:00:00 2001 From: Kumar Anirudha Date: Tue, 7 Jan 2020 14:38:33 +0530 Subject: [PATCH 040/128] cmd/puppeth: change dashboard title to not use "testnet" (#20513) --- cmd/puppeth/module_dashboard.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/puppeth/module_dashboard.go b/cmd/puppeth/module_dashboard.go index 9a77587b4a..39ccdd9aa0 100644 --- a/cmd/puppeth/module_dashboard.go +++ b/cmd/puppeth/module_dashboard.go @@ -41,7 +41,7 @@ var dashboardContent = ` - {{.NetworkTitle}}: Ethereum Testnet + {{.NetworkTitle}}: Network Dashboard From c6285e6437cd2d732721e170a5e4170af1328435 Mon Sep 17 00:00:00 2001 From: gary rong Date: Tue, 7 Jan 2020 17:24:21 +0800 Subject: [PATCH 041/128] les/checkpointoracle: move oracle into its own package (#20508) * les: move the checkpoint oracle into its own package It's first step of refactor LES package. LES package basically can be divided into LES client and LES server. However both sides will use checkpoint package for status retrieval and verification. So this PR moves checkpoint oracle into a separate package * les: address comments --- contracts/checkpointoracle/oracle.go | 10 ++- go.sum | 1 + les/api.go | 2 +- .../oracle.go} | 67 ++++++++++--------- les/client.go | 5 +- les/commons.go | 3 +- les/peer.go | 4 +- les/server.go | 5 +- les/sync.go | 6 +- les/sync_test.go | 8 +-- les/test_helper.go | 13 ++-- 11 files changed, 71 insertions(+), 53 deletions(-) rename les/{checkpointoracle.go => checkpointoracle/oracle.go} (65%) diff --git a/contracts/checkpointoracle/oracle.go b/contracts/checkpointoracle/oracle.go index 2d725397bd..1f273272ab 100644 --- a/contracts/checkpointoracle/oracle.go +++ b/contracts/checkpointoracle/oracle.go @@ -29,8 +29,9 @@ import ( "github.com/ethereum/go-ethereum/core/types" ) -// CheckpointOracle is a Go wrapper around an on-chain light client checkpoint oracle. +// CheckpointOracle is a Go wrapper around an on-chain checkpoint oracle contract. type CheckpointOracle struct { + address common.Address contract *contract.CheckpointOracle } @@ -40,7 +41,12 @@ func NewCheckpointOracle(contractAddr common.Address, backend bind.ContractBacke if err != nil { return nil, err } - return &CheckpointOracle{contract: c}, nil + return &CheckpointOracle{address: contractAddr, contract: c}, nil +} + +// ContractAddr returns the address of contract. +func (oracle *CheckpointOracle) ContractAddr() common.Address { + return oracle.address } // Contract returns the underlying contract instance. diff --git a/go.sum b/go.sum index edbb5ea2e0..4d18c8c20e 100644 --- a/go.sum +++ b/go.sum @@ -101,6 +101,7 @@ github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883 h1:FSeK4fZCo github.com/influxdata/influxdb v1.2.3-0.20180221223340-01288bdb0883/go.mod h1:qZna6X/4elxqT3yI9iZYdZrWWdeFOOprn86kgg4+IzY= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458 h1:6OvNmYgJyexcZ3pYbTI9jWx5tHo1Dee/tWbLMfPe2TA= github.com/jackpal/go-nat-pmp v1.0.2-0.20160603034137-1fa385a6f458/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21 h1:F/iKcka0K2LgnKy/fgSBf235AETtm1n1TvBzqu40LE0= github.com/julienschmidt/httprouter v1.1.1-0.20170430222011-975b5c4c7c21/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= diff --git a/les/api.go b/les/api.go index ad511c9d6b..f9b8c34458 100644 --- a/les/api.go +++ b/les/api.go @@ -350,5 +350,5 @@ func (api *PrivateLightAPI) GetCheckpointContractAddress() (string, error) { if api.backend.oracle == nil { return "", errNotActivated } - return api.backend.oracle.config.Address.Hex(), nil + return api.backend.oracle.Contract().ContractAddr().Hex(), nil } diff --git a/les/checkpointoracle.go b/les/checkpointoracle/oracle.go similarity index 65% rename from les/checkpointoracle.go rename to les/checkpointoracle/oracle.go index 5494e3d6d9..c3983e1a95 100644 --- a/les/checkpointoracle.go +++ b/les/checkpointoracle/oracle.go @@ -14,7 +14,10 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -package les +// Package checkpointoracle is a wrapper of checkpoint oracle contract with +// additional rules defined. This package can be used both in LES client or +// server side for offering oracle related APIs. +package checkpointoracle import ( "encoding/binary" @@ -28,10 +31,10 @@ import ( "github.com/ethereum/go-ethereum/params" ) -// checkpointOracle is responsible for offering the latest stable checkpoint -// generated and announced by the contract admins on-chain. The checkpoint is -// verified by clients locally during the checkpoint syncing. -type checkpointOracle struct { +// CheckpointOracle is responsible for offering the latest stable checkpoint +// generated and announced by the contract admins on-chain. The checkpoint can +// be verified by clients locally during the checkpoint syncing. +type CheckpointOracle struct { config *params.CheckpointOracleConfig contract *checkpointoracle.CheckpointOracle @@ -39,8 +42,8 @@ type checkpointOracle struct { getLocal func(uint64) params.TrustedCheckpoint // Function used to retrieve local checkpoint } -// newCheckpointOracle returns a checkpoint registrar handler. -func newCheckpointOracle(config *params.CheckpointOracleConfig, getLocal func(uint64) params.TrustedCheckpoint) *checkpointOracle { +// New creates a checkpoint oracle handler with given configs and callback. +func New(config *params.CheckpointOracleConfig, getLocal func(uint64) params.TrustedCheckpoint) *CheckpointOracle { if config == nil { log.Info("Checkpoint registrar is not enabled") return nil @@ -51,41 +54,46 @@ func newCheckpointOracle(config *params.CheckpointOracleConfig, getLocal func(ui } log.Info("Configured checkpoint registrar", "address", config.Address, "signers", len(config.Signers), "threshold", config.Threshold) - return &checkpointOracle{ + return &CheckpointOracle{ config: config, getLocal: getLocal, } } -// start binds the registrar contract and start listening to the -// newCheckpointEvent for the server side. -func (reg *checkpointOracle) start(backend bind.ContractBackend) { - contract, err := checkpointoracle.NewCheckpointOracle(reg.config.Address, backend) +// Start binds the contract backend, initializes the oracle instance +// and marks the status as available. +func (oracle *CheckpointOracle) Start(backend bind.ContractBackend) { + contract, err := checkpointoracle.NewCheckpointOracle(oracle.config.Address, backend) if err != nil { log.Error("Oracle contract binding failed", "err", err) return } - if !atomic.CompareAndSwapInt32(®.running, 0, 1) { + if !atomic.CompareAndSwapInt32(&oracle.running, 0, 1) { log.Error("Already bound and listening to registrar") return } - reg.contract = contract + oracle.contract = contract } -// isRunning returns an indicator whether the registrar is running. -func (reg *checkpointOracle) isRunning() bool { - return atomic.LoadInt32(®.running) == 1 +// IsRunning returns an indicator whether the oracle is running. +func (oracle *CheckpointOracle) IsRunning() bool { + return atomic.LoadInt32(&oracle.running) == 1 } -// stableCheckpoint returns the stable checkpoint which was generated by local +// Contract returns the underlying raw checkpoint oracle contract. +func (oracle *CheckpointOracle) Contract() *checkpointoracle.CheckpointOracle { + return oracle.contract +} + +// StableCheckpoint returns the stable checkpoint which was generated by local // indexers and announced by trusted signers. -func (reg *checkpointOracle) stableCheckpoint() (*params.TrustedCheckpoint, uint64) { +func (oracle *CheckpointOracle) StableCheckpoint() (*params.TrustedCheckpoint, uint64) { // Retrieve the latest checkpoint from the contract, abort if empty - latest, hash, height, err := reg.contract.Contract().GetLatestCheckpoint(nil) + latest, hash, height, err := oracle.contract.Contract().GetLatestCheckpoint(nil) if err != nil || (latest == 0 && hash == [32]byte{}) { return nil, 0 } - local := reg.getLocal(latest) + local := oracle.getLocal(latest) // The following scenarios may occur: // @@ -93,19 +101,18 @@ func (reg *checkpointOracle) stableCheckpoint() (*params.TrustedCheckpoint, uint // checkpoint which registered in the contract. // * local checkpoint doesn't match with the registered one. // - // In both cases, server won't send the **stable** checkpoint - // to the client(no worry, client can use hardcoded one instead). - if local.HashEqual(common.Hash(hash)) { + // In both cases, no stable checkpoint will be returned. + if local.HashEqual(hash) { return &local, height.Uint64() } return nil, 0 } -// verifySigners recovers the signer addresses according to the signature and +// VerifySigners recovers the signer addresses according to the signature and // checks whether there are enough approvals to finalize the checkpoint. -func (reg *checkpointOracle) verifySigners(index uint64, hash [32]byte, signatures [][]byte) (bool, []common.Address) { +func (oracle *CheckpointOracle) VerifySigners(index uint64, hash [32]byte, signatures [][]byte) (bool, []common.Address) { // Short circuit if the given signatures doesn't reach the threshold. - if len(signatures) < int(reg.config.Threshold) { + if len(signatures) < int(oracle.config.Threshold) { return false, nil } var ( @@ -128,7 +135,7 @@ func (reg *checkpointOracle) verifySigners(index uint64, hash [32]byte, signatur // hash = keccak256(checkpoint_index, section_head, cht_root, bloom_root) buf := make([]byte, 8) binary.BigEndian.PutUint64(buf, index) - data := append([]byte{0x19, 0x00}, append(reg.config.Address.Bytes(), append(buf, hash[:]...)...)...) + data := append([]byte{0x19, 0x00}, append(oracle.config.Address.Bytes(), append(buf, hash[:]...)...)...) signatures[i][64] -= 27 // Transform V from 27/28 to 0/1 according to the yellow paper for verification. pubkey, err := crypto.Ecrecover(crypto.Keccak256(data), signatures[i]) if err != nil { @@ -139,14 +146,14 @@ func (reg *checkpointOracle) verifySigners(index uint64, hash [32]byte, signatur if _, exist := checked[signer]; exist { continue } - for _, s := range reg.config.Signers { + for _, s := range oracle.config.Signers { if s == signer { signers = append(signers, signer) checked[signer] = struct{}{} } } } - threshold := reg.config.Threshold + threshold := oracle.config.Threshold if uint64(len(signers)) < threshold { log.Warn("Not enough signers to approve checkpoint", "signers", len(signers), "threshold", threshold) return false, nil diff --git a/les/client.go b/les/client.go index c460f4c09d..36d5a8d2fe 100644 --- a/les/client.go +++ b/les/client.go @@ -36,6 +36,7 @@ import ( "github.com/ethereum/go-ethereum/eth/gasprice" "github.com/ethereum/go-ethereum/event" "github.com/ethereum/go-ethereum/internal/ethapi" + "github.com/ethereum/go-ethereum/les/checkpointoracle" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/node" @@ -123,7 +124,7 @@ func New(ctx *node.ServiceContext, config *eth.Config) (*LightEthereum, error) { if oracle == nil { oracle = params.CheckpointOracles[genesisHash] } - leth.oracle = newCheckpointOracle(oracle, leth.localCheckpoint) + leth.oracle = checkpointoracle.New(oracle, leth.localCheckpoint) // Note: AddChildIndexer starts the update process for the child leth.bloomIndexer.AddChildIndexer(leth.bloomTrieIndexer) @@ -275,5 +276,5 @@ func (s *LightEthereum) SetContractBackend(backend bind.ContractBackend) { if s.oracle == nil { return } - s.oracle.start(backend) + s.oracle.Start(backend) } diff --git a/les/commons.go b/les/commons.go index ad3c5aef3d..b402c51769 100644 --- a/les/commons.go +++ b/les/commons.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/ethdb" + "github.com/ethereum/go-ethereum/les/checkpointoracle" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/p2p/discv5" @@ -63,7 +64,7 @@ type lesCommons struct { peers *peerSet chainReader chainReader chtIndexer, bloomTrieIndexer *core.ChainIndexer - oracle *checkpointOracle + oracle *checkpointoracle.CheckpointOracle closeCh chan struct{} wg sync.WaitGroup diff --git a/les/peer.go b/les/peer.go index 801147df6b..feb3910beb 100644 --- a/les/peer.go +++ b/les/peer.go @@ -616,8 +616,8 @@ func (p *peer) Handshake(td *big.Int, head common.Hash, headNum uint64, genesis // Add advertised checkpoint and register block height which // client can verify the checkpoint validity. - if server.oracle != nil && server.oracle.isRunning() { - cp, height := server.oracle.stableCheckpoint() + if server.oracle != nil && server.oracle.IsRunning() { + cp, height := server.oracle.StableCheckpoint() if cp != nil { send = send.add("checkpoint/value", cp) send = send.add("checkpoint/registerHeight", height) diff --git a/les/server.go b/les/server.go index e68903dd81..664eba9717 100644 --- a/les/server.go +++ b/les/server.go @@ -24,6 +24,7 @@ import ( "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/eth" + "github.com/ethereum/go-ethereum/les/checkpointoracle" "github.com/ethereum/go-ethereum/les/flowcontrol" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/log" @@ -96,7 +97,7 @@ func NewLesServer(e *eth.Ethereum, config *eth.Config) (*LesServer, error) { if oracle == nil { oracle = params.CheckpointOracles[e.BlockChain().Genesis().Hash()] } - srv.oracle = newCheckpointOracle(oracle, srv.localCheckpoint) + srv.oracle = checkpointoracle.New(oracle, srv.localCheckpoint) // Initialize server capacity management fields. srv.defParams = flowcontrol.ServerParams{ @@ -216,7 +217,7 @@ func (s *LesServer) SetContractBackend(backend bind.ContractBackend) { if s.oracle == nil { return } - s.oracle.start(backend) + s.oracle.Start(backend) } // capacityManagement starts an event handler loop that updates the recharge curve of diff --git a/les/sync.go b/les/sync.go index 1214fefcaf..207686403f 100644 --- a/les/sync.go +++ b/les/sync.go @@ -66,7 +66,7 @@ func (h *clientHandler) validateCheckpoint(peer *peer) error { if err != nil { return err } - events := h.backend.oracle.contract.LookupCheckpointEvents(logs, cp.SectionIndex, cp.Hash()) + events := h.backend.oracle.Contract().LookupCheckpointEvents(logs, cp.SectionIndex, cp.Hash()) if len(events) == 0 { return errInvalidCheckpoint } @@ -78,7 +78,7 @@ func (h *clientHandler) validateCheckpoint(peer *peer) error { for _, event := range events { signatures = append(signatures, append(event.R[:], append(event.S[:], event.V)...)) } - valid, signers := h.backend.oracle.verifySigners(index, hash, signatures) + valid, signers := h.backend.oracle.VerifySigners(index, hash, signatures) if !valid { return errInvalidCheckpoint } @@ -134,7 +134,7 @@ func (h *clientHandler) synchronise(peer *peer) { case hardcoded: mode = legacyCheckpointSync log.Debug("Disable checkpoint syncing", "reason", "checkpoint is hardcoded") - case h.backend.oracle == nil || !h.backend.oracle.isRunning(): + case h.backend.oracle == nil || !h.backend.oracle.IsRunning(): if h.checkpoint == nil { mode = lightSync // Downgrade to light sync unfortunately. } else { diff --git a/les/sync_test.go b/les/sync_test.go index 8df6223b84..1c157b4fbf 100644 --- a/les/sync_test.go +++ b/les/sync_test.go @@ -80,14 +80,14 @@ func testCheckpointSyncing(t *testing.T, protocol int, syncMode int) { data := append([]byte{0x19, 0x00}, append(registrarAddr.Bytes(), append([]byte{0, 0, 0, 0, 0, 0, 0, 0}, cp.Hash().Bytes()...)...)...) sig, _ := crypto.Sign(crypto.Keccak256(data), signerKey) sig[64] += 27 // Transform V from 0/1 to 27/28 according to the yellow paper - if _, err := server.handler.server.oracle.contract.RegisterCheckpoint(bind.NewKeyedTransactor(signerKey), cp.SectionIndex, cp.Hash().Bytes(), new(big.Int).Sub(header.Number, big.NewInt(1)), header.ParentHash, [][]byte{sig}); err != nil { + if _, err := server.handler.server.oracle.Contract().RegisterCheckpoint(bind.NewKeyedTransactor(signerKey), cp.SectionIndex, cp.Hash().Bytes(), new(big.Int).Sub(header.Number, big.NewInt(1)), header.ParentHash, [][]byte{sig}); err != nil { t.Error("register checkpoint failed", err) } server.backend.Commit() // Wait for the checkpoint registration for { - _, hash, _, err := server.handler.server.oracle.contract.Contract().GetLatestCheckpoint(nil) + _, hash, _, err := server.handler.server.oracle.Contract().Contract().GetLatestCheckpoint(nil) if err != nil || hash == [32]byte{} { time.Sleep(10 * time.Millisecond) continue @@ -164,14 +164,14 @@ func testMissOracleBackend(t *testing.T, hasCheckpoint bool) { data := append([]byte{0x19, 0x00}, append(registrarAddr.Bytes(), append([]byte{0, 0, 0, 0, 0, 0, 0, 0}, cp.Hash().Bytes()...)...)...) sig, _ := crypto.Sign(crypto.Keccak256(data), signerKey) sig[64] += 27 // Transform V from 0/1 to 27/28 according to the yellow paper - if _, err := server.handler.server.oracle.contract.RegisterCheckpoint(bind.NewKeyedTransactor(signerKey), cp.SectionIndex, cp.Hash().Bytes(), new(big.Int).Sub(header.Number, big.NewInt(1)), header.ParentHash, [][]byte{sig}); err != nil { + if _, err := server.handler.server.oracle.Contract().RegisterCheckpoint(bind.NewKeyedTransactor(signerKey), cp.SectionIndex, cp.Hash().Bytes(), new(big.Int).Sub(header.Number, big.NewInt(1)), header.ParentHash, [][]byte{sig}); err != nil { t.Error("register checkpoint failed", err) } server.backend.Commit() // Wait for the checkpoint registration for { - _, hash, _, err := server.handler.server.oracle.contract.Contract().GetLatestCheckpoint(nil) + _, hash, _, err := server.handler.server.oracle.Contract().Contract().GetLatestCheckpoint(nil) if err != nil || hash == [32]byte{} { time.Sleep(100 * time.Millisecond) continue diff --git a/les/test_helper.go b/les/test_helper.go index ee3d7a32e1..fb1965eebe 100644 --- a/les/test_helper.go +++ b/les/test_helper.go @@ -39,6 +39,7 @@ import ( "github.com/ethereum/go-ethereum/eth" "github.com/ethereum/go-ethereum/ethdb" "github.com/ethereum/go-ethereum/event" + "github.com/ethereum/go-ethereum/les/checkpointoracle" "github.com/ethereum/go-ethereum/les/flowcontrol" "github.com/ethereum/go-ethereum/light" "github.com/ethereum/go-ethereum/p2p" @@ -174,7 +175,7 @@ func newTestClientHandler(backend *backends.SimulatedBackend, odr *LesOdr, index Alloc: core.GenesisAlloc{bankAddr: {Balance: bankFunds}}, GasLimit: 100000000, } - oracle *checkpointOracle + oracle *checkpointoracle.CheckpointOracle ) genesis := gspec.MustCommit(db) chain, _ := light.NewLightChain(odr, gspec.Config, engine, nil) @@ -194,7 +195,7 @@ func newTestClientHandler(backend *backends.SimulatedBackend, odr *LesOdr, index BloomRoot: light.GetBloomTrieRoot(db, index, sectionHead), } } - oracle = newCheckpointOracle(checkpointConfig, getLocal) + oracle = checkpointoracle.New(checkpointConfig, getLocal) } client := &LightEthereum{ lesCommons: lesCommons{ @@ -218,7 +219,7 @@ func newTestClientHandler(backend *backends.SimulatedBackend, odr *LesOdr, index client.handler = newClientHandler(ulcServers, ulcFraction, nil, client) if client.oracle != nil { - client.oracle.start(backend) + client.oracle.Start(backend) } return client.handler } @@ -230,7 +231,7 @@ func newTestServerHandler(blocks int, indexers []*core.ChainIndexer, db ethdb.Da Alloc: core.GenesisAlloc{bankAddr: {Balance: bankFunds}}, GasLimit: 100000000, } - oracle *checkpointOracle + oracle *checkpointoracle.CheckpointOracle ) genesis := gspec.MustCommit(db) @@ -257,7 +258,7 @@ func newTestServerHandler(blocks int, indexers []*core.ChainIndexer, db ethdb.Da BloomRoot: light.GetBloomTrieRoot(db, index, sectionHead), } } - oracle = newCheckpointOracle(checkpointConfig, getLocal) + oracle = checkpointoracle.New(checkpointConfig, getLocal) } server := &LesServer{ lesCommons: lesCommons{ @@ -284,7 +285,7 @@ func newTestServerHandler(blocks int, indexers []*core.ChainIndexer, db ethdb.Da server.clientPool.setLimits(10000, 10000) // Assign enough capacity for clientpool server.handler = newServerHandler(server, simulation.Blockchain(), db, txpool, func() bool { return true }) if server.oracle != nil { - server.oracle.start(simulation) + server.oracle.Start(simulation) } server.servingQueue.setThreads(4) server.handler.start() From 4f7b7f84ae6bbb46001424f787dba93892dfc317 Mon Sep 17 00:00:00 2001 From: me020523 Date: Tue, 7 Jan 2020 17:31:20 +0800 Subject: [PATCH 042/128] add node.go unit test file node_test.go (#20028) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * add node.go unit test file node_test.go * add node_test.go file license and rollback trie_test.go * fix unused variable v * trie: fix license year Co-authored-by: PĂ©ter Szilágyi --- trie/node_test.go | 94 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 trie/node_test.go diff --git a/trie/node_test.go b/trie/node_test.go new file mode 100644 index 0000000000..52720f1c77 --- /dev/null +++ b/trie/node_test.go @@ -0,0 +1,94 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "bytes" + "testing" + + "github.com/ethereum/go-ethereum/rlp" +) + +func newTestFullNode(v []byte) []interface{} { + fullNodeData := []interface{}{} + for i := 0; i < 16; i++ { + k := bytes.Repeat([]byte{byte(i + 1)}, 32) + fullNodeData = append(fullNodeData, k) + } + fullNodeData = append(fullNodeData, v) + return fullNodeData +} + +func TestDecodeNestedNode(t *testing.T) { + fullNodeData := newTestFullNode([]byte("fullnode")) + + data := [][]byte{} + for i := 0; i < 16; i++ { + data = append(data, nil) + } + data = append(data, []byte("subnode")) + fullNodeData[15] = data + + buf := bytes.NewBuffer([]byte{}) + rlp.Encode(buf, fullNodeData) + + if _, err := decodeNode([]byte("testdecode"), buf.Bytes()); err != nil { + t.Fatalf("decode nested full node err: %v", err) + } +} + +func TestDecodeFullNodeWrongSizeChild(t *testing.T) { + fullNodeData := newTestFullNode([]byte("wrongsizechild")) + fullNodeData[0] = []byte("00") + buf := bytes.NewBuffer([]byte{}) + rlp.Encode(buf, fullNodeData) + + _, err := decodeNode([]byte("testdecode"), buf.Bytes()) + if _, ok := err.(*decodeError); !ok { + t.Fatalf("decodeNode returned wrong err: %v", err) + } +} + +func TestDecodeFullNodeWrongNestedFullNode(t *testing.T) { + fullNodeData := newTestFullNode([]byte("fullnode")) + + data := [][]byte{} + for i := 0; i < 16; i++ { + data = append(data, []byte("123456")) + } + data = append(data, []byte("subnode")) + fullNodeData[15] = data + + buf := bytes.NewBuffer([]byte{}) + rlp.Encode(buf, fullNodeData) + + _, err := decodeNode([]byte("testdecode"), buf.Bytes()) + if _, ok := err.(*decodeError); !ok { + t.Fatalf("decodeNode returned wrong err: %v", err) + } +} + +func TestDecodeFullNode(t *testing.T) { + fullNodeData := newTestFullNode([]byte("decodefullnode")) + buf := bytes.NewBuffer([]byte{}) + rlp.Encode(buf, fullNodeData) + + _, err := decodeNode([]byte("testdecode"), buf.Bytes()) + if err != nil { + t.Fatalf("decode full node err: %v", err) + } +} From 9e0f934e2b6ea28211d44e385ada14c8192622ad Mon Sep 17 00:00:00 2001 From: Yole <007yuyue@gmail.com> Date: Tue, 7 Jan 2020 17:56:16 +0800 Subject: [PATCH 043/128] cmd/geth: update copyright year (#20512) Update copyright from 2013-2019 to 2013-2020 --- cmd/geth/main.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 80ab0a44d8..d6ded24b31 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -194,7 +194,7 @@ func init() { // Initialize the CLI app and start Geth app.Action = geth app.HideVersion = true // we have a command to print the version - app.Copyright = "Copyright 2013-2019 The go-ethereum Authors" + app.Copyright = "Copyright 2013-2020 The go-ethereum Authors" app.Commands = []cli.Command{ // See chaincmd.go: initCommand, From 50be790869a294b2cbce9727232b6350a9883a6e Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Tue, 7 Jan 2020 18:06:44 +0100 Subject: [PATCH 044/128] README.md: Genoil fork has been discontinued (#20521) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 92a7125b49..757945e914 100644 --- a/README.md +++ b/README.md @@ -294,7 +294,7 @@ also need to configure a miner to process transactions and create new blocks for Mining on the public Ethereum network is a complex task as it's only feasible using GPUs, requiring an OpenCL or CUDA enabled `ethminer` instance. For information on such a setup, please consult the [EtherMining subreddit](https://www.reddit.com/r/EtherMining/) -and the [Genoil miner](https://github.com/Genoil/cpp-ethereum) repository. +and the [ethminer](https://github.com/ethereum-mining/ethminer) repository. In a private network setting, however a single CPU miner instance is more than enough for practical purposes as it can produce a stable stream of blocks at the correct intervals From a013f02df2329fafc08f23cd623df6afa7471d20 Mon Sep 17 00:00:00 2001 From: wangxiang Date: Wed, 8 Jan 2020 01:08:22 +0800 Subject: [PATCH 045/128] whisper/whisperv6: fix peer time.Ticker leak (#20520) --- whisper/whisperv6/peer.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/whisper/whisperv6/peer.go b/whisper/whisperv6/peer.go index 4451f14958..29d8bdf17e 100644 --- a/whisper/whisperv6/peer.go +++ b/whisper/whisperv6/peer.go @@ -146,7 +146,9 @@ func (peer *Peer) handshake() error { func (peer *Peer) update() { // Start the tickers for the updates expire := time.NewTicker(expirationCycle) + defer expire.Stop() transmit := time.NewTicker(transmissionCycle) + defer transmit.Stop() // Loop and transmit until termination is requested for { From a1bc0e3cb69d6080b884728a60d2f92392b4c392 Mon Sep 17 00:00:00 2001 From: Jonathan Gimeno Date: Tue, 7 Jan 2020 18:12:27 +0100 Subject: [PATCH 046/128] eth: refactor creation of EthAPIBackend (#20476) --- eth/backend.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eth/backend.go b/eth/backend.go index adde609de0..047583f5cc 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -213,12 +213,12 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { eth.miner = miner.New(eth, &config.Miner, chainConfig, eth.EventMux(), eth.engine, eth.isLocalBlock) eth.miner.SetExtra(makeExtraData(config.Miner.ExtraData)) - eth.APIBackend = &EthAPIBackend{ctx.ExtRPCEnabled(), eth, nil} gpoParams := config.GPO if gpoParams.Default == nil { gpoParams.Default = config.Miner.GasPrice } - eth.APIBackend.gpo = gasprice.NewOracle(eth.APIBackend, gpoParams) + + eth.APIBackend = &EthAPIBackend{ctx.ExtRPCEnabled(), eth, gasprice.NewOracle(eth.APIBackend, gpoParams)} return eth, nil } From c49a4165d074c2d08c362cfe1dac835b6a3e1251 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Tue, 7 Jan 2020 18:19:21 +0100 Subject: [PATCH 047/128] consensus/ethash: fix a typo and error message (#20503) --- consensus/ethash/consensus.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/consensus/ethash/consensus.go b/consensus/ethash/consensus.go index 270c5358dd..8ae99b1994 100644 --- a/consensus/ethash/consensus.go +++ b/consensus/ethash/consensus.go @@ -68,7 +68,7 @@ var ( // codebase, inherently breaking if the engine is swapped out. Please put common // error types into the consensus package. var ( - errZeroBlockTime = errors.New("timestamp equals parent's") + errOlderBlockTime = errors.New("timestamp older than parent") errTooManyUncles = errors.New("too many uncles") errDuplicateUncle = errors.New("duplicate uncle") errUncleIsAncestor = errors.New("uncle is ancestor") @@ -255,9 +255,9 @@ func (ethash *Ethash) verifyHeader(chain consensus.ChainReader, header, parent * } } if header.Time <= parent.Time { - return errZeroBlockTime + return errOlderBlockTime } - // Verify the block's difficulty based in its timestamp and parent's difficulty + // Verify the block's difficulty based on its timestamp and parent's difficulty expected := ethash.CalcDifficulty(chain, header.Time, parent) if expected.Cmp(header.Difficulty) != 0 { From 8a63f7f504b752b3da127b9df8152bce54fcaae5 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 8 Jan 2020 10:25:01 +0100 Subject: [PATCH 048/128] .travis.yml: use latest macOS 10.14 image (#20526) --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 9568c13ade..615b51157f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -57,6 +57,7 @@ jobs: - stage: build os: osx + osx_image: xcode11.3 go: 1.13.x script: - echo "Increase the maximum number of open file descriptors on macOS" From 4d663d57d663a342f05e690acb3f012e46010908 Mon Sep 17 00:00:00 2001 From: gary rong Date: Wed, 8 Jan 2020 21:08:56 +0800 Subject: [PATCH 049/128] les: fix request serving metrics (#20507) --- les/server_handler.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/les/server_handler.go b/les/server_handler.go index 4b505c2bc4..38aeac4e88 100644 --- a/les/server_handler.go +++ b/les/server_handler.go @@ -273,7 +273,6 @@ func (h *serverHandler) handleMsg(p *peer, wg *sync.WaitGroup) error { if metrics.EnabledExpensive { miscInHeaderPacketsMeter.Mark(1) miscInHeaderTrafficMeter.Mark(int64(msg.Size)) - defer func(start time.Time) { miscServingTimeHeaderTimer.UpdateSince(start) }(time.Now()) } var req struct { ReqID uint64 @@ -377,6 +376,7 @@ func (h *serverHandler) handleMsg(p *peer, wg *sync.WaitGroup) error { if metrics.EnabledExpensive { miscOutHeaderPacketsMeter.Mark(1) miscOutHeaderTrafficMeter.Mark(int64(reply.size())) + miscServingTimeHeaderTimer.Update(time.Duration(task.servingTime)) } }() } @@ -386,7 +386,6 @@ func (h *serverHandler) handleMsg(p *peer, wg *sync.WaitGroup) error { if metrics.EnabledExpensive { miscInBodyPacketsMeter.Mark(1) miscInBodyTrafficMeter.Mark(int64(msg.Size)) - defer func(start time.Time) { miscServingTimeBodyTimer.UpdateSince(start) }(time.Now()) } var req struct { ReqID uint64 @@ -426,6 +425,7 @@ func (h *serverHandler) handleMsg(p *peer, wg *sync.WaitGroup) error { if metrics.EnabledExpensive { miscOutBodyPacketsMeter.Mark(1) miscOutBodyTrafficMeter.Mark(int64(reply.size())) + miscServingTimeBodyTimer.Update(time.Duration(task.servingTime)) } }() } @@ -435,7 +435,6 @@ func (h *serverHandler) handleMsg(p *peer, wg *sync.WaitGroup) error { if metrics.EnabledExpensive { miscInCodePacketsMeter.Mark(1) miscInCodeTrafficMeter.Mark(int64(msg.Size)) - defer func(start time.Time) { miscServingTimeCodeTimer.UpdateSince(start) }(time.Now()) } var req struct { ReqID uint64 @@ -498,6 +497,7 @@ func (h *serverHandler) handleMsg(p *peer, wg *sync.WaitGroup) error { if metrics.EnabledExpensive { miscOutCodePacketsMeter.Mark(1) miscOutCodeTrafficMeter.Mark(int64(reply.size())) + miscServingTimeCodeTimer.Update(time.Duration(task.servingTime)) } }() } @@ -507,7 +507,6 @@ func (h *serverHandler) handleMsg(p *peer, wg *sync.WaitGroup) error { if metrics.EnabledExpensive { miscInReceiptPacketsMeter.Mark(1) miscInReceiptTrafficMeter.Mark(int64(msg.Size)) - defer func(start time.Time) { miscServingTimeReceiptTimer.UpdateSince(start) }(time.Now()) } var req struct { ReqID uint64 @@ -555,6 +554,7 @@ func (h *serverHandler) handleMsg(p *peer, wg *sync.WaitGroup) error { if metrics.EnabledExpensive { miscOutReceiptPacketsMeter.Mark(1) miscOutReceiptTrafficMeter.Mark(int64(reply.size())) + miscServingTimeReceiptTimer.Update(time.Duration(task.servingTime)) } }() } @@ -564,7 +564,6 @@ func (h *serverHandler) handleMsg(p *peer, wg *sync.WaitGroup) error { if metrics.EnabledExpensive { miscInTrieProofPacketsMeter.Mark(1) miscInTrieProofTrafficMeter.Mark(int64(msg.Size)) - defer func(start time.Time) { miscServingTimeTrieProofTimer.UpdateSince(start) }(time.Now()) } var req struct { ReqID uint64 @@ -658,6 +657,7 @@ func (h *serverHandler) handleMsg(p *peer, wg *sync.WaitGroup) error { if metrics.EnabledExpensive { miscOutTrieProofPacketsMeter.Mark(1) miscOutTrieProofTrafficMeter.Mark(int64(reply.size())) + miscServingTimeTrieProofTimer.Update(time.Duration(task.servingTime)) } }() } @@ -667,7 +667,6 @@ func (h *serverHandler) handleMsg(p *peer, wg *sync.WaitGroup) error { if metrics.EnabledExpensive { miscInHelperTriePacketsMeter.Mark(1) miscInHelperTrieTrafficMeter.Mark(int64(msg.Size)) - defer func(start time.Time) { miscServingTimeHelperTrieTimer.UpdateSince(start) }(time.Now()) } var req struct { ReqID uint64 @@ -733,6 +732,7 @@ func (h *serverHandler) handleMsg(p *peer, wg *sync.WaitGroup) error { if metrics.EnabledExpensive { miscOutHelperTriePacketsMeter.Mark(1) miscOutHelperTrieTrafficMeter.Mark(int64(reply.size())) + miscServingTimeHelperTrieTimer.Update(time.Duration(task.servingTime)) } }() } @@ -742,7 +742,6 @@ func (h *serverHandler) handleMsg(p *peer, wg *sync.WaitGroup) error { if metrics.EnabledExpensive { miscInTxsPacketsMeter.Mark(1) miscInTxsTrafficMeter.Mark(int64(msg.Size)) - defer func(start time.Time) { miscServingTimeTxTimer.UpdateSince(start) }(time.Now()) } var req struct { ReqID uint64 @@ -782,6 +781,7 @@ func (h *serverHandler) handleMsg(p *peer, wg *sync.WaitGroup) error { if metrics.EnabledExpensive { miscOutTxsPacketsMeter.Mark(1) miscOutTxsTrafficMeter.Mark(int64(reply.size())) + miscServingTimeTxTimer.Update(time.Duration(task.servingTime)) } }() } @@ -791,7 +791,6 @@ func (h *serverHandler) handleMsg(p *peer, wg *sync.WaitGroup) error { if metrics.EnabledExpensive { miscInTxStatusPacketsMeter.Mark(1) miscInTxStatusTrafficMeter.Mark(int64(msg.Size)) - defer func(start time.Time) { miscServingTimeTxStatusTimer.UpdateSince(start) }(time.Now()) } var req struct { ReqID uint64 @@ -819,6 +818,7 @@ func (h *serverHandler) handleMsg(p *peer, wg *sync.WaitGroup) error { if metrics.EnabledExpensive { miscOutTxStatusPacketsMeter.Mark(1) miscOutTxStatusTrafficMeter.Mark(int64(reply.size())) + miscServingTimeTxStatusTimer.Update(time.Duration(task.servingTime)) } }() } From 0218d7001d2a566b35072ee21b9f84f6b2711bbe Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 8 Jan 2020 16:11:51 +0100 Subject: [PATCH 050/128] internal/testlog: print file+line number of log call in test log (#20528) * internal/testlog: print file+line number of log call in test log This changes the unit test logger to print the actual file and line number of the logging call instead of "testlog.go:44". Output of 'go test -v -run TestServerListen ./p2p' before this change: === RUN TestServerListen --- PASS: TestServerListen (0.00s) testlog.go:44: DEBUG[01-08|15:16:31.651] UDP listener up addr=127.0.0.1:62678 testlog.go:44: DEBUG[01-08|15:16:31.651] TCP listener up addr=127.0.0.1:62678 testlog.go:44: TRACE[01-08|15:16:31.652] Accepted connection addr=127.0.0.1:62679 And after: === RUN TestServerListen --- PASS: TestServerListen (0.00s) server.go:868: DEBUG[01-08|15:25:35.679] TCP listener up addr=127.0.0.1:62712 server.go:557: DEBUG[01-08|15:25:35.679] UDP listener up addr=127.0.0.1:62712 server.go:912: TRACE[01-08|15:25:35.680] Accepted connection addr=127.0.0.1:62713 * internal/testlog: document use of t.Helper --- internal/testlog/testlog.go | 110 +++++++++++++++++++++++++++++++++--- 1 file changed, 103 insertions(+), 7 deletions(-) diff --git a/internal/testlog/testlog.go b/internal/testlog/testlog.go index c5a6114d16..684339f16d 100644 --- a/internal/testlog/testlog.go +++ b/internal/testlog/testlog.go @@ -18,18 +18,12 @@ package testlog import ( + "sync" "testing" "github.com/ethereum/go-ethereum/log" ) -// Logger returns a logger which logs to the unit test log of t. -func Logger(t *testing.T, level log.Lvl) log.Logger { - l := log.New() - l.SetHandler(Handler(t, level)) - return l -} - // Handler returns a log handler which logs to the unit test log of t. func Handler(t *testing.T, level log.Lvl) log.Handler { return log.LvlFilterHandler(level, &handler{t, log.TerminalFormat(false)}) @@ -44,3 +38,105 @@ func (h *handler) Log(r *log.Record) error { h.t.Logf("%s", h.fmt.Format(r)) return nil } + +// logger implements log.Logger such that all output goes to the unit test log via +// t.Logf(). All methods in between logger.Trace, logger.Debug, etc. are marked as test +// helpers, so the file and line number in unit test output correspond to the call site +// which emitted the log message. +type logger struct { + t *testing.T + l log.Logger + mu *sync.Mutex + h *bufHandler +} + +type bufHandler struct { + buf []*log.Record + fmt log.Format +} + +func (h *bufHandler) Log(r *log.Record) error { + h.buf = append(h.buf, r) + return nil +} + +// Logger returns a logger which logs to the unit test log of t. +func Logger(t *testing.T, level log.Lvl) log.Logger { + l := &logger{ + t: t, + l: log.New(), + mu: new(sync.Mutex), + h: &bufHandler{fmt: log.TerminalFormat(false)}, + } + l.l.SetHandler(log.LvlFilterHandler(level, l.h)) + return l +} + +func (l *logger) Trace(msg string, ctx ...interface{}) { + l.t.Helper() + l.mu.Lock() + defer l.mu.Unlock() + l.l.Trace(msg, ctx...) + l.flush() +} + +func (l *logger) Debug(msg string, ctx ...interface{}) { + l.t.Helper() + l.mu.Lock() + defer l.mu.Unlock() + l.l.Debug(msg, ctx...) + l.flush() +} + +func (l *logger) Info(msg string, ctx ...interface{}) { + l.t.Helper() + l.mu.Lock() + defer l.mu.Unlock() + l.l.Info(msg, ctx...) + l.flush() +} + +func (l *logger) Warn(msg string, ctx ...interface{}) { + l.t.Helper() + l.mu.Lock() + defer l.mu.Unlock() + l.l.Warn(msg, ctx...) + l.flush() +} + +func (l *logger) Error(msg string, ctx ...interface{}) { + l.t.Helper() + l.mu.Lock() + defer l.mu.Unlock() + l.l.Error(msg, ctx...) + l.flush() +} + +func (l *logger) Crit(msg string, ctx ...interface{}) { + l.t.Helper() + l.mu.Lock() + defer l.mu.Unlock() + l.l.Crit(msg, ctx...) + l.flush() +} + +func (l *logger) New(ctx ...interface{}) log.Logger { + return &logger{l.t, l.l.New(ctx...), l.mu, l.h} +} + +func (l *logger) GetHandler() log.Handler { + return l.l.GetHandler() +} + +func (l *logger) SetHandler(h log.Handler) { + l.l.SetHandler(h) +} + +// flush writes all buffered messages and clears the buffer. +func (l *logger) flush() { + l.t.Helper() + for _, r := range l.h.buf { + l.t.Logf("%s", l.h.fmt.Format(r)) + } + l.h.buf = nil +} From b211742e5f1ec87e59683e979dd914c1a5a2fade Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 9 Jan 2020 13:26:37 +0200 Subject: [PATCH 051/128] Revert "eth: refactor creation of EthAPIBackend (#20476)" (#20536) This reverts commit a1bc0e3cb69d6080b884728a60d2f92392b4c392. --- eth/backend.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eth/backend.go b/eth/backend.go index 047583f5cc..adde609de0 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -213,12 +213,12 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { eth.miner = miner.New(eth, &config.Miner, chainConfig, eth.EventMux(), eth.engine, eth.isLocalBlock) eth.miner.SetExtra(makeExtraData(config.Miner.ExtraData)) + eth.APIBackend = &EthAPIBackend{ctx.ExtRPCEnabled(), eth, nil} gpoParams := config.GPO if gpoParams.Default == nil { gpoParams.Default = config.Miner.GasPrice } - - eth.APIBackend = &EthAPIBackend{ctx.ExtRPCEnabled(), eth, gasprice.NewOracle(eth.APIBackend, gpoParams)} + eth.APIBackend.gpo = gasprice.NewOracle(eth.APIBackend, gpoParams) return eth, nil } From fc392395fbb7d26d8b2123b12402225b9d4324d3 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 10 Jan 2020 10:12:32 +0100 Subject: [PATCH 052/128] core/state: add more verbosity to panic --- core/state/statedb.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/state/statedb.go b/core/state/statedb.go index 03e118d117..085f2379fb 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -204,7 +204,7 @@ func (s *StateDB) AddRefund(gas uint64) { func (s *StateDB) SubRefund(gas uint64) { s.journal.append(refundChange{prev: s.refund}) if gas > s.refund { - panic("Refund counter below zero") + panic(fmt.Sprintf("Refund counter below zero (gas: %d > refund: %d)", gas, s.refund)) } s.refund -= gas } From 8bd37a1d919194d9a488d1cee823eaecfeb5ed9a Mon Sep 17 00:00:00 2001 From: MichaelRiabzev-StarkWare Date: Fri, 10 Jan 2020 11:40:03 +0200 Subject: [PATCH 053/128] core: count tx size in slots, bump max size ot 4x32KB (#20352) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * tests for tx size * alow multiple slots transactions * tests for tx size limit (32 KB) * change tx size tests to use addRemoteSync instead of validateTx (requested in pool request). * core: minor tx slotting polishes, add slot tracking metric Co-authored-by: Michael Riabzev Co-authored-by: PĂ©ter Szilágyi --- core/tx_list.go | 8 ++-- core/tx_pool.go | 43 ++++++++++++++++--- core/tx_pool_test.go | 98 ++++++++++++++++++++++++++++++++++++++++---- 3 files changed, 132 insertions(+), 17 deletions(-) diff --git a/core/tx_list.go b/core/tx_list.go index 75bfdaedac..6b22cbbebe 100644 --- a/core/tx_list.go +++ b/core/tx_list.go @@ -494,11 +494,11 @@ func (l *txPricedList) Underpriced(tx *types.Transaction, local *accountSet) boo // Discard finds a number of most underpriced transactions, removes them from the // priced list and returns them for further removal from the entire pool. -func (l *txPricedList) Discard(count int, local *accountSet) types.Transactions { - drop := make(types.Transactions, 0, count) // Remote underpriced transactions to drop +func (l *txPricedList) Discard(slots int, local *accountSet) types.Transactions { + drop := make(types.Transactions, 0, slots) // Remote underpriced transactions to drop save := make(types.Transactions, 0, 64) // Local underpriced transactions to keep - for len(*l.items) > 0 && count > 0 { + for len(*l.items) > 0 && slots > 0 { // Discard stale transactions if found during cleanup tx := heap.Pop(l.items).(*types.Transaction) if l.all.Get(tx.Hash()) == nil { @@ -510,7 +510,7 @@ func (l *txPricedList) Discard(count int, local *accountSet) types.Transactions save = append(save, tx) } else { drop = append(drop, tx) - count-- + slots -= numSlots(tx) } } for _, tx := range save { diff --git a/core/tx_pool.go b/core/tx_pool.go index f7032dbd1e..e95496b13b 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -38,6 +38,18 @@ import ( const ( // chainHeadChanSize is the size of channel listening to ChainHeadEvent. chainHeadChanSize = 10 + + // txSlotSize is used to calculate how many data slots a single transaction + // takes up based on its size. The slots are used as DoS protection, ensuring + // that validating a new transaction remains a constant operation (in reality + // O(maxslots), where max slots are 4 currently). + txSlotSize = 32 * 1024 + + // txMaxSize is the maximum size a single transaction can have. This field has + // non-trivial consequences: larger transactions are significantly harder and + // more expensive to propagate; larger transactions also take more resources + // to validate whether they fit into the pool or not. + txMaxSize = 4 * txSlotSize // 128KB, don't bump without chunking support ) var ( @@ -105,6 +117,7 @@ var ( pendingGauge = metrics.NewRegisteredGauge("txpool/pending", nil) queuedGauge = metrics.NewRegisteredGauge("txpool/queued", nil) localGauge = metrics.NewRegisteredGauge("txpool/local", nil) + slotsGauge = metrics.NewRegisteredGauge("txpool/slots", nil) ) // TxStatus is the current status of a transaction as seen by the pool. @@ -510,8 +523,8 @@ func (pool *TxPool) local() map[common.Address]types.Transactions { // validateTx checks whether a transaction is valid according to the consensus // rules and adheres to some heuristic limits of the local node (price and size). func (pool *TxPool) validateTx(tx *types.Transaction, local bool) error { - // Heuristic limit, reject transactions over 32KB to prevent DOS attacks - if tx.Size() > 32*1024 { + // Reject transactions over defined size to prevent DOS attacks + if uint64(tx.Size()) > txMaxSize { return ErrOversizedData } // Transactions can't be negative. This may never happen using RLP decoded @@ -583,7 +596,7 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (replaced bool, err e return false, ErrUnderpriced } // New transaction is better than our worse ones, make room for it - drop := pool.priced.Discard(pool.all.Count()-int(pool.config.GlobalSlots+pool.config.GlobalQueue-1), pool.locals) + drop := pool.priced.Discard(pool.all.Slots()-int(pool.config.GlobalSlots+pool.config.GlobalQueue)+numSlots(tx), pool.locals) for _, tx := range drop { log.Trace("Discarding freshly underpriced transaction", "hash", tx.Hash(), "price", tx.GasPrice()) underpricedTxMeter.Mark(1) @@ -1493,8 +1506,9 @@ func (as *accountSet) merge(other *accountSet) { // peeking into the pool in TxPool.Get without having to acquire the widely scoped // TxPool.mu mutex. type txLookup struct { - all map[common.Hash]*types.Transaction - lock sync.RWMutex + all map[common.Hash]*types.Transaction + slots int + lock sync.RWMutex } // newTxLookup returns a new txLookup structure. @@ -1532,11 +1546,22 @@ func (t *txLookup) Count() int { return len(t.all) } +// Slots returns the current number of slots used in the lookup. +func (t *txLookup) Slots() int { + t.lock.RLock() + defer t.lock.RUnlock() + + return t.slots +} + // Add adds a transaction to the lookup. func (t *txLookup) Add(tx *types.Transaction) { t.lock.Lock() defer t.lock.Unlock() + t.slots += numSlots(tx) + slotsGauge.Update(int64(t.slots)) + t.all[tx.Hash()] = tx } @@ -1545,5 +1570,13 @@ func (t *txLookup) Remove(hash common.Hash) { t.lock.Lock() defer t.lock.Unlock() + t.slots -= numSlots(t.all[hash]) + slotsGauge.Update(int64(t.slots)) + delete(t.all, hash) } + +// numSlots calculates the number of slots needed for a single transaction. +func numSlots(tx *types.Transaction) int { + return int((tx.Size() + txSlotSize - 1) / txSlotSize) +} diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index 0f1e7ac8f8..4db3e6deef 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -77,9 +77,17 @@ func pricedTransaction(nonce uint64, gaslimit uint64, gasprice *big.Int, key *ec return tx } +func pricedDataTransaction(nonce uint64, gaslimit uint64, gasprice *big.Int, key *ecdsa.PrivateKey, bytes uint64) *types.Transaction { + data := make([]byte, bytes) + rand.Read(data) + + tx, _ := types.SignTx(types.NewTransaction(nonce, common.Address{}, big.NewInt(0), gaslimit, gasprice, data), types.HomesteadSigner{}, key) + return tx +} + func setupTxPool() (*TxPool, *ecdsa.PrivateKey) { statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) - blockchain := &testBlockChain{statedb, 1000000, new(event.Feed)} + blockchain := &testBlockChain{statedb, 10000000, new(event.Feed)} key, _ := crypto.GenerateKey() pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) @@ -465,7 +473,7 @@ func TestTransactionDropping(t *testing.T) { pool, key := setupTxPool() defer pool.Stop() - account, _ := deriveSender(transaction(0, 0, key)) + account := crypto.PubkeyToAddress(key.PublicKey) pool.currentState.AddBalance(account, big.NewInt(1000)) // Add some pending and some queued transactions @@ -674,7 +682,7 @@ func TestTransactionGapFilling(t *testing.T) { pool, key := setupTxPool() defer pool.Stop() - account, _ := deriveSender(transaction(0, 0, key)) + account := crypto.PubkeyToAddress(key.PublicKey) pool.currentState.AddBalance(account, big.NewInt(1000000)) // Keep track of transaction events to ensure all executables get announced @@ -728,7 +736,7 @@ func TestTransactionQueueAccountLimiting(t *testing.T) { pool, key := setupTxPool() defer pool.Stop() - account, _ := deriveSender(transaction(0, 0, key)) + account := crypto.PubkeyToAddress(key.PublicKey) pool.currentState.AddBalance(account, big.NewInt(1000000)) // Keep queuing up transactions and make sure all above a limit are dropped @@ -923,7 +931,7 @@ func TestTransactionPendingLimiting(t *testing.T) { pool, key := setupTxPool() defer pool.Stop() - account, _ := deriveSender(transaction(0, 0, key)) + account := crypto.PubkeyToAddress(key.PublicKey) pool.currentState.AddBalance(account, big.NewInt(1000000)) // Keep track of transaction events to ensure all executables get announced @@ -1002,6 +1010,62 @@ func TestTransactionPendingGlobalLimiting(t *testing.T) { } } +// Test the limit on transaction size is enforced correctly. +// This test verifies every transaction having allowed size +// is added to the pool, and longer transactions are rejected. +func TestTransactionAllowedTxSize(t *testing.T) { + t.Parallel() + + // Create a test account and fund it + pool, key := setupTxPool() + defer pool.Stop() + + account := crypto.PubkeyToAddress(key.PublicKey) + pool.currentState.AddBalance(account, big.NewInt(1000000000)) + + // Compute maximal data size for transactions (lower bound). + // + // It is assumed the fields in the transaction (except of the data) are: + // - nonce <= 32 bytes + // - gasPrice <= 32 bytes + // - gasLimit <= 32 bytes + // - recipient == 20 bytes + // - value <= 32 bytes + // - signature == 65 bytes + // All those fields are summed up to at most 213 bytes. + baseSize := uint64(213) + dataSize := txMaxSize - baseSize + + // Try adding a transaction with maximal allowed size + tx := pricedDataTransaction(0, pool.currentMaxGas, big.NewInt(1), key, dataSize) + if err := pool.addRemoteSync(tx); err != nil { + t.Fatalf("failed to add transaction of size %d, close to maximal: %v", int(tx.Size()), err) + } + // Try adding a transaction with random allowed size + if err := pool.addRemoteSync(pricedDataTransaction(1, pool.currentMaxGas, big.NewInt(1), key, uint64(rand.Intn(int(dataSize))))); err != nil { + t.Fatalf("failed to add transaction of random allowed size: %v", err) + } + // Try adding a transaction of minimal not allowed size + if err := pool.addRemoteSync(pricedDataTransaction(2, pool.currentMaxGas, big.NewInt(1), key, txMaxSize)); err == nil { + t.Fatalf("expected rejection on slightly oversize transaction") + } + // Try adding a transaction of random not allowed size + if err := pool.addRemoteSync(pricedDataTransaction(2, pool.currentMaxGas, big.NewInt(1), key, dataSize+1+uint64(rand.Intn(int(10*txMaxSize))))); err == nil { + t.Fatalf("expected rejection on oversize transaction") + } + // Run some sanity checks on the pool internals + pending, queued := pool.Stats() + if pending != 2 { + t.Fatalf("pending transactions mismatched: have %d, want %d", pending, 2) + } + if queued != 0 { + t.Fatalf("queued transactions mismatched: have %d, want %d", queued, 0) + } + if err := validateTxPoolInternals(pool); err != nil { + t.Fatalf("pool internal state corrupted: %v", err) + } +} + // Tests that if transactions start being capped, transactions are also removed from 'all' func TestTransactionCapClearsFromAll(t *testing.T) { t.Parallel() @@ -1752,6 +1816,24 @@ func TestTransactionStatusCheck(t *testing.T) { } } +// Test the transaction slots consumption is computed correctly +func TestTransactionSlotCount(t *testing.T) { + t.Parallel() + + key, _ := crypto.GenerateKey() + + // Check that an empty transaction consumes a single slot + smallTx := pricedDataTransaction(0, 0, big.NewInt(0), key, 0) + if slots := numSlots(smallTx); slots != 1 { + t.Fatalf("small transactions slot count mismatch: have %d want %d", slots, 1) + } + // Check that a large transaction consumes the correct number of slots + bigTx := pricedDataTransaction(0, 0, big.NewInt(0), key, uint64(10*txSlotSize)) + if slots := numSlots(bigTx); slots != 11 { + t.Fatalf("big transactions slot count mismatch: have %d want %d", slots, 11) + } +} + // Benchmarks the speed of validating the contents of the pending queue of the // transaction pool. func BenchmarkPendingDemotion100(b *testing.B) { benchmarkPendingDemotion(b, 100) } @@ -1763,7 +1845,7 @@ func benchmarkPendingDemotion(b *testing.B, size int) { pool, key := setupTxPool() defer pool.Stop() - account, _ := deriveSender(transaction(0, 0, key)) + account := crypto.PubkeyToAddress(key.PublicKey) pool.currentState.AddBalance(account, big.NewInt(1000000)) for i := 0; i < size; i++ { @@ -1788,7 +1870,7 @@ func benchmarkFuturePromotion(b *testing.B, size int) { pool, key := setupTxPool() defer pool.Stop() - account, _ := deriveSender(transaction(0, 0, key)) + account := crypto.PubkeyToAddress(key.PublicKey) pool.currentState.AddBalance(account, big.NewInt(1000000)) for i := 0; i < size; i++ { @@ -1812,7 +1894,7 @@ func benchmarkPoolBatchInsert(b *testing.B, size int) { pool, key := setupTxPool() defer pool.Stop() - account, _ := deriveSender(transaction(0, 0, key)) + account := crypto.PubkeyToAddress(key.PublicKey) pool.currentState.AddBalance(account, big.NewInt(1000000)) batches := make([]types.Transactions, b.N) From a90cc66f3c9635ce93b8df3fd8dcd6418063f97e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 13 Jan 2020 14:23:54 +0200 Subject: [PATCH 054/128] eth: check propagated block malformation on receiption --- eth/handler.go | 8 ++++++ eth/handler_test.go | 62 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 70 insertions(+) diff --git a/eth/handler.go b/eth/handler.go index d2355a8768..e18fa61241 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -687,6 +687,14 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { if err := msg.Decode(&request); err != nil { return errResp(ErrDecode, "%v: %v", msg, err) } + if hash := types.CalcUncleHash(request.Block.Uncles()); hash != request.Block.UncleHash() { + log.Warn("Propagated block has invalid uncles", "have", hash, "exp", request.Block.UncleHash()) + break // TODO(karalabe): return error eventually, but wait a few releases + } + if hash := types.DeriveSha(request.Block.Transactions()); hash != request.Block.TxHash() { + log.Warn("Propagated block has invalid body", "have", hash, "exp", request.Block.TxHash()) + break // TODO(karalabe): return error eventually, but wait a few releases + } if err := request.sanityCheck(); err != nil { return err } diff --git a/eth/handler_test.go b/eth/handler_test.go index 05ddca3efd..893bfffa42 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -634,3 +634,65 @@ outer: t.Errorf("block broadcast to %d peers, expected %d", receivedCount, broadcastExpected) } } + +// Tests that a propagated malformed block (uncles or transactions don't match +// with the hashes in the header) gets discarded and not broadcast forward. +func TestBroadcastMalformedBlock(t *testing.T) { + // Create a live node to test propagation with + var ( + engine = ethash.NewFaker() + db = rawdb.NewMemoryDatabase() + config = ¶ms.ChainConfig{} + gspec = &core.Genesis{Config: config} + genesis = gspec.MustCommit(db) + ) + blockchain, err := core.NewBlockChain(db, nil, config, engine, vm.Config{}, nil) + if err != nil { + t.Fatalf("failed to create new blockchain: %v", err) + } + pm, err := NewProtocolManager(config, nil, downloader.FullSync, DefaultConfig.NetworkId, new(event.TypeMux), new(testTxPool), engine, blockchain, db, 1, nil) + if err != nil { + t.Fatalf("failed to start test protocol manager: %v", err) + } + pm.Start(2) + defer pm.Stop() + + // Create two peers, one to send the malformed block with and one to check + // propagation + source, _ := newTestPeer("source", eth63, pm, true) + defer source.close() + + sink, _ := newTestPeer("sink", eth63, pm, true) + defer sink.close() + + // Create various combinations of malformed blocks + chain, _ := core.GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 1, func(i int, gen *core.BlockGen) {}) + + malformedUncles := chain[0].Header() + malformedUncles.UncleHash[0]++ + malformedTransactions := chain[0].Header() + malformedTransactions.TxHash[0]++ + malformedEverything := chain[0].Header() + malformedEverything.UncleHash[0]++ + malformedEverything.TxHash[0]++ + + // Keep listening to broadcasts and notify if any arrives + notify := make(chan struct{}) + go func() { + if _, err := sink.app.ReadMsg(); err == nil { + notify <- struct{}{} + } + }() + // Try to broadcast all malformations and ensure they all get discarded + for _, header := range []*types.Header{malformedUncles, malformedTransactions, malformedEverything} { + block := types.NewBlockWithHeader(header).WithBody(chain[0].Transactions(), chain[0].Uncles()) + if err := p2p.Send(source.app, NewBlockMsg, []interface{}{block, big.NewInt(131136)}); err != nil { + t.Fatalf("failed to broadcast block: %v", err) + } + select { + case <-notify: + t.Fatalf("malformed block forwarded") + case <-time.After(100 * time.Millisecond): + } + } +} From b2de0bd87beea1f8b0f6787fb76c4dc0538db37b Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 14 Jan 2020 11:49:36 +0100 Subject: [PATCH 055/128] core: set max tx size down to 2 slots (64KB) --- core/tx_pool.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/tx_pool.go b/core/tx_pool.go index e95496b13b..ae6962c5d9 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -49,7 +49,7 @@ const ( // non-trivial consequences: larger transactions are significantly harder and // more expensive to propagate; larger transactions also take more resources // to validate whether they fit into the pool or not. - txMaxSize = 4 * txSlotSize // 128KB, don't bump without chunking support + txMaxSize = 2 * txSlotSize // 64KB, don't bump without EIP-2464 support ) var ( From feda78e052edb862b0e666f8d3af68f00ed7c0df Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 14 Jan 2020 13:13:14 +0100 Subject: [PATCH 056/128] build: remove env.sh (#20541) * build: remove env.sh This removes the dirty symlink-to-self hack we've had for years. The script was added to enable building without GOPATH and did that job reliably for all this time. We can remove the workaround because modern Go supports building without GOPATH natively. * Makefile: add GO111MODULE=on to environment --- Makefile | 43 ++++++++++++++++++------------------ build/ci.go | 14 ++---------- build/deb/ethereum/deb.rules | 2 +- build/env.sh | 30 ------------------------- internal/build/util.go | 9 -------- 5 files changed, 25 insertions(+), 73 deletions(-) delete mode 100755 build/env.sh diff --git a/Makefile b/Makefile index 474fe61ed5..67095f4d00 100644 --- a/Makefile +++ b/Makefile @@ -10,33 +10,34 @@ GOBIN = ./build/bin GO ?= latest +GORUN = env GO111MODULE=on go run geth: - build/env.sh go run build/ci.go install ./cmd/geth + $(GORUN) build/ci.go install ./cmd/geth @echo "Done building." @echo "Run \"$(GOBIN)/geth\" to launch geth." all: - build/env.sh go run build/ci.go install + $(GORUN) build/ci.go install android: - build/env.sh go run build/ci.go aar --local + $(GORUN) build/ci.go aar --local @echo "Done building." @echo "Import \"$(GOBIN)/geth.aar\" to use the library." ios: - build/env.sh go run build/ci.go xcode --local + $(GORUN) build/ci.go xcode --local @echo "Done building." @echo "Import \"$(GOBIN)/Geth.framework\" to use the library." test: all - build/env.sh go run build/ci.go test + $(GORUN) build/ci.go test lint: ## Run linters. - build/env.sh go run build/ci.go lint + $(GORUN) build/ci.go lint clean: - go clean -cache + env GO111MODULE=on go clean -cache rm -fr build/_workspace/pkg/ $(GOBIN)/* # The devtools target installs tools required for 'go generate'. @@ -63,12 +64,12 @@ geth-linux: geth-linux-386 geth-linux-amd64 geth-linux-arm geth-linux-mips64 get @ls -ld $(GOBIN)/geth-linux-* geth-linux-386: - build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/386 -v ./cmd/geth + $(GORUN) build/ci.go xgo -- --go=$(GO) --targets=linux/386 -v ./cmd/geth @echo "Linux 386 cross compilation done:" @ls -ld $(GOBIN)/geth-linux-* | grep 386 geth-linux-amd64: - build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/amd64 -v ./cmd/geth + $(GORUN) build/ci.go xgo -- --go=$(GO) --targets=linux/amd64 -v ./cmd/geth @echo "Linux amd64 cross compilation done:" @ls -ld $(GOBIN)/geth-linux-* | grep amd64 @@ -77,42 +78,42 @@ geth-linux-arm: geth-linux-arm-5 geth-linux-arm-6 geth-linux-arm-7 geth-linux-ar @ls -ld $(GOBIN)/geth-linux-* | grep arm geth-linux-arm-5: - build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/arm-5 -v ./cmd/geth + $(GORUN) build/ci.go xgo -- --go=$(GO) --targets=linux/arm-5 -v ./cmd/geth @echo "Linux ARMv5 cross compilation done:" @ls -ld $(GOBIN)/geth-linux-* | grep arm-5 geth-linux-arm-6: - build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/arm-6 -v ./cmd/geth + $(GORUN) build/ci.go xgo -- --go=$(GO) --targets=linux/arm-6 -v ./cmd/geth @echo "Linux ARMv6 cross compilation done:" @ls -ld $(GOBIN)/geth-linux-* | grep arm-6 geth-linux-arm-7: - build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/arm-7 -v ./cmd/geth + $(GORUN) build/ci.go xgo -- --go=$(GO) --targets=linux/arm-7 -v ./cmd/geth @echo "Linux ARMv7 cross compilation done:" @ls -ld $(GOBIN)/geth-linux-* | grep arm-7 geth-linux-arm64: - build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/arm64 -v ./cmd/geth + $(GORUN) build/ci.go xgo -- --go=$(GO) --targets=linux/arm64 -v ./cmd/geth @echo "Linux ARM64 cross compilation done:" @ls -ld $(GOBIN)/geth-linux-* | grep arm64 geth-linux-mips: - build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/mips --ldflags '-extldflags "-static"' -v ./cmd/geth + $(GORUN) build/ci.go xgo -- --go=$(GO) --targets=linux/mips --ldflags '-extldflags "-static"' -v ./cmd/geth @echo "Linux MIPS cross compilation done:" @ls -ld $(GOBIN)/geth-linux-* | grep mips geth-linux-mipsle: - build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/mipsle --ldflags '-extldflags "-static"' -v ./cmd/geth + $(GORUN) build/ci.go xgo -- --go=$(GO) --targets=linux/mipsle --ldflags '-extldflags "-static"' -v ./cmd/geth @echo "Linux MIPSle cross compilation done:" @ls -ld $(GOBIN)/geth-linux-* | grep mipsle geth-linux-mips64: - build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/mips64 --ldflags '-extldflags "-static"' -v ./cmd/geth + $(GORUN) build/ci.go xgo -- --go=$(GO) --targets=linux/mips64 --ldflags '-extldflags "-static"' -v ./cmd/geth @echo "Linux MIPS64 cross compilation done:" @ls -ld $(GOBIN)/geth-linux-* | grep mips64 geth-linux-mips64le: - build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=linux/mips64le --ldflags '-extldflags "-static"' -v ./cmd/geth + $(GORUN) build/ci.go xgo -- --go=$(GO) --targets=linux/mips64le --ldflags '-extldflags "-static"' -v ./cmd/geth @echo "Linux MIPS64le cross compilation done:" @ls -ld $(GOBIN)/geth-linux-* | grep mips64le @@ -121,12 +122,12 @@ geth-darwin: geth-darwin-386 geth-darwin-amd64 @ls -ld $(GOBIN)/geth-darwin-* geth-darwin-386: - build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=darwin/386 -v ./cmd/geth + $(GORUN) build/ci.go xgo -- --go=$(GO) --targets=darwin/386 -v ./cmd/geth @echo "Darwin 386 cross compilation done:" @ls -ld $(GOBIN)/geth-darwin-* | grep 386 geth-darwin-amd64: - build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=darwin/amd64 -v ./cmd/geth + $(GORUN) build/ci.go xgo -- --go=$(GO) --targets=darwin/amd64 -v ./cmd/geth @echo "Darwin amd64 cross compilation done:" @ls -ld $(GOBIN)/geth-darwin-* | grep amd64 @@ -135,11 +136,11 @@ geth-windows: geth-windows-386 geth-windows-amd64 @ls -ld $(GOBIN)/geth-windows-* geth-windows-386: - build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=windows/386 -v ./cmd/geth + $(GORUN) build/ci.go xgo -- --go=$(GO) --targets=windows/386 -v ./cmd/geth @echo "Windows 386 cross compilation done:" @ls -ld $(GOBIN)/geth-windows-* | grep 386 geth-windows-amd64: - build/env.sh go run build/ci.go xgo -- --go=$(GO) --targets=windows/amd64 -v ./cmd/geth + $(GORUN) build/ci.go xgo -- --go=$(GO) --targets=windows/amd64 -v ./cmd/geth @echo "Windows amd64 cross compilation done:" @ls -ld $(GOBIN)/geth-windows-* | grep amd64 diff --git a/build/ci.go b/build/ci.go index 1ec8e01832..257c479e30 100644 --- a/build/ci.go +++ b/build/ci.go @@ -214,9 +214,9 @@ func doInstall(cmdline []string) { var minor int fmt.Sscanf(strings.TrimPrefix(runtime.Version(), "go1."), "%d", &minor) - if minor < 9 { + if minor < 11 { log.Println("You have Go version", runtime.Version()) - log.Println("go-ethereum requires at least Go version 1.9 and cannot") + log.Println("go-ethereum requires at least Go version 1.11 and cannot") log.Println("be compiled with an earlier version. Please upgrade your Go installation.") os.Exit(1) } @@ -237,13 +237,6 @@ func doInstall(cmdline []string) { build.MustRun(goinstall) return } - // If we are cross compiling to ARMv5 ARMv6 or ARMv7, clean any previous builds - if *arch == "arm" { - os.RemoveAll(filepath.Join(runtime.GOROOT(), "pkg", runtime.GOOS+"_arm")) - for _, path := range filepath.SplitList(build.GOPATH()) { - os.RemoveAll(filepath.Join(path, "pkg", runtime.GOOS+"_arm")) - } - } // Seems we are cross compiling, work around forbidden GOBIN goinstall := goToolArch(*arch, *cc, "install", buildFlags(env)...) @@ -294,7 +287,6 @@ func goTool(subcmd string, args ...string) *exec.Cmd { func goToolArch(arch string, cc string, subcmd string, args ...string) *exec.Cmd { cmd := build.GoTool(subcmd, args...) - cmd.Env = []string{"GOPATH=" + build.GOPATH()} if arch == "" || arch == runtime.GOARCH { cmd.Env = append(cmd.Env, "GOBIN="+GOBIN) } else { @@ -888,7 +880,6 @@ func gomobileTool(subcmd string, args ...string) *exec.Cmd { cmd := exec.Command(filepath.Join(GOBIN, "gomobile"), subcmd) cmd.Args = append(cmd.Args, args...) cmd.Env = []string{ - "GOPATH=" + build.GOPATH(), "PATH=" + GOBIN + string(os.PathListSeparator) + os.Getenv("PATH"), } for _, e := range os.Environ() { @@ -1078,7 +1069,6 @@ func xgoTool(args []string) *exec.Cmd { cmd := exec.Command(filepath.Join(GOBIN, "xgo"), args...) cmd.Env = os.Environ() cmd.Env = append(cmd.Env, []string{ - "GOPATH=" + build.GOPATH(), "GOBIN=" + GOBIN, }...) return cmd diff --git a/build/deb/ethereum/deb.rules b/build/deb/ethereum/deb.rules index 983b87af16..760e1f0e25 100644 --- a/build/deb/ethereum/deb.rules +++ b/build/deb/ethereum/deb.rules @@ -22,7 +22,7 @@ override_dh_auto_build: (mkdir -p build/_workspace/pkg/mod && mv .mod/* build/_workspace/pkg/mod) # A fresh Go was built, all dependency downloads faked, hope build works now - build/env.sh ../.go/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}} + ../.go/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}} override_dh_auto_test: diff --git a/build/env.sh b/build/env.sh deleted file mode 100755 index 3914555d1b..0000000000 --- a/build/env.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/sh - -set -e - -if [ ! -f "build/env.sh" ]; then - echo "$0 must be run from the root of the repository." - exit 2 -fi - -# Create fake Go workspace if it doesn't exist yet. -workspace="$PWD/build/_workspace" -root="$PWD" -ethdir="$workspace/src/github.com/ethereum" -if [ ! -L "$ethdir/go-ethereum" ]; then - mkdir -p "$ethdir" - cd "$ethdir" - ln -s ../../../../../. go-ethereum - cd "$root" -fi - -# Set up the environment to use the workspace. -GOPATH="$workspace" -export GOPATH - -# Run the command inside the workspace. -cd "$ethdir/go-ethereum" -PWD="$ethdir/go-ethereum" - -# Launch the arguments with the configured environment. -exec "$@" diff --git a/internal/build/util.go b/internal/build/util.go index 08a71d8777..fc559760b2 100644 --- a/internal/build/util.go +++ b/internal/build/util.go @@ -51,15 +51,6 @@ func MustRunCommand(cmd string, args ...string) { MustRun(exec.Command(cmd, args...)) } -// GOPATH returns the value that the GOPATH environment -// variable should be set to. -func GOPATH() string { - if os.Getenv("GOPATH") == "" { - log.Fatal("GOPATH is not set") - } - return os.Getenv("GOPATH") -} - var warnedAboutGit bool // RunGit runs a git subcommand and returns its output. From 94e8418939c66233180e9c79a2f716a3b4be70dc Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 15 Jan 2020 10:37:31 +0100 Subject: [PATCH 057/128] build: attempt to fix debian build failure without GOPATH (#20561) --- build/deb/ethereum/deb.rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/deb/ethereum/deb.rules b/build/deb/ethereum/deb.rules index 760e1f0e25..e9c2fd8143 100644 --- a/build/deb/ethereum/deb.rules +++ b/build/deb/ethereum/deb.rules @@ -19,7 +19,7 @@ override_dh_auto_build: # We can't download external go modules within Launchpad, so we're shipping the # entire dependency source cache with go-ethereum. - (mkdir -p build/_workspace/pkg/mod && mv .mod/* build/_workspace/pkg/mod) + (mkdir -p $HOME/go/pkg/mod && mv .mod/* $HOME/go/pkg/mod) # A fresh Go was built, all dependency downloads faked, hope build works now ../.go/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}} From 8704e8a8fc1ae7f70478a344518a864391b13657 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 15 Jan 2020 11:38:35 +0100 Subject: [PATCH 058/128] build: fix makefile HOME reference (#20562) --- build/deb/ethereum/deb.rules | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/build/deb/ethereum/deb.rules b/build/deb/ethereum/deb.rules index e9c2fd8143..e791381689 100644 --- a/build/deb/ethereum/deb.rules +++ b/build/deb/ethereum/deb.rules @@ -19,7 +19,7 @@ override_dh_auto_build: # We can't download external go modules within Launchpad, so we're shipping the # entire dependency source cache with go-ethereum. - (mkdir -p $HOME/go/pkg/mod && mv .mod/* $HOME/go/pkg/mod) + (mkdir -p $(HOME)/go/pkg/mod && mv .mod/* $(HOME)/go/pkg/mod) # A fresh Go was built, all dependency downloads faked, hope build works now ../.go/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}} From f20c8d495a60db86149ebe3e13e995d661bc7958 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Wed, 15 Jan 2020 15:30:50 +0100 Subject: [PATCH 059/128] eth: increase timeout to fix a spurious travis test failure (#20560) --- eth/handler_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/handler_test.go b/eth/handler_test.go index 893bfffa42..354cbc068c 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -608,7 +608,7 @@ func testBroadcastBlock(t *testing.T, totalPeers, broadcastExpected int) { } }(peer) } - timeout := time.After(time.Second) + timeout := time.After(2 * time.Second) var receivedCount int outer: for { From 3e97b04a3df8a175b7cc61ae8f6f7e0d40f5cd4b Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 16 Jan 2020 12:03:41 +0100 Subject: [PATCH 060/128] build: put GOPATH in /tmp on launchpad (#20564) * build: put GOPATH in /tmp on launchpad * build: don't remove GOPATH from go tool environment --- build/ci.go | 2 +- build/deb/ethereum/deb.rules | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/build/ci.go b/build/ci.go index 257c479e30..c96ab1f379 100644 --- a/build/ci.go +++ b/build/ci.go @@ -297,7 +297,7 @@ func goToolArch(arch string, cc string, subcmd string, args ...string) *exec.Cmd cmd.Env = append(cmd.Env, "CC="+cc) } for _, e := range os.Environ() { - if strings.HasPrefix(e, "GOPATH=") || strings.HasPrefix(e, "GOBIN=") { + if strings.HasPrefix(e, "GOBIN=") { continue } cmd.Env = append(cmd.Env, e) diff --git a/build/deb/ethereum/deb.rules b/build/deb/ethereum/deb.rules index e791381689..0677ef91e4 100644 --- a/build/deb/ethereum/deb.rules +++ b/build/deb/ethereum/deb.rules @@ -6,6 +6,7 @@ # Launchpad rejects Go's access to $HOME, use custom folders export GOCACHE=/tmp/go-build +export GOPATH=/tmp/gopath export GOROOT_BOOTSTRAP={{.GoBootPath}} override_dh_auto_clean: @@ -19,7 +20,8 @@ override_dh_auto_build: # We can't download external go modules within Launchpad, so we're shipping the # entire dependency source cache with go-ethereum. - (mkdir -p $(HOME)/go/pkg/mod && mv .mod/* $(HOME)/go/pkg/mod) + mkdir -p $(GOPATH)/pkg + mv .mod $(GOPATH)/pkg/mod # A fresh Go was built, all dependency downloads faked, hope build works now ../.go/bin/go run build/ci.go install -git-commit={{.Env.Commit}} -git-branch={{.Env.Branch}} -git-tag={{.Env.Tag}} -buildnum={{.Env.Buildnum}} -pull-request={{.Env.IsPullRequest}} From 9b32f592dc2f3c20f0705cdc7346a6315440a39d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Thu, 16 Jan 2020 13:28:32 +0200 Subject: [PATCH 061/128] travis, build: enable Ubuntu Focal and Go 1.13.6 on PPA --- .travis.yml | 2 +- build/checksums.txt | 2 +- build/ci.go | 1 + 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 615b51157f..31cc398c57 100644 --- a/.travis.yml +++ b/.travis.yml @@ -93,7 +93,7 @@ jobs: - python-paramiko script: - echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts - - go run build/ci.go debsrc -goversion 1.13.4 -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder " + - go run build/ci.go debsrc -goversion 1.13.6 -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder " # This builder does the Linux Azure uploads - stage: build diff --git a/build/checksums.txt b/build/checksums.txt index bb814e3392..4dc07a10fe 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -1,6 +1,6 @@ # This file contains sha256 checksums of optional build dependencies. -95dbeab442ee2746b9acf0934c8e2fc26414a0565c008631b04addb8c02e7624 go1.13.4.src.tar.gz +aae5be954bdc40bcf8006eb77e8d8a5dde412722bc8effcdaf9772620d06420c go1.13.6.src.tar.gz 1fcbc9e36f4319eeed02beb8cfd1b3d425ffc2f90ddf09a80f18d5064c51e0cb golangci-lint-1.21.0-linux-386.tar.gz 267b4066e67139a38d29499331a002d6a29ad5be7aafc83db3b1e88f1b027f90 golangci-lint-1.21.0-linux-armv6.tar.gz diff --git a/build/ci.go b/build/ci.go index c96ab1f379..dfdc85653e 100644 --- a/build/ci.go +++ b/build/ci.go @@ -145,6 +145,7 @@ var ( "bionic": "golang-go", "disco": "golang-go", "eoan": "golang-go", + "focal": "golang-go", } debGoBootPaths = map[string]string{ From 1ee754b05647ab28a9269238ec0f52f095c7c131 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Thu, 16 Jan 2020 13:09:38 +0100 Subject: [PATCH 062/128] build: upgrade golangci to 1.22.2 (#20566) * build: upgrade golangci to 1.22.2 * .golangci.yml: don't fail on asset deadcode --- .golangci.yml | 5 +++++ build/checksums.txt | 30 +++++++++++++++--------------- build/ci.go | 2 +- 3 files changed, 21 insertions(+), 16 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 44fce8bea3..24d00da6ec 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -6,6 +6,8 @@ run: # default is true. Enables skipping of directories: # vendor$, third_party$, testdata$, examples$, Godeps$, builtin$ skip-dirs-use-default: true + skip-files: + - core/genesis_alloc.go linters: disable-all: true @@ -43,3 +45,6 @@ issues: - path: core/vm/instructions_test.go linters: - goconst + - path: cmd/faucet/ + linters: + - deadcode diff --git a/build/checksums.txt b/build/checksums.txt index 4dc07a10fe..44530ce4be 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -2,18 +2,18 @@ aae5be954bdc40bcf8006eb77e8d8a5dde412722bc8effcdaf9772620d06420c go1.13.6.src.tar.gz -1fcbc9e36f4319eeed02beb8cfd1b3d425ffc2f90ddf09a80f18d5064c51e0cb golangci-lint-1.21.0-linux-386.tar.gz -267b4066e67139a38d29499331a002d6a29ad5be7aafc83db3b1e88f1b027f90 golangci-lint-1.21.0-linux-armv6.tar.gz -a602c1f25f90e46e621019cff0a8cb3f4e1837011f3537f15e730d6a9ebf507b golangci-lint-1.21.0-freebsd-armv7.tar.gz -2c861f8dc56b560474aa27cab0c075991628cc01af3451e27ac82f5d10d5106b golangci-lint-1.21.0-linux-amd64.tar.gz -a1c39e055280e755acaa906e7abfc20b99a5c28be8af541c57fbc44abbb20dde golangci-lint-1.21.0-linux-arm64.tar.gz -a8f8bda8c6a4136acf858091077830b1e83ad5612606cb69d5dced869ce00bd8 golangci-lint-1.21.0-linux-ppc64le.tar.gz -0a8a8c3bc660ccbca668897ab520f7ee9878f16cc8e4dd24fe46236ceec97ba3 golangci-lint-1.21.0-freebsd-armv6.tar.gz -699b07f45e216571f54002bcbd83b511c4801464a422162158e299587b095b18 golangci-lint-1.21.0-freebsd-amd64.tar.gz -980fb4993942154bb5c8129ea3b86de09574fe81b24384ebb58cd7a9d2f04483 golangci-lint-1.21.0-linux-armv7.tar.gz -f15b689088a47f20d5d3c1d945e9ee7c6238f2b84ea468b5f886cf8713dce62e golangci-lint-1.21.0-windows-386.zip -2e40ded7adcf11e59013cb15c24438b15a86526ca241edfcfdf1abd73a5280a8 golangci-lint-1.21.0-windows-amd64.zip -6052c7cfea4d6dc2fc722f6c12792a5ec087420198db495afffbc22052653bf7 golangci-lint-1.21.0-freebsd-386.tar.gz -ca00b8eacf9af14a71b908b4149606c762aa5c0eac781e74ca0abedfdfdf6c8c golangci-lint-1.21.0-linux-s390x.tar.gz -1365455940c342f95718159d89d66ad2eef19f0846c3e87023e915a3527b929f golangci-lint-1.21.0-darwin-386.tar.gz -2b2713ec5007e67883aa501eebb81f22abfab0cf0909134ba90f60a066db3760 golangci-lint-1.21.0-darwin-amd64.tar.gz +478994633b0f5121a7a8d4f368078093e21014fdc7fb2c0ceeae63668c13c5b6 golangci-lint-1.22.2-freebsd-amd64.tar.gz +fcf80824c21567eb0871055711bf9bdca91cf9a081122e2a45f1d11fed754600 golangci-lint-1.22.2-darwin-amd64.tar.gz +cda85c72fc128b2ea0ae05baea7b91172c63aea34064829f65285f1dd536f1e0 golangci-lint-1.22.2-windows-386.zip +94f04899f620aadc9c1524e5482e415efdbd993fa2b2918c4fec2798f030ac1c golangci-lint-1.22.2-linux-armv7.tar.gz +0e72a87d71edde00b6e37e84a99841833ad55fee83e20d21130a7a622b2860bb golangci-lint-1.22.2-freebsd-386.tar.gz +86def2f31fe8fd7c05674104ed2a4bef3e44b7132b93c6ad2f52f198b3d01801 golangci-lint-1.22.2-linux-s390x.tar.gz +b0df4546d36be94e8107733ba290b98dd9b7e41a42d3fb202e87fc7e4ee800c3 golangci-lint-1.22.2-freebsd-armv6.tar.gz +3d45958dcf6a8d195086d2fced1a21db42a90815dfd156d180efa62dbdda6724 golangci-lint-1.22.2-darwin-386.tar.gz +7ee29f35c74fab017a454237990c74d984ce3855960f2c10509238992bb781f9 golangci-lint-1.22.2-linux-arm64.tar.gz +52086ac52a502b68578e58e35d3964f127c16d7a90b9ffcb399a004d055ded51 golangci-lint-1.22.2-linux-386.tar.gz +c2e4df1fab2ae53762f9baac6041503eeeaa968ce38ea41779f7cb526751c667 golangci-lint-1.22.2-windows-amd64.zip +109d38cdc89f271392f5a138d6782657157f9f496fd4801956efa2d0428e0cbe golangci-lint-1.22.2-linux-amd64.tar.gz +f08aae4868d4828c8f07deb0dcd941a1da695b97e58d15e9f3d1d07dcc7a0c84 golangci-lint-1.22.2-linux-armv6.tar.gz +37af03d9c144d527cb15c46a07e6a22d3f62b5491e34ad6f3bfe6bb0b0b597d4 golangci-lint-1.22.2-linux-ppc64le.tar.gz +251a1081d53944f1d5f86216d752837b23079f90605c9d1cc628da1ffcd2e749 golangci-lint-1.22.2-freebsd-armv7.tar.gz diff --git a/build/ci.go b/build/ci.go index dfdc85653e..8c89aa83ae 100644 --- a/build/ci.go +++ b/build/ci.go @@ -356,7 +356,7 @@ func doLint(cmdline []string) { // downloadLinter downloads and unpacks golangci-lint. func downloadLinter(cachedir string) string { - const version = "1.21.0" + const version = "1.22.2" csdb := build.MustLoadChecksums("build/checksums.txt") base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, runtime.GOARCH) From fcafa0baa53a5e0db69100c05da04c61d4810768 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 16 Jan 2020 13:10:15 +0100 Subject: [PATCH 063/128] p2p: wait for listener goroutines on shutdown (#20569) * p2p: wait for goroutine exit, fixes #20558 * p2p: wait for all slots on exit Co-authored-by: Martin Holst Swende --- p2p/server.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/p2p/server.go b/p2p/server.go index 9b9effaf6c..9242961764 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -864,9 +864,9 @@ func (srv *Server) maxDialedConns() int { // listenLoop runs in its own goroutine and accepts // inbound connections. func (srv *Server) listenLoop() { - defer srv.loopWG.Done() srv.log.Debug("TCP listener up", "addr", srv.listener.Addr()) + // The slots channel limits accepts of new connections. tokens := defaultMaxPendingPeers if srv.MaxPendingPeers > 0 { tokens = srv.MaxPendingPeers @@ -876,6 +876,15 @@ func (srv *Server) listenLoop() { slots <- struct{}{} } + // Wait for slots to be returned on exit. This ensures all connection goroutines + // are down before listenLoop returns. + defer srv.loopWG.Done() + defer func() { + for i := 0; i < cap(slots); i++ { + <-slots + } + }() + for { // Wait for a free slot before accepting. <-slots @@ -891,6 +900,7 @@ func (srv *Server) listenLoop() { continue } else if err != nil { srv.log.Debug("Read error", "err", err) + slots <- struct{}{} return } break From d5acc5ed9e74bf5e23b50ca176332028211e3581 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 17 Jan 2020 11:29:16 +0100 Subject: [PATCH 064/128] p2p: ensure Server.loop is ticking even if discovery hangs (#20573) This is a temporary fix for a problem which started happening when the dialer was changed to read nodes from an enode.Iterator. Before the iterator change, discovery queries would always return within a couple seconds even if there was no Internet access. Since the iterator won't return unless a node is actually found, discoverTask can take much longer. This means that the 'emergency connect' logic might not execute in time, leading to a stuck node. --- p2p/server.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/p2p/server.go b/p2p/server.go index 9242961764..fda72e3793 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -650,9 +650,12 @@ func (srv *Server) run(dialstate dialer) { inboundCount = 0 trusted = make(map[enode.ID]bool, len(srv.TrustedNodes)) taskdone = make(chan task, maxActiveDialTasks) + tick = time.NewTicker(30 * time.Second) runningTasks []task queuedTasks []task // tasks that can't run yet ) + defer tick.Stop() + // Put trusted nodes into a map to speed up checks. // Trusted peers are loaded on startup or added via AddTrustedPeer RPC. for _, n := range srv.TrustedNodes { @@ -694,6 +697,9 @@ running: scheduleTasks() select { + case <-tick.C: + // This is just here to ensure the dial scheduler runs occasionally. + case <-srv.quit: // The server was stopped. Run the cleanup logic. break running From 0af96d2556de8eaa41bd2002b028b3740504b734 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Fri, 17 Jan 2020 11:32:29 +0100 Subject: [PATCH 065/128] cmd/devp2p: submit Route53 changes in batches (#20524) This change works around the 32k RDATA character limit per change request and fixes several issues in the deployer which prevented it from working for our production trees. --- cmd/devp2p/dns_cloudflare.go | 4 +- cmd/devp2p/dns_route53.go | 129 +++++++++++++++++++-------- cmd/devp2p/dns_route53_test.go | 156 +++++++++++++++++++++++++++++++++ cmd/devp2p/dnscmd.go | 5 ++ 4 files changed, 255 insertions(+), 39 deletions(-) create mode 100644 cmd/devp2p/dns_route53_test.go diff --git a/cmd/devp2p/dns_cloudflare.go b/cmd/devp2p/dns_cloudflare.go index 83279168cc..a4d10dcfdd 100644 --- a/cmd/devp2p/dns_cloudflare.go +++ b/cmd/devp2p/dns_cloudflare.go @@ -130,9 +130,9 @@ func (c *cloudflareClient) uploadRecords(name string, records map[string]string) if !exists { // Entry is unknown, push a new one to Cloudflare. log.Info(fmt.Sprintf("Creating %s = %q", path, val)) - ttl := 1 + ttl := rootTTL if path != name { - ttl = 2147483647 // Max TTL permitted by Cloudflare + ttl = treeNodeTTL // Max TTL permitted by Cloudflare } _, err = c.CreateDNSRecord(c.zoneID, cloudflare.DNSRecord{Type: "TXT", Name: path, Content: val, TTL: ttl}) } else if old.Content != val { diff --git a/cmd/devp2p/dns_route53.go b/cmd/devp2p/dns_route53.go index 1e9b39b0ea..bdad5c2e9c 100644 --- a/cmd/devp2p/dns_route53.go +++ b/cmd/devp2p/dns_route53.go @@ -19,6 +19,7 @@ package main import ( "errors" "fmt" + "sort" "strconv" "strings" @@ -31,6 +32,10 @@ import ( "gopkg.in/urfave/cli.v1" ) +// The Route53 limits change sets to this size. DNS changes need to be split +// up into multiple batches to work around the limit. +const route53ChangeLimit = 30000 + var ( route53AccessKeyFlag = cli.StringFlag{ Name: "access-key-id", @@ -53,6 +58,11 @@ type route53Client struct { zoneID string } +type recordSet struct { + values []string + ttl int64 +} + // newRoute53Client sets up a Route53 API client from command line flags. func newRoute53Client(ctx *cli.Context) *route53Client { akey := ctx.String(route53AccessKeyFlag.Name) @@ -78,31 +88,39 @@ func (c *route53Client) deploy(name string, t *dnsdisc.Tree) error { } // Compute DNS changes. - records := t.ToTXT(name) - changes, err := c.computeChanges(name, records) + existing, err := c.collectRecords(name) if err != nil { return err } + log.Info(fmt.Sprintf("Found %d TXT records", len(existing))) + + records := t.ToTXT(name) + changes := c.computeChanges(name, records, existing) if len(changes) == 0 { log.Info("No DNS changes needed") return nil } - // Submit change request. - log.Info(fmt.Sprintf("Submitting %d changes to Route53", len(changes))) - batch := new(route53.ChangeBatch) - batch.SetChanges(changes) - batch.SetComment(fmt.Sprintf("enrtree update of %s at seq %d", name, t.Seq())) - req := &route53.ChangeResourceRecordSetsInput{HostedZoneId: &c.zoneID, ChangeBatch: batch} - resp, err := c.api.ChangeResourceRecordSets(req) - if err != nil { - return err - } + // Submit change batches. + batches := splitChanges(changes, route53ChangeLimit) + for i, changes := range batches { + log.Info(fmt.Sprintf("Submitting %d changes to Route53", len(changes))) + batch := new(route53.ChangeBatch) + batch.SetChanges(changes) + batch.SetComment(fmt.Sprintf("enrtree update %d/%d of %s at seq %d", i+1, len(batches), name, t.Seq())) + req := &route53.ChangeResourceRecordSetsInput{HostedZoneId: &c.zoneID, ChangeBatch: batch} + resp, err := c.api.ChangeResourceRecordSets(req) + if err != nil { + return err + } - // Wait for the change to be applied. - log.Info(fmt.Sprintf("Waiting for change request %s", *resp.ChangeInfo.Id)) - wreq := &route53.GetChangeInput{Id: resp.ChangeInfo.Id} - return c.api.WaitUntilResourceRecordSetsChanged(wreq) + log.Info(fmt.Sprintf("Waiting for change request %s", *resp.ChangeInfo.Id)) + wreq := &route53.GetChangeInput{Id: resp.ChangeInfo.Id} + if err := c.api.WaitUntilResourceRecordSetsChanged(wreq); err != nil { + return err + } + } + return nil } // checkZone verifies zone information for the given domain. @@ -137,7 +155,7 @@ func (c *route53Client) findZoneID(name string) (string, error) { } // computeChanges creates DNS changes for the given record. -func (c *route53Client) computeChanges(name string, records map[string]string) ([]*route53.Change, error) { +func (c *route53Client) computeChanges(name string, records map[string]string, existing map[string]recordSet) []*route53.Change { // Convert all names to lowercase. lrecords := make(map[string]string, len(records)) for name, r := range records { @@ -145,22 +163,15 @@ func (c *route53Client) computeChanges(name string, records map[string]string) ( } records = lrecords - // Get existing records. - existing, err := c.collectRecords(name) - if err != nil { - return nil, err - } - log.Info(fmt.Sprintf("Found %d TXT records", len(existing))) - var changes []*route53.Change for path, val := range records { - ttl := 1 + ttl := int64(rootTTL) if path != name { - ttl = 2147483647 + ttl = int64(treeNodeTTL) } prevRecords, exists := existing[path] - prevValue := combineTXT(prevRecords) + prevValue := combineTXT(prevRecords.values) if !exists { // Entry is unknown, push a new one log.Info(fmt.Sprintf("Creating %s = %q", path, val)) @@ -175,32 +186,76 @@ func (c *route53Client) computeChanges(name string, records map[string]string) ( } // Iterate over the old records and delete anything stale. - for path, values := range existing { + for path, set := range existing { if _, ok := records[path]; ok { continue } // Stale entry, nuke it. - log.Info(fmt.Sprintf("Deleting %s = %q", path, combineTXT(values))) - changes = append(changes, newTXTChange("DELETE", path, 1, values)) + log.Info(fmt.Sprintf("Deleting %s = %q", path, combineTXT(set.values))) + changes = append(changes, newTXTChange("DELETE", path, set.ttl, set.values)) } - return changes, nil + + sortChanges(changes) + return changes +} + +// sortChanges ensures DNS changes are in leaf-added -> root-changed -> leaf-deleted order. +func sortChanges(changes []*route53.Change) { + score := map[string]int{"CREATE": 1, "UPSERT": 2, "DELETE": 3} + sort.Slice(changes, func(i, j int) bool { + if *changes[i].Action == *changes[j].Action { + return *changes[i].ResourceRecordSet.Name < *changes[j].ResourceRecordSet.Name + } + return score[*changes[i].Action] < score[*changes[j].Action] + }) +} + +// splitChanges splits up DNS changes such that each change batch +// is smaller than the given RDATA limit. +func splitChanges(changes []*route53.Change, limit int) [][]*route53.Change { + var batches [][]*route53.Change + var batchSize int + for _, ch := range changes { + // Start new batch if this change pushes the current one over the limit. + size := changeSize(ch) + if len(batches) == 0 || batchSize+size > limit { + batches = append(batches, nil) + batchSize = 0 + } + batches[len(batches)-1] = append(batches[len(batches)-1], ch) + batchSize += size + } + return batches +} + +// changeSize returns the RDATA size of a DNS change. +func changeSize(ch *route53.Change) int { + size := 0 + for _, rr := range ch.ResourceRecordSet.ResourceRecords { + if rr.Value != nil { + size += len(*rr.Value) + } + } + return size } // collectRecords collects all TXT records below the given name. -func (c *route53Client) collectRecords(name string) (map[string][]string, error) { +func (c *route53Client) collectRecords(name string) (map[string]recordSet, error) { log.Info(fmt.Sprintf("Retrieving existing TXT records on %s (%s)", name, c.zoneID)) var req route53.ListResourceRecordSetsInput req.SetHostedZoneId(c.zoneID) - existing := make(map[string][]string) + existing := make(map[string]recordSet) err := c.api.ListResourceRecordSetsPages(&req, func(resp *route53.ListResourceRecordSetsOutput, last bool) bool { for _, set := range resp.ResourceRecordSets { if !isSubdomain(*set.Name, name) || *set.Type != "TXT" { continue } - name := strings.TrimSuffix(*set.Name, ".") + s := recordSet{ttl: *set.TTL} for _, rec := range set.ResourceRecords { - existing[name] = append(existing[name], *rec.Value) + s.values = append(s.values, *rec.Value) } + name := strings.TrimSuffix(*set.Name, ".") + existing[name] = s } return true }) @@ -208,7 +263,7 @@ func (c *route53Client) collectRecords(name string) (map[string][]string, error) } // newTXTChange creates a change to a TXT record. -func newTXTChange(action, name string, ttl int, values []string) *route53.Change { +func newTXTChange(action, name string, ttl int64, values []string) *route53.Change { var c route53.Change var r route53.ResourceRecordSet var rrs []*route53.ResourceRecord @@ -219,7 +274,7 @@ func newTXTChange(action, name string, ttl int, values []string) *route53.Change } r.SetType("TXT") r.SetName(name) - r.SetTTL(int64(ttl)) + r.SetTTL(ttl) r.SetResourceRecords(rrs) c.SetAction(action) c.SetResourceRecordSet(&r) diff --git a/cmd/devp2p/dns_route53_test.go b/cmd/devp2p/dns_route53_test.go new file mode 100644 index 0000000000..c64f1d1698 --- /dev/null +++ b/cmd/devp2p/dns_route53_test.go @@ -0,0 +1,156 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of go-ethereum. +// +// go-ethereum is free software: you can redistribute it and/or modify +// it under the terms of the GNU General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// go-ethereum is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU General Public License for more details. +// +// You should have received a copy of the GNU General Public License +// along with go-ethereum. If not, see . + +package main + +import ( + "reflect" + "testing" + + "github.com/aws/aws-sdk-go/service/route53" +) + +// This test checks that computeChanges/splitChanges create DNS changes in +// leaf-added -> root-changed -> leaf-deleted order. +func TestRoute53ChangeSort(t *testing.T) { + testTree0 := map[string]recordSet{ + "2kfjogvxdqtxxugbh7gs7naaai.n": {ttl: 3333, values: []string{ + `"enr:-HW4QO1ml1DdXLeZLsUxewnthhUy8eROqkDyoMTyavfks9JlYQIlMFEUoM78PovJDPQrAkrb3LRJ-"`, + `"vtrymDguKCOIAWAgmlkgnY0iXNlY3AyNTZrMaEDffaGfJzgGhUif1JqFruZlYmA31HzathLSWxfbq_QoQ4"`, + }}, + "fdxn3sn67na5dka4j2gok7bvqi.n": {ttl: treeNodeTTL, values: []string{`"enrtree-branch:"`}}, + "n": {ttl: rootTTL, values: []string{`"enrtree-root:v1 e=2KFJOGVXDQTXXUGBH7GS7NAAAI l=FDXN3SN67NA5DKA4J2GOK7BVQI seq=0 sig=v_-J_q_9ICQg5ztExFvLQhDBGMb0lZPJLhe3ts9LAcgqhOhtT3YFJsl8BWNDSwGtamUdR-9xl88_w-X42SVpjwE"`}}, + } + + testTree1 := map[string]string{ + "n": "enrtree-root:v1 e=JWXYDBPXYWG6FX3GMDIBFA6CJ4 l=C7HRFPF3BLGF3YR4DY5KX3SMBE seq=1 sig=o908WmNp7LibOfPsr4btQwatZJ5URBr2ZAuxvK4UWHlsB9sUOTJQaGAlLPVAhM__XJesCHxLISo94z5Z2a463gA", + "C7HRFPF3BLGF3YR4DY5KX3SMBE.n": "enrtree://AM5FCQLWIZX2QFPNJAP7VUERCCRNGRHWZG3YYHIUV7BVDQ5FDPRT2@morenodes.example.org", + "JWXYDBPXYWG6FX3GMDIBFA6CJ4.n": "enrtree-branch:2XS2367YHAXJFGLZHVAWLQD4ZY,H4FHT4B454P6UXFD7JCYQ5PWDY,MHTDO6TMUBRIA2XWG5LUDACK24", + "2XS2367YHAXJFGLZHVAWLQD4ZY.n": "enr:-HW4QOFzoVLaFJnNhbgMoDXPnOvcdVuj7pDpqRvh6BRDO68aVi5ZcjB3vzQRZH2IcLBGHzo8uUN3snqmgTiE56CH3AMBgmlkgnY0iXNlY3AyNTZrMaECC2_24YYkYHEgdzxlSNKQEnHhuNAbNlMlWJxrJxbAFvA", + "H4FHT4B454P6UXFD7JCYQ5PWDY.n": "enr:-HW4QAggRauloj2SDLtIHN1XBkvhFZ1vtf1raYQp9TBW2RD5EEawDzbtSmlXUfnaHcvwOizhVYLtr7e6vw7NAf6mTuoCgmlkgnY0iXNlY3AyNTZrMaECjrXI8TLNXU0f8cthpAMxEshUyQlK-AM0PW2wfrnacNI", + "MHTDO6TMUBRIA2XWG5LUDACK24.n": "enr:-HW4QLAYqmrwllBEnzWWs7I5Ev2IAs7x_dZlbYdRdMUx5EyKHDXp7AV5CkuPGUPdvbv1_Ms1CPfhcGCvSElSosZmyoqAgmlkgnY0iXNlY3AyNTZrMaECriawHKWdDRk2xeZkrOXBQ0dfMFLHY4eENZwdufn1S1o", + } + + wantChanges := []*route53.Change{ + { + Action: sp("CREATE"), + ResourceRecordSet: &route53.ResourceRecordSet{ + Name: sp("2xs2367yhaxjfglzhvawlqd4zy.n"), + ResourceRecords: []*route53.ResourceRecord{{ + Value: sp(`"enr:-HW4QOFzoVLaFJnNhbgMoDXPnOvcdVuj7pDpqRvh6BRDO68aVi5ZcjB3vzQRZH2IcLBGHzo8uUN3snqmgTiE56CH3AMBgmlkgnY0iXNlY3AyNTZrMaECC2_24YYkYHEgdzxlSNKQEnHhuNAbNlMlWJxrJxbAFvA"`), + }}, + TTL: ip(treeNodeTTL), + Type: sp("TXT"), + }, + }, + { + Action: sp("CREATE"), + ResourceRecordSet: &route53.ResourceRecordSet{ + Name: sp("c7hrfpf3blgf3yr4dy5kx3smbe.n"), + ResourceRecords: []*route53.ResourceRecord{{ + Value: sp(`"enrtree://AM5FCQLWIZX2QFPNJAP7VUERCCRNGRHWZG3YYHIUV7BVDQ5FDPRT2@morenodes.example.org"`), + }}, + TTL: ip(treeNodeTTL), + Type: sp("TXT"), + }, + }, + { + Action: sp("CREATE"), + ResourceRecordSet: &route53.ResourceRecordSet{ + Name: sp("h4fht4b454p6uxfd7jcyq5pwdy.n"), + ResourceRecords: []*route53.ResourceRecord{{ + Value: sp(`"enr:-HW4QAggRauloj2SDLtIHN1XBkvhFZ1vtf1raYQp9TBW2RD5EEawDzbtSmlXUfnaHcvwOizhVYLtr7e6vw7NAf6mTuoCgmlkgnY0iXNlY3AyNTZrMaECjrXI8TLNXU0f8cthpAMxEshUyQlK-AM0PW2wfrnacNI"`), + }}, + TTL: ip(treeNodeTTL), + Type: sp("TXT"), + }, + }, + { + Action: sp("CREATE"), + ResourceRecordSet: &route53.ResourceRecordSet{ + Name: sp("jwxydbpxywg6fx3gmdibfa6cj4.n"), + ResourceRecords: []*route53.ResourceRecord{{ + Value: sp(`"enrtree-branch:2XS2367YHAXJFGLZHVAWLQD4ZY,H4FHT4B454P6UXFD7JCYQ5PWDY,MHTDO6TMUBRIA2XWG5LUDACK24"`), + }}, + TTL: ip(treeNodeTTL), + Type: sp("TXT"), + }, + }, + { + Action: sp("CREATE"), + ResourceRecordSet: &route53.ResourceRecordSet{ + Name: sp("mhtdo6tmubria2xwg5ludack24.n"), + ResourceRecords: []*route53.ResourceRecord{{ + Value: sp(`"enr:-HW4QLAYqmrwllBEnzWWs7I5Ev2IAs7x_dZlbYdRdMUx5EyKHDXp7AV5CkuPGUPdvbv1_Ms1CPfhcGCvSElSosZmyoqAgmlkgnY0iXNlY3AyNTZrMaECriawHKWdDRk2xeZkrOXBQ0dfMFLHY4eENZwdufn1S1o"`), + }}, + TTL: ip(treeNodeTTL), + Type: sp("TXT"), + }, + }, + { + Action: sp("UPSERT"), + ResourceRecordSet: &route53.ResourceRecordSet{ + Name: sp("n"), + ResourceRecords: []*route53.ResourceRecord{{ + Value: sp(`"enrtree-root:v1 e=JWXYDBPXYWG6FX3GMDIBFA6CJ4 l=C7HRFPF3BLGF3YR4DY5KX3SMBE seq=1 sig=o908WmNp7LibOfPsr4btQwatZJ5URBr2ZAuxvK4UWHlsB9sUOTJQaGAlLPVAhM__XJesCHxLISo94z5Z2a463gA"`), + }}, + TTL: ip(rootTTL), + Type: sp("TXT"), + }, + }, + { + Action: sp("DELETE"), + ResourceRecordSet: &route53.ResourceRecordSet{ + Name: sp("2kfjogvxdqtxxugbh7gs7naaai.n"), + ResourceRecords: []*route53.ResourceRecord{ + {Value: sp(`"enr:-HW4QO1ml1DdXLeZLsUxewnthhUy8eROqkDyoMTyavfks9JlYQIlMFEUoM78PovJDPQrAkrb3LRJ-"`)}, + {Value: sp(`"vtrymDguKCOIAWAgmlkgnY0iXNlY3AyNTZrMaEDffaGfJzgGhUif1JqFruZlYmA31HzathLSWxfbq_QoQ4"`)}, + }, + TTL: ip(3333), + Type: sp("TXT"), + }, + }, + { + Action: sp("DELETE"), + ResourceRecordSet: &route53.ResourceRecordSet{ + Name: sp("fdxn3sn67na5dka4j2gok7bvqi.n"), + ResourceRecords: []*route53.ResourceRecord{{ + Value: sp(`"enrtree-branch:"`), + }}, + TTL: ip(treeNodeTTL), + Type: sp("TXT"), + }, + }, + } + + var client route53Client + changes := client.computeChanges("n", testTree1, testTree0) + if !reflect.DeepEqual(changes, wantChanges) { + t.Fatalf("wrong changes (got %d, want %d)", len(changes), len(wantChanges)) + } + + wantSplit := [][]*route53.Change{ + wantChanges[:4], + wantChanges[4:8], + } + split := splitChanges(changes, 600) + if !reflect.DeepEqual(split, wantSplit) { + t.Fatalf("wrong split batches: got %d, want %d", len(split), len(wantSplit)) + } +} + +func sp(s string) *string { return &s } +func ip(i int64) *int64 { return &i } diff --git a/cmd/devp2p/dnscmd.go b/cmd/devp2p/dnscmd.go index 287d6e6c76..7c9ccd31f4 100644 --- a/cmd/devp2p/dnscmd.go +++ b/cmd/devp2p/dnscmd.go @@ -96,6 +96,11 @@ var ( } ) +const ( + rootTTL = 1 + treeNodeTTL = 2147483647 +) + // dnsSync performs dnsSyncCommand. func dnsSync(ctx *cli.Context) error { var ( From 770316dc206618c97afaac9deceb1f9b878488ea Mon Sep 17 00:00:00 2001 From: gary rong Date: Fri, 17 Jan 2020 18:49:32 +0800 Subject: [PATCH 066/128] core, light: write chain data in atomic way (#20287) * core: write chain data in atomic way * core, light: address comments * core, light: fix linter * core, light: address comments --- core/blockchain.go | 142 ++++++++++++++++++++++++++------------------ core/headerchain.go | 85 +++++++++++++++----------- light/lightchain.go | 21 +++++-- 3 files changed, 151 insertions(+), 97 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index ebab7a3ab5..f083f53e04 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -407,6 +407,11 @@ func (bc *BlockChain) SetHead(head uint64) error { } } rawdb.WriteHeadBlockHash(db, newHeadBlock.Hash()) + + // Degrade the chain markers if they are explicitly reverted. + // In theory we should update all in-memory markers in the + // last step, however the direction of SetHead is from high + // to low, so it's safe the update in-memory markers directly. bc.currentBlock.Store(newHeadBlock) headBlockGauge.Update(int64(newHeadBlock.NumberU64())) } @@ -419,6 +424,11 @@ func (bc *BlockChain) SetHead(head uint64) error { newHeadFastBlock = bc.genesisBlock } rawdb.WriteHeadFastBlockHash(db, newHeadFastBlock.Hash()) + + // Degrade the chain markers if they are explicitly reverted. + // In theory we should update all in-memory markers in the + // last step, however the direction of SetHead is from high + // to low, so it's safe the update in-memory markers directly. bc.currentFastBlock.Store(newHeadFastBlock) headFastBlockGauge.Update(int64(newHeadFastBlock.NumberU64())) } @@ -538,21 +548,22 @@ func (bc *BlockChain) ResetWithGenesisBlock(genesis *types.Block) error { defer bc.chainmu.Unlock() // Prepare the genesis block and reinitialise the chain - if err := bc.hc.WriteTd(genesis.Hash(), genesis.NumberU64(), genesis.Difficulty()); err != nil { - log.Crit("Failed to write genesis block TD", "err", err) + batch := bc.db.NewBatch() + rawdb.WriteTd(batch, genesis.Hash(), genesis.NumberU64(), genesis.Difficulty()) + rawdb.WriteBlock(batch, genesis) + if err := batch.Write(); err != nil { + log.Crit("Failed to write genesis block", "err", err) } - rawdb.WriteBlock(bc.db, genesis) + bc.writeHeadBlock(genesis) + // Last update all in-memory chain markers bc.genesisBlock = genesis - bc.insert(bc.genesisBlock) bc.currentBlock.Store(bc.genesisBlock) headBlockGauge.Update(int64(bc.genesisBlock.NumberU64())) - bc.hc.SetGenesis(bc.genesisBlock.Header()) bc.hc.SetCurrentHeader(bc.genesisBlock.Header()) bc.currentFastBlock.Store(bc.genesisBlock) headFastBlockGauge.Update(int64(bc.genesisBlock.NumberU64())) - return nil } @@ -610,31 +621,39 @@ func (bc *BlockChain) ExportN(w io.Writer, first uint64, last uint64) error { return nil } -// insert injects a new head block into the current block chain. This method +// writeHeadBlock injects a new head block into the current block chain. This method // assumes that the block is indeed a true head. It will also reset the head // header and the head fast sync block to this very same block if they are older // or if they are on a different side chain. // // Note, this function assumes that the `mu` mutex is held! -func (bc *BlockChain) insert(block *types.Block) { +func (bc *BlockChain) writeHeadBlock(block *types.Block) { // If the block is on a side chain or an unknown one, force other heads onto it too updateHeads := rawdb.ReadCanonicalHash(bc.db, block.NumberU64()) != block.Hash() // Add the block to the canonical chain number scheme and mark as the head - rawdb.WriteCanonicalHash(bc.db, block.Hash(), block.NumberU64()) - rawdb.WriteHeadBlockHash(bc.db, block.Hash()) - - bc.currentBlock.Store(block) - headBlockGauge.Update(int64(block.NumberU64())) + batch := bc.db.NewBatch() + rawdb.WriteCanonicalHash(batch, block.Hash(), block.NumberU64()) + rawdb.WriteTxLookupEntries(batch, block) + rawdb.WriteHeadBlockHash(batch, block.Hash()) // If the block is better than our head or is on a different chain, force update heads + if updateHeads { + rawdb.WriteHeadHeaderHash(batch, block.Hash()) + rawdb.WriteHeadFastBlockHash(batch, block.Hash()) + } + // Flush the whole batch into the disk, exit the node if failed + if err := batch.Write(); err != nil { + log.Crit("Failed to update chain indexes and markers", "err", err) + } + // Update all in-memory chain markers in the last step if updateHeads { bc.hc.SetCurrentHeader(block.Header()) - rawdb.WriteHeadFastBlockHash(bc.db, block.Hash()) - bc.currentFastBlock.Store(block) headFastBlockGauge.Update(int64(block.NumberU64())) } + bc.currentBlock.Store(block) + headBlockGauge.Update(int64(block.NumberU64())) } // Genesis retrieves the chain's genesis block. @@ -881,26 +900,36 @@ func (bc *BlockChain) Rollback(chain []common.Hash) { bc.chainmu.Lock() defer bc.chainmu.Unlock() + batch := bc.db.NewBatch() for i := len(chain) - 1; i >= 0; i-- { hash := chain[i] + // Degrade the chain markers if they are explicitly reverted. + // In theory we should update all in-memory markers in the + // last step, however the direction of rollback is from high + // to low, so it's safe the update in-memory markers directly. currentHeader := bc.hc.CurrentHeader() if currentHeader.Hash() == hash { - bc.hc.SetCurrentHeader(bc.GetHeader(currentHeader.ParentHash, currentHeader.Number.Uint64()-1)) + newHeadHeader := bc.GetHeader(currentHeader.ParentHash, currentHeader.Number.Uint64()-1) + rawdb.WriteHeadHeaderHash(batch, currentHeader.ParentHash) + bc.hc.SetCurrentHeader(newHeadHeader) } if currentFastBlock := bc.CurrentFastBlock(); currentFastBlock.Hash() == hash { newFastBlock := bc.GetBlock(currentFastBlock.ParentHash(), currentFastBlock.NumberU64()-1) - rawdb.WriteHeadFastBlockHash(bc.db, newFastBlock.Hash()) + rawdb.WriteHeadFastBlockHash(batch, currentFastBlock.ParentHash()) bc.currentFastBlock.Store(newFastBlock) headFastBlockGauge.Update(int64(newFastBlock.NumberU64())) } if currentBlock := bc.CurrentBlock(); currentBlock.Hash() == hash { newBlock := bc.GetBlock(currentBlock.ParentHash(), currentBlock.NumberU64()-1) - rawdb.WriteHeadBlockHash(bc.db, newBlock.Hash()) + rawdb.WriteHeadBlockHash(batch, currentBlock.ParentHash()) bc.currentBlock.Store(newBlock) headBlockGauge.Update(int64(newBlock.NumberU64())) } } + if err := batch.Write(); err != nil { + log.Crit("Failed to rollback chain markers", "err", err) + } // Truncate ancient data which exceeds the current header. // // Notably, it can happen that system crashes without truncating the ancient data @@ -1063,7 +1092,6 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ } // Don't collect too much in-memory, write it out every 100K blocks if len(deleted) > 100000 { - // Sync the ancient store explicitly to ensure all data has been flushed to disk. if err := bc.db.Sync(); err != nil { return 0, err @@ -1172,7 +1200,9 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ rawdb.WriteReceipts(batch, block.Hash(), block.NumberU64(), receiptChain[i]) rawdb.WriteTxLookupEntries(batch, block) - stats.processed++ + // Write everything belongs to the blocks into the database. So that + // we can ensure all components of body is completed(body, receipts, + // tx indexes) if batch.ValueSize() >= ethdb.IdealBatchSize { if err := batch.Write(); err != nil { return 0, err @@ -1180,7 +1210,11 @@ func (bc *BlockChain) InsertReceiptChain(blockChain types.Blocks, receiptChain [ size += batch.ValueSize() batch.Reset() } + stats.processed++ } + // Write everything belongs to the blocks into the database. So that + // we can ensure all components of body is completed(body, receipts, + // tx indexes) if batch.ValueSize() > 0 { size += batch.ValueSize() if err := batch.Write(); err != nil { @@ -1231,11 +1265,12 @@ func (bc *BlockChain) writeBlockWithoutState(block *types.Block, td *big.Int) (e bc.wg.Add(1) defer bc.wg.Done() - if err := bc.hc.WriteTd(block.Hash(), block.NumberU64(), td); err != nil { - return err + batch := bc.db.NewBatch() + rawdb.WriteTd(batch, block.Hash(), block.NumberU64(), td) + rawdb.WriteBlock(batch, block) + if err := batch.Write(); err != nil { + log.Crit("Failed to write block into disk", "err", err) } - rawdb.WriteBlock(bc.db, block) - return nil } @@ -1251,11 +1286,7 @@ func (bc *BlockChain) writeKnownBlock(block *types.Block) error { return err } } - // Write the positional metadata for transaction/receipt lookups. - // Preimages here is empty, ignore it. - rawdb.WriteTxLookupEntries(bc.db, block) - - bc.insert(block) + bc.writeHeadBlock(block) return nil } @@ -1283,12 +1314,19 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. localTd := bc.GetTd(currentBlock.Hash(), currentBlock.NumberU64()) externTd := new(big.Int).Add(block.Difficulty(), ptd) - // Irrelevant of the canonical status, write the block itself to the database - if err := bc.hc.WriteTd(block.Hash(), block.NumberU64(), externTd); err != nil { - return NonStatTy, err - } - rawdb.WriteBlock(bc.db, block) - + // Irrelevant of the canonical status, write the block itself to the database. + // + // Note all the components of block(td, hash->number map, header, body, receipts) + // should be written atomically. BlockBatch is used for containing all components. + blockBatch := bc.db.NewBatch() + rawdb.WriteTd(blockBatch, block.Hash(), block.NumberU64(), externTd) + rawdb.WriteBlock(blockBatch, block) + rawdb.WriteReceipts(blockBatch, block.Hash(), block.NumberU64(), receipts) + rawdb.WritePreimages(blockBatch, state.Preimages()) + if err := blockBatch.Write(); err != nil { + log.Crit("Failed to write block into disk", "err", err) + } + // Commit all cached state changes into underlying memory database. root, err := state.Commit(bc.chainConfig.IsEIP158(block.Number())) if err != nil { return NonStatTy, err @@ -1347,11 +1385,6 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. } } } - - // Write other block data using a batch. - batch := bc.db.NewBatch() - rawdb.WriteReceipts(batch, block.Hash(), block.NumberU64(), receipts) - // If the total difficulty is higher than our known, add it to the canonical chain // Second clause in the if statement reduces the vulnerability to selfish mining. // Please refer to http://www.cs.cornell.edu/~ie53/publications/btcProcFC.pdf @@ -1377,21 +1410,13 @@ func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types. return NonStatTy, err } } - // Write the positional metadata for transaction/receipt lookups and preimages - rawdb.WriteTxLookupEntries(batch, block) - rawdb.WritePreimages(batch, state.Preimages()) - status = CanonStatTy } else { status = SideStatTy } - if err := batch.Write(); err != nil { - return NonStatTy, err - } - // Set new head. if status == CanonStatTy { - bc.insert(block) + bc.writeHeadBlock(block) } bc.futureBlocks.Remove(block.Hash()) @@ -1991,20 +2016,19 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { // taking care of the proper incremental order. for i := len(newChain) - 1; i >= 1; i-- { // Insert the block in the canonical way, re-writing history - bc.insert(newChain[i]) + bc.writeHeadBlock(newChain[i]) // Collect reborn logs due to chain reorg collectLogs(newChain[i].Hash(), false) - // Write lookup entries for hash based transaction/receipt searches - rawdb.WriteTxLookupEntries(bc.db, newChain[i]) + // Collect the new added transactions. addedTxs = append(addedTxs, newChain[i].Transactions()...) } - // When transactions get deleted from the database, the receipts that were - // created in the fork must also be deleted - batch := bc.db.NewBatch() + // Delete useless indexes right now which includes the non-canonical + // transaction indexes, canonical chain indexes which above the head. + indexesBatch := bc.db.NewBatch() for _, tx := range types.TxDifference(deletedTxs, addedTxs) { - rawdb.DeleteTxLookupEntry(batch, tx.Hash()) + rawdb.DeleteTxLookupEntry(indexesBatch, tx.Hash()) } // Delete any canonical number assignments above the new head number := bc.CurrentBlock().NumberU64() @@ -2013,9 +2037,11 @@ func (bc *BlockChain) reorg(oldBlock, newBlock *types.Block) error { if hash == (common.Hash{}) { break } - rawdb.DeleteCanonicalHash(batch, i) + rawdb.DeleteCanonicalHash(indexesBatch, i) + } + if err := indexesBatch.Write(); err != nil { + log.Crit("Failed to delete useless indexes", "err", err) } - batch.Write() // If any logs need to be fired, do it now. In theory we could avoid creating // this goroutine if there are no events to fire, but realistcally that only // ever happens if we're reorging empty blocks, which will only happen on idle diff --git a/core/headerchain.go b/core/headerchain.go index 4682069cff..f21dcf537e 100644 --- a/core/headerchain.go +++ b/core/headerchain.go @@ -45,6 +45,14 @@ const ( // HeaderChain implements the basic block header chain logic that is shared by // core.BlockChain and light.LightChain. It is not usable in itself, only as // a part of either structure. +// +// HeaderChain is responsible for maintaining the header chain including the +// header query and updating. +// +// The components maintained by headerchain includes: (1) total difficult +// (2) header (3) block hash -> number mapping (4) canonical number -> hash mapping +// and (5) head header flag. +// // It is not thread safe either, the encapsulating chain structures should do // the necessary mutex locking/unlocking. type HeaderChain struct { @@ -66,10 +74,8 @@ type HeaderChain struct { engine consensus.Engine } -// NewHeaderChain creates a new HeaderChain structure. -// getValidator should return the parent's validator -// procInterrupt points to the parent's interrupt semaphore -// wg points to the parent's shutdown wait group +// NewHeaderChain creates a new HeaderChain structure. ProcInterrupt points +// to the parent's interrupt semaphore. func NewHeaderChain(chainDb ethdb.Database, config *params.ChainConfig, engine consensus.Engine, procInterrupt func() bool) (*HeaderChain, error) { headerCache, _ := lru.New(headerCacheLimit) tdCache, _ := lru.New(tdCacheLimit) @@ -147,25 +153,33 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er externTd := new(big.Int).Add(header.Difficulty, ptd) // Irrelevant of the canonical status, write the td and header to the database - if err := hc.WriteTd(hash, number, externTd); err != nil { - log.Crit("Failed to write header total difficulty", "err", err) + // + // Note all the components of header(td, hash->number index and header) should + // be written atomically. + headerBatch := hc.chainDb.NewBatch() + rawdb.WriteTd(headerBatch, hash, number, externTd) + rawdb.WriteHeader(headerBatch, header) + if err := headerBatch.Write(); err != nil { + log.Crit("Failed to write header into disk", "err", err) } - rawdb.WriteHeader(hc.chainDb, header) - // If the total difficulty is higher than our known, add it to the canonical chain // Second clause in the if statement reduces the vulnerability to selfish mining. // Please refer to http://www.cs.cornell.edu/~ie53/publications/btcProcFC.pdf if externTd.Cmp(localTd) > 0 || (externTd.Cmp(localTd) == 0 && mrand.Float64() < 0.5) { + // If the header can be added into canonical chain, adjust the + // header chain markers(canonical indexes and head header flag). + // + // Note all markers should be written atomically. + // Delete any canonical number assignments above the new head - batch := hc.chainDb.NewBatch() + markerBatch := hc.chainDb.NewBatch() for i := number + 1; ; i++ { hash := rawdb.ReadCanonicalHash(hc.chainDb, i) if hash == (common.Hash{}) { break } - rawdb.DeleteCanonicalHash(batch, i) + rawdb.DeleteCanonicalHash(markerBatch, i) } - batch.Write() // Overwrite any stale canonical number assignments var ( @@ -174,16 +188,19 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er headHeader = hc.GetHeader(headHash, headNumber) ) for rawdb.ReadCanonicalHash(hc.chainDb, headNumber) != headHash { - rawdb.WriteCanonicalHash(hc.chainDb, headHash, headNumber) + rawdb.WriteCanonicalHash(markerBatch, headHash, headNumber) headHash = headHeader.ParentHash headNumber = headHeader.Number.Uint64() - 1 headHeader = hc.GetHeader(headHash, headNumber) } // Extend the canonical chain with the new header - rawdb.WriteCanonicalHash(hc.chainDb, hash, number) - rawdb.WriteHeadHeaderHash(hc.chainDb, hash) - + rawdb.WriteCanonicalHash(markerBatch, hash, number) + rawdb.WriteHeadHeaderHash(markerBatch, hash) + if err := markerBatch.Write(); err != nil { + log.Crit("Failed to write header markers into disk", "err", err) + } + // Last step update all in-memory head header markers hc.currentHeaderHash = hash hc.currentHeader.Store(types.CopyHeader(header)) headHeaderGauge.Update(header.Number.Int64()) @@ -192,9 +209,9 @@ func (hc *HeaderChain) WriteHeader(header *types.Header) (status WriteStatus, er } else { status = SideStatTy } + hc.tdCache.Add(hash, externTd) hc.headerCache.Add(hash, header) hc.numberCache.Add(hash, number) - return } @@ -396,14 +413,6 @@ func (hc *HeaderChain) GetTdByHash(hash common.Hash) *big.Int { return hc.GetTd(hash, *number) } -// WriteTd stores a block's total difficulty into the database, also caching it -// along the way. -func (hc *HeaderChain) WriteTd(hash common.Hash, number uint64, td *big.Int) error { - rawdb.WriteTd(hc.chainDb, hash, number, td) - hc.tdCache.Add(hash, new(big.Int).Set(td)) - return nil -} - // GetHeader retrieves a block header from the database by hash and number, // caching it if found. func (hc *HeaderChain) GetHeader(hash common.Hash, number uint64) *types.Header { @@ -431,6 +440,8 @@ func (hc *HeaderChain) GetHeaderByHash(hash common.Hash) *types.Header { } // HasHeader checks if a block header is present in the database or not. +// In theory, if header is present in the database, all relative components +// like td and hash->number should be present too. func (hc *HeaderChain) HasHeader(hash common.Hash, number uint64) bool { if hc.numberCache.Contains(hash) || hc.headerCache.Contains(hash) { return true @@ -458,10 +469,9 @@ func (hc *HeaderChain) CurrentHeader() *types.Header { return hc.currentHeader.Load().(*types.Header) } -// SetCurrentHeader sets the current head header of the canonical chain. +// SetCurrentHeader sets the in-memory head header marker of the canonical chan +// as the given header. func (hc *HeaderChain) SetCurrentHeader(head *types.Header) { - rawdb.WriteHeadHeaderHash(hc.chainDb, head.Hash()) - hc.currentHeader.Store(head) hc.currentHeaderHash = head.Hash() headHeaderGauge.Update(head.Number.Int64()) @@ -500,11 +510,18 @@ func (hc *HeaderChain) SetHead(head uint64, updateFn UpdateHeadBlocksCallback, d // first then remove the relative data from the database. // // Update head first(head fast block, head full block) before deleting the data. + markerBatch := hc.chainDb.NewBatch() if updateFn != nil { - updateFn(hc.chainDb, parent) + updateFn(markerBatch, parent) } // Update head header then. - rawdb.WriteHeadHeaderHash(hc.chainDb, parentHash) + rawdb.WriteHeadHeaderHash(markerBatch, parentHash) + if err := markerBatch.Write(); err != nil { + log.Crit("Failed to update chain markers", "error", err) + } + hc.currentHeader.Store(parent) + hc.currentHeaderHash = parentHash + headHeaderGauge.Update(parent.Number.Int64()) // Remove the relative data from the database. if delFn != nil { @@ -514,13 +531,11 @@ func (hc *HeaderChain) SetHead(head uint64, updateFn UpdateHeadBlocksCallback, d rawdb.DeleteHeader(batch, hash, num) rawdb.DeleteTd(batch, hash, num) rawdb.DeleteCanonicalHash(batch, num) - - hc.currentHeader.Store(parent) - hc.currentHeaderHash = parentHash - headHeaderGauge.Update(parent.Number.Int64()) } - batch.Write() - + // Flush all accumulated deletions. + if err := batch.Write(); err != nil { + log.Crit("Failed to rewind block", "error", err) + } // Clear out any stale content from the caches hc.headerCache.Purge() hc.tdCache.Purge() diff --git a/light/lightchain.go b/light/lightchain.go index 90ea263237..636e06f518 100644 --- a/light/lightchain.go +++ b/light/lightchain.go @@ -159,7 +159,6 @@ func (lc *LightChain) loadLastState() error { lc.hc.SetCurrentHeader(header) } } - // Issue a status log and return header := lc.hc.CurrentHeader() headerTd := lc.GetTd(header.Hash(), header.Number.Uint64()) @@ -198,9 +197,13 @@ func (lc *LightChain) ResetWithGenesisBlock(genesis *types.Block) { defer lc.chainmu.Unlock() // Prepare the genesis block and reinitialise the chain - rawdb.WriteTd(lc.chainDb, genesis.Hash(), genesis.NumberU64(), genesis.Difficulty()) - rawdb.WriteBlock(lc.chainDb, genesis) - + batch := lc.chainDb.NewBatch() + rawdb.WriteTd(batch, genesis.Hash(), genesis.NumberU64(), genesis.Difficulty()) + rawdb.WriteBlock(batch, genesis) + rawdb.WriteHeadHeaderHash(batch, genesis.Hash()) + if err := batch.Write(); err != nil { + log.Crit("Failed to reset genesis block", "err", err) + } lc.genesisBlock = genesis lc.hc.SetGenesis(lc.genesisBlock.Header()) lc.hc.SetCurrentHeader(lc.genesisBlock.Header()) @@ -323,13 +326,22 @@ func (lc *LightChain) Rollback(chain []common.Hash) { lc.chainmu.Lock() defer lc.chainmu.Unlock() + batch := lc.chainDb.NewBatch() for i := len(chain) - 1; i >= 0; i-- { hash := chain[i] + // Degrade the chain markers if they are explicitly reverted. + // In theory we should update all in-memory markers in the + // last step, however the direction of rollback is from high + // to low, so it's safe the update in-memory markers directly. if head := lc.hc.CurrentHeader(); head.Hash() == hash { + rawdb.WriteHeadHeaderHash(batch, head.ParentHash) lc.hc.SetCurrentHeader(lc.GetHeader(head.ParentHash, head.Number.Uint64()-1)) } } + if err := batch.Write(); err != nil { + log.Crit("Failed to rollback light chain", "error", err) + } } // postChainEvents iterates over the events generated by a chain insertion and @@ -493,6 +505,7 @@ func (lc *LightChain) SyncCheckpoint(ctx context.Context, checkpoint *params.Tru // Ensure the chain didn't move past the latest block while retrieving it if lc.hc.CurrentHeader().Number.Uint64() < header.Number.Uint64() { log.Info("Updated latest header based on CHT", "number", header.Number, "hash", header.Hash(), "age", common.PrettyAge(time.Unix(int64(header.Time), 0))) + rawdb.WriteHeadHeaderHash(lc.chainDb, header.Hash()) lc.hc.SetCurrentHeader(header) } return true From 9b09c0fc8314e6ee355cc11a81ed0fc40b8a1b4a Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 17 Jan 2020 12:59:45 +0100 Subject: [PATCH 067/128] * trie: utilize callbacks instead of amassing lists in ref/unref (#20529) * trie/tests: add benchmarks and update trie tests * trie: update benchmark tests * trie: utilize callbacks instead of amassing lists of hashes in database ref/unref * trie: replace remaining non-callback based accesses --- trie/database.go | 46 +++---- trie/trie_test.go | 322 ++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 337 insertions(+), 31 deletions(-) diff --git a/trie/database.go b/trie/database.go index b0fd787444..dee9f7844b 100644 --- a/trie/database.go +++ b/trie/database.go @@ -180,35 +180,31 @@ func (n *cachedNode) obj(hash common.Hash) node { return expandNode(hash[:], n.node) } -// childs returns all the tracked children of this node, both the implicit ones -// from inside the node as well as the explicit ones from outside the node. -func (n *cachedNode) childs() []common.Hash { - children := make([]common.Hash, 0, 16) +// forChilds invokes the callback for all the tracked children of this node, +// both the implicit ones from inside the node as well as the explicit ones +//from outside the node. +func (n *cachedNode) forChilds(onChild func(hash common.Hash)) { for child := range n.children { - children = append(children, child) + onChild(child) } if _, ok := n.node.(rawNode); !ok { - gatherChildren(n.node, &children) + forGatherChildren(n.node, onChild) } - return children } -// gatherChildren traverses the node hierarchy of a collapsed storage node and -// retrieves all the hashnode children. -func gatherChildren(n node, children *[]common.Hash) { +// forGatherChildren traverses the node hierarchy of a collapsed storage node and +// invokes the callback for all the hashnode children. +func forGatherChildren(n node, onChild func(hash common.Hash)) { switch n := n.(type) { case *rawShortNode: - gatherChildren(n.Val, children) - + forGatherChildren(n.Val, onChild) case rawFullNode: for i := 0; i < 16; i++ { - gatherChildren(n[i], children) + forGatherChildren(n[i], onChild) } case hashNode: - *children = append(*children, common.BytesToHash(n)) - + onChild(common.BytesToHash(n)) case valueNode, nil: - default: panic(fmt.Sprintf("unknown node type: %T", n)) } @@ -334,11 +330,11 @@ func (db *Database) insert(hash common.Hash, blob []byte, node node) { size: uint16(len(blob)), flushPrev: db.newest, } - for _, child := range entry.childs() { + entry.forChilds(func(child common.Hash) { if c := db.dirties[child]; c != nil { c.parents++ } - } + }) db.dirties[hash] = entry // Update the flush-list endpoints @@ -570,9 +566,9 @@ func (db *Database) dereference(child common.Hash, parent common.Hash) { db.dirties[node.flushNext].flushPrev = node.flushPrev } // Dereference all children and delete the node - for _, hash := range node.childs() { + node.forChilds(func(hash common.Hash) { db.dereference(hash, child) - } + }) delete(db.dirties, child) db.dirtiesSize -= common.StorageSize(common.HashLength + int(node.size)) if node.children != nil { @@ -766,10 +762,14 @@ func (db *Database) commit(hash common.Hash, batch ethdb.Batch, uncacher *cleane if !ok { return nil } - for _, child := range node.childs() { - if err := db.commit(child, batch, uncacher); err != nil { - return err + var err error + node.forChilds(func(child common.Hash) { + if err == nil { + err = db.commit(child, batch, uncacher) } + }) + if err != nil { + return err } if err := batch.Put(hash[:], node.rlp()); err != nil { return err diff --git a/trie/trie_test.go b/trie/trie_test.go index e53ac568e9..172572dddc 100644 --- a/trie/trie_test.go +++ b/trie/trie_test.go @@ -161,7 +161,7 @@ func TestInsert(t *testing.T) { exp := common.HexToHash("8aad789dff2f538bca5d8ea56e8abe10f4c7ba3a5dea95fea4cd6e7c3a1168d3") root := trie.Hash() if root != exp { - t.Errorf("exp %x got %x", exp, root) + t.Errorf("case 1: exp %x got %x", exp, root) } trie = newEmpty() @@ -173,7 +173,7 @@ func TestInsert(t *testing.T) { t.Fatalf("commit error: %v", err) } if root != exp { - t.Errorf("exp %x got %x", exp, root) + t.Errorf("case 2: exp %x got %x", exp, root) } } @@ -316,6 +316,40 @@ func TestLargeValue(t *testing.T) { trie.Hash() } +// TestRandomCases tests som cases that were found via random fuzzing +func TestRandomCases(t *testing.T) { + var rt []randTestStep = []randTestStep{ + {op: 6, key: common.Hex2Bytes(""), value: common.Hex2Bytes("")}, // step 0 + {op: 6, key: common.Hex2Bytes(""), value: common.Hex2Bytes("")}, // step 1 + {op: 0, key: common.Hex2Bytes("d51b182b95d677e5f1c82508c0228de96b73092d78ce78b2230cd948674f66fd1483bd"), value: common.Hex2Bytes("0000000000000002")}, // step 2 + {op: 2, key: common.Hex2Bytes("c2a38512b83107d665c65235b0250002882ac2022eb00711552354832c5f1d030d0e408e"), value: common.Hex2Bytes("")}, // step 3 + {op: 3, key: common.Hex2Bytes(""), value: common.Hex2Bytes("")}, // step 4 + {op: 3, key: common.Hex2Bytes(""), value: common.Hex2Bytes("")}, // step 5 + {op: 6, key: common.Hex2Bytes(""), value: common.Hex2Bytes("")}, // step 6 + {op: 3, key: common.Hex2Bytes(""), value: common.Hex2Bytes("")}, // step 7 + {op: 0, key: common.Hex2Bytes("c2a38512b83107d665c65235b0250002882ac2022eb00711552354832c5f1d030d0e408e"), value: common.Hex2Bytes("0000000000000008")}, // step 8 + {op: 0, key: common.Hex2Bytes("d51b182b95d677e5f1c82508c0228de96b73092d78ce78b2230cd948674f66fd1483bd"), value: common.Hex2Bytes("0000000000000009")}, // step 9 + {op: 2, key: common.Hex2Bytes("fd"), value: common.Hex2Bytes("")}, // step 10 + {op: 6, key: common.Hex2Bytes(""), value: common.Hex2Bytes("")}, // step 11 + {op: 6, key: common.Hex2Bytes(""), value: common.Hex2Bytes("")}, // step 12 + {op: 0, key: common.Hex2Bytes("fd"), value: common.Hex2Bytes("000000000000000d")}, // step 13 + {op: 6, key: common.Hex2Bytes(""), value: common.Hex2Bytes("")}, // step 14 + {op: 1, key: common.Hex2Bytes("c2a38512b83107d665c65235b0250002882ac2022eb00711552354832c5f1d030d0e408e"), value: common.Hex2Bytes("")}, // step 15 + {op: 3, key: common.Hex2Bytes(""), value: common.Hex2Bytes("")}, // step 16 + {op: 0, key: common.Hex2Bytes("c2a38512b83107d665c65235b0250002882ac2022eb00711552354832c5f1d030d0e408e"), value: common.Hex2Bytes("0000000000000011")}, // step 17 + {op: 5, key: common.Hex2Bytes(""), value: common.Hex2Bytes("")}, // step 18 + {op: 3, key: common.Hex2Bytes(""), value: common.Hex2Bytes("")}, // step 19 + {op: 0, key: common.Hex2Bytes("d51b182b95d677e5f1c82508c0228de96b73092d78ce78b2230cd948674f66fd1483bd"), value: common.Hex2Bytes("0000000000000014")}, // step 20 + {op: 0, key: common.Hex2Bytes("d51b182b95d677e5f1c82508c0228de96b73092d78ce78b2230cd948674f66fd1483bd"), value: common.Hex2Bytes("0000000000000015")}, // step 21 + {op: 0, key: common.Hex2Bytes("c2a38512b83107d665c65235b0250002882ac2022eb00711552354832c5f1d030d0e408e"), value: common.Hex2Bytes("0000000000000016")}, // step 22 + {op: 5, key: common.Hex2Bytes(""), value: common.Hex2Bytes("")}, // step 23 + {op: 1, key: common.Hex2Bytes("980c393656413a15c8da01978ed9f89feb80b502f58f2d640e3a2f5f7a99a7018f1b573befd92053ac6f78fca4a87268"), value: common.Hex2Bytes("")}, // step 24 + {op: 1, key: common.Hex2Bytes("fd"), value: common.Hex2Bytes("")}, // step 25 + } + runRandTest(rt) + +} + // randTest performs random trie operations. // Instances of this test are created by Generate. type randTest []randTestStep @@ -375,6 +409,8 @@ func runRandTest(rt randTest) bool { values := make(map[string]string) // tracks content of the trie for i, step := range rt { + fmt.Printf("{op: %d, key: common.Hex2Bytes(\"%x\"), value: common.Hex2Bytes(\"%x\")}, // step %d\n", + step.op, step.key, step.value, i) switch step.op { case opUpdate: tr.Update(step.key, step.value) @@ -470,6 +506,7 @@ func benchGet(b *testing.B, commit bool) { func benchUpdate(b *testing.B, e binary.ByteOrder) *Trie { trie := newEmpty() k := make([]byte, 32) + b.ReportAllocs() for i := 0; i < b.N; i++ { e.PutUint64(k, uint64(i)) trie.Update(k, k) @@ -481,18 +518,135 @@ func benchUpdate(b *testing.B, e binary.ByteOrder) *Trie { // we cannot use b.N as the number of hashing rouns, since all rounds apart from // the first one will be NOOP. As such, we'll use b.N as the number of account to // insert into the trie before measuring the hashing. +// BenchmarkHash-6 288680 4561 ns/op 682 B/op 9 allocs/op +// BenchmarkHash-6 275095 4800 ns/op 685 B/op 9 allocs/op +// pure hasher: +// BenchmarkHash-6 319362 4230 ns/op 675 B/op 9 allocs/op +// BenchmarkHash-6 257460 4674 ns/op 689 B/op 9 allocs/op +// With hashing in-between and pure hasher: +// BenchmarkHash-6 225417 7150 ns/op 982 B/op 12 allocs/op +// BenchmarkHash-6 220378 6197 ns/op 983 B/op 12 allocs/op +// same with old hasher +// BenchmarkHash-6 229758 6437 ns/op 981 B/op 12 allocs/op +// BenchmarkHash-6 212610 7137 ns/op 986 B/op 12 allocs/op func BenchmarkHash(b *testing.B) { + // Create a realistic account trie to hash. We're first adding and hashing N + // entries, then adding N more. + addresses, accounts := makeAccounts(2 * b.N) + // Insert the accounts into the trie and hash it + trie := newEmpty() + i := 0 + for ; i < len(addresses)/2; i++ { + trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) + } + trie.Hash() + for ; i < len(addresses); i++ { + trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) + } + b.ResetTimer() + b.ReportAllocs() + //trie.hashRoot(nil, nil) + trie.Hash() +} + +type account struct { + Nonce uint64 + Balance *big.Int + Root common.Hash + Code []byte +} + +// Benchmarks the trie Commit following a Hash. Since the trie caches the result of any operation, +// we cannot use b.N as the number of hashing rouns, since all rounds apart from +// the first one will be NOOP. As such, we'll use b.N as the number of account to +// insert into the trie before measuring the hashing. +func BenchmarkCommitAfterHash(b *testing.B) { + b.Run("no-onleaf", func(b *testing.B) { + benchmarkCommitAfterHash(b, nil) + }) + var a account + onleaf := func(leaf []byte, parent common.Hash) error { + rlp.DecodeBytes(leaf, &a) + return nil + } + b.Run("with-onleaf", func(b *testing.B) { + benchmarkCommitAfterHash(b, onleaf) + }) +} + +func benchmarkCommitAfterHash(b *testing.B, onleaf LeafCallback) { // Make the random benchmark deterministic - random := rand.New(rand.NewSource(0)) + addresses, accounts := makeAccounts(b.N) + trie := newEmpty() + for i := 0; i < len(addresses); i++ { + trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) + } + // Insert the accounts into the trie and hash it + trie.Hash() + b.ResetTimer() + b.ReportAllocs() + trie.Commit(onleaf) +} + +func TestTinyTrie(t *testing.T) { + // Create a realistic account trie to hash + _, accounts := makeAccounts(10000) + trie := newEmpty() + trie.Update(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001337"), accounts[3]) + if exp, root := common.HexToHash("4fa6efd292cffa2db0083b8bedd23add2798ae73802442f52486e95c3df7111c"), trie.Hash(); exp != root { + t.Fatalf("1: got %x, exp %x", root, exp) + } + trie.Update(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001338"), accounts[4]) + if exp, root := common.HexToHash("cb5fb1213826dad9e604f095f8ceb5258fe6b5c01805ce6ef019a50699d2d479"), trie.Hash(); exp != root { + t.Fatalf("2: got %x, exp %x", root, exp) + } + trie.Update(common.Hex2Bytes("0000000000000000000000000000000000000000000000000000000000001339"), accounts[4]) + if exp, root := common.HexToHash("ed7e06b4010057d8703e7b9a160a6d42cf4021f9020da3c8891030349a646987"), trie.Hash(); exp != root { + t.Fatalf("3: got %x, exp %x", root, exp) + } + + checktr, _ := New(common.Hash{}, trie.db) + it := NewIterator(trie.NodeIterator(nil)) + for it.Next() { + checktr.Update(it.Key, it.Value) + } + if troot, itroot := trie.Hash(), checktr.Hash(); troot != itroot { + t.Fatalf("hash mismatch in opItercheckhash, trie: %x, check: %x", troot, itroot) + } +} +func TestCommitAfterHash(t *testing.T) { // Create a realistic account trie to hash - addresses := make([][20]byte, b.N) + addresses, accounts := makeAccounts(1000) + trie := newEmpty() + for i := 0; i < len(addresses); i++ { + trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) + } + // Insert the accounts into the trie and hash it + trie.Hash() + trie.Commit(nil) + root := trie.Hash() + exp := common.HexToHash("e5e9c29bb50446a4081e6d1d748d2892c6101c1e883a1f77cf21d4094b697822") + if exp != root { + t.Errorf("got %x, exp %x", root, exp) + } + root, _ = trie.Commit(nil) + if exp != root { + t.Errorf("got %x, exp %x", root, exp) + } +} + +func makeAccounts(size int) (addresses [][20]byte, accounts [][]byte) { + // Make the random benchmark deterministic + random := rand.New(rand.NewSource(0)) + // Create a realistic account trie to hash + addresses = make([][20]byte, size) for i := 0; i < len(addresses); i++ { for j := 0; j < len(addresses[i]); j++ { addresses[i][j] = byte(random.Intn(256)) } } - accounts := make([][]byte, len(addresses)) + accounts = make([][]byte, len(addresses)) for i := 0; i < len(accounts); i++ { var ( nonce = uint64(random.Int63()) @@ -500,16 +654,168 @@ func BenchmarkHash(b *testing.B) { root = emptyRoot code = crypto.Keccak256(nil) ) - accounts[i], _ = rlp.EncodeToBytes([]interface{}{nonce, balance, root, code}) + accounts[i], _ = rlp.EncodeToBytes(&account{nonce, balance, root, code}) } - // Insert the accounts into the trie and hash it + return addresses, accounts +} + +// BenchmarkCommitAfterHashFixedSize benchmarks the Commit (after Hash) of a fixed number of updates to a trie. +// This benchmark is meant to capture the difference on efficiency of small versus large changes. Typically, +// storage tries are small (a couple of entries), whereas the full post-block account trie update is large (a couple +// of thousand entries) +func BenchmarkHashFixedSize(b *testing.B) { + b.Run("10", func(b *testing.B) { + b.StopTimer() + acc, add := makeAccounts(20) + for i := 0; i < b.N; i++ { + benchmarkHashFixedSize(b, acc, add) + } + }) + b.Run("100", func(b *testing.B) { + b.StopTimer() + acc, add := makeAccounts(100) + for i := 0; i < b.N; i++ { + benchmarkHashFixedSize(b, acc, add) + } + }) + + b.Run("1K", func(b *testing.B) { + b.StopTimer() + acc, add := makeAccounts(1000) + for i := 0; i < b.N; i++ { + benchmarkHashFixedSize(b, acc, add) + } + }) + b.Run("10K", func(b *testing.B) { + b.StopTimer() + acc, add := makeAccounts(10000) + for i := 0; i < b.N; i++ { + benchmarkHashFixedSize(b, acc, add) + } + }) + b.Run("100K", func(b *testing.B) { + b.StopTimer() + acc, add := makeAccounts(100000) + for i := 0; i < b.N; i++ { + benchmarkHashFixedSize(b, acc, add) + } + }) +} + +func benchmarkHashFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { + b.ReportAllocs() trie := newEmpty() for i := 0; i < len(addresses); i++ { trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) } - b.ResetTimer() + // Insert the accounts into the trie and hash it + b.StartTimer() + trie.Hash() + b.StopTimer() +} + +func BenchmarkCommitAfterHashFixedSize(b *testing.B) { + b.Run("10", func(b *testing.B) { + b.StopTimer() + acc, add := makeAccounts(20) + for i := 0; i < b.N; i++ { + benchmarkCommitAfterHashFixedSize(b, acc, add) + } + }) + b.Run("100", func(b *testing.B) { + b.StopTimer() + acc, add := makeAccounts(100) + for i := 0; i < b.N; i++ { + benchmarkCommitAfterHashFixedSize(b, acc, add) + } + }) + + b.Run("1K", func(b *testing.B) { + b.StopTimer() + acc, add := makeAccounts(1000) + for i := 0; i < b.N; i++ { + benchmarkCommitAfterHashFixedSize(b, acc, add) + } + }) + b.Run("10K", func(b *testing.B) { + b.StopTimer() + acc, add := makeAccounts(10000) + for i := 0; i < b.N; i++ { + benchmarkCommitAfterHashFixedSize(b, acc, add) + } + }) + b.Run("100K", func(b *testing.B) { + b.StopTimer() + acc, add := makeAccounts(100000) + for i := 0; i < b.N; i++ { + benchmarkCommitAfterHashFixedSize(b, acc, add) + } + }) +} + +func benchmarkCommitAfterHashFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { b.ReportAllocs() + trie := newEmpty() + for i := 0; i < len(addresses); i++ { + trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) + } + // Insert the accounts into the trie and hash it trie.Hash() + b.StartTimer() + trie.Commit(nil) + b.StopTimer() +} + +func BenchmarkDerefRootFixedSize(b *testing.B) { + b.Run("10", func(b *testing.B) { + b.StopTimer() + acc, add := makeAccounts(20) + for i := 0; i < b.N; i++ { + benchmarkDerefRootFixedSize(b, acc, add) + } + }) + b.Run("100", func(b *testing.B) { + b.StopTimer() + acc, add := makeAccounts(100) + for i := 0; i < b.N; i++ { + benchmarkDerefRootFixedSize(b, acc, add) + } + }) + + b.Run("1K", func(b *testing.B) { + b.StopTimer() + acc, add := makeAccounts(1000) + for i := 0; i < b.N; i++ { + benchmarkDerefRootFixedSize(b, acc, add) + } + }) + b.Run("10K", func(b *testing.B) { + b.StopTimer() + acc, add := makeAccounts(10000) + for i := 0; i < b.N; i++ { + benchmarkDerefRootFixedSize(b, acc, add) + } + }) + b.Run("100K", func(b *testing.B) { + b.StopTimer() + acc, add := makeAccounts(100000) + for i := 0; i < b.N; i++ { + benchmarkDerefRootFixedSize(b, acc, add) + } + }) +} + +func benchmarkDerefRootFixedSize(b *testing.B, addresses [][20]byte, accounts [][]byte) { + b.ReportAllocs() + trie := newEmpty() + for i := 0; i < len(addresses); i++ { + trie.Update(crypto.Keccak256(addresses[i][:]), accounts[i]) + } + h := trie.Hash() + trie.Commit(nil) + b.StartTimer() + trie.db.Dereference(h) + b.StopTimer() } func tempDB() (string, *Database) { From 92956e29305c5c89a29bf69d5831c8b863be3ddb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 20 Jan 2020 00:50:59 +0200 Subject: [PATCH 068/128] appveyor: bump Go to 1.13.6 on Windows --- appveyor.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/appveyor.yml b/appveyor.yml index 0f230bac14..8fcacfd5fe 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,8 +23,8 @@ environment: install: - git submodule update --init - rmdir C:\go /s /q - - appveyor DownloadFile https://dl.google.com/go/go1.13.4.windows-%GETH_ARCH%.zip - - 7z x go1.13.4.windows-%GETH_ARCH%.zip -y -oC:\ > NUL + - appveyor DownloadFile https://dl.google.com/go/go1.13.6.windows-%GETH_ARCH%.zip + - 7z x go1.13.6.windows-%GETH_ARCH%.zip -y -oC:\ > NUL - go version - gcc --version From fcc84c38ddb6514059fec2da1c70376a551905aa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 20 Jan 2020 00:54:20 +0200 Subject: [PATCH 069/128] travis: bump Android builder to Go 1.13.6 --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 31cc398c57..416a83018d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -183,7 +183,7 @@ jobs: git: submodules: false # avoid cloning ethereum/tests before_install: - - curl https://dl.google.com/go/go1.13.linux-amd64.tar.gz | tar -xz + - curl https://dl.google.com/go/go1.13.6.linux-amd64.tar.gz | tar -xz - export PATH=`pwd`/go/bin:$PATH - export GOROOT=`pwd`/go - export GOPATH=$HOME/go From 1f1cefc036e23ab71f5b259e64f22df6de03ec34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 20 Jan 2020 10:28:49 +0200 Subject: [PATCH 070/128] params: update CHTs for v1.9.10 release --- params/config.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/params/config.go b/params/config.go index dac56b1bd7..5a2078c42c 100644 --- a/params/config.go +++ b/params/config.go @@ -72,10 +72,10 @@ var ( // MainnetTrustedCheckpoint contains the light client trusted checkpoint for the main network. MainnetTrustedCheckpoint = &TrustedCheckpoint{ - SectionIndex: 275, - SectionHead: common.HexToHash("0x03159234a3699e31d27e5d83a55cbcf8ceb1f2d90855c219c55d79089b61abd4"), - CHTRoot: common.HexToHash("0xd0c1f3828a4dcb2ee76625fdbea85afeabfb61c04adf07439d2fc1cf00469f76"), - BloomRoot: common.HexToHash("0xab8ea2be8aa24703208fee3fc0afdbb536301013f412a7282b2692d6d68f92c5"), + SectionIndex: 283, + SectionHead: common.HexToHash("0x7da9639e5b378421f2cabd1d3bdae02dbf6d4ba79fc9b3c3916c66412ef7d0b6"), + CHTRoot: common.HexToHash("0xb8c6f06e1d5a4fddf593d1ff787e6b89eff4183d0b40d276f19e7d41178a53cf"), + BloomRoot: common.HexToHash("0xc47d7d6924ba46090d568296bb667a3ea4b6a00dd69bd5fb4f4f78be2fe05ec2"), } // MainnetCheckpointOracle contains a set of configs for the main network oracle. @@ -111,10 +111,10 @@ var ( // TestnetTrustedCheckpoint contains the light client trusted checkpoint for the Ropsten test network. TestnetTrustedCheckpoint = &TrustedCheckpoint{ - SectionIndex: 209, - SectionHead: common.HexToHash("0x8037eb6872b69397d434121424ed8d6ab74be32bf3cb3f12dc5d9657fc146860"), - CHTRoot: common.HexToHash("0xe64b7d6324e5cbdcbbc250adf4cf24a639a665aa83ccfd6a0b84a80faaaa0d41"), - BloomRoot: common.HexToHash("0x80fedbef680cd70d3dc4b50b14480fba82c74361a35e8dc7be9f11e03077c840"), + SectionIndex: 217, + SectionHead: common.HexToHash("0x1895e3cceb6fb201044679db2b9f4f9df4233b52e8d3c5ec4b75ae0ae75c90fa"), + CHTRoot: common.HexToHash("0x8f2016fb336b64bd8ef4e9a73659a0a99476ea8789aacad695d65295a50fdb8d"), + BloomRoot: common.HexToHash("0x57f5b8ecfa10ada7509a45f7e0f2283c6b2dc08d8771163ffbb4ff0e3e6bca1c"), } // TestnetCheckpointOracle contains a set of configs for the Ropsten test network oracle. @@ -152,10 +152,10 @@ var ( // RinkebyTrustedCheckpoint contains the light client trusted checkpoint for the Rinkeby test network. RinkebyTrustedCheckpoint = &TrustedCheckpoint{ - SectionIndex: 168, - SectionHead: common.HexToHash("0x87301279595b16ac59360c839ef86b159e21fedbfcc8847d727ef446a14cf334"), - CHTRoot: common.HexToHash("0x00f522dd0705ff647cebdd36707d6779caaf77f5fe8f958aae85f36aa88e3f9c"), - BloomRoot: common.HexToHash("0xc908547a6b01c47c65a4581c68090e5602308d39e893f7c0ae3e16c52ce2abf2"), + SectionIndex: 176, + SectionHead: common.HexToHash("0xb2cbd3396f25647fd80598fc4f7b015fb4131c0831630d878742b33dd286b641"), + CHTRoot: common.HexToHash("0xd81776495227babd75d8a519cb55e506e7b24265de5aa1847d38cb8f216ac09e"), + BloomRoot: common.HexToHash("0x0ea52b610139fcd6b1bc75351787e21c72922b18ea8a27091a51eb29b6723cd6"), } // RinkebyCheckpointOracle contains a set of configs for the Rinkeby test network oracle. @@ -191,10 +191,10 @@ var ( // GoerliTrustedCheckpoint contains the light client trusted checkpoint for the Görli test network. GoerliTrustedCheckpoint = &TrustedCheckpoint{ - SectionIndex: 52, - SectionHead: common.HexToHash("0x64c3bbc896578cbf782e343db48e334177e87fb8b16106b75e1dcebf59ca59dc"), - CHTRoot: common.HexToHash("0x5d092e644f3815de40b8c4196698d3e34a9097cf3066a499c96e83e3927d8b8d"), - BloomRoot: common.HexToHash("0xb2ceb966b499dd9e6e5bf6adbf35440a0e15cbccc0f527f89a1c522a9f36250a"), + SectionIndex: 60, + SectionHead: common.HexToHash("0x9bc784b6ad0c944f6aebf930fb5e811497059788b616e211db655a885e65f1cb"), + CHTRoot: common.HexToHash("0x961a811e2843f6196903edc1d04adbf13dee92627a89c21b3e0cdf69e0100638"), + BloomRoot: common.HexToHash("0x1564889bbe8c68011d29d3966439956303283c0fb8b08daa8376028e82dcd763"), } // GoerliCheckpointOracle contains a set of configs for the Goerli test network oracle. From b88b4632c2d1beba72dda31d072902c503962591 Mon Sep 17 00:00:00 2001 From: gary rong Date: Mon, 20 Jan 2020 16:38:08 +0800 Subject: [PATCH 071/128] core: fix chain indexer unit test (#20506) --- core/chain_indexer_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/chain_indexer_test.go b/core/chain_indexer_test.go index abf5b3cc14..ff7548e7bd 100644 --- a/core/chain_indexer_test.go +++ b/core/chain_indexer_test.go @@ -203,7 +203,7 @@ func (b *testChainIndexBackend) assertBlocks(headNum, failNum uint64) (uint64, b } func (b *testChainIndexBackend) reorg(headNum uint64) uint64 { - firstChanged := headNum / b.indexer.sectionSize + firstChanged := (headNum + 1) / b.indexer.sectionSize if firstChanged < b.stored { b.stored = firstChanged } From 58cf5686eab9019cc01e202e846a6bbc70a3301d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 20 Jan 2020 12:27:51 +0200 Subject: [PATCH 072/128] params: release Geth v1.9.10 --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index d37ea9d995..254f897daa 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 9 // Minor version component of the current release - VersionPatch = 10 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 9 // Minor version component of the current release + VersionPatch = 10 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From ad2fc7c6a6487d9864b47b29e7686f0afb765ede Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 20 Jan 2020 12:32:47 +0200 Subject: [PATCH 073/128] params: begin Geth v1.9.11 release cycle --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 254f897daa..54e006f480 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 9 // Minor version component of the current release - VersionPatch = 10 // Patch version component of the current release - VersionMeta = "stable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 9 // Minor version component of the current release + VersionPatch = 11 // Patch version component of the current release + VersionMeta = "unstable" // Version metadata to append to the version string ) // Version holds the textual version string. From 31baf3a9afbca0ca08cc9487444d31f1530fa1b0 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 21 Jan 2020 13:57:33 +0100 Subject: [PATCH 074/128] log, internal/debug: delete RotatingFileHandler (#20586) * log: delete RotatingFileHandler We added this for the dashboard, which is gone now. The handler never really worked well and had data race and file handling issues. * internal/debug: remove unused RotatingFileHandler setup code --- cmd/devp2p/main.go | 2 +- cmd/geth/main.go | 2 +- internal/debug/flags.go | 13 +---- log/handler.go | 110 ---------------------------------------- 4 files changed, 3 insertions(+), 124 deletions(-) diff --git a/cmd/devp2p/main.go b/cmd/devp2p/main.go index 6faa650937..b895941f25 100644 --- a/cmd/devp2p/main.go +++ b/cmd/devp2p/main.go @@ -45,7 +45,7 @@ func init() { // Set up the CLI app. app.Flags = append(app.Flags, debug.Flags...) app.Before = func(ctx *cli.Context) error { - return debug.Setup(ctx, "") + return debug.Setup(ctx) } app.After = func(ctx *cli.Context) error { debug.Exit() diff --git a/cmd/geth/main.go b/cmd/geth/main.go index d6ded24b31..8db19077a8 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -233,7 +233,7 @@ func init() { app.Flags = append(app.Flags, metricsFlags...) app.Before = func(ctx *cli.Context) error { - return debug.Setup(ctx, "") + return debug.Setup(ctx) } app.After = func(ctx *cli.Context) error { debug.Exit() diff --git a/internal/debug/flags.go b/internal/debug/flags.go index 46c8fe9f80..3c85749402 100644 --- a/internal/debug/flags.go +++ b/internal/debug/flags.go @@ -112,20 +112,9 @@ func init() { // Setup initializes profiling and logging based on the CLI flags. // It should be called as early as possible in the program. -func Setup(ctx *cli.Context, logdir string) error { +func Setup(ctx *cli.Context) error { // logging log.PrintOrigins(ctx.GlobalBool(debugFlag.Name)) - if logdir != "" { - rfh, err := log.RotatingFileHandler( - logdir, - 262144, - log.JSONFormatOrderedEx(false, true), - ) - if err != nil { - return err - } - glogger.SetHandler(log.MultiHandler(ostream, rfh)) - } glogger.Verbosity(log.Lvl(ctx.GlobalInt(verbosityFlag.Name))) glogger.Vmodule(ctx.GlobalString(vmoduleFlag.Name)) glogger.BacktraceAt(ctx.GlobalString(backtraceAtFlag.Name)) diff --git a/log/handler.go b/log/handler.go index 2f01b5dc6f..3c99114dcb 100644 --- a/log/handler.go +++ b/log/handler.go @@ -8,11 +8,6 @@ import ( "reflect" "sync" - "io/ioutil" - "path/filepath" - "regexp" - "strings" - "github.com/go-stack/stack" ) @@ -75,111 +70,6 @@ func FileHandler(path string, fmtr Format) (Handler, error) { return closingHandler{f, StreamHandler(f, fmtr)}, nil } -// countingWriter wraps a WriteCloser object in order to count the written bytes. -type countingWriter struct { - w io.WriteCloser // the wrapped object - count uint // number of bytes written -} - -// Write increments the byte counter by the number of bytes written. -// Implements the WriteCloser interface. -func (w *countingWriter) Write(p []byte) (n int, err error) { - n, err = w.w.Write(p) - w.count += uint(n) - return n, err -} - -// Close implements the WriteCloser interface. -func (w *countingWriter) Close() error { - return w.w.Close() -} - -// prepFile opens the log file at the given path, and cuts off the invalid part -// from the end, because the previous execution could have been finished by interruption. -// Assumes that every line ended by '\n' contains a valid log record. -func prepFile(path string) (*countingWriter, error) { - f, err := os.OpenFile(path, os.O_RDWR|os.O_APPEND, 0600) - if err != nil { - return nil, err - } - _, err = f.Seek(-1, io.SeekEnd) - if err != nil { - return nil, err - } - buf := make([]byte, 1) - var cut int64 - for { - if _, err := f.Read(buf); err != nil { - return nil, err - } - if buf[0] == '\n' { - break - } - if _, err = f.Seek(-2, io.SeekCurrent); err != nil { - return nil, err - } - cut++ - } - fi, err := f.Stat() - if err != nil { - return nil, err - } - ns := fi.Size() - cut - if err = f.Truncate(ns); err != nil { - return nil, err - } - return &countingWriter{w: f, count: uint(ns)}, nil -} - -// RotatingFileHandler returns a handler which writes log records to file chunks -// at the given path. When a file's size reaches the limit, the handler creates -// a new file named after the timestamp of the first log record it will contain. -func RotatingFileHandler(path string, limit uint, formatter Format) (Handler, error) { - if err := os.MkdirAll(path, 0700); err != nil { - return nil, err - } - files, err := ioutil.ReadDir(path) - if err != nil { - return nil, err - } - re := regexp.MustCompile(`\.log$`) - last := len(files) - 1 - for last >= 0 && (!files[last].Mode().IsRegular() || !re.MatchString(files[last].Name())) { - last-- - } - var counter *countingWriter - if last >= 0 && files[last].Size() < int64(limit) { - // Open the last file, and continue to write into it until it's size reaches the limit. - if counter, err = prepFile(filepath.Join(path, files[last].Name())); err != nil { - return nil, err - } - } - if counter == nil { - counter = new(countingWriter) - } - h := StreamHandler(counter, formatter) - - return FuncHandler(func(r *Record) error { - if counter.count > limit { - counter.Close() - counter.w = nil - } - if counter.w == nil { - f, err := os.OpenFile( - filepath.Join(path, fmt.Sprintf("%s.log", strings.Replace(r.Time.Format("060102150405.00"), ".", "", 1))), - os.O_CREATE|os.O_APPEND|os.O_WRONLY, - 0600, - ) - if err != nil { - return err - } - counter.w = f - counter.count = 0 - } - return h.Log(r) - }), nil -} - // NetHandler opens a socket to the given address and writes records // over the connection. func NetHandler(network, addr string, fmtr Format) (Handler, error) { From 33c56ebc6757fd50e7a7ea1d594754e33d904887 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 21 Jan 2020 15:51:36 +0100 Subject: [PATCH 075/128] cmd: implement abidump (#19958) * abidump: implement abi dump command * cmd/abidump: add license --- cmd/abidump/main.go | 74 +++++++++++++++++++++++++++++++++++ signer/fourbyte/abi.go | 2 +- signer/fourbyte/validation.go | 8 ++-- 3 files changed, 79 insertions(+), 5 deletions(-) create mode 100644 cmd/abidump/main.go diff --git a/cmd/abidump/main.go b/cmd/abidump/main.go new file mode 100644 index 0000000000..35cbcbb0ed --- /dev/null +++ b/cmd/abidump/main.go @@ -0,0 +1,74 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package main + +import ( + "encoding/hex" + "flag" + "fmt" + "os" + "strings" + + "github.com/ethereum/go-ethereum/signer/core" + "github.com/ethereum/go-ethereum/signer/fourbyte" +) + +func init() { + flag.Usage = func() { + fmt.Fprintln(os.Stderr, "Usage:", os.Args[0], "") + flag.PrintDefaults() + fmt.Fprintln(os.Stderr, ` +Parses the given ABI data and tries to interpret it from the fourbyte database.`) + } +} + +func parse(data []byte) { + db, err := fourbyte.New() + if err != nil { + die(err) + } + messages := core.ValidationMessages{} + db.ValidateCallData(nil, data, &messages) + for _, m := range messages.Messages { + fmt.Printf("%v: %v\n", m.Typ, m.Message) + } +} + +// Example +// ./abidump a9059cbb000000000000000000000000ea0e2dc7d65a50e77fc7e84bff3fd2a9e781ff5c0000000000000000000000000000000000000000000000015af1d78b58c40000 +func main() { + flag.Parse() + + switch { + case flag.NArg() == 1: + hexdata := flag.Arg(0) + data, err := hex.DecodeString(strings.TrimPrefix(hexdata, "0x")) + if err != nil { + die(err) + } + parse(data) + default: + fmt.Fprintln(os.Stderr, "Error: one argument needed") + flag.Usage() + os.Exit(2) + } +} + +func die(args ...interface{}) { + fmt.Fprintln(os.Stderr, args...) + os.Exit(1) +} diff --git a/signer/fourbyte/abi.go b/signer/fourbyte/abi.go index 585eae1cd8..796086d415 100644 --- a/signer/fourbyte/abi.go +++ b/signer/fourbyte/abi.go @@ -137,7 +137,7 @@ func parseCallData(calldata []byte, abidata string) (*decodedCallData, error) { } values, err := method.Inputs.UnpackValues(argdata) if err != nil { - return nil, err + return nil, fmt.Errorf("signature %q matches, but arguments mismatch: %v", method.String(), err) } // Everything valid, assemble the call infos for the signer decoded := decodedCallData{signature: method.Sig(), name: method.RawName} diff --git a/signer/fourbyte/validation.go b/signer/fourbyte/validation.go index 4d042d240f..fd13e0a630 100644 --- a/signer/fourbyte/validation.go +++ b/signer/fourbyte/validation.go @@ -74,13 +74,13 @@ func (db *Database) ValidateTransaction(selector *string, tx *core.SendTxArgs) ( messages.Crit("Transaction recipient is the zero address") } // Semantic fields validated, try to make heads or tails of the call data - db.validateCallData(selector, data, messages) + db.ValidateCallData(selector, data, messages) return messages, nil } -// validateCallData checks if the ABI call-data + method selector (if given) can +// ValidateCallData checks if the ABI call-data + method selector (if given) can // be parsed and seems to match. -func (db *Database) validateCallData(selector *string, data []byte, messages *core.ValidationMessages) { +func (db *Database) ValidateCallData(selector *string, data []byte, messages *core.ValidationMessages) { // If the data is empty, we have a plain value transfer, nothing more to do if len(data) == 0 { return @@ -110,7 +110,7 @@ func (db *Database) validateCallData(selector *string, data []byte, messages *co return } if info, err := verifySelector(embedded, data); err != nil { - messages.Warn(fmt.Sprintf("Transaction contains data, but provided ABI signature could not be varified: %v", err)) + messages.Warn(fmt.Sprintf("Transaction contains data, but provided ABI signature could not be verified: %v", err)) } else { messages.Info(info.String()) } From 8a5c81349ead2dc3208ba09517f7ee9f6866a7db Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Thu, 23 Jan 2020 16:08:06 +0100 Subject: [PATCH 076/128] eth: fix comment typo in handler.go (#20575) --- eth/handler.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eth/handler.go b/eth/handler.go index e18fa61241..ae2b764cf6 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -749,8 +749,8 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { return nil } -// BroadcastBlock will either propagate a block to a subset of it's peers, or -// will only announce it's availability (depending what's requested). +// BroadcastBlock will either propagate a block to a subset of its peers, or +// will only announce its availability (depending what's requested). func (pm *ProtocolManager) BroadcastBlock(block *types.Block, propagate bool) { hash := block.Hash() peers := pm.peers.PeersWithoutBlock(hash) From 0b284f6c6cfc6df452ca23f9454ee16a6330cb8e Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Thu, 23 Jan 2020 20:55:56 +0100 Subject: [PATCH 077/128] cmd/geth/retesteth: use canon head instead of keeping alternate count (#20572) --- cmd/geth/retesteth.go | 25 ++++++++++++++----------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/cmd/geth/retesteth.go b/cmd/geth/retesteth.go index 2c3f8be78d..f4f7bda00b 100644 --- a/cmd/geth/retesteth.go +++ b/cmd/geth/retesteth.go @@ -110,7 +110,6 @@ type RetestethAPI struct { genesisHash common.Hash engine *NoRewardEngine blockchain *core.BlockChain - blockNumber uint64 txMap map[common.Address]map[uint64]*types.Transaction // Sender -> Nonce -> Transaction txSenders map[common.Address]struct{} // Set of transaction senders blockInterval uint64 @@ -411,7 +410,6 @@ func (api *RetestethAPI) SetChainParams(ctx context.Context, chainParams ChainPa api.engine = engine api.blockchain = blockchain api.db = state.NewDatabase(api.ethDb) - api.blockNumber = 0 api.txMap = make(map[common.Address]map[uint64]*types.Transaction) api.txSenders = make(map[common.Address]struct{}) api.blockInterval = 0 @@ -424,7 +422,7 @@ func (api *RetestethAPI) SendRawTransaction(ctx context.Context, rawTx hexutil.B // Return nil is not by mistake - some tests include sending transaction where gasLimit overflows uint64 return common.Hash{}, nil } - signer := types.MakeSigner(api.chainConfig, big.NewInt(int64(api.blockNumber))) + signer := types.MakeSigner(api.chainConfig, big.NewInt(int64(api.currentNumber()))) sender, err := types.Sender(signer, tx) if err != nil { return common.Hash{}, err @@ -450,9 +448,17 @@ func (api *RetestethAPI) MineBlocks(ctx context.Context, number uint64) (bool, e return true, nil } +func (api *RetestethAPI) currentNumber() uint64 { + if current := api.blockchain.CurrentBlock(); current != nil { + return current.NumberU64() + } + return 0 +} + func (api *RetestethAPI) mineBlock() error { - parentHash := rawdb.ReadCanonicalHash(api.ethDb, api.blockNumber) - parent := rawdb.ReadBlock(api.ethDb, parentHash, api.blockNumber) + number := api.currentNumber() + parentHash := rawdb.ReadCanonicalHash(api.ethDb, number) + parent := rawdb.ReadBlock(api.ethDb, parentHash, number) var timestamp uint64 if api.blockInterval == 0 { timestamp = uint64(time.Now().Unix()) @@ -462,7 +468,7 @@ func (api *RetestethAPI) mineBlock() error { gasLimit := core.CalcGasLimit(parent, 9223372036854775807, 9223372036854775807) header := &types.Header{ ParentHash: parent.Hash(), - Number: big.NewInt(int64(api.blockNumber + 1)), + Number: big.NewInt(int64(number + 1)), GasLimit: gasLimit, Extra: api.extraData, Time: timestamp, @@ -548,8 +554,7 @@ func (api *RetestethAPI) importBlock(block *types.Block) error { if _, err := api.blockchain.InsertChain([]*types.Block{block}); err != nil { return err } - api.blockNumber = block.NumberU64() - fmt.Printf("Imported block %d\n", block.NumberU64()) + fmt.Printf("Imported block %d, head is %d\n", block.NumberU64(), api.currentNumber()) return nil } @@ -574,7 +579,6 @@ func (api *RetestethAPI) RewindToBlock(ctx context.Context, newHead uint64) (boo if err := api.blockchain.SetHead(newHead); err != nil { return false, err } - api.blockNumber = newHead return true, nil } @@ -594,8 +598,7 @@ func (api *RetestethAPI) GetLogHash(ctx context.Context, txHash common.Hash) (co } func (api *RetestethAPI) BlockNumber(ctx context.Context) (uint64, error) { - //fmt.Printf("BlockNumber, response: %d\n", api.blockNumber) - return api.blockNumber, nil + return api.currentNumber(), nil } func (api *RetestethAPI) GetBlockByNumber(ctx context.Context, blockNr math.HexOrDecimal64, fullTx bool) (map[string]interface{}, error) { From 60deeb103e0a3c9d414e71cc47c9adb1d46fe0da Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Mon, 27 Jan 2020 10:05:21 +0100 Subject: [PATCH 078/128] cmd/evm: accept --input for disasm command (#20548) --- cmd/evm/disasm.go | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/cmd/evm/disasm.go b/cmd/evm/disasm.go index 69f611e39b..5bc743aa88 100644 --- a/cmd/evm/disasm.go +++ b/cmd/evm/disasm.go @@ -34,17 +34,22 @@ var disasmCommand = cli.Command{ } func disasmCmd(ctx *cli.Context) error { - if len(ctx.Args().First()) == 0 { - return errors.New("filename required") + var in string + switch { + case len(ctx.Args().First()) > 0: + fn := ctx.Args().First() + input, err := ioutil.ReadFile(fn) + if err != nil { + return err + } + in = string(input) + case ctx.GlobalIsSet(InputFlag.Name): + in = ctx.GlobalString(InputFlag.Name) + default: + return errors.New("Missing filename or --input value") } - fn := ctx.Args().First() - in, err := ioutil.ReadFile(fn) - if err != nil { - return err - } - - code := strings.TrimSpace(string(in)) + code := strings.TrimSpace(in) fmt.Printf("%v\n", code) return asm.PrintDisassembled(code) } From 7b68975a00ee914d91f9c6a56c55a357012ab844 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Mon, 27 Jan 2020 11:50:48 +0100 Subject: [PATCH 079/128] console, internal/jsre: use github.com/dop251/goja (#20470) This replaces the JavaScript interpreter used by the console with goja, which is actively maintained and a lot faster than otto. Clef still uses otto and eth/tracers still uses duktape, so we are currently dependent on three different JS interpreters. We're looking to replace the remaining uses of otto soon though. --- cmd/geth/consolecmd_test.go | 8 +- console/bridge.go | 373 +++++++++++++++++-------------- console/console.go | 207 +++++++++-------- console/console_test.go | 2 +- go.mod | 3 + go.sum | 8 + internal/jsre/completion.go | 36 +-- internal/jsre/completion_test.go | 4 + internal/jsre/jsre.go | 187 +++++++--------- internal/jsre/jsre_test.go | 34 +-- internal/jsre/pretty.go | 174 ++++++++------ 11 files changed, 559 insertions(+), 477 deletions(-) diff --git a/cmd/geth/consolecmd_test.go b/cmd/geth/consolecmd_test.go index 45d4daff07..8f6a102d73 100644 --- a/cmd/geth/consolecmd_test.go +++ b/cmd/geth/consolecmd_test.go @@ -51,7 +51,9 @@ func TestConsoleWelcome(t *testing.T) { geth.SetTemplateFunc("goarch", func() string { return runtime.GOARCH }) geth.SetTemplateFunc("gover", runtime.Version) geth.SetTemplateFunc("gethver", func() string { return params.VersionWithCommit("", "") }) - geth.SetTemplateFunc("niltime", func() string { return time.Unix(0, 0).Format(time.RFC1123) }) + geth.SetTemplateFunc("niltime", func() string { + return time.Unix(0, 0).Format("Mon Jan 02 2006 15:04:05 GMT-0700 (MST)") + }) geth.SetTemplateFunc("apis", func() string { return ipcAPIs }) // Verify the actual welcome message to the required template @@ -142,7 +144,9 @@ func testAttachWelcome(t *testing.T, geth *testgeth, endpoint, apis string) { attach.SetTemplateFunc("gover", runtime.Version) attach.SetTemplateFunc("gethver", func() string { return params.VersionWithCommit("", "") }) attach.SetTemplateFunc("etherbase", func() string { return geth.Etherbase }) - attach.SetTemplateFunc("niltime", func() string { return time.Unix(0, 0).Format(time.RFC1123) }) + attach.SetTemplateFunc("niltime", func() string { + return time.Unix(0, 0).Format("Mon Jan 02 2006 15:04:05 GMT-0700 (MST)") + }) attach.SetTemplateFunc("ipc", func() bool { return strings.HasPrefix(endpoint, "ipc") }) attach.SetTemplateFunc("datadir", func() string { return geth.Datadir }) attach.SetTemplateFunc("apis", func() string { return apis }) diff --git a/console/bridge.go b/console/bridge.go index c7a67a6850..2625c481d5 100644 --- a/console/bridge.go +++ b/console/bridge.go @@ -20,14 +20,16 @@ import ( "encoding/json" "fmt" "io" + "reflect" "strings" "time" + "github.com/dop251/goja" "github.com/ethereum/go-ethereum/accounts/scwallet" "github.com/ethereum/go-ethereum/accounts/usbwallet" - "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/common/hexutil" + "github.com/ethereum/go-ethereum/internal/jsre" "github.com/ethereum/go-ethereum/rpc" - "github.com/robertkrimen/otto" ) // bridge is a collection of JavaScript utility methods to bride the .js runtime @@ -47,10 +49,18 @@ func newBridge(client *rpc.Client, prompter UserPrompter, printer io.Writer) *br } } +func getJeth(vm *goja.Runtime) *goja.Object { + jeth := vm.Get("jeth") + if jeth == nil { + panic(vm.ToValue("jeth object does not exist")) + } + return jeth.ToObject(vm) +} + // NewAccount is a wrapper around the personal.newAccount RPC method that uses a // non-echoing password prompt to acquire the passphrase and executes the original // RPC method (saved in jeth.newAccount) with it to actually execute the RPC call. -func (b *bridge) NewAccount(call otto.FunctionCall) (response otto.Value) { +func (b *bridge) NewAccount(call jsre.Call) (goja.Value, error) { var ( password string confirm string @@ -58,52 +68,57 @@ func (b *bridge) NewAccount(call otto.FunctionCall) (response otto.Value) { ) switch { // No password was specified, prompt the user for it - case len(call.ArgumentList) == 0: - if password, err = b.prompter.PromptPassword("Password: "); err != nil { - throwJSException(err.Error()) + case len(call.Arguments) == 0: + if password, err = b.prompter.PromptPassword("Passphrase: "); err != nil { + return nil, err } - if confirm, err = b.prompter.PromptPassword("Repeat password: "); err != nil { - throwJSException(err.Error()) + if confirm, err = b.prompter.PromptPassword("Repeat passphrase: "); err != nil { + return nil, err } if password != confirm { - throwJSException("passwords don't match!") + return nil, fmt.Errorf("passwords don't match!") } - // A single string password was specified, use that - case len(call.ArgumentList) == 1 && call.Argument(0).IsString(): - password, _ = call.Argument(0).ToString() - - // Otherwise fail with some error + case len(call.Arguments) == 1 && call.Argument(0).ToString() != nil: + password = call.Argument(0).ToString().String() default: - throwJSException("expected 0 or 1 string argument") + return nil, fmt.Errorf("expected 0 or 1 string argument") } // Password acquired, execute the call and return - ret, err := call.Otto.Call("jeth.newAccount", nil, password) + newAccount, callable := goja.AssertFunction(getJeth(call.VM).Get("newAccount")) + if !callable { + return nil, fmt.Errorf("jeth.newAccount is not callable") + } + ret, err := newAccount(goja.Null(), call.VM.ToValue(password)) if err != nil { - throwJSException(err.Error()) + return nil, err } - return ret + return ret, nil } // OpenWallet is a wrapper around personal.openWallet which can interpret and // react to certain error messages, such as the Trezor PIN matrix request. -func (b *bridge) OpenWallet(call otto.FunctionCall) (response otto.Value) { +func (b *bridge) OpenWallet(call jsre.Call) (goja.Value, error) { // Make sure we have a wallet specified to open - if !call.Argument(0).IsString() { - throwJSException("first argument must be the wallet URL to open") + if call.Argument(0).ToObject(call.VM).ClassName() != "String" { + return nil, fmt.Errorf("first argument must be the wallet URL to open") } wallet := call.Argument(0) - var passwd otto.Value - if call.Argument(1).IsUndefined() || call.Argument(1).IsNull() { - passwd, _ = otto.ToValue("") + var passwd goja.Value + if goja.IsUndefined(call.Argument(1)) || goja.IsNull(call.Argument(1)) { + passwd = call.VM.ToValue("") } else { passwd = call.Argument(1) } // Open the wallet and return if successful in itself - val, err := call.Otto.Call("jeth.openWallet", nil, wallet, passwd) + openWallet, callable := goja.AssertFunction(getJeth(call.VM).Get("openWallet")) + if !callable { + return nil, fmt.Errorf("jeth.openWallet is not callable") + } + val, err := openWallet(goja.Null(), wallet, passwd) if err == nil { - return val + return val, nil } // Wallet open failed, report error unless it's a PIN or PUK entry @@ -111,32 +126,31 @@ func (b *bridge) OpenWallet(call otto.FunctionCall) (response otto.Value) { case strings.HasSuffix(err.Error(), usbwallet.ErrTrezorPINNeeded.Error()): val, err = b.readPinAndReopenWallet(call) if err == nil { - return val + return val, nil } val, err = b.readPassphraseAndReopenWallet(call) if err != nil { - throwJSException(err.Error()) + return nil, err } case strings.HasSuffix(err.Error(), scwallet.ErrPairingPasswordNeeded.Error()): // PUK input requested, fetch from the user and call open again - if input, err := b.prompter.PromptPassword("Please enter the pairing password: "); err != nil { - throwJSException(err.Error()) - } else { - passwd, _ = otto.ToValue(input) + input, err := b.prompter.PromptPassword("Please enter the pairing password: ") + if err != nil { + return nil, err } - if val, err = call.Otto.Call("jeth.openWallet", nil, wallet, passwd); err != nil { + passwd = call.VM.ToValue(input) + if val, err = openWallet(goja.Null(), wallet, passwd); err != nil { if !strings.HasSuffix(err.Error(), scwallet.ErrPINNeeded.Error()) { - throwJSException(err.Error()) + return nil, err } else { // PIN input requested, fetch from the user and call open again - if input, err := b.prompter.PromptPassword("Please enter current PIN: "); err != nil { - throwJSException(err.Error()) - } else { - passwd, _ = otto.ToValue(input) + input, err := b.prompter.PromptPassword("Please enter current PIN: ") + if err != nil { + return nil, err } - if val, err = call.Otto.Call("jeth.openWallet", nil, wallet, passwd); err != nil { - throwJSException(err.Error()) + if val, err = openWallet(goja.Null(), wallet, call.VM.ToValue(input)); err != nil { + return nil, err } } } @@ -144,52 +158,52 @@ func (b *bridge) OpenWallet(call otto.FunctionCall) (response otto.Value) { case strings.HasSuffix(err.Error(), scwallet.ErrPINUnblockNeeded.Error()): // PIN unblock requested, fetch PUK and new PIN from the user var pukpin string - if input, err := b.prompter.PromptPassword("Please enter current PUK: "); err != nil { - throwJSException(err.Error()) - } else { - pukpin = input + input, err := b.prompter.PromptPassword("Please enter current PUK: ") + if err != nil { + return nil, err } - if input, err := b.prompter.PromptPassword("Please enter new PIN: "); err != nil { - throwJSException(err.Error()) - } else { - pukpin += input + pukpin = input + input, err = b.prompter.PromptPassword("Please enter new PIN: ") + if err != nil { + return nil, err } - passwd, _ = otto.ToValue(pukpin) - if val, err = call.Otto.Call("jeth.openWallet", nil, wallet, passwd); err != nil { - throwJSException(err.Error()) + pukpin += input + + if val, err = openWallet(goja.Null(), wallet, call.VM.ToValue(pukpin)); err != nil { + return nil, err } case strings.HasSuffix(err.Error(), scwallet.ErrPINNeeded.Error()): // PIN input requested, fetch from the user and call open again - if input, err := b.prompter.PromptPassword("Please enter current PIN: "); err != nil { - throwJSException(err.Error()) - } else { - passwd, _ = otto.ToValue(input) + input, err := b.prompter.PromptPassword("Please enter current PIN: ") + if err != nil { + return nil, err } - if val, err = call.Otto.Call("jeth.openWallet", nil, wallet, passwd); err != nil { - throwJSException(err.Error()) + if val, err = openWallet(goja.Null(), wallet, call.VM.ToValue(input)); err != nil { + return nil, err } default: // Unknown error occurred, drop to the user - throwJSException(err.Error()) + return nil, err } - return val + return val, nil } -func (b *bridge) readPassphraseAndReopenWallet(call otto.FunctionCall) (otto.Value, error) { - var passwd otto.Value +func (b *bridge) readPassphraseAndReopenWallet(call jsre.Call) (goja.Value, error) { wallet := call.Argument(0) - if input, err := b.prompter.PromptPassword("Please enter your password: "); err != nil { - throwJSException(err.Error()) - } else { - passwd, _ = otto.ToValue(input) + input, err := b.prompter.PromptPassword("Please enter your passphrase: ") + if err != nil { + return nil, err + } + openWallet, callable := goja.AssertFunction(getJeth(call.VM).Get("openWallet")) + if !callable { + return nil, fmt.Errorf("jeth.openWallet is not callable") } - return call.Otto.Call("jeth.openWallet", nil, wallet, passwd) + return openWallet(goja.Null(), wallet, call.VM.ToValue(input)) } -func (b *bridge) readPinAndReopenWallet(call otto.FunctionCall) (otto.Value, error) { - var passwd otto.Value +func (b *bridge) readPinAndReopenWallet(call jsre.Call) (goja.Value, error) { wallet := call.Argument(0) // Trezor PIN matrix input requested, display the matrix to the user and fetch the data fmt.Fprintf(b.printer, "Look at the device for number positions\n\n") @@ -199,155 +213,154 @@ func (b *bridge) readPinAndReopenWallet(call otto.FunctionCall) (otto.Value, err fmt.Fprintf(b.printer, "--+---+--\n") fmt.Fprintf(b.printer, "1 | 2 | 3\n\n") - if input, err := b.prompter.PromptPassword("Please enter current PIN: "); err != nil { - throwJSException(err.Error()) - } else { - passwd, _ = otto.ToValue(input) + input, err := b.prompter.PromptPassword("Please enter current PIN: ") + if err != nil { + return nil, err + } + openWallet, callable := goja.AssertFunction(getJeth(call.VM).Get("openWallet")) + if !callable { + return nil, fmt.Errorf("jeth.openWallet is not callable") } - return call.Otto.Call("jeth.openWallet", nil, wallet, passwd) + return openWallet(goja.Null(), wallet, call.VM.ToValue(input)) } // UnlockAccount is a wrapper around the personal.unlockAccount RPC method that // uses a non-echoing password prompt to acquire the passphrase and executes the // original RPC method (saved in jeth.unlockAccount) with it to actually execute // the RPC call. -func (b *bridge) UnlockAccount(call otto.FunctionCall) (response otto.Value) { - // Make sure we have an account specified to unlock - if !call.Argument(0).IsString() { - throwJSException("first argument must be the account to unlock") +func (b *bridge) UnlockAccount(call jsre.Call) (goja.Value, error) { + // Make sure we have an account specified to unlock. + if call.Argument(0).ExportType().Kind() != reflect.String { + return nil, fmt.Errorf("first argument must be the account to unlock") } account := call.Argument(0) - // If password is not given or is the null value, prompt the user for it - var passwd otto.Value - - if call.Argument(1).IsUndefined() || call.Argument(1).IsNull() { + // If password is not given or is the null value, prompt the user for it. + var passwd goja.Value + if goja.IsUndefined(call.Argument(1)) || goja.IsNull(call.Argument(1)) { fmt.Fprintf(b.printer, "Unlock account %s\n", account) - if input, err := b.prompter.PromptPassword("Password: "); err != nil { - throwJSException(err.Error()) - } else { - passwd, _ = otto.ToValue(input) + input, err := b.prompter.PromptPassword("Passphrase: ") + if err != nil { + return nil, err } + passwd = call.VM.ToValue(input) } else { - if !call.Argument(1).IsString() { - throwJSException("password must be a string") + if call.Argument(1).ExportType().Kind() != reflect.String { + return nil, fmt.Errorf("password must be a string") } passwd = call.Argument(1) } - // Third argument is the duration how long the account must be unlocked. - duration := otto.NullValue() - if call.Argument(2).IsDefined() && !call.Argument(2).IsNull() { - if !call.Argument(2).IsNumber() { - throwJSException("unlock duration must be a number") + + // Third argument is the duration how long the account should be unlocked. + duration := goja.Null() + if !goja.IsUndefined(call.Argument(2)) && !goja.IsNull(call.Argument(2)) { + if !isNumber(call.Argument(2)) { + return nil, fmt.Errorf("unlock duration must be a number") } duration = call.Argument(2) } - // Send the request to the backend and return - val, err := call.Otto.Call("jeth.unlockAccount", nil, account, passwd, duration) - if err != nil { - throwJSException(err.Error()) + + // Send the request to the backend and return. + unlockAccount, callable := goja.AssertFunction(getJeth(call.VM).Get("unlockAccount")) + if !callable { + return nil, fmt.Errorf("jeth.unlockAccount is not callable") } - return val + return unlockAccount(goja.Null(), account, passwd, duration) } // Sign is a wrapper around the personal.sign RPC method that uses a non-echoing password // prompt to acquire the passphrase and executes the original RPC method (saved in // jeth.sign) with it to actually execute the RPC call. -func (b *bridge) Sign(call otto.FunctionCall) (response otto.Value) { +func (b *bridge) Sign(call jsre.Call) (goja.Value, error) { var ( message = call.Argument(0) account = call.Argument(1) passwd = call.Argument(2) ) - if !message.IsString() { - throwJSException("first argument must be the message to sign") + if message.ExportType().Kind() != reflect.String { + return nil, fmt.Errorf("first argument must be the message to sign") } - if !account.IsString() { - throwJSException("second argument must be the account to sign with") + if account.ExportType().Kind() != reflect.String { + return nil, fmt.Errorf("second argument must be the account to sign with") } // if the password is not given or null ask the user and ensure password is a string - if passwd.IsUndefined() || passwd.IsNull() { + if goja.IsUndefined(passwd) || goja.IsNull(passwd) { fmt.Fprintf(b.printer, "Give password for account %s\n", account) - if input, err := b.prompter.PromptPassword("Password: "); err != nil { - throwJSException(err.Error()) - } else { - passwd, _ = otto.ToValue(input) + input, err := b.prompter.PromptPassword("Password: ") + if err != nil { + return nil, err } - } - if !passwd.IsString() { - throwJSException("third argument must be the password to unlock the account") + passwd = call.VM.ToValue(input) + } else if passwd.ExportType().Kind() != reflect.String { + return nil, fmt.Errorf("third argument must be the password to unlock the account") } // Send the request to the backend and return - val, err := call.Otto.Call("jeth.sign", nil, message, account, passwd) - if err != nil { - throwJSException(err.Error()) + sign, callable := goja.AssertFunction(getJeth(call.VM).Get("unlockAccount")) + if !callable { + return nil, fmt.Errorf("jeth.unlockAccount is not callable") } - return val + return sign(goja.Null(), message, account, passwd) } // Sleep will block the console for the specified number of seconds. -func (b *bridge) Sleep(call otto.FunctionCall) (response otto.Value) { - if call.Argument(0).IsNumber() { - sleep, _ := call.Argument(0).ToInteger() - time.Sleep(time.Duration(sleep) * time.Second) - return otto.TrueValue() +func (b *bridge) Sleep(call jsre.Call) (goja.Value, error) { + if !isNumber(call.Argument(0)) { + return nil, fmt.Errorf("usage: sleep()") } - return throwJSException("usage: sleep()") + sleep := call.Argument(0).ToFloat() + time.Sleep(time.Duration(sleep * float64(time.Second))) + return call.VM.ToValue(true), nil } // SleepBlocks will block the console for a specified number of new blocks optionally // until the given timeout is reached. -func (b *bridge) SleepBlocks(call otto.FunctionCall) (response otto.Value) { +func (b *bridge) SleepBlocks(call jsre.Call) (goja.Value, error) { + // Parse the input parameters for the sleep. var ( blocks = int64(0) sleep = int64(9999999999999999) // indefinitely ) - // Parse the input parameters for the sleep - nArgs := len(call.ArgumentList) + nArgs := len(call.Arguments) if nArgs == 0 { - throwJSException("usage: sleepBlocks([, max sleep in seconds])") + return nil, fmt.Errorf("usage: sleepBlocks([, max sleep in seconds])") } if nArgs >= 1 { - if call.Argument(0).IsNumber() { - blocks, _ = call.Argument(0).ToInteger() - } else { - throwJSException("expected number as first argument") + if !isNumber(call.Argument(0)) { + return nil, fmt.Errorf("expected number as first argument") } + blocks = call.Argument(0).ToInteger() } if nArgs >= 2 { - if call.Argument(1).IsNumber() { - sleep, _ = call.Argument(1).ToInteger() - } else { - throwJSException("expected number as second argument") + if isNumber(call.Argument(1)) { + return nil, fmt.Errorf("expected number as second argument") } + sleep = call.Argument(1).ToInteger() } - // go through the console, this will allow web3 to call the appropriate - // callbacks if a delayed response or notification is received. - blockNumber := func() int64 { - result, err := call.Otto.Run("eth.blockNumber") + + // Poll the current block number until either it or a timeout is reached. + var ( + deadline = time.Now().Add(time.Duration(sleep) * time.Second) + lastNumber = ^hexutil.Uint64(0) + ) + for time.Now().Before(deadline) { + var number hexutil.Uint64 + err := b.client.Call(&number, "eth_blockNumber") if err != nil { - throwJSException(err.Error()) + return nil, err } - block, err := result.ToInteger() - if err != nil { - throwJSException(err.Error()) + if number != lastNumber { + lastNumber = number + blocks-- } - return block - } - // Poll the current block number until either it ot a timeout is reached - targetBlockNr := blockNumber() + blocks - deadline := time.Now().Add(time.Duration(sleep) * time.Second) - - for time.Now().Before(deadline) { - if blockNumber() >= targetBlockNr { - return otto.TrueValue() + if blocks <= 0 { + break } time.Sleep(time.Second) } - return otto.FalseValue() + return call.VM.ToValue(true), nil } type jsonrpcCall struct { @@ -357,15 +370,15 @@ type jsonrpcCall struct { } // Send implements the web3 provider "send" method. -func (b *bridge) Send(call otto.FunctionCall) (response otto.Value) { +func (b *bridge) Send(call jsre.Call) (goja.Value, error) { // Remarshal the request into a Go value. - JSON, _ := call.Otto.Object("JSON") - reqVal, err := JSON.Call("stringify", call.Argument(0)) + reqVal, err := call.Argument(0).ToObject(call.VM).MarshalJSON() if err != nil { - throwJSException(err.Error()) + return nil, err } + var ( - rawReq = reqVal.String() + rawReq = string(reqVal) dec = json.NewDecoder(strings.NewReader(rawReq)) reqs []jsonrpcCall batch bool @@ -381,10 +394,12 @@ func (b *bridge) Send(call otto.FunctionCall) (response otto.Value) { } // Execute the requests. - resps, _ := call.Otto.Object("new Array()") + var resps []*goja.Object for _, req := range reqs { - resp, _ := call.Otto.Object(`({"jsonrpc":"2.0"})`) + resp := call.VM.NewObject() + resp.Set("jsonrpc", "2.0") resp.Set("id", req.ID) + var result json.RawMessage err = b.client.Call(&result, req.Method, req.Params...) switch err := err.(type) { @@ -392,9 +407,14 @@ func (b *bridge) Send(call otto.FunctionCall) (response otto.Value) { if result == nil { // Special case null because it is decoded as an empty // raw message for some reason. - resp.Set("result", otto.NullValue()) + resp.Set("result", goja.Null()) } else { - resultVal, err := JSON.Call("parse", string(result)) + JSON := call.VM.Get("JSON").ToObject(call.VM) + parse, callable := goja.AssertFunction(JSON.Get("parse")) + if !callable { + return nil, fmt.Errorf("JSON.parse is not a function") + } + resultVal, err := parse(goja.Null(), call.VM.ToValue(string(result))) if err != nil { setError(resp, -32603, err.Error()) } else { @@ -406,33 +426,38 @@ func (b *bridge) Send(call otto.FunctionCall) (response otto.Value) { default: setError(resp, -32603, err.Error()) } - resps.Call("push", resp) + resps = append(resps, resp) } // Return the responses either to the callback (if supplied) // or directly as the return value. + var result goja.Value if batch { - response = resps.Value() + result = call.VM.ToValue(resps) } else { - response, _ = resps.Get("0") + result = resps[0] } - if fn := call.Argument(1); fn.Class() == "Function" { - fn.Call(otto.NullValue(), otto.NullValue(), response) - return otto.UndefinedValue() + if fn, isFunc := goja.AssertFunction(call.Argument(1)); isFunc { + fn(goja.Null(), goja.Null(), result) + return goja.Undefined(), nil } - return response + return result, nil } -func setError(resp *otto.Object, code int, msg string) { +func setError(resp *goja.Object, code int, msg string) { resp.Set("error", map[string]interface{}{"code": code, "message": msg}) } -// throwJSException panics on an otto.Value. The Otto VM will recover from the -// Go panic and throw msg as a JavaScript error. -func throwJSException(msg interface{}) otto.Value { - val, err := otto.ToValue(msg) - if err != nil { - log.Error("Failed to serialize JavaScript exception", "exception", msg, "err", err) +// isNumber returns true if input value is a JS number. +func isNumber(v goja.Value) bool { + k := v.ExportType().Kind() + return k >= reflect.Int && k <= reflect.Float64 +} + +func getObject(vm *goja.Runtime, name string) *goja.Object { + v := vm.Get(name) + if v == nil { + return nil } - panic(val) + return v.ToObject(vm) } diff --git a/console/console.go b/console/console.go index 5326ed2c87..a2f12d8ede 100644 --- a/console/console.go +++ b/console/console.go @@ -28,12 +28,13 @@ import ( "strings" "syscall" + "github.com/dop251/goja" "github.com/ethereum/go-ethereum/internal/jsre" + "github.com/ethereum/go-ethereum/internal/jsre/deps" "github.com/ethereum/go-ethereum/internal/web3ext" "github.com/ethereum/go-ethereum/rpc" "github.com/mattn/go-colorable" "github.com/peterh/liner" - "github.com/robertkrimen/otto" ) var ( @@ -86,6 +87,7 @@ func New(config Config) (*Console, error) { if config.Printer == nil { config.Printer = colorable.NewColorableStdout() } + // Initialize the console and return console := &Console{ client: config.Client, @@ -107,120 +109,141 @@ func New(config Config) (*Console, error) { // init retrieves the available APIs from the remote RPC provider and initializes // the console's JavaScript namespaces based on the exposed modules. func (c *Console) init(preload []string) error { - // Initialize the JavaScript <-> Go RPC bridge + c.initConsoleObject() + + // Initialize the JavaScript <-> Go RPC bridge. bridge := newBridge(c.client, c.prompter, c.printer) - c.jsre.Set("jeth", struct{}{}) + if err := c.initWeb3(bridge); err != nil { + return err + } + if err := c.initExtensions(); err != nil { + return err + } + + // Add bridge overrides for web3.js functionality. + c.jsre.Do(func(vm *goja.Runtime) { + c.initAdmin(vm, bridge) + c.initPersonal(vm, bridge) + }) + + // Preload JavaScript files. + for _, path := range preload { + if err := c.jsre.Exec(path); err != nil { + failure := err.Error() + if gojaErr, ok := err.(*goja.Exception); ok { + failure = gojaErr.String() + } + return fmt.Errorf("%s: %v", path, failure) + } + } - jethObj, _ := c.jsre.Get("jeth") - jethObj.Object().Set("send", bridge.Send) - jethObj.Object().Set("sendAsync", bridge.Send) + // Configure the input prompter for history and tab completion. + if c.prompter != nil { + if content, err := ioutil.ReadFile(c.histPath); err != nil { + c.prompter.SetHistory(nil) + } else { + c.history = strings.Split(string(content), "\n") + c.prompter.SetHistory(c.history) + } + c.prompter.SetWordCompleter(c.AutoCompleteInput) + } + return nil +} - consoleObj, _ := c.jsre.Get("console") - consoleObj.Object().Set("log", c.consoleOutput) - consoleObj.Object().Set("error", c.consoleOutput) +func (c *Console) initConsoleObject() { + c.jsre.Do(func(vm *goja.Runtime) { + console := vm.NewObject() + console.Set("log", c.consoleOutput) + console.Set("error", c.consoleOutput) + vm.Set("console", console) + }) +} - // Load all the internal utility JavaScript libraries - if err := c.jsre.Compile("bignumber.js", jsre.BignumberJs); err != nil { +func (c *Console) initWeb3(bridge *bridge) error { + bnJS := string(deps.MustAsset("bignumber.js")) + web3JS := string(deps.MustAsset("web3.js")) + if err := c.jsre.Compile("bignumber.js", bnJS); err != nil { return fmt.Errorf("bignumber.js: %v", err) } - if err := c.jsre.Compile("web3.js", jsre.Web3Js); err != nil { + if err := c.jsre.Compile("web3.js", web3JS); err != nil { return fmt.Errorf("web3.js: %v", err) } if _, err := c.jsre.Run("var Web3 = require('web3');"); err != nil { return fmt.Errorf("web3 require: %v", err) } - if _, err := c.jsre.Run("var web3 = new Web3(jeth);"); err != nil { - return fmt.Errorf("web3 provider: %v", err) - } - // Load the supported APIs into the JavaScript runtime environment + var err error + c.jsre.Do(func(vm *goja.Runtime) { + transport := vm.NewObject() + transport.Set("send", jsre.MakeCallback(vm, bridge.Send)) + transport.Set("sendAsync", jsre.MakeCallback(vm, bridge.Send)) + vm.Set("_consoleWeb3Transport", transport) + _, err = vm.RunString("var web3 = new Web3(_consoleWeb3Transport)") + }) + return err +} + +// initExtensions loads and registers web3.js extensions. +func (c *Console) initExtensions() error { + // Compute aliases from server-provided modules. apis, err := c.client.SupportedModules() if err != nil { return fmt.Errorf("api modules: %v", err) } - flatten := "var eth = web3.eth; var personal = web3.personal; " + aliases := map[string]struct{}{"eth": {}, "personal": {}} for api := range apis { if api == "web3" { - continue // manually mapped or ignore + continue } + aliases[api] = struct{}{} if file, ok := web3ext.Modules[api]; ok { - // Load our extension for the module. - if err = c.jsre.Compile(fmt.Sprintf("%s.js", api), file); err != nil { + if err = c.jsre.Compile(api+".js", file); err != nil { return fmt.Errorf("%s.js: %v", api, err) } - flatten += fmt.Sprintf("var %s = web3.%s; ", api, api) - } else if obj, err := c.jsre.Run("web3." + api); err == nil && obj.IsObject() { - // Enable web3.js built-in extension if available. - flatten += fmt.Sprintf("var %s = web3.%s; ", api, api) } } - if _, err = c.jsre.Run(flatten); err != nil { - return fmt.Errorf("namespace flattening: %v", err) - } - // Initialize the global name register (disabled for now) - //c.jsre.Run(`var GlobalRegistrar = eth.contract(` + registrar.GlobalRegistrarAbi + `); registrar = GlobalRegistrar.at("` + registrar.GlobalRegistrarAddr + `");`) - // If the console is in interactive mode, instrument password related methods to query the user - if c.prompter != nil { - // Retrieve the account management object to instrument - personal, err := c.jsre.Get("personal") - if err != nil { - return err - } - // Override the openWallet, unlockAccount, newAccount and sign methods since - // these require user interaction. Assign these method in the Console the - // original web3 callbacks. These will be called by the jeth.* methods after - // they got the password from the user and send the original web3 request to - // the backend. - if obj := personal.Object(); obj != nil { // make sure the personal api is enabled over the interface - if _, err = c.jsre.Run(`jeth.openWallet = personal.openWallet;`); err != nil { - return fmt.Errorf("personal.openWallet: %v", err) - } - if _, err = c.jsre.Run(`jeth.unlockAccount = personal.unlockAccount;`); err != nil { - return fmt.Errorf("personal.unlockAccount: %v", err) - } - if _, err = c.jsre.Run(`jeth.newAccount = personal.newAccount;`); err != nil { - return fmt.Errorf("personal.newAccount: %v", err) - } - if _, err = c.jsre.Run(`jeth.sign = personal.sign;`); err != nil { - return fmt.Errorf("personal.sign: %v", err) - } - obj.Set("openWallet", bridge.OpenWallet) - obj.Set("unlockAccount", bridge.UnlockAccount) - obj.Set("newAccount", bridge.NewAccount) - obj.Set("sign", bridge.Sign) - } - } - // The admin.sleep and admin.sleepBlocks are offered by the console and not by the RPC layer. - admin, err := c.jsre.Get("admin") - if err != nil { - return err - } - if obj := admin.Object(); obj != nil { // make sure the admin api is enabled over the interface - obj.Set("sleepBlocks", bridge.SleepBlocks) - obj.Set("sleep", bridge.Sleep) - obj.Set("clearHistory", c.clearHistory) - } - // Preload any JavaScript files before starting the console - for _, path := range preload { - if err := c.jsre.Exec(path); err != nil { - failure := err.Error() - if ottoErr, ok := err.(*otto.Error); ok { - failure = ottoErr.String() + // Apply aliases. + c.jsre.Do(func(vm *goja.Runtime) { + web3 := getObject(vm, "web3") + for name := range aliases { + if v := web3.Get(name); v != nil { + vm.Set(name, v) } - return fmt.Errorf("%s: %v", path, failure) } + }) + return nil +} + +// initAdmin creates additional admin APIs implemented by the bridge. +func (c *Console) initAdmin(vm *goja.Runtime, bridge *bridge) { + if admin := getObject(vm, "admin"); admin != nil { + admin.Set("sleepBlocks", jsre.MakeCallback(vm, bridge.SleepBlocks)) + admin.Set("sleep", jsre.MakeCallback(vm, bridge.Sleep)) + admin.Set("clearHistory", c.clearHistory) } - // Configure the console's input prompter for scrollback and tab completion - if c.prompter != nil { - if content, err := ioutil.ReadFile(c.histPath); err != nil { - c.prompter.SetHistory(nil) - } else { - c.history = strings.Split(string(content), "\n") - c.prompter.SetHistory(c.history) - } - c.prompter.SetWordCompleter(c.AutoCompleteInput) +} + +// initPersonal redirects account-related API methods through the bridge. +// +// If the console is in interactive mode and the 'personal' API is available, override +// the openWallet, unlockAccount, newAccount and sign methods since these require user +// interaction. The original web3 callbacks are stored in 'jeth'. These will be called +// by the bridge after the prompt and send the original web3 request to the backend. +func (c *Console) initPersonal(vm *goja.Runtime, bridge *bridge) { + personal := getObject(vm, "personal") + if personal == nil || c.prompter == nil { + return } - return nil + jeth := vm.NewObject() + vm.Set("jeth", jeth) + jeth.Set("openWallet", personal.Get("openWallet")) + jeth.Set("unlockAccount", personal.Get("unlockAccount")) + jeth.Set("newAccount", personal.Get("newAccount")) + jeth.Set("sign", personal.Get("sign")) + personal.Set("openWallet", jsre.MakeCallback(vm, bridge.OpenWallet)) + personal.Set("unlockAccount", jsre.MakeCallback(vm, bridge.UnlockAccount)) + personal.Set("newAccount", jsre.MakeCallback(vm, bridge.NewAccount)) + personal.Set("sign", jsre.MakeCallback(vm, bridge.Sign)) } func (c *Console) clearHistory() { @@ -235,13 +258,13 @@ func (c *Console) clearHistory() { // consoleOutput is an override for the console.log and console.error methods to // stream the output into the configured output stream instead of stdout. -func (c *Console) consoleOutput(call otto.FunctionCall) otto.Value { +func (c *Console) consoleOutput(call goja.FunctionCall) goja.Value { var output []string - for _, argument := range call.ArgumentList { + for _, argument := range call.Arguments { output = append(output, fmt.Sprintf("%v", argument)) } fmt.Fprintln(c.printer, strings.Join(output, " ")) - return otto.Value{} + return goja.Null() } // AutoCompleteInput is a pre-assembled word completer to be used by the user @@ -304,13 +327,13 @@ func (c *Console) Welcome() { // Evaluate executes code and pretty prints the result to the specified output // stream. -func (c *Console) Evaluate(statement string) error { +func (c *Console) Evaluate(statement string) { defer func() { if r := recover(); r != nil { fmt.Fprintf(c.printer, "[native] error: %v\n", r) } }() - return c.jsre.Evaluate(statement, c.printer) + c.jsre.Evaluate(statement, c.printer) } // Interactive starts an interactive user session, where input is propted from diff --git a/console/console_test.go b/console/console_test.go index 89dd7cd838..9a2b474442 100644 --- a/console/console_test.go +++ b/console/console_test.go @@ -289,7 +289,7 @@ func TestPrettyError(t *testing.T) { defer tester.Close(t) tester.console.Evaluate("throw 'hello'") - want := jsre.ErrorColor("hello") + "\n" + want := jsre.ErrorColor("hello") + "\n\tat :1:7(1)\n\n" if output := tester.output.String(); output != want { t.Fatalf("pretty error mismatch: have %s, want %s", output, want) } diff --git a/go.mod b/go.mod index 223086f8c5..c3b5b6fd02 100644 --- a/go.mod +++ b/go.mod @@ -16,13 +16,16 @@ require ( github.com/cloudflare/cloudflare-go v0.10.2-0.20190916151808-a80f83b9add9 github.com/davecgh/go-spew v1.1.1 github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea + github.com/dlclark/regexp2 v1.2.0 // indirect github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf + github.com/dop251/goja v0.0.0-20200106141417-aaec0e7bde29 github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c github.com/elastic/gosigar v0.8.1-0.20180330100440-37f05ff46ffa github.com/fatih/color v1.3.0 github.com/fjl/memsize v0.0.0-20180418122429-ca190fb6ffbc github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff github.com/go-ole/go-ole v1.2.1 // indirect + github.com/go-sourcemap/sourcemap v2.1.2+incompatible // indirect github.com/go-stack/stack v1.8.0 github.com/golang/protobuf v1.3.2-0.20190517061210-b285ee9cfc6c github.com/golang/snappy v0.0.1 diff --git a/go.sum b/go.sum index 4d18c8c20e..81c718b024 100644 --- a/go.sum +++ b/go.sum @@ -57,8 +57,14 @@ github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea/go.mod h1:93vs github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-sip13 v0.0.0-20181026042036-e10d5fee7954/go.mod h1:vAd38F8PWV+bWy6jNmig1y/TA+kYO4g3RSRF0IAv0no= +github.com/dlclark/regexp2 v1.2.0 h1:8sAhBGEM0dRWogWqWyQeIJnxjWO6oIjl8FKqREDsGfk= +github.com/dlclark/regexp2 v1.2.0/go.mod h1:2pZnwuY/m+8K6iRw6wQdMtk+rH5tNGR1i55kozfMjCc= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf h1:sh8rkQZavChcmakYiSlqu2425CHyFXLZZnvm7PDpU8M= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/dop251/goja v0.0.0-20191203121440-007eef3bc40f h1:vtCDQseO/Sbu5IZSoc2uzZ7CkSoai7OtpcwGFK5FlyE= +github.com/dop251/goja v0.0.0-20191203121440-007eef3bc40f/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= +github.com/dop251/goja v0.0.0-20200106141417-aaec0e7bde29 h1:Ewd9K+mC725sITA12QQHRqWj78NU4t7EhlFVVgdlzJg= +github.com/dop251/goja v0.0.0-20200106141417-aaec0e7bde29/go.mod h1:Mw6PkjjMXWbTj+nnj4s3QPXq1jaT0s5pC0iFD4+BOAA= github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c h1:JHHhtb9XWJrGNMcrVP6vyzO4dusgi/HnceHTgxSejUM= github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elastic/gosigar v0.8.1-0.20180330100440-37f05ff46ffa h1:XKAhUk/dtp+CV0VO6mhG2V7jA9vbcGcnYF/Ay9NjZrY= @@ -77,6 +83,8 @@ github.com/go-logfmt/logfmt v0.3.0 h1:8HUsc87TaSWLKwrnumgC8/YconD2fJQsRJAsWaPg2i github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-ole/go-ole v1.2.1 h1:2lOsA72HgjxAuMlKpFiCbHTvu44PIVkZ5hqm3RSdI/E= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= +github.com/go-sourcemap/sourcemap v2.1.2+incompatible h1:0b/xya7BKGhXuqFESKM4oIiRo9WOt2ebz7KxfreD6ug= +github.com/go-sourcemap/sourcemap v2.1.2+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= diff --git a/internal/jsre/completion.go b/internal/jsre/completion.go index 7f484bbbb8..2c105184cb 100644 --- a/internal/jsre/completion.go +++ b/internal/jsre/completion.go @@ -20,35 +20,43 @@ import ( "sort" "strings" - "github.com/robertkrimen/otto" + "github.com/dop251/goja" ) // CompleteKeywords returns potential continuations for the given line. Since line is // evaluated, callers need to make sure that evaluating line does not have side effects. func (jsre *JSRE) CompleteKeywords(line string) []string { var results []string - jsre.Do(func(vm *otto.Otto) { + jsre.Do(func(vm *goja.Runtime) { results = getCompletions(vm, line) }) return results } -func getCompletions(vm *otto.Otto, line string) (results []string) { +func getCompletions(vm *goja.Runtime, line string) (results []string) { parts := strings.Split(line, ".") - objRef := "this" - prefix := line - if len(parts) > 1 { - objRef = strings.Join(parts[0:len(parts)-1], ".") - prefix = parts[len(parts)-1] + if len(parts) == 0 { + return nil } - obj, _ := vm.Object(objRef) - if obj == nil { - return nil + // Find the right-most fully named object in the line. e.g. if line = "x.y.z" + // and "x.y" is an object, obj will reference "x.y". + obj := vm.GlobalObject() + for i := 0; i < len(parts)-1; i++ { + v := obj.Get(parts[i]) + if v == nil { + return nil // No object was found + } + obj = v.ToObject(vm) } + + // Go over the keys of the object and retain the keys matching prefix. + // Example: if line = "x.y.z" and "x.y" exists and has keys "zebu", "zebra" + // and "platypus", then "x.y.zebu" and "x.y.zebra" will be added to results. + prefix := parts[len(parts)-1] iterOwnAndConstructorKeys(vm, obj, func(k string) { if strings.HasPrefix(k, prefix) { - if objRef == "this" { + if len(parts) == 1 { results = append(results, k) } else { results = append(results, strings.Join(parts[:len(parts)-1], ".")+"."+k) @@ -59,9 +67,9 @@ func getCompletions(vm *otto.Otto, line string) (results []string) { // Append opening parenthesis (for functions) or dot (for objects) // if the line itself is the only completion. if len(results) == 1 && results[0] == line { - obj, _ := vm.Object(line) + obj := obj.Get(parts[len(parts)-1]) if obj != nil { - if obj.Class() == "Function" { + if _, isfunc := goja.AssertFunction(obj); isfunc { results[0] += "(" } else { results[0] += "." diff --git a/internal/jsre/completion_test.go b/internal/jsre/completion_test.go index ccbd73dccc..2d05547d12 100644 --- a/internal/jsre/completion_test.go +++ b/internal/jsre/completion_test.go @@ -39,6 +39,10 @@ func TestCompleteKeywords(t *testing.T) { input string want []string }{ + { + input: "St", + want: []string{"String"}, + }, { input: "x", want: []string{"x."}, diff --git a/internal/jsre/jsre.go b/internal/jsre/jsre.go index 1b3528a036..bc8869b254 100644 --- a/internal/jsre/jsre.go +++ b/internal/jsre/jsre.go @@ -26,30 +26,30 @@ import ( "math/rand" "time" + "github.com/dop251/goja" "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/internal/jsre/deps" - "github.com/robertkrimen/otto" ) -var ( - BignumberJs = deps.MustAsset("bignumber.js") - Web3Js = deps.MustAsset("web3.js") -) - -/* -JSRE is a generic JS runtime environment embedding the otto JS interpreter. -It provides some helper functions to -- load code from files -- run code snippets -- require libraries -- bind native go objects -*/ +// JSRE is a JS runtime environment embedding the goja interpreter. +// It provides helper functions to load code from files, run code snippets +// and bind native go objects to JS. +// +// The runtime runs all code on a dedicated event loop and does not expose the underlying +// goja runtime directly. To use the runtime, call JSRE.Do. When binding a Go function, +// use the Call type to gain access to the runtime. type JSRE struct { assetPath string output io.Writer evalQueue chan *evalReq stopEventLoop chan bool closed chan struct{} + vm *goja.Runtime +} + +// Call is the argument type of Go functions which are callable from JS. +type Call struct { + goja.FunctionCall + VM *goja.Runtime } // jsTimer is a single timer instance with a callback function @@ -57,12 +57,12 @@ type jsTimer struct { timer *time.Timer duration time.Duration interval bool - call otto.FunctionCall + call goja.FunctionCall } // evalReq is a serialized vm execution request processed by runEventLoop. type evalReq struct { - fn func(vm *otto.Otto) + fn func(vm *goja.Runtime) done chan bool } @@ -74,9 +74,10 @@ func New(assetPath string, output io.Writer) *JSRE { closed: make(chan struct{}), evalQueue: make(chan *evalReq), stopEventLoop: make(chan bool), + vm: goja.New(), } go re.runEventLoop() - re.Set("loadScript", re.loadScript) + re.Set("loadScript", MakeCallback(re.vm, re.loadScript)) re.Set("inspect", re.prettyPrintJS) return re } @@ -99,21 +100,20 @@ func randomSource() *rand.Rand { // serialized way and calls timer callback functions at the appropriate time. // Exported functions always access the vm through the event queue. You can -// call the functions of the otto vm directly to circumvent the queue. These +// call the functions of the goja vm directly to circumvent the queue. These // functions should be used if and only if running a routine that was already // called from JS through an RPC call. func (re *JSRE) runEventLoop() { defer close(re.closed) - vm := otto.New() r := randomSource() - vm.SetRandomSource(r.Float64) + re.vm.SetRandSource(r.Float64) registry := map[*jsTimer]*jsTimer{} ready := make(chan *jsTimer) - newTimer := func(call otto.FunctionCall, interval bool) (*jsTimer, otto.Value) { - delay, _ := call.Argument(1).ToInteger() + newTimer := func(call goja.FunctionCall, interval bool) (*jsTimer, goja.Value) { + delay := call.Argument(1).ToInteger() if 0 >= delay { delay = 1 } @@ -128,47 +128,43 @@ func (re *JSRE) runEventLoop() { ready <- timer }) - value, err := call.Otto.ToValue(timer) - if err != nil { - panic(err) - } - return timer, value + return timer, re.vm.ToValue(timer) } - setTimeout := func(call otto.FunctionCall) otto.Value { + setTimeout := func(call goja.FunctionCall) goja.Value { _, value := newTimer(call, false) return value } - setInterval := func(call otto.FunctionCall) otto.Value { + setInterval := func(call goja.FunctionCall) goja.Value { _, value := newTimer(call, true) return value } - clearTimeout := func(call otto.FunctionCall) otto.Value { - timer, _ := call.Argument(0).Export() + clearTimeout := func(call goja.FunctionCall) goja.Value { + timer := call.Argument(0).Export() if timer, ok := timer.(*jsTimer); ok { timer.timer.Stop() delete(registry, timer) } - return otto.UndefinedValue() + return goja.Undefined() } - vm.Set("_setTimeout", setTimeout) - vm.Set("_setInterval", setInterval) - vm.Run(`var setTimeout = function(args) { + re.vm.Set("_setTimeout", setTimeout) + re.vm.Set("_setInterval", setInterval) + re.vm.RunString(`var setTimeout = function(args) { if (arguments.length < 1) { throw TypeError("Failed to execute 'setTimeout': 1 argument required, but only 0 present."); } return _setTimeout.apply(this, arguments); }`) - vm.Run(`var setInterval = function(args) { + re.vm.RunString(`var setInterval = function(args) { if (arguments.length < 1) { throw TypeError("Failed to execute 'setInterval': 1 argument required, but only 0 present."); } return _setInterval.apply(this, arguments); }`) - vm.Set("clearTimeout", clearTimeout) - vm.Set("clearInterval", clearTimeout) + re.vm.Set("clearTimeout", clearTimeout) + re.vm.Set("clearInterval", clearTimeout) var waitForCallbacks bool @@ -178,8 +174,8 @@ loop: case timer := <-ready: // execute callback, remove/reschedule the timer var arguments []interface{} - if len(timer.call.ArgumentList) > 2 { - tmp := timer.call.ArgumentList[2:] + if len(timer.call.Arguments) > 2 { + tmp := timer.call.Arguments[2:] arguments = make([]interface{}, 2+len(tmp)) for i, value := range tmp { arguments[i+2] = value @@ -187,11 +183,12 @@ loop: } else { arguments = make([]interface{}, 1) } - arguments[0] = timer.call.ArgumentList[0] - _, err := vm.Call(`Function.call.call`, nil, arguments...) - if err != nil { - fmt.Println("js error:", err, arguments) + arguments[0] = timer.call.Arguments[0] + call, isFunc := goja.AssertFunction(timer.call.Arguments[0]) + if !isFunc { + panic(re.vm.ToValue("js error: timer/timeout callback is not a function")) } + call(goja.Null(), timer.call.Arguments...) _, inreg := registry[timer] // when clearInterval is called from within the callback don't reset it if timer.interval && inreg { @@ -204,7 +201,7 @@ loop: } case req := <-re.evalQueue: // run the code, send the result back - req.fn(vm) + req.fn(re.vm) close(req.done) if waitForCallbacks && (len(registry) == 0) { break loop @@ -223,7 +220,7 @@ loop: } // Do executes the given function on the JS event loop. -func (re *JSRE) Do(fn func(*otto.Otto)) { +func (re *JSRE) Do(fn func(*goja.Runtime)) { done := make(chan bool) req := &evalReq{fn, done} re.evalQueue <- req @@ -246,70 +243,36 @@ func (re *JSRE) Exec(file string) error { if err != nil { return err } - var script *otto.Script - re.Do(func(vm *otto.Otto) { - script, err = vm.Compile(file, code) - if err != nil { - return - } - _, err = vm.Run(script) - }) - return err -} - -// Bind assigns value v to a variable in the JS environment -// This method is deprecated, use Set. -func (re *JSRE) Bind(name string, v interface{}) error { - return re.Set(name, v) + return re.Compile(file, string(code)) } // Run runs a piece of JS code. -func (re *JSRE) Run(code string) (v otto.Value, err error) { - re.Do(func(vm *otto.Otto) { v, err = vm.Run(code) }) - return v, err -} - -// Get returns the value of a variable in the JS environment. -func (re *JSRE) Get(ns string) (v otto.Value, err error) { - re.Do(func(vm *otto.Otto) { v, err = vm.Get(ns) }) +func (re *JSRE) Run(code string) (v goja.Value, err error) { + re.Do(func(vm *goja.Runtime) { v, err = vm.RunString(code) }) return v, err } // Set assigns value v to a variable in the JS environment. func (re *JSRE) Set(ns string, v interface{}) (err error) { - re.Do(func(vm *otto.Otto) { err = vm.Set(ns, v) }) + re.Do(func(vm *goja.Runtime) { vm.Set(ns, v) }) return err } -// loadScript executes a JS script from inside the currently executing JS code. -func (re *JSRE) loadScript(call otto.FunctionCall) otto.Value { - file, err := call.Argument(0).ToString() - if err != nil { - // TODO: throw exception - return otto.FalseValue() - } - file = common.AbsolutePath(re.assetPath, file) - source, err := ioutil.ReadFile(file) - if err != nil { - // TODO: throw exception - return otto.FalseValue() - } - if _, err := compileAndRun(call.Otto, file, source); err != nil { - // TODO: throw exception - fmt.Println("err:", err) - return otto.FalseValue() - } - // TODO: return evaluation result - return otto.TrueValue() +// MakeCallback turns the given function into a function that's callable by JS. +func MakeCallback(vm *goja.Runtime, fn func(Call) (goja.Value, error)) goja.Value { + return vm.ToValue(func(call goja.FunctionCall) goja.Value { + result, err := fn(Call{call, vm}) + if err != nil { + panic(vm.NewGoError(err)) + } + return result + }) } -// Evaluate executes code and pretty prints the result to the specified output -// stream. -func (re *JSRE) Evaluate(code string, w io.Writer) error { - var fail error - - re.Do(func(vm *otto.Otto) { - val, err := vm.Run(code) +// Evaluate executes code and pretty prints the result to the specified output stream. +func (re *JSRE) Evaluate(code string, w io.Writer) { + re.Do(func(vm *goja.Runtime) { + val, err := vm.RunString(code) if err != nil { prettyError(vm, err, w) } else { @@ -317,19 +280,33 @@ func (re *JSRE) Evaluate(code string, w io.Writer) error { } fmt.Fprintln(w) }) - return fail } // Compile compiles and then runs a piece of JS code. -func (re *JSRE) Compile(filename string, src interface{}) (err error) { - re.Do(func(vm *otto.Otto) { _, err = compileAndRun(vm, filename, src) }) +func (re *JSRE) Compile(filename string, src string) (err error) { + re.Do(func(vm *goja.Runtime) { _, err = compileAndRun(vm, filename, src) }) return err } -func compileAndRun(vm *otto.Otto, filename string, src interface{}) (otto.Value, error) { - script, err := vm.Compile(filename, src) +// loadScript loads and executes a JS file. +func (re *JSRE) loadScript(call Call) (goja.Value, error) { + file := call.Argument(0).ToString().String() + file = common.AbsolutePath(re.assetPath, file) + source, err := ioutil.ReadFile(file) + if err != nil { + return nil, fmt.Errorf("Could not read file %s: %v", file, err) + } + value, err := compileAndRun(re.vm, file, string(source)) + if err != nil { + return nil, fmt.Errorf("Error while compiling or running script: %v", err) + } + return value, nil +} + +func compileAndRun(vm *goja.Runtime, filename string, src string) (goja.Value, error) { + script, err := goja.Compile(filename, src, false) if err != nil { - return otto.Value{}, err + return goja.Null(), err } - return vm.Run(script) + return vm.RunProgram(script) } diff --git a/internal/jsre/jsre_test.go b/internal/jsre/jsre_test.go index bcb6e0dd23..bc38f7a44a 100644 --- a/internal/jsre/jsre_test.go +++ b/internal/jsre/jsre_test.go @@ -20,25 +20,24 @@ import ( "io/ioutil" "os" "path" + "reflect" "testing" "time" - "github.com/robertkrimen/otto" + "github.com/dop251/goja" ) -type testNativeObjectBinding struct{} +type testNativeObjectBinding struct { + vm *goja.Runtime +} type msg struct { Msg string } -func (no *testNativeObjectBinding) TestMethod(call otto.FunctionCall) otto.Value { - m, err := call.Argument(0).ToString() - if err != nil { - return otto.UndefinedValue() - } - v, _ := call.Otto.ToValue(&msg{m}) - return v +func (no *testNativeObjectBinding) TestMethod(call goja.FunctionCall) goja.Value { + m := call.Argument(0).ToString().String() + return no.vm.ToValue(&msg{m}) } func newWithTestJS(t *testing.T, testjs string) (*JSRE, string) { @@ -51,7 +50,8 @@ func newWithTestJS(t *testing.T, testjs string) (*JSRE, string) { t.Fatal("cannot create test.js:", err) } } - return New(dir, os.Stdout), dir + jsre := New(dir, os.Stdout) + return jsre, dir } func TestExec(t *testing.T) { @@ -66,11 +66,11 @@ func TestExec(t *testing.T) { if err != nil { t.Errorf("expected no error, got %v", err) } - if !val.IsString() { + if val.ExportType().Kind() != reflect.String { t.Errorf("expected string value, got %v", val) } exp := "testMsg" - got, _ := val.ToString() + got := val.ToString().String() if exp != got { t.Errorf("expected '%v', got '%v'", exp, got) } @@ -90,11 +90,11 @@ func TestNatto(t *testing.T) { if err != nil { t.Errorf("expected no error, got %v", err) } - if !val.IsString() { + if val.ExportType().Kind() != reflect.String { t.Errorf("expected string value, got %v", val) } exp := "testMsg" - got, _ := val.ToString() + got := val.ToString().String() if exp != got { t.Errorf("expected '%v', got '%v'", exp, got) } @@ -105,7 +105,7 @@ func TestBind(t *testing.T) { jsre := New("", os.Stdout) defer jsre.Stop(false) - jsre.Bind("no", &testNativeObjectBinding{}) + jsre.Set("no", &testNativeObjectBinding{vm: jsre.vm}) _, err := jsre.Run(`no.TestMethod("testMsg")`) if err != nil { @@ -125,11 +125,11 @@ func TestLoadScript(t *testing.T) { if err != nil { t.Errorf("expected no error, got %v", err) } - if !val.IsString() { + if val.ExportType().Kind() != reflect.String { t.Errorf("expected string value, got %v", val) } exp := "testMsg" - got, _ := val.ToString() + got := val.ToString().String() if exp != got { t.Errorf("expected '%v', got '%v'", exp, got) } diff --git a/internal/jsre/pretty.go b/internal/jsre/pretty.go index 16fa91b67d..4171e00906 100644 --- a/internal/jsre/pretty.go +++ b/internal/jsre/pretty.go @@ -19,12 +19,13 @@ package jsre import ( "fmt" "io" + "reflect" "sort" "strconv" "strings" + "github.com/dop251/goja" "github.com/fatih/color" - "github.com/robertkrimen/otto" ) const ( @@ -52,29 +53,29 @@ var boringKeys = map[string]bool{ } // prettyPrint writes value to standard output. -func prettyPrint(vm *otto.Otto, value otto.Value, w io.Writer) { +func prettyPrint(vm *goja.Runtime, value goja.Value, w io.Writer) { ppctx{vm: vm, w: w}.printValue(value, 0, false) } // prettyError writes err to standard output. -func prettyError(vm *otto.Otto, err error, w io.Writer) { +func prettyError(vm *goja.Runtime, err error, w io.Writer) { failure := err.Error() - if ottoErr, ok := err.(*otto.Error); ok { - failure = ottoErr.String() + if gojaErr, ok := err.(*goja.Exception); ok { + failure = gojaErr.String() } fmt.Fprint(w, ErrorColor("%s", failure)) } -func (re *JSRE) prettyPrintJS(call otto.FunctionCall) otto.Value { - for _, v := range call.ArgumentList { - prettyPrint(call.Otto, v, re.output) +func (re *JSRE) prettyPrintJS(call goja.FunctionCall) goja.Value { + for _, v := range call.Arguments { + prettyPrint(re.vm, v, re.output) fmt.Fprintln(re.output) } - return otto.UndefinedValue() + return goja.Undefined() } type ppctx struct { - vm *otto.Otto + vm *goja.Runtime w io.Writer } @@ -82,35 +83,47 @@ func (ctx ppctx) indent(level int) string { return strings.Repeat(indentString, level) } -func (ctx ppctx) printValue(v otto.Value, level int, inArray bool) { +func (ctx ppctx) printValue(v goja.Value, level int, inArray bool) { + if goja.IsNull(v) || goja.IsUndefined(v) { + fmt.Fprint(ctx.w, SpecialColor(v.String())) + return + } + kind := v.ExportType().Kind() switch { - case v.IsObject(): - ctx.printObject(v.Object(), level, inArray) - case v.IsNull(): - fmt.Fprint(ctx.w, SpecialColor("null")) - case v.IsUndefined(): - fmt.Fprint(ctx.w, SpecialColor("undefined")) - case v.IsString(): - s, _ := v.ToString() - fmt.Fprint(ctx.w, StringColor("%q", s)) - case v.IsBoolean(): - b, _ := v.ToBoolean() - fmt.Fprint(ctx.w, SpecialColor("%t", b)) - case v.IsNaN(): - fmt.Fprint(ctx.w, NumberColor("NaN")) - case v.IsNumber(): - s, _ := v.ToString() - fmt.Fprint(ctx.w, NumberColor("%s", s)) + case kind == reflect.Bool: + fmt.Fprint(ctx.w, SpecialColor("%t", v.ToBoolean())) + case kind == reflect.String: + fmt.Fprint(ctx.w, StringColor("%q", v.String())) + case kind >= reflect.Int && kind <= reflect.Complex128: + fmt.Fprint(ctx.w, NumberColor("%s", v.String())) default: - fmt.Fprint(ctx.w, "") + if obj, ok := v.(*goja.Object); ok { + ctx.printObject(obj, level, inArray) + } else { + fmt.Fprintf(ctx.w, "", v) + } } } -func (ctx ppctx) printObject(obj *otto.Object, level int, inArray bool) { - switch obj.Class() { +// SafeGet attempt to get the value associated to `key`, and +// catches the panic that goja creates if an error occurs in +// key. +func SafeGet(obj *goja.Object, key string) (ret goja.Value) { + defer func() { + if r := recover(); r != nil { + ret = goja.Undefined() + } + }() + ret = obj.Get(key) + + return ret +} + +func (ctx ppctx) printObject(obj *goja.Object, level int, inArray bool) { + switch obj.ClassName() { case "Array", "GoArray": - lv, _ := obj.Get("length") - len, _ := lv.ToInteger() + lv := obj.Get("length") + len := lv.ToInteger() if len == 0 { fmt.Fprintf(ctx.w, "[]") return @@ -121,8 +134,8 @@ func (ctx ppctx) printObject(obj *otto.Object, level int, inArray bool) { } fmt.Fprint(ctx.w, "[") for i := int64(0); i < len; i++ { - el, err := obj.Get(strconv.FormatInt(i, 10)) - if err == nil { + el := obj.Get(strconv.FormatInt(i, 10)) + if el != nil { ctx.printValue(el, level+1, true) } if i < len-1 { @@ -149,7 +162,7 @@ func (ctx ppctx) printObject(obj *otto.Object, level int, inArray bool) { } fmt.Fprintln(ctx.w, "{") for i, k := range keys { - v, _ := obj.Get(k) + v := SafeGet(obj, k) fmt.Fprintf(ctx.w, "%s%s: ", ctx.indent(level+1), k) ctx.printValue(v, level+1, false) if i < len(keys)-1 { @@ -163,29 +176,25 @@ func (ctx ppctx) printObject(obj *otto.Object, level int, inArray bool) { fmt.Fprintf(ctx.w, "%s}", ctx.indent(level)) case "Function": - // Use toString() to display the argument list if possible. - if robj, err := obj.Call("toString"); err != nil { - fmt.Fprint(ctx.w, FunctionColor("function()")) - } else { - desc := strings.Trim(strings.Split(robj.String(), "{")[0], " \t\n") - desc = strings.Replace(desc, " (", "(", 1) - fmt.Fprint(ctx.w, FunctionColor("%s", desc)) - } + robj := obj.ToString() + desc := strings.Trim(strings.Split(robj.String(), "{")[0], " \t\n") + desc = strings.Replace(desc, " (", "(", 1) + fmt.Fprint(ctx.w, FunctionColor("%s", desc)) case "RegExp": fmt.Fprint(ctx.w, StringColor("%s", toString(obj))) default: - if v, _ := obj.Get("toString"); v.IsFunction() && level <= maxPrettyPrintLevel { - s, _ := obj.Call("toString") - fmt.Fprintf(ctx.w, "<%s %s>", obj.Class(), s.String()) + if level <= maxPrettyPrintLevel { + s := obj.ToString().String() + fmt.Fprintf(ctx.w, "<%s %s>", obj.ClassName(), s) } else { - fmt.Fprintf(ctx.w, "<%s>", obj.Class()) + fmt.Fprintf(ctx.w, "<%s>", obj.ClassName()) } } } -func (ctx ppctx) fields(obj *otto.Object) []string { +func (ctx ppctx) fields(obj *goja.Object) []string { var ( vals, methods []string seen = make(map[string]bool) @@ -195,11 +204,22 @@ func (ctx ppctx) fields(obj *otto.Object) []string { return } seen[k] = true - if v, _ := obj.Get(k); v.IsFunction() { - methods = append(methods, k) - } else { + + key := SafeGet(obj, k) + if key == nil { + // The value corresponding to that key could not be found + // (typically because it is backed by an RPC call that is + // not supported by this instance. Add it to the list of + // values so that it appears as `undefined` to the user. vals = append(vals, k) + } else { + if _, callable := goja.AssertFunction(key); callable { + methods = append(methods, k) + } else { + vals = append(vals, k) + } } + } iterOwnAndConstructorKeys(ctx.vm, obj, add) sort.Strings(vals) @@ -207,13 +227,13 @@ func (ctx ppctx) fields(obj *otto.Object) []string { return append(vals, methods...) } -func iterOwnAndConstructorKeys(vm *otto.Otto, obj *otto.Object, f func(string)) { +func iterOwnAndConstructorKeys(vm *goja.Runtime, obj *goja.Object, f func(string)) { seen := make(map[string]bool) iterOwnKeys(vm, obj, func(prop string) { seen[prop] = true f(prop) }) - if cp := constructorPrototype(obj); cp != nil { + if cp := constructorPrototype(vm, obj); cp != nil { iterOwnKeys(vm, cp, func(prop string) { if !seen[prop] { f(prop) @@ -222,10 +242,17 @@ func iterOwnAndConstructorKeys(vm *otto.Otto, obj *otto.Object, f func(string)) } } -func iterOwnKeys(vm *otto.Otto, obj *otto.Object, f func(string)) { - Object, _ := vm.Object("Object") - rv, _ := Object.Call("getOwnPropertyNames", obj.Value()) - gv, _ := rv.Export() +func iterOwnKeys(vm *goja.Runtime, obj *goja.Object, f func(string)) { + Object := vm.Get("Object").ToObject(vm) + getOwnPropertyNames, isFunc := goja.AssertFunction(Object.Get("getOwnPropertyNames")) + if !isFunc { + panic(vm.ToValue("Object.getOwnPropertyNames isn't a function")) + } + rv, err := getOwnPropertyNames(goja.Null(), obj) + if err != nil { + panic(vm.ToValue(fmt.Sprintf("Error getting object properties: %v", err))) + } + gv := rv.Export() switch gv := gv.(type) { case []interface{}: for _, v := range gv { @@ -240,32 +267,35 @@ func iterOwnKeys(vm *otto.Otto, obj *otto.Object, f func(string)) { } } -func (ctx ppctx) isBigNumber(v *otto.Object) bool { +func (ctx ppctx) isBigNumber(v *goja.Object) bool { // Handle numbers with custom constructor. - if v, _ := v.Get("constructor"); v.Object() != nil { - if strings.HasPrefix(toString(v.Object()), "function BigNumber") { + if obj := v.Get("constructor").ToObject(ctx.vm); obj != nil { + if strings.HasPrefix(toString(obj), "function BigNumber") { return true } } // Handle default constructor. - BigNumber, _ := ctx.vm.Object("BigNumber.prototype") + BigNumber := ctx.vm.Get("BigNumber").ToObject(ctx.vm) if BigNumber == nil { return false } - bv, _ := BigNumber.Call("isPrototypeOf", v) - b, _ := bv.ToBoolean() - return b + prototype := BigNumber.Get("prototype").ToObject(ctx.vm) + isPrototypeOf, callable := goja.AssertFunction(prototype.Get("isPrototypeOf")) + if !callable { + return false + } + bv, _ := isPrototypeOf(prototype, v) + return bv.ToBoolean() } -func toString(obj *otto.Object) string { - s, _ := obj.Call("toString") - return s.String() +func toString(obj *goja.Object) string { + return obj.ToString().String() } -func constructorPrototype(obj *otto.Object) *otto.Object { - if v, _ := obj.Get("constructor"); v.Object() != nil { - if v, _ = v.Object().Get("prototype"); v.Object() != nil { - return v.Object() +func constructorPrototype(vm *goja.Runtime, obj *goja.Object) *goja.Object { + if v := obj.Get("constructor"); v != nil { + if v := v.ToObject(vm).Get("prototype"); v != nil { + return v.ToObject(vm) } } return nil From 44c365c3e2a3ab0f9f6f9c8ae1fa22e51cd3424f Mon Sep 17 00:00:00 2001 From: Zhou Zhiyao Date: Mon, 27 Jan 2020 21:03:15 +0800 Subject: [PATCH 080/128] rpc: reset writeConn when conn is closed on readErr (#20414) This change makes the client attempt to reconnect when a write fails. We already had reconnect support, but the reconnect would previously happen on the next call after an error. Being more eager leads to a smoother experience overall. --- rpc/client.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/rpc/client.go b/rpc/client.go index a04198ad87..dd06b9469e 100644 --- a/rpc/client.go +++ b/rpc/client.go @@ -465,7 +465,7 @@ func (c *Client) newMessage(method string, paramsIn ...interface{}) (*jsonrpcMes func (c *Client) send(ctx context.Context, op *requestOp, msg interface{}) error { select { case c.reqInit <- op: - err := c.write(ctx, msg) + err := c.write(ctx, msg, false) c.reqSent <- err return err case <-ctx.Done(): @@ -477,7 +477,7 @@ func (c *Client) send(ctx context.Context, op *requestOp, msg interface{}) error } } -func (c *Client) write(ctx context.Context, msg interface{}) error { +func (c *Client) write(ctx context.Context, msg interface{}, retry bool) error { // The previous write failed. Try to establish a new connection. if c.writeConn == nil { if err := c.reconnect(ctx); err != nil { @@ -487,6 +487,9 @@ func (c *Client) write(ctx context.Context, msg interface{}) error { err := c.writeConn.writeJSON(ctx, msg) if err != nil { c.writeConn = nil + if !retry { + return c.write(ctx, msg, true) + } } return err } From a903912b9622780a7b6d852a21307a8a81962fd7 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 28 Jan 2020 10:37:08 +0100 Subject: [PATCH 081/128] rpc: check module availability at startup (#20597) Fixes #20467 Co-authored-by: meowsbits <45600330+meowsbits@users.noreply.github.com> --- cmd/geth/config.go | 4 ++-- rpc/endpoints.go | 34 +++++++++++++++++++++++++++++++--- 2 files changed, 33 insertions(+), 5 deletions(-) diff --git a/cmd/geth/config.go b/cmd/geth/config.go index b450dd9c8b..71e8355348 100644 --- a/cmd/geth/config.go +++ b/cmd/geth/config.go @@ -99,8 +99,8 @@ func defaultNodeConfig() node.Config { cfg := node.DefaultConfig cfg.Name = clientIdentifier cfg.Version = params.VersionWithCommit(gitCommit, gitDate) - cfg.HTTPModules = append(cfg.HTTPModules, "eth", "shh") - cfg.WSModules = append(cfg.WSModules, "eth", "shh") + cfg.HTTPModules = append(cfg.HTTPModules, "eth") + cfg.WSModules = append(cfg.WSModules, "eth") cfg.IPCPath = "geth.ipc" return cfg } diff --git a/rpc/endpoints.go b/rpc/endpoints.go index 8ca6d4eb0c..2cf51aedfc 100644 --- a/rpc/endpoints.go +++ b/rpc/endpoints.go @@ -17,13 +17,39 @@ package rpc import ( + "fmt" "net" "github.com/ethereum/go-ethereum/log" ) -// StartHTTPEndpoint starts the HTTP RPC endpoint, configured with cors/vhosts/modules +// checkModuleAvailability check that all names given in modules are actually +// available API services. +func checkModuleAvailability(modules []string, apis []API) error { + available := make(map[string]struct{}) + var availableNames string + for i, api := range apis { + if _, ok := available[api.Namespace]; !ok { + available[api.Namespace] = struct{}{} + if i > 0 { + availableNames += ", " + } + availableNames += api.Namespace + } + } + for _, name := range modules { + if _, ok := available[name]; !ok { + return fmt.Errorf("invalid API %q in whitelist (available: %s)", name, availableNames) + } + } + return nil +} + +// StartHTTPEndpoint starts the HTTP RPC endpoint, configured with cors/vhosts/modules. func StartHTTPEndpoint(endpoint string, apis []API, modules []string, cors []string, vhosts []string, timeouts HTTPTimeouts) (net.Listener, *Server, error) { + if err := checkModuleAvailability(modules, apis); err != nil { + return nil, nil, err + } // Generate the whitelist based on the allowed modules whitelist := make(map[string]bool) for _, module := range modules { @@ -51,9 +77,11 @@ func StartHTTPEndpoint(endpoint string, apis []API, modules []string, cors []str return listener, handler, err } -// StartWSEndpoint starts a websocket endpoint +// StartWSEndpoint starts a websocket endpoint. func StartWSEndpoint(endpoint string, apis []API, modules []string, wsOrigins []string, exposeAll bool) (net.Listener, *Server, error) { - + if err := checkModuleAvailability(modules, apis); err != nil { + return nil, nil, err + } // Generate the whitelist based on the allowed modules whitelist := make(map[string]bool) for _, module := range modules { From 594e038e75a1acf3d922d48afdcc2b7bdae40368 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Wed, 29 Jan 2020 13:47:56 +0100 Subject: [PATCH 082/128] signer/rules: use goja and remove otto (#20599) * signer: replace otto with goja * go.mod: remove Otto --- go.mod | 1 - go.sum | 2 -- signer/rules/rules.go | 52 ++++++++++++++++++-------------------- signer/rules/rules_test.go | 2 +- 4 files changed, 26 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index c3b5b6fd02..b496b871ba 100644 --- a/go.mod +++ b/go.mod @@ -49,7 +49,6 @@ require ( github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7 github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150 github.com/rjeczalik/notify v0.9.1 - github.com/robertkrimen/otto v0.0.0-20170205013659-6a77b7cbc37d github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00 github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521 // indirect github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 diff --git a/go.sum b/go.sum index 81c718b024..f3ddb7ba09 100644 --- a/go.sum +++ b/go.sum @@ -167,8 +167,6 @@ github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150 h1:ZeU+auZj1iNzN github.com/prometheus/tsdb v0.6.2-0.20190402121629-4f204dcbc150/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= -github.com/robertkrimen/otto v0.0.0-20170205013659-6a77b7cbc37d h1:ouzpe+YhpIfnjR40gSkJHWsvXmB6TiPKqMtMpfyU9DE= -github.com/robertkrimen/otto v0.0.0-20170205013659-6a77b7cbc37d/go.mod h1:xvqspoSXJTIpemEonrMDFq6XzwHYYgToXWj5eRX1OtY= github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00 h1:8DPul/X0IT/1TNMIxoKLwdemEOBBHDC/K4EB16Cw5WE= github.com/rs/cors v0.0.0-20160617231935-a62a804a8a00/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/xhandler v0.0.0-20160618193221-ed27b6fd6521 h1:3hxavr+IHMsQBrYUPQM5v0CgENFktkkbg1sfpgM3h20= diff --git a/signer/rules/rules.go b/signer/rules/rules.go index cb375a62ad..03e5136730 100644 --- a/signer/rules/rules.go +++ b/signer/rules/rules.go @@ -22,12 +22,12 @@ import ( "os" "strings" + "github.com/dop251/goja" "github.com/ethereum/go-ethereum/internal/ethapi" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/signer/core" "github.com/ethereum/go-ethereum/signer/rules/deps" "github.com/ethereum/go-ethereum/signer/storage" - "github.com/robertkrimen/otto" ) var ( @@ -36,13 +36,13 @@ var ( // consoleOutput is an override for the console.log and console.error methods to // stream the output into the configured output stream instead of stdout. -func consoleOutput(call otto.FunctionCall) otto.Value { +func consoleOutput(call goja.FunctionCall) goja.Value { output := []string{"JS:> "} - for _, argument := range call.ArgumentList { + for _, argument := range call.Arguments { output = append(output, fmt.Sprintf("%v", argument)) } fmt.Fprintln(os.Stderr, strings.Join(output, " ")) - return otto.Value{} + return goja.Undefined() } // rulesetUI provides an implementation of UIClientAPI that evaluates a javascript @@ -70,45 +70,47 @@ func (r *rulesetUI) Init(javascriptRules string) error { r.jsRules = javascriptRules return nil } -func (r *rulesetUI) execute(jsfunc string, jsarg interface{}) (otto.Value, error) { +func (r *rulesetUI) execute(jsfunc string, jsarg interface{}) (goja.Value, error) { // Instantiate a fresh vm engine every time - vm := otto.New() + vm := goja.New() // Set the native callbacks - consoleObj, _ := vm.Get("console") - consoleObj.Object().Set("log", consoleOutput) - consoleObj.Object().Set("error", consoleOutput) + consoleObj := vm.NewObject() + consoleObj.Set("log", consoleOutput) + consoleObj.Set("error", consoleOutput) + vm.Set("console", consoleObj) - vm.Set("storage", struct{}{}) - storageObj, _ := vm.Get("storage") - storageObj.Object().Set("put", func(call otto.FunctionCall) otto.Value { + storageObj := vm.NewObject() + storageObj.Set("put", func(call goja.FunctionCall) goja.Value { key, val := call.Argument(0).String(), call.Argument(1).String() if val == "" { r.storage.Del(key) } else { r.storage.Put(key, val) } - return otto.NullValue() + return goja.Null() }) - storageObj.Object().Set("get", func(call otto.FunctionCall) otto.Value { + storageObj.Set("get", func(call goja.FunctionCall) goja.Value { goval, _ := r.storage.Get(call.Argument(0).String()) - jsval, _ := otto.ToValue(goval) + jsval := vm.ToValue(goval) return jsval }) + vm.Set("storage", storageObj) + // Load bootstrap libraries - script, err := vm.Compile("bignumber.js", BigNumber_JS) + script, err := goja.Compile("bignumber.js", string(BigNumber_JS), true) if err != nil { log.Warn("Failed loading libraries", "err", err) - return otto.UndefinedValue(), err + return goja.Undefined(), err } - vm.Run(script) + vm.RunProgram(script) // Run the actual rule implementation - _, err = vm.Run(r.jsRules) + _, err = vm.RunString(r.jsRules) if err != nil { log.Warn("Execution failed", "err", err) - return otto.UndefinedValue(), err + return goja.Undefined(), err } // And the actual call @@ -119,7 +121,7 @@ func (r *rulesetUI) execute(jsfunc string, jsarg interface{}) (otto.Value, error jsonbytes, err := json.Marshal(jsarg) if err != nil { log.Warn("failed marshalling data", "data", jsarg) - return otto.UndefinedValue(), err + return goja.Undefined(), err } // Now, we call foobar(JSON.parse()). var call string @@ -128,7 +130,7 @@ func (r *rulesetUI) execute(jsfunc string, jsarg interface{}) (otto.Value, error } else { call = fmt.Sprintf("%v()", jsfunc) } - return vm.Run(call) + return vm.RunString(call) } func (r *rulesetUI) checkApproval(jsfunc string, jsarg []byte, err error) (bool, error) { @@ -140,11 +142,7 @@ func (r *rulesetUI) checkApproval(jsfunc string, jsarg []byte, err error) (bool, log.Info("error occurred during execution", "error", err) return false, err } - result, err := v.ToString() - if err != nil { - log.Info("error occurred during response unmarshalling", "error", err) - return false, err - } + result := v.ToString().String() if result == "Approve" { log.Info("Op approved") return true, nil diff --git a/signer/rules/rules_test.go b/signer/rules/rules_test.go index c030ed47ce..510c57e67f 100644 --- a/signer/rules/rules_test.go +++ b/signer/rules/rules_test.go @@ -337,7 +337,7 @@ func TestStorage(t *testing.T) { if err != nil { t.Errorf("Unexpected error %v", err) } - retval, err := v.ToString() + retval := v.ToString().String() if err != nil { t.Errorf("Unexpected error %v", err) From 24cab2d5357ed968aa5a4752f7afc27c8320b064 Mon Sep 17 00:00:00 2001 From: Guillaume Ballet Date: Thu, 30 Jan 2020 11:21:10 +0100 Subject: [PATCH 083/128] core/vm/runtime: fix typos in comment (#20608) --- core/vm/runtime/runtime.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/core/vm/runtime/runtime.go b/core/vm/runtime/runtime.go index db1f6f3822..dd5dba66f0 100644 --- a/core/vm/runtime/runtime.go +++ b/core/vm/runtime/runtime.go @@ -90,8 +90,8 @@ func setDefaults(cfg *Config) { // Execute executes the code using the input as call data during the execution. // It returns the EVM's return value, the new state and an error if it failed. // -// Executes sets up a in memory, temporarily, environment for the execution of -// the given code. It makes sure that it's restored to it's original state afterwards. +// Execute sets up an in-memory, temporary, environment for the execution of +// the given code. It makes sure that it's restored to its original state afterwards. func Execute(code, input []byte, cfg *Config) ([]byte, *state.StateDB, error) { if cfg == nil { cfg = new(Config) From 3c776c71994d7efde8793172cc8c09573a006dcc Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 31 Jan 2020 12:00:37 +0100 Subject: [PATCH 084/128] retesteth: clean txpool on rewind, default dao support (#20596) --- cmd/geth/retesteth.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/geth/retesteth.go b/cmd/geth/retesteth.go index f4f7bda00b..28e792bd77 100644 --- a/cmd/geth/retesteth.go +++ b/cmd/geth/retesteth.go @@ -355,7 +355,7 @@ func (api *RetestethAPI) SetChainParams(ctx context.Context, chainParams ChainPa ChainID: chainId, HomesteadBlock: homesteadBlock, DAOForkBlock: daoForkBlock, - DAOForkSupport: false, + DAOForkSupport: true, EIP150Block: eip150Block, EIP155Block: eip155Block, EIP158Block: eip158Block, @@ -579,6 +579,9 @@ func (api *RetestethAPI) RewindToBlock(ctx context.Context, newHead uint64) (boo if err := api.blockchain.SetHead(newHead); err != nil { return false, err } + // When we rewind, the transaction pool should be cleaned out. + api.txMap = make(map[common.Address]map[uint64]*types.Transaction) + api.txSenders = make(map[common.Address]struct{}) return true, nil } From 15d09038a6b1d533bc78662f2f0b77ccf03aaab0 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 31 Jan 2020 14:12:19 +0100 Subject: [PATCH 085/128] params: update bootnodes (#20610) --- params/bootnodes.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/params/bootnodes.go b/params/bootnodes.go index 967cba5bc4..f92b487ff5 100644 --- a/params/bootnodes.go +++ b/params/bootnodes.go @@ -28,9 +28,6 @@ var MainnetBootnodes = []string{ "enode://103858bdb88756c71f15e9b5e09b56dc1be52f0a5021d46301dbbfb7e130029cc9d0d6f73f693bc29b665770fff7da4d34f3c6379fe12721b5d7a0bcb5ca1fc1@191.234.162.198:30303", // bootnode-azure-brazilsouth-001 "enode://715171f50508aba88aecd1250af392a45a330af91d7b90701c436b618c86aaa1589c9184561907bebbb56439b8f8787bc01f49a7c77276c58c1b09822d75e8e8@52.231.165.108:30303", // bootnode-azure-koreasouth-001 "enode://5d6d7cd20d6da4bb83a1d28cadb5d409b64edf314c0335df658c1a54e32c7c4a7ab7823d57c39b6a757556e68ff1df17c748b698544a55cb488b52479a92b60f@104.42.217.25:30303", // bootnode-azure-westus-001 - - // Ethereum Foundation C++ Bootnodes - "enode://979b7fa28feeb35a4741660a16076f1943202cb72b6af70d327f053e248bab9ba81760f39d0701ef1d8f89cc1fbd2cacba0710a12cd5314d5e0c9021aa3637f9@5.1.83.226:30303", // DE } // TestnetBootnodes are the enode URLs of the P2P bootstrap nodes running on the @@ -61,7 +58,7 @@ var GoerliBootnodes = []string{ "enode://f4a9c6ee28586009fb5a96c8af13a58ed6d8315a9eee4772212c1d4d9cebe5a8b8a78ea4434f318726317d04a3f531a1ef0420cf9752605a562cfe858c46e263@213.186.16.82:30303", // Ethereum Foundation bootnode - "enode://573b6607cd59f241e30e4c4943fd50e99e2b6f42f9bd5ca111659d309c06741247f4f1e93843ad3e8c8c18b6e2d94c161b7ef67479b3938780a97134b618b5ce@52.56.136.200:30303", + "enode://a61215641fb8714a373c80edbfa0ea8878243193f57c96eeb44d0bc019ef295abd4e044fd619bfc4c59731a73fb79afe84e9ab6da0c743ceb479cbb6d263fa91@3.11.147.67:30303", } // DiscoveryV5Bootnodes are the enode URLs of the P2P bootstrap nodes for the From 4cc89a5a32bd3623a065765637d5c08641e2a251 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 3 Feb 2020 16:22:46 +0100 Subject: [PATCH 086/128] internal/build: don't crash in DownloadFile when offline (#20595) --- internal/build/download.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/internal/build/download.go b/internal/build/download.go index c506800295..0ed0b5e130 100644 --- a/internal/build/download.go +++ b/internal/build/download.go @@ -83,8 +83,10 @@ func (db *ChecksumDB) DownloadFile(url, dstPath string) error { fmt.Printf("downloading from %s\n", url) resp, err := http.Get(url) - if err != nil || resp.StatusCode != http.StatusOK { - return fmt.Errorf("download error: code %d, err %v", resp.StatusCode, err) + if err != nil { + return fmt.Errorf("download error: %v", err) + } else if resp.StatusCode != http.StatusOK { + return fmt.Errorf("download error: status %d", resp.StatusCode) } defer resp.Body.Close() if err := os.MkdirAll(filepath.Dir(dstPath), 0755); err != nil { From 5a9c96454e04673124eedf681ae3aa0a58ffc843 Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Mon, 3 Feb 2020 16:28:30 +0100 Subject: [PATCH 087/128] trie: separate hashes and committer, collapse on commit * trie: make db insert use size instead of full data * core/state: minor optimization in state onleaf allocation * trie: implement dedicated committer and hasher * trie: use dedicated committer/hasher * trie: linter nitpicks * core/state, trie: avoid unnecessary storage trie load+commit * trie: review feedback, mainly docs + minor changes * trie: start deprecating old hasher * trie: fix misspell+lint * trie: deprecate hasher.go, make proof framework use new hasher * trie: rename pure_committer/hasher to committer/hasher * trie, core/state: fix review concerns * trie: more review concerns * trie: make commit collapse into hashnode, don't touch dirtyness * trie: goimports fixes * trie: remove panics --- core/state/state_object.go | 16 ++- core/state/statedb.go | 7 +- trie/committer.go | 279 +++++++++++++++++++++++++++++++++++++ trie/database.go | 10 +- trie/hasher.go | 225 ++++++++++++++---------------- trie/iterator.go | 6 +- trie/proof.go | 24 ++-- trie/secure_trie.go | 2 +- trie/trie.go | 44 +++++- 9 files changed, 457 insertions(+), 156 deletions(-) create mode 100644 trie/committer.go diff --git a/core/state/state_object.go b/core/state/state_object.go index 8680de021f..667d5ec02e 100644 --- a/core/state/state_object.go +++ b/core/state/state_object.go @@ -272,10 +272,13 @@ func (s *stateObject) finalise() { } // updateTrie writes cached storage modifications into the object's storage trie. +// It will return nil if the trie has not been loaded and no changes have been made func (s *stateObject) updateTrie(db Database) Trie { // Make sure all dirty slots are finalized into the pending storage area s.finalise() - + if len(s.pendingStorage) == 0 { + return s.trie + } // Track the amount of time wasted on updating the storge trie if metrics.EnabledExpensive { defer func(start time.Time) { s.db.StorageUpdates += time.Since(start) }(time.Now()) @@ -305,8 +308,10 @@ func (s *stateObject) updateTrie(db Database) Trie { // UpdateRoot sets the trie root to the current root hash of func (s *stateObject) updateRoot(db Database) { - s.updateTrie(db) - + // If nothing changed, don't bother with hashing anything + if s.updateTrie(db) == nil { + return + } // Track the amount of time wasted on hashing the storge trie if metrics.EnabledExpensive { defer func(start time.Time) { s.db.StorageHashes += time.Since(start) }(time.Now()) @@ -317,7 +322,10 @@ func (s *stateObject) updateRoot(db Database) { // CommitTrie the storage trie of the object to db. // This updates the trie root. func (s *stateObject) CommitTrie(db Database) error { - s.updateTrie(db) + // If nothing changed, don't bother with hashing anything + if s.updateTrie(db) == nil { + return nil + } if s.dbErr != nil { return s.dbErr } diff --git a/core/state/statedb.go b/core/state/statedb.go index 085f2379fb..5d40f59c65 100644 --- a/core/state/statedb.go +++ b/core/state/statedb.go @@ -330,7 +330,8 @@ func (s *StateDB) StorageTrie(addr common.Address) Trie { return nil } cpy := stateObject.deepCopy(s) - return cpy.updateTrie(s.db) + cpy.updateTrie(s.db) + return cpy.getTrie(s.db) } func (s *StateDB) HasSuicided(addr common.Address) bool { @@ -750,8 +751,10 @@ func (s *StateDB) Commit(deleteEmptyObjects bool) (common.Hash, error) { if metrics.EnabledExpensive { defer func(start time.Time) { s.AccountCommits += time.Since(start) }(time.Now()) } + // The onleaf func is called _serially_, so we can reuse the same account + // for unmarshalling every time. + var account Account return s.trie.Commit(func(leaf []byte, parent common.Hash) error { - var account Account if err := rlp.DecodeBytes(leaf, &account); err != nil { return nil } diff --git a/trie/committer.go b/trie/committer.go new file mode 100644 index 0000000000..eacefdff11 --- /dev/null +++ b/trie/committer.go @@ -0,0 +1,279 @@ +// Copyright 2019 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package trie + +import ( + "errors" + "fmt" + "sync" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/rlp" + "golang.org/x/crypto/sha3" +) + +// leafChanSize is the size of the leafCh. It's a pretty arbitrary number, to allow +// some paralellism but not incur too much memory overhead. +const leafChanSize = 200 + +// leaf represents a trie leaf value +type leaf struct { + size int // size of the rlp data (estimate) + hash common.Hash // hash of rlp data + node node // the node to commit + vnodes bool // set to true if the node (possibly) contains a valueNode +} + +// committer is a type used for the trie Commit operation. A committer has some +// internal preallocated temp space, and also a callback that is invoked when +// leaves are committed. The leafs are passed through the `leafCh`, to allow +// some level of paralellism. +// By 'some level' of parallelism, it's still the case that all leaves will be +// processed sequentially - onleaf will never be called in parallel or out of order. +type committer struct { + tmp sliceBuffer + sha keccakState + + onleaf LeafCallback + leafCh chan *leaf +} + +// committers live in a global sync.Pool +var committerPool = sync.Pool{ + New: func() interface{} { + return &committer{ + tmp: make(sliceBuffer, 0, 550), // cap is as large as a full fullNode. + sha: sha3.NewLegacyKeccak256().(keccakState), + } + }, +} + +// newCommitter creates a new committer or picks one from the pool. +func newCommitter() *committer { + return committerPool.Get().(*committer) +} + +func returnCommitterToPool(h *committer) { + h.onleaf = nil + h.leafCh = nil + committerPool.Put(h) +} + +// commitNeeded returns 'false' if the given node is already in sync with db +func (c *committer) commitNeeded(n node) bool { + hash, dirty := n.cache() + return hash == nil || dirty +} + +// commit collapses a node down into a hash node and inserts it into the database +func (c *committer) Commit(n node, db *Database) (hashNode, error) { + if db == nil { + return nil, errors.New("no db provided") + } + h, err := c.commit(n, db, true) + if err != nil { + return nil, err + } + return h.(hashNode), nil +} + +// commit collapses a node down into a hash node and inserts it into the database +func (c *committer) commit(n node, db *Database, force bool) (node, error) { + // if this path is clean, use available cached data + hash, dirty := n.cache() + if hash != nil && !dirty { + return hash, nil + } + // Commit children, then parent, and remove remove the dirty flag. + switch cn := n.(type) { + case *shortNode: + // Commit child + collapsed := cn.copy() + if _, ok := cn.Val.(valueNode); !ok { + if childV, err := c.commit(cn.Val, db, false); err != nil { + return nil, err + } else { + collapsed.Val = childV + } + } + // The key needs to be copied, since we're delivering it to database + collapsed.Key = hexToCompact(cn.Key) + hashedNode := c.store(collapsed, db, force, true) + if hn, ok := hashedNode.(hashNode); ok { + return hn, nil + } else { + return collapsed, nil + } + case *fullNode: + hashedKids, hasVnodes, err := c.commitChildren(cn, db, force) + if err != nil { + return nil, err + } + collapsed := cn.copy() + collapsed.Children = hashedKids + + hashedNode := c.store(collapsed, db, force, hasVnodes) + if hn, ok := hashedNode.(hashNode); ok { + return hn, nil + } else { + return collapsed, nil + } + case valueNode: + return c.store(cn, db, force, false), nil + // hashnodes aren't stored + case hashNode: + return cn, nil + } + return hash, nil +} + +// commitChildren commits the children of the given fullnode +func (c *committer) commitChildren(n *fullNode, db *Database, force bool) ([17]node, bool, error) { + var children [17]node + var hasValueNodeChildren = false + for i, child := range n.Children { + if child == nil { + continue + } + hnode, err := c.commit(child, db, false) + if err != nil { + return children, false, err + } + children[i] = hnode + if _, ok := hnode.(valueNode); ok { + hasValueNodeChildren = true + } + } + return children, hasValueNodeChildren, nil +} + +// store hashes the node n and if we have a storage layer specified, it writes +// the key/value pair to it and tracks any node->child references as well as any +// node->external trie references. +func (c *committer) store(n node, db *Database, force bool, hasVnodeChildren bool) node { + // Larger nodes are replaced by their hash and stored in the database. + var ( + hash, _ = n.cache() + size int + ) + if hash == nil { + if vn, ok := n.(valueNode); ok { + c.tmp.Reset() + if err := rlp.Encode(&c.tmp, vn); err != nil { + panic("encode error: " + err.Error()) + } + size = len(c.tmp) + if size < 32 && !force { + return n // Nodes smaller than 32 bytes are stored inside their parent + } + hash = c.makeHashNode(c.tmp) + } else { + // This was not generated - must be a small node stored in the parent + // No need to do anything here + return n + } + } else { + // We have the hash already, estimate the RLP encoding-size of the node. + // The size is used for mem tracking, does not need to be exact + size = estimateSize(n) + } + // If we're using channel-based leaf-reporting, send to channel. + // The leaf channel will be active only when there an active leaf-callback + if c.leafCh != nil { + c.leafCh <- &leaf{ + size: size, + hash: common.BytesToHash(hash), + node: n, + vnodes: hasVnodeChildren, + } + } else if db != nil { + // No leaf-callback used, but there's still a database. Do serial + // insertion + db.lock.Lock() + db.insert(common.BytesToHash(hash), size, n) + db.lock.Unlock() + } + return hash +} + +// commitLoop does the actual insert + leaf callback for nodes +func (c *committer) commitLoop(db *Database) { + for item := range c.leafCh { + var ( + hash = item.hash + size = item.size + n = item.node + hasVnodes = item.vnodes + ) + // We are pooling the trie nodes into an intermediate memory cache + db.lock.Lock() + db.insert(hash, size, n) + db.lock.Unlock() + if c.onleaf != nil && hasVnodes { + switch n := n.(type) { + case *shortNode: + if child, ok := n.Val.(valueNode); ok { + c.onleaf(child, hash) + } + case *fullNode: + for i := 0; i < 16; i++ { + if child, ok := n.Children[i].(valueNode); ok { + c.onleaf(child, hash) + } + } + } + } + } +} + +func (c *committer) makeHashNode(data []byte) hashNode { + n := make(hashNode, c.sha.Size()) + c.sha.Reset() + c.sha.Write(data) + c.sha.Read(n) + return n +} + +// estimateSize estimates the size of an rlp-encoded node, without actually +// rlp-encoding it (zero allocs). This method has been experimentally tried, and with a trie +// with 1000 leafs, the only errors above 1% are on small shortnodes, where this +// method overestimates by 2 or 3 bytes (e.g. 37 instead of 35) +func estimateSize(n node) int { + switch n := n.(type) { + case *shortNode: + // A short node contains a compacted key, and a value. + return 3 + len(n.Key) + estimateSize(n.Val) + case *fullNode: + // A full node contains up to 16 hashes (some nils), and a key + s := 3 + for i := 0; i < 16; i++ { + if child := n.Children[i]; child != nil { + s += estimateSize(child) + } else { + s += 1 + } + } + return s + case valueNode: + return 1 + len(n) + case hashNode: + return 1 + len(n) + default: + panic(fmt.Sprintf("node type %T", n)) + + } +} diff --git a/trie/database.go b/trie/database.go index dee9f7844b..5b673938f0 100644 --- a/trie/database.go +++ b/trie/database.go @@ -310,24 +310,24 @@ func (db *Database) InsertBlob(hash common.Hash, blob []byte) { db.lock.Lock() defer db.lock.Unlock() - db.insert(hash, blob, rawNode(blob)) + db.insert(hash, len(blob), rawNode(blob)) } // insert inserts a collapsed trie node into the memory database. This method is // a more generic version of InsertBlob, supporting both raw blob insertions as -// well ex trie node insertions. The blob must always be specified to allow proper +// well ex trie node insertions. The blob size must be specified to allow proper // size tracking. -func (db *Database) insert(hash common.Hash, blob []byte, node node) { +func (db *Database) insert(hash common.Hash, size int, node node) { // If the node's already cached, skip if _, ok := db.dirties[hash]; ok { return } - memcacheDirtyWriteMeter.Mark(int64(len(blob))) + memcacheDirtyWriteMeter.Mark(int64(size)) // Create the cached entry for this node entry := &cachedNode{ node: simplifyNode(node), - size: uint16(len(blob)), + size: uint16(size), flushPrev: db.newest, } entry.forChilds(func(child common.Hash) { diff --git a/trie/hasher.go b/trie/hasher.go index 54f6a9de2b..71a3aec3b2 100644 --- a/trie/hasher.go +++ b/trie/hasher.go @@ -1,4 +1,4 @@ -// Copyright 2016 The go-ethereum Authors +// Copyright 2019 The go-ethereum Authors // This file is part of the go-ethereum library. // // The go-ethereum library is free software: you can redistribute it and/or modify @@ -20,17 +20,10 @@ import ( "hash" "sync" - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/rlp" "golang.org/x/crypto/sha3" ) -type hasher struct { - tmp sliceBuffer - sha keccakState - onleaf LeafCallback -} - // keccakState wraps sha3.state. In addition to the usual hash methods, it also supports // Read to get a variable amount of data from the hash state. Read is faster than Sum // because it doesn't copy the internal state, but also modifies the internal state. @@ -50,7 +43,14 @@ func (b *sliceBuffer) Reset() { *b = (*b)[:0] } -// hashers live in a global db. +// hasher is a type used for the trie Hash operation. A hasher has some +// internal preallocated temp space +type hasher struct { + sha keccakState + tmp sliceBuffer +} + +// hasherPool holds pureHashers var hasherPool = sync.Pool{ New: func() interface{} { return &hasher{ @@ -60,9 +60,8 @@ var hasherPool = sync.Pool{ }, } -func newHasher(onleaf LeafCallback) *hasher { +func newHasher() *hasher { h := hasherPool.Get().(*hasher) - h.onleaf = onleaf return h } @@ -72,144 +71,126 @@ func returnHasherToPool(h *hasher) { // hash collapses a node down into a hash node, also returning a copy of the // original node initialized with the computed hash to replace the original one. -func (h *hasher) hash(n node, db *Database, force bool) (node, node, error) { - // If we're not storing the node, just hashing, use available cached data - if hash, dirty := n.cache(); hash != nil { - if db == nil { - return hash, n, nil - } - if !dirty { - switch n.(type) { - case *fullNode, *shortNode: - return hash, hash, nil - default: - return hash, n, nil - } - } +func (h *hasher) hash(n node, force bool) (hashed node, cached node) { + // We're not storing the node, just hashing, use available cached data + if hash, _ := n.cache(); hash != nil { + return hash, n } // Trie not processed yet or needs storage, walk the children - collapsed, cached, err := h.hashChildren(n, db) - if err != nil { - return hashNode{}, n, err - } - hashed, err := h.store(collapsed, db, force) - if err != nil { - return hashNode{}, n, err - } - // Cache the hash of the node for later reuse and remove - // the dirty flag in commit mode. It's fine to assign these values directly - // without copying the node first because hashChildren copies it. - cachedHash, _ := hashed.(hashNode) - switch cn := cached.(type) { + switch n := n.(type) { case *shortNode: - cn.flags.hash = cachedHash - if db != nil { - cn.flags.dirty = false + collapsed, cached := h.hashShortNodeChildren(n) + hashed := h.shortnodeToHash(collapsed, force) + // We need to retain the possibly _not_ hashed node, in case it was too + // small to be hashed + if hn, ok := hashed.(hashNode); ok { + cached.flags.hash = hn + } else { + cached.flags.hash = nil } + return hashed, cached case *fullNode: - cn.flags.hash = cachedHash - if db != nil { - cn.flags.dirty = false + collapsed, cached := h.hashFullNodeChildren(n) + hashed = h.fullnodeToHash(collapsed, force) + if hn, ok := hashed.(hashNode); ok { + cached.flags.hash = hn + } else { + cached.flags.hash = nil } + return hashed, cached + default: + // Value and hash nodes don't have children so they're left as were + return n, n } - return hashed, cached, nil } -// hashChildren replaces the children of a node with their hashes if the encoded -// size of the child is larger than a hash, returning the collapsed node as well -// as a replacement for the original node with the child hashes cached in. -func (h *hasher) hashChildren(original node, db *Database) (node, node, error) { - var err error - - switch n := original.(type) { - case *shortNode: - // Hash the short node's child, caching the newly hashed subtree - collapsed, cached := n.copy(), n.copy() - collapsed.Key = hexToCompact(n.Key) - cached.Key = common.CopyBytes(n.Key) - - if _, ok := n.Val.(valueNode); !ok { - collapsed.Val, cached.Val, err = h.hash(n.Val, db, false) - if err != nil { - return original, original, err - } - } - return collapsed, cached, nil +// hashShortNodeChildren collapses the short node. The returned collapsed node +// holds a live reference to the Key, and must not be modified. +// The cached +func (h *hasher) hashShortNodeChildren(n *shortNode) (collapsed, cached *shortNode) { + // Hash the short node's child, caching the newly hashed subtree + collapsed, cached = n.copy(), n.copy() + // Previously, we did copy this one. We don't seem to need to actually + // do that, since we don't overwrite/reuse keys + //cached.Key = common.CopyBytes(n.Key) + collapsed.Key = hexToCompact(n.Key) + // Unless the child is a valuenode or hashnode, hash it + switch n.Val.(type) { + case *fullNode, *shortNode: + collapsed.Val, cached.Val = h.hash(n.Val, false) + } + return collapsed, cached +} - case *fullNode: - // Hash the full node's children, caching the newly hashed subtrees - collapsed, cached := n.copy(), n.copy() - - for i := 0; i < 16; i++ { - if n.Children[i] != nil { - collapsed.Children[i], cached.Children[i], err = h.hash(n.Children[i], db, false) - if err != nil { - return original, original, err - } - } +func (h *hasher) hashFullNodeChildren(n *fullNode) (collapsed *fullNode, cached *fullNode) { + // Hash the full node's children, caching the newly hashed subtrees + cached = n.copy() + collapsed = n.copy() + for i := 0; i < 16; i++ { + if child := n.Children[i]; child != nil { + collapsed.Children[i], cached.Children[i] = h.hash(child, false) + } else { + collapsed.Children[i] = nilValueNode } - cached.Children[16] = n.Children[16] - return collapsed, cached, nil - - default: - // Value and hash nodes don't have children so they're left as were - return n, original, nil } + cached.Children[16] = n.Children[16] + return collapsed, cached } -// store hashes the node n and if we have a storage layer specified, it writes -// the key/value pair to it and tracks any node->child references as well as any -// node->external trie references. -func (h *hasher) store(n node, db *Database, force bool) (node, error) { - // Don't store hashes or empty nodes. - if _, isHash := n.(hashNode); n == nil || isHash { - return n, nil - } - // Generate the RLP encoding of the node +// shortnodeToHash creates a hashNode from a shortNode. The supplied shortnode +// should have hex-type Key, which will be converted (without modification) +// into compact form for RLP encoding. +// If the rlp data is smaller than 32 bytes, `nil` is returned. +func (h *hasher) shortnodeToHash(n *shortNode, force bool) node { h.tmp.Reset() if err := rlp.Encode(&h.tmp, n); err != nil { panic("encode error: " + err.Error()) } + if len(h.tmp) < 32 && !force { - return n, nil // Nodes smaller than 32 bytes are stored inside their parent + return n // Nodes smaller than 32 bytes are stored inside their parent } - // Larger nodes are replaced by their hash and stored in the database. - hash, _ := n.cache() - if hash == nil { - hash = h.makeHashNode(h.tmp) + return h.hashData(h.tmp) +} + +// shortnodeToHash is used to creates a hashNode from a set of hashNodes, (which +// may contain nil values) +func (h *hasher) fullnodeToHash(n *fullNode, force bool) node { + h.tmp.Reset() + // Generate the RLP encoding of the node + if err := n.EncodeRLP(&h.tmp); err != nil { + panic("encode error: " + err.Error()) } - if db != nil { - // We are pooling the trie nodes into an intermediate memory cache - hash := common.BytesToHash(hash) - - db.lock.Lock() - db.insert(hash, h.tmp, n) - db.lock.Unlock() - - // Track external references from account->storage trie - if h.onleaf != nil { - switch n := n.(type) { - case *shortNode: - if child, ok := n.Val.(valueNode); ok { - h.onleaf(child, hash) - } - case *fullNode: - for i := 0; i < 16; i++ { - if child, ok := n.Children[i].(valueNode); ok { - h.onleaf(child, hash) - } - } - } - } + if len(h.tmp) < 32 && !force { + return n // Nodes smaller than 32 bytes are stored inside their parent } - return hash, nil + return h.hashData(h.tmp) } -func (h *hasher) makeHashNode(data []byte) hashNode { - n := make(hashNode, h.sha.Size()) +// hashData hashes the provided data +func (h *hasher) hashData(data []byte) hashNode { + n := make(hashNode, 32) h.sha.Reset() h.sha.Write(data) h.sha.Read(n) return n } + +// proofHash is used to construct trie proofs, and returns the 'collapsed' +// node (for later RLP encoding) aswell as the hashed node -- unless the +// node is smaller than 32 bytes, in which case it will be returned as is. +// This method does not do anything on value- or hash-nodes. +func (h *hasher) proofHash(original node) (collapsed, hashed node) { + switch n := original.(type) { + case *shortNode: + sn, _ := h.hashShortNodeChildren(n) + return sn, h.shortnodeToHash(sn, false) + case *fullNode: + fn, _ := h.hashFullNodeChildren(n) + return fn, h.fullnodeToHash(fn, false) + default: + // Value and hash nodes don't have children so they're left as were + return n, n + } +} diff --git a/trie/iterator.go b/trie/iterator.go index 8e84dee3b6..94b36a0183 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -182,15 +182,13 @@ func (it *nodeIterator) LeafBlob() []byte { func (it *nodeIterator) LeafProof() [][]byte { if len(it.stack) > 0 { if _, ok := it.stack[len(it.stack)-1].node.(valueNode); ok { - hasher := newHasher(nil) + hasher := newHasher() defer returnHasherToPool(hasher) - proofs := make([][]byte, 0, len(it.stack)) for i, item := range it.stack[:len(it.stack)-1] { // Gather nodes that end up as hash nodes (or the root) - node, _, _ := hasher.hashChildren(item.node, nil) - hashed, _ := hasher.store(node, nil, false) + node, hashed := hasher.proofHash(item.node) if _, ok := hashed.(hashNode); ok || i == 0 { enc, _ := rlp.EncodeToBytes(node) proofs = append(proofs, enc) diff --git a/trie/proof.go b/trie/proof.go index 9985e730dd..f2c4658c49 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -64,26 +64,24 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) e panic(fmt.Sprintf("%T: invalid node: %v", tn, tn)) } } - hasher := newHasher(nil) + hasher := newHasher() defer returnHasherToPool(hasher) for i, n := range nodes { - // Don't bother checking for errors here since hasher panics - // if encoding doesn't work and we're not writing to any database. - n, _, _ = hasher.hashChildren(n, nil) - hn, _ := hasher.store(n, nil, false) + if fromLevel > 0 { + fromLevel-- + continue + } + var hn node + n, hn = hasher.proofHash(n) if hash, ok := hn.(hashNode); ok || i == 0 { // If the node's database encoding is a hash (or is the // root node), it becomes a proof element. - if fromLevel > 0 { - fromLevel-- - } else { - enc, _ := rlp.EncodeToBytes(n) - if !ok { - hash = hasher.makeHashNode(enc) - } - proofDb.Put(hash, enc) + enc, _ := rlp.EncodeToBytes(n) + if !ok { + hash = hasher.hashData(enc) } + proofDb.Put(hash, enc) } } return nil diff --git a/trie/secure_trie.go b/trie/secure_trie.go index fbc591ed10..b76a1dc8ab 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -176,7 +176,7 @@ func (t *SecureTrie) NodeIterator(start []byte) NodeIterator { // The caller must not hold onto the return value because it will become // invalid on the next call to hashKey or secKey. func (t *SecureTrie) hashKey(key []byte) []byte { - h := newHasher(nil) + h := newHasher() h.sha.Reset() h.sha.Write(key) buf := h.sha.Sum(t.hashKeyBuf[:0]) diff --git a/trie/trie.go b/trie/trie.go index 920e331fd6..dd26f9b340 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -20,6 +20,7 @@ package trie import ( "bytes" "fmt" + "sync" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/crypto" @@ -415,19 +416,52 @@ func (t *Trie) Commit(onleaf LeafCallback) (root common.Hash, err error) { if t.db == nil { panic("commit called on trie with nil database") } - hash, cached, err := t.hashRoot(t.db, onleaf) + if t.root == nil { + return emptyRoot, nil + } + rootHash := t.Hash() + h := newCommitter() + defer returnCommitterToPool(h) + // Do a quick check if we really need to commit, before we spin + // up goroutines. This can happen e.g. if we load a trie for reading storage + // values, but don't write to it. + if !h.commitNeeded(t.root) { + return rootHash, nil + } + var wg sync.WaitGroup + if onleaf != nil { + h.onleaf = onleaf + h.leafCh = make(chan *leaf, leafChanSize) + wg.Add(1) + go func() { + defer wg.Done() + h.commitLoop(t.db) + }() + } + var newRoot hashNode + newRoot, err = h.Commit(t.root, t.db) + if onleaf != nil { + // The leafch is created in newCommitter if there was an onleaf callback + // provided. The commitLoop only _reads_ from it, and the commit + // operation was the sole writer. Therefore, it's safe to close this + // channel here. + close(h.leafCh) + wg.Wait() + } if err != nil { return common.Hash{}, err } - t.root = cached - return common.BytesToHash(hash.(hashNode)), nil + t.root = newRoot + return rootHash, nil } +// hashRoot calculates the root hash of the given trie func (t *Trie) hashRoot(db *Database, onleaf LeafCallback) (node, node, error) { if t.root == nil { return hashNode(emptyRoot.Bytes()), nil, nil } - h := newHasher(onleaf) + h := newHasher() defer returnHasherToPool(h) - return h.hash(t.root, db, true) + hashed, cached := h.hash(t.root, true) + return hashed, cached, nil } From 33791dbeb51ce910983d15dda53fd83c05be9bb3 Mon Sep 17 00:00:00 2001 From: tintin Date: Tue, 4 Feb 2020 09:55:07 +0100 Subject: [PATCH 088/128] tracers: avoid panic on invalid arguments (#20612) * add regression tests for #20611 * eth/tracers: fix panics occurring for invalid params in js-tracers Co-authored-by: Martin Holst Swende --- eth/tracers/tracer.go | 13 +++++++++++-- eth/tracers/tracer_test.go | 33 +++++++++++++++++++++++++++++++++ 2 files changed, 44 insertions(+), 2 deletions(-) diff --git a/eth/tracers/tracer.go b/eth/tracers/tracer.go index 63d38ed32d..a74ed51e1a 100644 --- a/eth/tracers/tracer.go +++ b/eth/tracers/tracer.go @@ -93,6 +93,15 @@ type memoryWrapper struct { // slice returns the requested range of memory as a byte slice. func (mw *memoryWrapper) slice(begin, end int64) []byte { + if end == begin { + return []byte{} + } + if end < begin || begin < 0 { + // TODO(karalabe): We can't js-throw from Go inside duktape inside Go. The Go + // runtime goes belly up https://github.com/golang/go/issues/15639. + log.Warn("Tracer accessed out of bound memory", "offset", begin, "end", end) + return nil + } if mw.memory.Len() < int(end) { // TODO(karalabe): We can't js-throw from Go inside duktape inside Go. The Go // runtime goes belly up https://github.com/golang/go/issues/15639. @@ -104,7 +113,7 @@ func (mw *memoryWrapper) slice(begin, end int64) []byte { // getUint returns the 32 bytes at the specified address interpreted as a uint. func (mw *memoryWrapper) getUint(addr int64) *big.Int { - if mw.memory.Len() < int(addr)+32 { + if mw.memory.Len() < int(addr)+32 || addr < 0 { // TODO(karalabe): We can't js-throw from Go inside duktape inside Go. The Go // runtime goes belly up https://github.com/golang/go/issues/15639. log.Warn("Tracer accessed out of bound memory", "available", mw.memory.Len(), "offset", addr, "size", 32) @@ -147,7 +156,7 @@ type stackWrapper struct { // peek returns the nth-from-the-top element of the stack. func (sw *stackWrapper) peek(idx int) *big.Int { - if len(sw.stack.Data()) <= idx { + if len(sw.stack.Data()) <= idx || idx < 0 { // TODO(karalabe): We can't js-throw from Go inside duktape inside Go. The Go // runtime goes belly up https://github.com/golang/go/issues/15639. log.Warn("Tracer accessed out of bound stack", "size", len(sw.stack.Data()), "index", idx) diff --git a/eth/tracers/tracer_test.go b/eth/tracers/tracer_test.go index a45a121159..0f52370675 100644 --- a/eth/tracers/tracer_test.go +++ b/eth/tracers/tracer_test.go @@ -63,6 +63,39 @@ func runTrace(tracer *Tracer) (json.RawMessage, error) { return tracer.GetResult() } +// TestRegressionPanicSlice tests that we don't panic on bad arguments to memory access +func TestRegressionPanicSlice(t *testing.T) { + tracer, err := New("{depths: [], step: function(log) { this.depths.push(log.memory.slice(-1,-2)); }, fault: function() {}, result: function() { return this.depths; }}") + if err != nil { + t.Fatal(err) + } + if _, err = runTrace(tracer); err != nil { + t.Fatal(err) + } +} + +// TestRegressionPanicSlice tests that we don't panic on bad arguments to stack peeks +func TestRegressionPanicPeek(t *testing.T) { + tracer, err := New("{depths: [], step: function(log) { this.depths.push(log.stack.peek(-1)); }, fault: function() {}, result: function() { return this.depths; }}") + if err != nil { + t.Fatal(err) + } + if _, err = runTrace(tracer); err != nil { + t.Fatal(err) + } +} + +// TestRegressionPanicSlice tests that we don't panic on bad arguments to memory getUint +func TestRegressionPanicGetUint(t *testing.T) { + tracer, err := New("{ depths: [], step: function(log, db) { this.depths.push(log.memory.getUint(-64));}, fault: function() {}, result: function() { return this.depths; }}") + if err != nil { + t.Fatal(err) + } + if _, err = runTrace(tracer); err != nil { + t.Fatal(err) + } +} + func TestTracing(t *testing.T) { tracer, err := New("{count: 0, step: function() { this.count += 1; }, fault: function() {}, result: function() { return this.count; }}") if err != nil { From 058a4ac5f13f49a67d4b28111dd0ad3eea387c8c Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 4 Feb 2020 11:32:31 +0100 Subject: [PATCH 089/128] core/evm: less iteration in blockhash (#20589) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * core/vm/runtime: add test for blockhash * core/evm: less iteration in blockhash * core/vm/runtime: nitpickfix Co-authored-by: PĂ©ter Szilágyi --- core/evm.go | 34 ++++++---- core/vm/runtime/env.go | 4 +- core/vm/runtime/runtime_test.go | 113 ++++++++++++++++++++++++++++++++ 3 files changed, 135 insertions(+), 16 deletions(-) diff --git a/core/evm.go b/core/evm.go index b654bbd479..8abe5a0477 100644 --- a/core/evm.go +++ b/core/evm.go @@ -60,24 +60,32 @@ func NewEVMContext(msg Message, header *types.Header, chain ChainContext, author // GetHashFn returns a GetHashFunc which retrieves header hashes by number func GetHashFn(ref *types.Header, chain ChainContext) func(n uint64) common.Hash { - var cache map[uint64]common.Hash + // Cache will initially contain [refHash.parent], + // Then fill up with [refHash.p, refHash.pp, refHash.ppp, ...] + var cache []common.Hash return func(n uint64) common.Hash { // If there's no hash cache yet, make one - if cache == nil { - cache = map[uint64]common.Hash{ - ref.Number.Uint64() - 1: ref.ParentHash, - } + if len(cache) == 0 { + cache = append(cache, ref.ParentHash) } - // Try to fulfill the request from the cache - if hash, ok := cache[n]; ok { - return hash + if idx := ref.Number.Uint64() - n - 1; idx < uint64(len(cache)) { + return cache[idx] } - // Not cached, iterate the blocks and cache the hashes - for header := chain.GetHeader(ref.ParentHash, ref.Number.Uint64()-1); header != nil; header = chain.GetHeader(header.ParentHash, header.Number.Uint64()-1) { - cache[header.Number.Uint64()-1] = header.ParentHash - if n == header.Number.Uint64()-1 { - return header.ParentHash + // No luck in the cache, but we can start iterating from the last element we already know + lastKnownHash := cache[len(cache)-1] + lastKnownNumber := ref.Number.Uint64() - uint64(len(cache)) + + for { + header := chain.GetHeader(lastKnownHash, lastKnownNumber) + if header == nil { + break + } + cache = append(cache, header.ParentHash) + lastKnownHash = header.ParentHash + lastKnownNumber = header.Number.Uint64() - 1 + if n == lastKnownNumber { + return lastKnownHash } } return common.Hash{} diff --git a/core/vm/runtime/env.go b/core/vm/runtime/env.go index 31c9b9cf9d..38ee448904 100644 --- a/core/vm/runtime/env.go +++ b/core/vm/runtime/env.go @@ -17,7 +17,6 @@ package runtime import ( - "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/vm" ) @@ -26,8 +25,7 @@ func NewEnv(cfg *Config) *vm.EVM { context := vm.Context{ CanTransfer: core.CanTransfer, Transfer: core.Transfer, - GetHash: func(uint64) common.Hash { return common.Hash{} }, - + GetHash: cfg.GetHashFn, Origin: cfg.Origin, Coinbase: cfg.Coinbase, BlockNumber: cfg.BlockNumber, diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index 15f545ddca..f2d05118ce 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -23,8 +23,11 @@ import ( "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/consensus" + "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/rawdb" "github.com/ethereum/go-ethereum/core/state" + "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/core/vm" "github.com/ethereum/go-ethereum/params" ) @@ -203,3 +206,113 @@ func BenchmarkEVM_CREATE2_1200(bench *testing.B) { // initcode size 1200K, repeatedly calls CREATE2 and then modifies the mem contents benchmarkEVM_Create(bench, "5b5862124f80600080f5600152600056") } + +func fakeHeader(n uint64, parentHash common.Hash) *types.Header { + header := types.Header{ + Coinbase: common.HexToAddress("0x00000000000000000000000000000000deadbeef"), + Number: big.NewInt(int64(n)), + ParentHash: parentHash, + Time: 1000, + Nonce: types.BlockNonce{0x1}, + Extra: []byte{}, + Difficulty: big.NewInt(0), + GasLimit: 100000, + } + return &header +} + +type dummyChain struct { + counter int +} + +// Engine retrieves the chain's consensus engine. +func (d *dummyChain) Engine() consensus.Engine { + return nil +} + +// GetHeader returns the hash corresponding to their hash. +func (d *dummyChain) GetHeader(h common.Hash, n uint64) *types.Header { + d.counter++ + parentHash := common.Hash{} + s := common.LeftPadBytes(big.NewInt(int64(n-1)).Bytes(), 32) + copy(parentHash[:], s) + + //parentHash := common.Hash{byte(n - 1)} + //fmt.Printf("GetHeader(%x, %d) => header with parent %x\n", h, n, parentHash) + return fakeHeader(n, parentHash) +} + +// TestBlockhash tests the blockhash operation. It's a bit special, since it internally +// requires access to a chain reader. +func TestBlockhash(t *testing.T) { + // Current head + n := uint64(1000) + parentHash := common.Hash{} + s := common.LeftPadBytes(big.NewInt(int64(n-1)).Bytes(), 32) + copy(parentHash[:], s) + header := fakeHeader(n, parentHash) + + // This is the contract we're using. It requests the blockhash for current num (should be all zeroes), + // then iteratively fetches all blockhashes back to n-260. + // It returns + // 1. the first (should be zero) + // 2. the second (should be the parent hash) + // 3. the last non-zero hash + // By making the chain reader return hashes which correlate to the number, we can + // verify that it obtained the right hashes where it should + + /* + + pragma solidity ^0.5.3; + contract Hasher{ + + function test() public view returns (bytes32, bytes32, bytes32){ + uint256 x = block.number; + bytes32 first; + bytes32 last; + bytes32 zero; + zero = blockhash(x); // Should be zeroes + first = blockhash(x-1); + for(uint256 i = 2 ; i < 260; i++){ + bytes32 hash = blockhash(x - i); + if (uint256(hash) != 0){ + last = hash; + } + } + return (zero, first, last); + } + } + + */ + // The contract above + data := common.Hex2Bytes("6080604052348015600f57600080fd5b50600436106045576000357c010000000000000000000000000000000000000000000000000000000090048063f8a8fd6d14604a575b600080fd5b60506074565b60405180848152602001838152602001828152602001935050505060405180910390f35b600080600080439050600080600083409050600184034092506000600290505b61010481101560c35760008186034090506000816001900414151560b6578093505b5080806001019150506094565b508083839650965096505050505090919256fea165627a7a72305820462d71b510c1725ff35946c20b415b0d50b468ea157c8c77dff9466c9cb85f560029") + // The method call to 'test()' + input := common.Hex2Bytes("f8a8fd6d") + chain := &dummyChain{} + ret, _, err := Execute(data, input, &Config{ + GetHashFn: core.GetHashFn(header, chain), + BlockNumber: new(big.Int).Set(header.Number), + }) + if err != nil { + t.Fatalf("expected no error, got %v", err) + } + if len(ret) != 96 { + t.Fatalf("expected returndata to be 96 bytes, got %d", len(ret)) + } + + zero := new(big.Int).SetBytes(ret[0:32]) + first := new(big.Int).SetBytes(ret[32:64]) + last := new(big.Int).SetBytes(ret[64:96]) + if zero.BitLen() != 0 { + t.Fatalf("expected zeroes, got %x", ret[0:32]) + } + if first.Uint64() != 999 { + t.Fatalf("second block should be 999, got %d (%x)", first, ret[32:64]) + } + if last.Uint64() != 744 { + t.Fatalf("last block should be 744, got %d (%x)", last, ret[64:96]) + } + if exp, got := 255, chain.counter; exp != got { + t.Errorf("suboptimal; too much chain iteration, expected %d, got %d", exp, got) + } +} From 711ed74e091f2ada6e2aa84e5e62b9696b589d8a Mon Sep 17 00:00:00 2001 From: meowsbits <45600330+meowsbits@users.noreply.github.com> Date: Tue, 4 Feb 2020 04:49:13 -0600 Subject: [PATCH 090/128] cmd/geth: add 'dumpgenesis' command (#20191) Adds the 'geth dumpgenesis' command, which writes the configured genesis in JSON format to stdout. This provides a way to generate the data (structure and content) that can then be used with the 'geth init' command. --- cmd/geth/chaincmd.go | 23 +++++++++++++++++++++++ cmd/geth/main.go | 1 + 2 files changed, 24 insertions(+) diff --git a/cmd/geth/chaincmd.go b/cmd/geth/chaincmd.go index 49e6a05949..5b176b6da1 100644 --- a/cmd/geth/chaincmd.go +++ b/cmd/geth/chaincmd.go @@ -56,6 +56,18 @@ This is a destructive action and changes the network in which you will be participating. It expects the genesis file as argument.`, + } + dumpGenesisCommand = cli.Command{ + Action: utils.MigrateFlags(dumpGenesis), + Name: "dumpgenesis", + Usage: "Dumps genesis block JSON configuration to stdout", + ArgsUsage: "", + Flags: []cli.Flag{ + utils.DataDirFlag, + }, + Category: "BLOCKCHAIN COMMANDS", + Description: ` +The dumpgenesis command dumps the genesis block configuration in JSON format to stdout.`, } importCommand = cli.Command{ Action: utils.MigrateFlags(importChain), @@ -227,6 +239,17 @@ func initGenesis(ctx *cli.Context) error { return nil } +func dumpGenesis(ctx *cli.Context) error { + genesis := utils.MakeGenesis(ctx) + if genesis == nil { + genesis = core.DefaultGenesisBlock() + } + if err := json.NewEncoder(os.Stdout).Encode(genesis); err != nil { + utils.Fatalf("could not encode genesis") + } + return nil +} + func importChain(ctx *cli.Context) error { if len(ctx.Args()) < 1 { utils.Fatalf("This command requires an argument.") diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 8db19077a8..0369e527b5 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -205,6 +205,7 @@ func init() { copydbCommand, removedbCommand, dumpCommand, + dumpGenesisCommand, inspectCommand, // See accountcmd.go: accountCommand, From a1313b5b1e9713e7e10f5ef29b7981c9c49d688b Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Tue, 4 Feb 2020 13:02:38 +0100 Subject: [PATCH 091/128] trie: make hasher parallel when number of changes are large (#20488) * trie: make hasher parallel when number of changes are large * trie: remove unused field dirtyCount * trie: rename unhashedCount/unhashed --- trie/hasher.go | 37 ++++++++++++++++++++++++++++--------- trie/iterator.go | 2 +- trie/proof.go | 2 +- trie/secure_trie.go | 2 +- trie/trie.go | 14 +++++++++++--- 5 files changed, 42 insertions(+), 15 deletions(-) diff --git a/trie/hasher.go b/trie/hasher.go index 71a3aec3b2..8e8eec9f61 100644 --- a/trie/hasher.go +++ b/trie/hasher.go @@ -46,8 +46,9 @@ func (b *sliceBuffer) Reset() { // hasher is a type used for the trie Hash operation. A hasher has some // internal preallocated temp space type hasher struct { - sha keccakState - tmp sliceBuffer + sha keccakState + tmp sliceBuffer + parallel bool // Whether to use paralallel threads when hashing } // hasherPool holds pureHashers @@ -60,8 +61,9 @@ var hasherPool = sync.Pool{ }, } -func newHasher() *hasher { +func newHasher(parallel bool) *hasher { h := hasherPool.Get().(*hasher) + h.parallel = parallel return h } @@ -126,14 +128,31 @@ func (h *hasher) hashFullNodeChildren(n *fullNode) (collapsed *fullNode, cached // Hash the full node's children, caching the newly hashed subtrees cached = n.copy() collapsed = n.copy() - for i := 0; i < 16; i++ { - if child := n.Children[i]; child != nil { - collapsed.Children[i], cached.Children[i] = h.hash(child, false) - } else { - collapsed.Children[i] = nilValueNode + if h.parallel { + var wg sync.WaitGroup + wg.Add(16) + for i := 0; i < 16; i++ { + go func(i int) { + hasher := newHasher(false) + if child := n.Children[i]; child != nil { + collapsed.Children[i], cached.Children[i] = hasher.hash(child, false) + } else { + collapsed.Children[i] = nilValueNode + } + returnHasherToPool(hasher) + wg.Done() + }(i) + } + wg.Wait() + } else { + for i := 0; i < 16; i++ { + if child := n.Children[i]; child != nil { + collapsed.Children[i], cached.Children[i] = h.hash(child, false) + } else { + collapsed.Children[i] = nilValueNode + } } } - cached.Children[16] = n.Children[16] return collapsed, cached } diff --git a/trie/iterator.go b/trie/iterator.go index 94b36a0183..bb4025d8f3 100644 --- a/trie/iterator.go +++ b/trie/iterator.go @@ -182,7 +182,7 @@ func (it *nodeIterator) LeafBlob() []byte { func (it *nodeIterator) LeafProof() [][]byte { if len(it.stack) > 0 { if _, ok := it.stack[len(it.stack)-1].node.(valueNode); ok { - hasher := newHasher() + hasher := newHasher(false) defer returnHasherToPool(hasher) proofs := make([][]byte, 0, len(it.stack)) diff --git a/trie/proof.go b/trie/proof.go index f2c4658c49..58ca69c680 100644 --- a/trie/proof.go +++ b/trie/proof.go @@ -64,7 +64,7 @@ func (t *Trie) Prove(key []byte, fromLevel uint, proofDb ethdb.KeyValueWriter) e panic(fmt.Sprintf("%T: invalid node: %v", tn, tn)) } } - hasher := newHasher() + hasher := newHasher(false) defer returnHasherToPool(hasher) for i, n := range nodes { diff --git a/trie/secure_trie.go b/trie/secure_trie.go index b76a1dc8ab..955771495b 100644 --- a/trie/secure_trie.go +++ b/trie/secure_trie.go @@ -176,7 +176,7 @@ func (t *SecureTrie) NodeIterator(start []byte) NodeIterator { // The caller must not hold onto the return value because it will become // invalid on the next call to hashKey or secKey. func (t *SecureTrie) hashKey(key []byte) []byte { - h := newHasher() + h := newHasher(false) h.sha.Reset() h.sha.Write(key) buf := h.sha.Sum(t.hashKeyBuf[:0]) diff --git a/trie/trie.go b/trie/trie.go index dd26f9b340..78e2eff534 100644 --- a/trie/trie.go +++ b/trie/trie.go @@ -48,6 +48,10 @@ type LeafCallback func(leaf []byte, parent common.Hash) error type Trie struct { db *Database root node + // Keep track of the number leafs which have been inserted since the last + // hashing operation. This number will not directly map to the number of + // actually unhashed nodes + unhashed int } // newFlag returns the cache flag value for a newly created node. @@ -163,6 +167,7 @@ func (t *Trie) Update(key, value []byte) { // // If a node was not found in the database, a MissingNodeError is returned. func (t *Trie) TryUpdate(key, value []byte) error { + t.unhashed++ k := keybytesToHex(key) if len(value) != 0 { _, n, err := t.insert(t.root, nil, k, valueNode(value)) @@ -259,6 +264,7 @@ func (t *Trie) Delete(key []byte) { // TryDelete removes any existing value for key from the trie. // If a node was not found in the database, a MissingNodeError is returned. func (t *Trie) TryDelete(key []byte) error { + t.unhashed++ k := keybytesToHex(key) _, n, err := t.delete(t.root, nil, k) if err != nil { @@ -405,7 +411,7 @@ func (t *Trie) resolveHash(n hashNode, prefix []byte) (node, error) { // Hash returns the root hash of the trie. It does not write to the // database and can be used even if the trie doesn't have one. func (t *Trie) Hash() common.Hash { - hash, cached, _ := t.hashRoot(nil, nil) + hash, cached, _ := t.hashRoot(nil) t.root = cached return common.BytesToHash(hash.(hashNode)) } @@ -456,12 +462,14 @@ func (t *Trie) Commit(onleaf LeafCallback) (root common.Hash, err error) { } // hashRoot calculates the root hash of the given trie -func (t *Trie) hashRoot(db *Database, onleaf LeafCallback) (node, node, error) { +func (t *Trie) hashRoot(db *Database) (node, node, error) { if t.root == nil { return hashNode(emptyRoot.Bytes()), nil, nil } - h := newHasher() + // If the number of changes is below 100, we let one thread handle it + h := newHasher(t.unhashed >= 100) defer returnHasherToPool(h) hashed, cached := h.hash(t.root, true) + t.unhashed = 0 return hashed, cached, nil } From 976a0f5558e20ed7cb7ba2cd68d7429d1ef01db9 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Wed, 5 Feb 2020 15:29:59 +0100 Subject: [PATCH 092/128] cmd/devp2p: fix Route53 TXT record splitting (#20626) For longer records and subtree entries, the deployer created two separate TXT records. This doesn't work as intended because the client will receive the two records in arbitrary order. The fix is to encode longer values as "string1""string2" instead of "string1", "string2". This encoding creates a single record on AWS Route53. --- cmd/devp2p/dns_route53.go | 28 ++++++++-------------------- cmd/devp2p/dns_route53_test.go | 6 ++---- 2 files changed, 10 insertions(+), 24 deletions(-) diff --git a/cmd/devp2p/dns_route53.go b/cmd/devp2p/dns_route53.go index bdad5c2e9c..71118be543 100644 --- a/cmd/devp2p/dns_route53.go +++ b/cmd/devp2p/dns_route53.go @@ -171,7 +171,7 @@ func (c *route53Client) computeChanges(name string, records map[string]string, e } prevRecords, exists := existing[path] - prevValue := combineTXT(prevRecords.values) + prevValue := strings.Join(prevRecords.values, "") if !exists { // Entry is unknown, push a new one log.Info(fmt.Sprintf("Creating %s = %q", path, val)) @@ -191,8 +191,8 @@ func (c *route53Client) computeChanges(name string, records map[string]string, e continue } // Stale entry, nuke it. - log.Info(fmt.Sprintf("Deleting %s = %q", path, combineTXT(set.values))) - changes = append(changes, newTXTChange("DELETE", path, set.ttl, set.values)) + log.Info(fmt.Sprintf("Deleting %s = %q", path, strings.Join(set.values, ""))) + changes = append(changes, newTXTChange("DELETE", path, set.ttl, set.values...)) } sortChanges(changes) @@ -263,7 +263,7 @@ func (c *route53Client) collectRecords(name string) (map[string]recordSet, error } // newTXTChange creates a change to a TXT record. -func newTXTChange(action, name string, ttl int64, values []string) *route53.Change { +func newTXTChange(action, name string, ttl int64, values ...string) *route53.Change { var c route53.Change var r route53.ResourceRecordSet var rrs []*route53.ResourceRecord @@ -288,28 +288,16 @@ func isSubdomain(name, domain string) bool { return strings.HasSuffix("."+name, "."+domain) } -// combineTXT concatenates the given quoted strings into a single unquoted string. -func combineTXT(values []string) string { - result := "" - for _, v := range values { - if v[0] == '"' { - v = v[1 : len(v)-1] - } - result += v - } - return result -} - // splitTXT splits value into a list of quoted 255-character strings. -func splitTXT(value string) []string { - var result []string +func splitTXT(value string) string { + var result strings.Builder for len(value) > 0 { rlen := len(value) if rlen > 253 { rlen = 253 } - result = append(result, strconv.Quote(value[:rlen])) + result.WriteString(strconv.Quote(value[:rlen])) value = value[rlen:] } - return result + return result.String() } diff --git a/cmd/devp2p/dns_route53_test.go b/cmd/devp2p/dns_route53_test.go index c64f1d1698..f6ab6c1762 100644 --- a/cmd/devp2p/dns_route53_test.go +++ b/cmd/devp2p/dns_route53_test.go @@ -28,8 +28,7 @@ import ( func TestRoute53ChangeSort(t *testing.T) { testTree0 := map[string]recordSet{ "2kfjogvxdqtxxugbh7gs7naaai.n": {ttl: 3333, values: []string{ - `"enr:-HW4QO1ml1DdXLeZLsUxewnthhUy8eROqkDyoMTyavfks9JlYQIlMFEUoM78PovJDPQrAkrb3LRJ-"`, - `"vtrymDguKCOIAWAgmlkgnY0iXNlY3AyNTZrMaEDffaGfJzgGhUif1JqFruZlYmA31HzathLSWxfbq_QoQ4"`, + `"enr:-HW4QO1ml1DdXLeZLsUxewnthhUy8eROqkDyoMTyavfks9JlYQIlMFEUoM78PovJDPQrAkrb3LRJ-""vtrymDguKCOIAWAgmlkgnY0iXNlY3AyNTZrMaEDffaGfJzgGhUif1JqFruZlYmA31HzathLSWxfbq_QoQ4"`, }}, "fdxn3sn67na5dka4j2gok7bvqi.n": {ttl: treeNodeTTL, values: []string{`"enrtree-branch:"`}}, "n": {ttl: rootTTL, values: []string{`"enrtree-root:v1 e=2KFJOGVXDQTXXUGBH7GS7NAAAI l=FDXN3SN67NA5DKA4J2GOK7BVQI seq=0 sig=v_-J_q_9ICQg5ztExFvLQhDBGMb0lZPJLhe3ts9LAcgqhOhtT3YFJsl8BWNDSwGtamUdR-9xl88_w-X42SVpjwE"`}}, @@ -116,8 +115,7 @@ func TestRoute53ChangeSort(t *testing.T) { ResourceRecordSet: &route53.ResourceRecordSet{ Name: sp("2kfjogvxdqtxxugbh7gs7naaai.n"), ResourceRecords: []*route53.ResourceRecord{ - {Value: sp(`"enr:-HW4QO1ml1DdXLeZLsUxewnthhUy8eROqkDyoMTyavfks9JlYQIlMFEUoM78PovJDPQrAkrb3LRJ-"`)}, - {Value: sp(`"vtrymDguKCOIAWAgmlkgnY0iXNlY3AyNTZrMaEDffaGfJzgGhUif1JqFruZlYmA31HzathLSWxfbq_QoQ4"`)}, + {Value: sp(`"enr:-HW4QO1ml1DdXLeZLsUxewnthhUy8eROqkDyoMTyavfks9JlYQIlMFEUoM78PovJDPQrAkrb3LRJ-""vtrymDguKCOIAWAgmlkgnY0iXNlY3AyNTZrMaEDffaGfJzgGhUif1JqFruZlYmA31HzathLSWxfbq_QoQ4"`)}, }, TTL: ip(3333), Type: sp("TXT"), From 4a231cd951faaf011cdec8fdde66a948b637718e Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 7 Feb 2020 10:44:32 +0100 Subject: [PATCH 093/128] internal/ethapi: return non-null "number" for pending block (#20616) Fixes: #20587, ethereum/web3.py#1572 --- internal/ethapi/api.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/ethapi/api.go b/internal/ethapi/api.go index 37433f335a..bec62c1290 100644 --- a/internal/ethapi/api.go +++ b/internal/ethapi/api.go @@ -642,7 +642,7 @@ func (s *PublicBlockChainAPI) GetBlockByNumber(ctx context.Context, number rpc.B response, err := s.rpcMarshalBlock(block, true, fullTx) if err == nil && number == rpc.PendingBlockNumber { // Pending blocks need to nil out a few fields - for _, field := range []string{"hash", "nonce", "miner", "number"} { + for _, field := range []string{"hash", "nonce", "miner"} { response[field] = nil } } From 675f4e75b868289993fa2cb9eb543532a3c80610 Mon Sep 17 00:00:00 2001 From: Nick Ward Date: Sun, 9 Feb 2020 20:18:47 +0400 Subject: [PATCH 094/128] README.md: update evm usage example (#20635) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 757945e914..72479b5a1e 100644 --- a/README.md +++ b/README.md @@ -39,7 +39,7 @@ directory. | **`geth`** | Our main Ethereum CLI client. It is the entry point into the Ethereum network (main-, test- or private net), capable of running as a full node (default), archive node (retaining all historical state) or a light node (retrieving data live). It can be used by other processes as a gateway into the Ethereum network via JSON RPC endpoints exposed on top of HTTP, WebSocket and/or IPC transports. `geth --help` and the [CLI Wiki page](https://github.com/ethereum/go-ethereum/wiki/Command-Line-Options) for command line options. | | `abigen` | Source code generator to convert Ethereum contract definitions into easy to use, compile-time type-safe Go packages. It operates on plain [Ethereum contract ABIs](https://github.com/ethereum/wiki/wiki/Ethereum-Contract-ABI) with expanded functionality if the contract bytecode is also available. However, it also accepts Solidity source files, making development much more streamlined. Please see our [Native DApps](https://github.com/ethereum/go-ethereum/wiki/Native-DApps:-Go-bindings-to-Ethereum-contracts) wiki page for details. | | `bootnode` | Stripped down version of our Ethereum client implementation that only takes part in the network node discovery protocol, but does not run any of the higher level application protocols. It can be used as a lightweight bootstrap node to aid in finding peers in private networks. | -| `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug`). | +| `evm` | Developer utility version of the EVM (Ethereum Virtual Machine) that is capable of running bytecode snippets within a configurable environment and execution mode. Its purpose is to allow isolated, fine-grained debugging of EVM opcodes (e.g. `evm --code 60ff60ff --debug run`). | | `gethrpctest` | Developer utility tool to support our [ethereum/rpc-test](https://github.com/ethereum/rpc-tests) test suite which validates baseline conformity to the [Ethereum JSON RPC](https://github.com/ethereum/wiki/wiki/JSON-RPC) specs. Please see the [test suite's readme](https://github.com/ethereum/rpc-tests/blob/master/README.md) for details. | | `rlpdump` | Developer utility tool to convert binary RLP ([Recursive Length Prefix](https://github.com/ethereum/wiki/wiki/RLP)) dumps (data encoding used by the Ethereum protocol both network as well as consensus wise) to user-friendlier hierarchical representation (e.g. `rlpdump --hex CE0183FFFFFFC4C304050583616263`). | | `puppeth` | a CLI wizard that aids in creating a new Ethereum network. | From 34bb132b108e0c17c079d6b24524c2958306a009 Mon Sep 17 00:00:00 2001 From: AmitBRD <60668103+AmitBRD@users.noreply.github.com> Date: Sun, 9 Feb 2020 15:50:44 -0500 Subject: [PATCH 095/128] graphql: add transaction signature values (#20623) The feature update allows the GraphQL API endpoint to retrieve transaction signature R,S,V parameters. Co-authored-by: amitshah Co-authored-by: Felix Lange --- graphql/graphql.go | 27 +++++++++++++++++++++++++++ graphql/schema.go | 3 +++ 2 files changed, 30 insertions(+) diff --git a/graphql/graphql.go b/graphql/graphql.go index ddd928dff1..0f7bba8657 100644 --- a/graphql/graphql.go +++ b/graphql/graphql.go @@ -314,6 +314,33 @@ func (t *Transaction) Logs(ctx context.Context) (*[]*Log, error) { return &ret, nil } +func (t *Transaction) R(ctx context.Context) (hexutil.Big, error) { + tx, err := t.resolve(ctx) + if err != nil || tx == nil { + return hexutil.Big{}, err + } + _, r, _ := tx.RawSignatureValues() + return hexutil.Big(*r), nil +} + +func (t *Transaction) S(ctx context.Context) (hexutil.Big, error) { + tx, err := t.resolve(ctx) + if err != nil || tx == nil { + return hexutil.Big{}, err + } + _, _, s := tx.RawSignatureValues() + return hexutil.Big(*s), nil +} + +func (t *Transaction) V(ctx context.Context) (hexutil.Big, error) { + tx, err := t.resolve(ctx) + if err != nil || tx == nil { + return hexutil.Big{}, err + } + v, _, _ := tx.RawSignatureValues() + return hexutil.Big(*v), nil +} + type BlockType int // Block represents an Ethereum block. diff --git a/graphql/schema.go b/graphql/schema.go index 525b9e1e5a..5dec10db20 100644 --- a/graphql/schema.go +++ b/graphql/schema.go @@ -115,6 +115,9 @@ const schema string = ` # Logs is a list of log entries emitted by this transaction. If the # transaction has not yet been mined, this field will be null. logs: [Log!] + r: BigInt! + s: BigInt! + v: BigInt! } # BlockFilterCriteria encapsulates log filter criteria for a filter applied From 172f7778fe498c383e0bdb234aaac4f61d4a2512 Mon Sep 17 00:00:00 2001 From: Adam Schmideg Date: Tue, 11 Feb 2020 09:48:58 +0100 Subject: [PATCH 096/128] rpc: add error when call result parameter is not addressable (#20638) --- rpc/client.go | 3 +++ rpc/client_test.go | 17 +++++++++++++++++ 2 files changed, 20 insertions(+) diff --git a/rpc/client.go b/rpc/client.go index dd06b9469e..984f56bc69 100644 --- a/rpc/client.go +++ b/rpc/client.go @@ -276,6 +276,9 @@ func (c *Client) Call(result interface{}, method string, args ...interface{}) er // The result must be a pointer so that package json can unmarshal into it. You // can also pass nil, in which case the result is ignored. func (c *Client) CallContext(ctx context.Context, result interface{}, method string, args ...interface{}) error { + if result != nil && reflect.TypeOf(result).Kind() != reflect.Ptr { + return fmt.Errorf("call result parameter must be pointer or nil interface: %v", result) + } msg, err := c.newMessage(method, args...) if err != nil { return err diff --git a/rpc/client_test.go b/rpc/client_test.go index 0d55140222..9c7f72053b 100644 --- a/rpc/client_test.go +++ b/rpc/client_test.go @@ -49,6 +49,23 @@ func TestClientRequest(t *testing.T) { } } +func TestClientResponseType(t *testing.T) { + server := newTestServer() + defer server.Stop() + client := DialInProc(server) + defer client.Close() + + if err := client.Call(nil, "test_echo", "hello", 10, &echoArgs{"world"}); err != nil { + t.Errorf("Passing nil as result should be fine, but got an error: %v", err) + } + var resultVar echoResult + // Note: passing the var, not a ref + err := client.Call(resultVar, "test_echo", "hello", 10, &echoArgs{"world"}) + if err == nil { + t.Error("Passing a var as result should be an error") + } +} + func TestClientBatchRequest(t *testing.T) { server := newTestServer() defer server.Stop() From 8694d14e655a20d70e51b7c4605d63d53de8d351 Mon Sep 17 00:00:00 2001 From: chabashilah Date: Tue, 11 Feb 2020 18:52:51 +0900 Subject: [PATCH 097/128] signer: add bytes32 as valid primitive (#20609) --- signer/core/signed_data.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/signer/core/signed_data.go b/signer/core/signed_data.go index f512be7cea..3a827afa2d 100644 --- a/signer/core/signed_data.go +++ b/signer/core/signed_data.go @@ -923,7 +923,9 @@ func isPrimitiveTypeValid(primitiveType string) bool { primitiveType == "bytes30" || primitiveType == "bytes30[]" || primitiveType == "bytes31" || - primitiveType == "bytes31[]" { + primitiveType == "bytes31[]" || + primitiveType == "bytes32" || + primitiveType == "bytes32[]" { return true } if primitiveType == "int" || From dcffb7777fc2978cf1bb243f1ebdaa5f75d1e542 Mon Sep 17 00:00:00 2001 From: winsvega Date: Tue, 11 Feb 2020 10:54:05 +0100 Subject: [PATCH 098/128] cmd/geth retesteth: add eth_getBlockByHash (#20621) --- cmd/geth/retesteth.go | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/cmd/geth/retesteth.go b/cmd/geth/retesteth.go index 28e792bd77..ab7981e518 100644 --- a/cmd/geth/retesteth.go +++ b/cmd/geth/retesteth.go @@ -80,6 +80,7 @@ type RetestethEthAPI interface { SendRawTransaction(ctx context.Context, rawTx hexutil.Bytes) (common.Hash, error) BlockNumber(ctx context.Context) (uint64, error) GetBlockByNumber(ctx context.Context, blockNr math.HexOrDecimal64, fullTx bool) (map[string]interface{}, error) + GetBlockByHash(ctx context.Context, blockHash common.Hash, fullTx bool) (map[string]interface{}, error) GetBalance(ctx context.Context, address common.Address, blockNr math.HexOrDecimal64) (*math.HexOrDecimal256, error) GetCode(ctx context.Context, address common.Address, blockNr math.HexOrDecimal64) (hexutil.Bytes, error) GetTransactionCount(ctx context.Context, address common.Address, blockNr math.HexOrDecimal64) (uint64, error) @@ -618,6 +619,20 @@ func (api *RetestethAPI) GetBlockByNumber(ctx context.Context, blockNr math.HexO return nil, fmt.Errorf("block %d not found", blockNr) } +func (api *RetestethAPI) GetBlockByHash(ctx context.Context, blockHash common.Hash, fullTx bool) (map[string]interface{}, error) { + block := api.blockchain.GetBlockByHash(blockHash) + if block != nil { + response, err := RPCMarshalBlock(block, true, fullTx) + if err != nil { + return nil, err + } + response["author"] = response["miner"] + response["totalDifficulty"] = (*hexutil.Big)(api.blockchain.GetTd(block.Hash(), block.Number().Uint64())) + return response, err + } + return nil, fmt.Errorf("block 0x%x not found", blockHash) +} + func (api *RetestethAPI) AccountRange(ctx context.Context, blockHashOrNumber *math.HexOrDecimal256, txIndex uint64, addressHash *math.HexOrDecimal256, maxResults uint64, From 049e17116e5e00a052384905a433fed5245ea5c4 Mon Sep 17 00:00:00 2001 From: rjl493456442 Date: Mon, 28 Oct 2019 19:59:07 +0800 Subject: [PATCH 099/128] core, eth: implement eth/65 transaction fetcher --- core/tx_pool.go | 6 + eth/downloader/peer.go | 8 +- eth/fetcher/{fetcher.go => block_fetcher.go} | 129 ++++--- ...{fetcher_test.go => block_fetcher_test.go} | 4 +- eth/fetcher/metrics.go | 27 +- eth/fetcher/tx_fetcher.go | 319 +++++++++++++++ eth/fetcher/tx_fetcher_test.go | 318 +++++++++++++++ eth/handler.go | 141 ++++++- eth/handler_test.go | 4 +- eth/helper_test.go | 33 +- eth/peer.go | 365 +++++++++++++++--- eth/protocol.go | 20 +- eth/protocol_test.go | 114 +++++- eth/sync.go | 59 ++- eth/sync_test.go | 10 +- 15 files changed, 1345 insertions(+), 212 deletions(-) rename eth/fetcher/{fetcher.go => block_fetcher.go} (84%) rename eth/fetcher/{fetcher_test.go => block_fetcher_test.go} (99%) create mode 100644 eth/fetcher/tx_fetcher.go create mode 100644 eth/fetcher/tx_fetcher_test.go diff --git a/core/tx_pool.go b/core/tx_pool.go index ae6962c5d9..16d80d644f 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -864,6 +864,12 @@ func (pool *TxPool) Get(hash common.Hash) *types.Transaction { return pool.all.Get(hash) } +// Has returns an indicator whether txpool has a transaction cached with the +// given hash. +func (pool *TxPool) Has(hash common.Hash) bool { + return pool.all.Get(hash) != nil +} + // removeTx removes a single transaction from the queue, moving all subsequent // transactions back to the future queue. func (pool *TxPool) removeTx(hash common.Hash, outofbound bool) { diff --git a/eth/downloader/peer.go b/eth/downloader/peer.go index 60f86d0e14..5c2020d7d8 100644 --- a/eth/downloader/peer.go +++ b/eth/downloader/peer.go @@ -470,7 +470,7 @@ func (ps *peerSet) HeaderIdlePeers() ([]*peerConnection, int) { defer p.lock.RUnlock() return p.headerThroughput } - return ps.idlePeers(62, 64, idle, throughput) + return ps.idlePeers(62, 65, idle, throughput) } // BodyIdlePeers retrieves a flat list of all the currently body-idle peers within @@ -484,7 +484,7 @@ func (ps *peerSet) BodyIdlePeers() ([]*peerConnection, int) { defer p.lock.RUnlock() return p.blockThroughput } - return ps.idlePeers(62, 64, idle, throughput) + return ps.idlePeers(62, 65, idle, throughput) } // ReceiptIdlePeers retrieves a flat list of all the currently receipt-idle peers @@ -498,7 +498,7 @@ func (ps *peerSet) ReceiptIdlePeers() ([]*peerConnection, int) { defer p.lock.RUnlock() return p.receiptThroughput } - return ps.idlePeers(63, 64, idle, throughput) + return ps.idlePeers(63, 65, idle, throughput) } // NodeDataIdlePeers retrieves a flat list of all the currently node-data-idle @@ -512,7 +512,7 @@ func (ps *peerSet) NodeDataIdlePeers() ([]*peerConnection, int) { defer p.lock.RUnlock() return p.stateThroughput } - return ps.idlePeers(63, 64, idle, throughput) + return ps.idlePeers(63, 65, idle, throughput) } // idlePeers retrieves a flat list of all currently idle peers satisfying the diff --git a/eth/fetcher/fetcher.go b/eth/fetcher/block_fetcher.go similarity index 84% rename from eth/fetcher/fetcher.go rename to eth/fetcher/block_fetcher.go index 28c532d9bd..7395ec83de 100644 --- a/eth/fetcher/fetcher.go +++ b/eth/fetcher/block_fetcher.go @@ -14,7 +14,7 @@ // You should have received a copy of the GNU Lesser General Public License // along with the go-ethereum library. If not, see . -// Package fetcher contains the block announcement based synchronisation. +// Package fetcher contains the announcement based blocks or transaction synchronisation. package fetcher import ( @@ -30,13 +30,16 @@ import ( ) const ( - arriveTimeout = 500 * time.Millisecond // Time allowance before an announced block is explicitly requested + arriveTimeout = 500 * time.Millisecond // Time allowance before an announced block/transaction is explicitly requested gatherSlack = 100 * time.Millisecond // Interval used to collate almost-expired announces with fetches - fetchTimeout = 5 * time.Second // Maximum allotted time to return an explicitly requested block - maxUncleDist = 7 // Maximum allowed backward distance from the chain head - maxQueueDist = 32 // Maximum allowed distance from the chain head to queue - hashLimit = 256 // Maximum number of unique blocks a peer may have announced - blockLimit = 64 // Maximum number of unique blocks a peer may have delivered + fetchTimeout = 5 * time.Second // Maximum allotted time to return an explicitly requested block/transaction +) + +const ( + maxUncleDist = 7 // Maximum allowed backward distance from the chain head + maxQueueDist = 32 // Maximum allowed distance from the chain head to queue + hashLimit = 256 // Maximum number of unique blocks a peer may have announced + blockLimit = 64 // Maximum number of unique blocks a peer may have delivered ) var ( @@ -67,9 +70,9 @@ type chainInsertFn func(types.Blocks) (int, error) // peerDropFn is a callback type for dropping a peer detected as malicious. type peerDropFn func(id string) -// announce is the hash notification of the availability of a new block in the +// blockAnnounce is the hash notification of the availability of a new block in the // network. -type announce struct { +type blockAnnounce struct { hash common.Hash // Hash of the block being announced number uint64 // Number of the block being announced (0 = unknown | old protocol) header *types.Header // Header of the block partially reassembled (new protocol) @@ -97,18 +100,18 @@ type bodyFilterTask struct { time time.Time // Arrival time of the blocks' contents } -// inject represents a schedules import operation. -type inject struct { +// blockInject represents a schedules import operation. +type blockInject struct { origin string block *types.Block } -// Fetcher is responsible for accumulating block announcements from various peers +// BlockFetcher is responsible for accumulating block announcements from various peers // and scheduling them for retrieval. -type Fetcher struct { +type BlockFetcher struct { // Various event channels - notify chan *announce - inject chan *inject + notify chan *blockAnnounce + inject chan *blockInject headerFilter chan chan *headerFilterTask bodyFilter chan chan *bodyFilterTask @@ -117,16 +120,16 @@ type Fetcher struct { quit chan struct{} // Announce states - announces map[string]int // Per peer announce counts to prevent memory exhaustion - announced map[common.Hash][]*announce // Announced blocks, scheduled for fetching - fetching map[common.Hash]*announce // Announced blocks, currently fetching - fetched map[common.Hash][]*announce // Blocks with headers fetched, scheduled for body retrieval - completing map[common.Hash]*announce // Blocks with headers, currently body-completing + announces map[string]int // Per peer blockAnnounce counts to prevent memory exhaustion + announced map[common.Hash][]*blockAnnounce // Announced blocks, scheduled for fetching + fetching map[common.Hash]*blockAnnounce // Announced blocks, currently fetching + fetched map[common.Hash][]*blockAnnounce // Blocks with headers fetched, scheduled for body retrieval + completing map[common.Hash]*blockAnnounce // Blocks with headers, currently body-completing // Block cache - queue *prque.Prque // Queue containing the import operations (block number sorted) - queues map[string]int // Per peer block counts to prevent memory exhaustion - queued map[common.Hash]*inject // Set of already queued blocks (to dedupe imports) + queue *prque.Prque // Queue containing the import operations (block number sorted) + queues map[string]int // Per peer block counts to prevent memory exhaustion + queued map[common.Hash]*blockInject // Set of already queued blocks (to dedupe imports) // Callbacks getBlock blockRetrievalFn // Retrieves a block from the local chain @@ -137,30 +140,30 @@ type Fetcher struct { dropPeer peerDropFn // Drops a peer for misbehaving // Testing hooks - announceChangeHook func(common.Hash, bool) // Method to call upon adding or deleting a hash from the announce list + announceChangeHook func(common.Hash, bool) // Method to call upon adding or deleting a hash from the blockAnnounce list queueChangeHook func(common.Hash, bool) // Method to call upon adding or deleting a block from the import queue fetchingHook func([]common.Hash) // Method to call upon starting a block (eth/61) or header (eth/62) fetch completingHook func([]common.Hash) // Method to call upon starting a block body fetch (eth/62) importedHook func(*types.Block) // Method to call upon successful block import (both eth/61 and eth/62) } -// New creates a block fetcher to retrieve blocks based on hash announcements. -func New(getBlock blockRetrievalFn, verifyHeader headerVerifierFn, broadcastBlock blockBroadcasterFn, chainHeight chainHeightFn, insertChain chainInsertFn, dropPeer peerDropFn) *Fetcher { - return &Fetcher{ - notify: make(chan *announce), - inject: make(chan *inject), +// NewBlockFetcher creates a block fetcher to retrieve blocks based on hash announcements. +func NewBlockFetcher(getBlock blockRetrievalFn, verifyHeader headerVerifierFn, broadcastBlock blockBroadcasterFn, chainHeight chainHeightFn, insertChain chainInsertFn, dropPeer peerDropFn) *BlockFetcher { + return &BlockFetcher{ + notify: make(chan *blockAnnounce), + inject: make(chan *blockInject), headerFilter: make(chan chan *headerFilterTask), bodyFilter: make(chan chan *bodyFilterTask), done: make(chan common.Hash), quit: make(chan struct{}), announces: make(map[string]int), - announced: make(map[common.Hash][]*announce), - fetching: make(map[common.Hash]*announce), - fetched: make(map[common.Hash][]*announce), - completing: make(map[common.Hash]*announce), + announced: make(map[common.Hash][]*blockAnnounce), + fetching: make(map[common.Hash]*blockAnnounce), + fetched: make(map[common.Hash][]*blockAnnounce), + completing: make(map[common.Hash]*blockAnnounce), queue: prque.New(nil), queues: make(map[string]int), - queued: make(map[common.Hash]*inject), + queued: make(map[common.Hash]*blockInject), getBlock: getBlock, verifyHeader: verifyHeader, broadcastBlock: broadcastBlock, @@ -172,21 +175,21 @@ func New(getBlock blockRetrievalFn, verifyHeader headerVerifierFn, broadcastBloc // Start boots up the announcement based synchroniser, accepting and processing // hash notifications and block fetches until termination requested. -func (f *Fetcher) Start() { +func (f *BlockFetcher) Start() { go f.loop() } // Stop terminates the announcement based synchroniser, canceling all pending // operations. -func (f *Fetcher) Stop() { +func (f *BlockFetcher) Stop() { close(f.quit) } // Notify announces the fetcher of the potential availability of a new block in // the network. -func (f *Fetcher) Notify(peer string, hash common.Hash, number uint64, time time.Time, +func (f *BlockFetcher) Notify(peer string, hash common.Hash, number uint64, time time.Time, headerFetcher headerRequesterFn, bodyFetcher bodyRequesterFn) error { - block := &announce{ + block := &blockAnnounce{ hash: hash, number: number, time: time, @@ -203,8 +206,8 @@ func (f *Fetcher) Notify(peer string, hash common.Hash, number uint64, time time } // Enqueue tries to fill gaps the fetcher's future import queue. -func (f *Fetcher) Enqueue(peer string, block *types.Block) error { - op := &inject{ +func (f *BlockFetcher) Enqueue(peer string, block *types.Block) error { + op := &blockInject{ origin: peer, block: block, } @@ -218,7 +221,7 @@ func (f *Fetcher) Enqueue(peer string, block *types.Block) error { // FilterHeaders extracts all the headers that were explicitly requested by the fetcher, // returning those that should be handled differently. -func (f *Fetcher) FilterHeaders(peer string, headers []*types.Header, time time.Time) []*types.Header { +func (f *BlockFetcher) FilterHeaders(peer string, headers []*types.Header, time time.Time) []*types.Header { log.Trace("Filtering headers", "peer", peer, "headers", len(headers)) // Send the filter channel to the fetcher @@ -246,7 +249,7 @@ func (f *Fetcher) FilterHeaders(peer string, headers []*types.Header, time time. // FilterBodies extracts all the block bodies that were explicitly requested by // the fetcher, returning those that should be handled differently. -func (f *Fetcher) FilterBodies(peer string, transactions [][]*types.Transaction, uncles [][]*types.Header, time time.Time) ([][]*types.Transaction, [][]*types.Header) { +func (f *BlockFetcher) FilterBodies(peer string, transactions [][]*types.Transaction, uncles [][]*types.Header, time time.Time) ([][]*types.Transaction, [][]*types.Header) { log.Trace("Filtering bodies", "peer", peer, "txs", len(transactions), "uncles", len(uncles)) // Send the filter channel to the fetcher @@ -274,7 +277,7 @@ func (f *Fetcher) FilterBodies(peer string, transactions [][]*types.Transaction, // Loop is the main fetcher loop, checking and processing various notification // events. -func (f *Fetcher) loop() { +func (f *BlockFetcher) loop() { // Iterate the block fetching until a quit is requested fetchTimer := time.NewTimer(0) completeTimer := time.NewTimer(0) @@ -289,7 +292,7 @@ func (f *Fetcher) loop() { // Import any queued blocks that could potentially fit height := f.chainHeight() for !f.queue.Empty() { - op := f.queue.PopItem().(*inject) + op := f.queue.PopItem().(*blockInject) hash := op.block.Hash() if f.queueChangeHook != nil { f.queueChangeHook(hash, false) @@ -313,24 +316,24 @@ func (f *Fetcher) loop() { // Wait for an outside event to occur select { case <-f.quit: - // Fetcher terminating, abort all operations + // BlockFetcher terminating, abort all operations return case notification := <-f.notify: // A block was announced, make sure the peer isn't DOSing us - propAnnounceInMeter.Mark(1) + blockAnnounceInMeter.Mark(1) count := f.announces[notification.origin] + 1 if count > hashLimit { log.Debug("Peer exceeded outstanding announces", "peer", notification.origin, "limit", hashLimit) - propAnnounceDOSMeter.Mark(1) + blockAnnounceDOSMeter.Mark(1) break } // If we have a valid block number, check that it's potentially useful if notification.number > 0 { if dist := int64(notification.number) - int64(f.chainHeight()); dist < -maxUncleDist || dist > maxQueueDist { log.Debug("Peer discarded announcement", "peer", notification.origin, "number", notification.number, "hash", notification.hash, "distance", dist) - propAnnounceDropMeter.Mark(1) + blockAnnounceDropMeter.Mark(1) break } } @@ -352,7 +355,7 @@ func (f *Fetcher) loop() { case op := <-f.inject: // A direct block insertion was requested, try and fill any pending gaps - propBroadcastInMeter.Mark(1) + blockBroadcastInMeter.Mark(1) f.enqueue(op.origin, op.block) case hash := <-f.done: @@ -439,7 +442,7 @@ func (f *Fetcher) loop() { // Split the batch of headers into unknown ones (to return to the caller), // known incomplete ones (requiring body retrievals) and completed blocks. - unknown, incomplete, complete := []*types.Header{}, []*announce{}, []*types.Block{} + unknown, incomplete, complete := []*types.Header{}, []*blockAnnounce{}, []*types.Block{} for _, header := range task.headers { hash := header.Hash() @@ -475,7 +478,7 @@ func (f *Fetcher) loop() { f.forgetHash(hash) } } else { - // Fetcher doesn't know about it, add to the return list + // BlockFetcher doesn't know about it, add to the return list unknown = append(unknown, header) } } @@ -562,8 +565,8 @@ func (f *Fetcher) loop() { } } -// rescheduleFetch resets the specified fetch timer to the next announce timeout. -func (f *Fetcher) rescheduleFetch(fetch *time.Timer) { +// rescheduleFetch resets the specified fetch timer to the next blockAnnounce timeout. +func (f *BlockFetcher) rescheduleFetch(fetch *time.Timer) { // Short circuit if no blocks are announced if len(f.announced) == 0 { return @@ -579,7 +582,7 @@ func (f *Fetcher) rescheduleFetch(fetch *time.Timer) { } // rescheduleComplete resets the specified completion timer to the next fetch timeout. -func (f *Fetcher) rescheduleComplete(complete *time.Timer) { +func (f *BlockFetcher) rescheduleComplete(complete *time.Timer) { // Short circuit if no headers are fetched if len(f.fetched) == 0 { return @@ -596,27 +599,27 @@ func (f *Fetcher) rescheduleComplete(complete *time.Timer) { // enqueue schedules a new future import operation, if the block to be imported // has not yet been seen. -func (f *Fetcher) enqueue(peer string, block *types.Block) { +func (f *BlockFetcher) enqueue(peer string, block *types.Block) { hash := block.Hash() // Ensure the peer isn't DOSing us count := f.queues[peer] + 1 if count > blockLimit { log.Debug("Discarded propagated block, exceeded allowance", "peer", peer, "number", block.Number(), "hash", hash, "limit", blockLimit) - propBroadcastDOSMeter.Mark(1) + blockBroadcastDOSMeter.Mark(1) f.forgetHash(hash) return } // Discard any past or too distant blocks if dist := int64(block.NumberU64()) - int64(f.chainHeight()); dist < -maxUncleDist || dist > maxQueueDist { log.Debug("Discarded propagated block, too far away", "peer", peer, "number", block.Number(), "hash", hash, "distance", dist) - propBroadcastDropMeter.Mark(1) + blockBroadcastDropMeter.Mark(1) f.forgetHash(hash) return } // Schedule the block for future importing if _, ok := f.queued[hash]; !ok { - op := &inject{ + op := &blockInject{ origin: peer, block: block, } @@ -633,7 +636,7 @@ func (f *Fetcher) enqueue(peer string, block *types.Block) { // insert spawns a new goroutine to run a block insertion into the chain. If the // block's number is at the same height as the current import phase, it updates // the phase states accordingly. -func (f *Fetcher) insert(peer string, block *types.Block) { +func (f *BlockFetcher) insert(peer string, block *types.Block) { hash := block.Hash() // Run the import on a new thread @@ -651,7 +654,7 @@ func (f *Fetcher) insert(peer string, block *types.Block) { switch err := f.verifyHeader(block.Header()); err { case nil: // All ok, quickly propagate to our peers - propBroadcastOutTimer.UpdateSince(block.ReceivedAt) + blockBroadcastOutTimer.UpdateSince(block.ReceivedAt) go f.broadcastBlock(block, true) case consensus.ErrFutureBlock: @@ -669,7 +672,7 @@ func (f *Fetcher) insert(peer string, block *types.Block) { return } // If import succeeded, broadcast the block - propAnnounceOutTimer.UpdateSince(block.ReceivedAt) + blockAnnounceOutTimer.UpdateSince(block.ReceivedAt) go f.broadcastBlock(block, false) // Invoke the testing hook if needed @@ -681,7 +684,7 @@ func (f *Fetcher) insert(peer string, block *types.Block) { // forgetHash removes all traces of a block announcement from the fetcher's // internal state. -func (f *Fetcher) forgetHash(hash common.Hash) { +func (f *BlockFetcher) forgetHash(hash common.Hash) { // Remove all pending announces and decrement DOS counters for _, announce := range f.announced[hash] { f.announces[announce.origin]-- @@ -723,7 +726,7 @@ func (f *Fetcher) forgetHash(hash common.Hash) { // forgetBlock removes all traces of a queued block from the fetcher's internal // state. -func (f *Fetcher) forgetBlock(hash common.Hash) { +func (f *BlockFetcher) forgetBlock(hash common.Hash) { if insert := f.queued[hash]; insert != nil { f.queues[insert.origin]-- if f.queues[insert.origin] == 0 { diff --git a/eth/fetcher/fetcher_test.go b/eth/fetcher/block_fetcher_test.go similarity index 99% rename from eth/fetcher/fetcher_test.go rename to eth/fetcher/block_fetcher_test.go index 83172c5348..038ead12e7 100644 --- a/eth/fetcher/fetcher_test.go +++ b/eth/fetcher/block_fetcher_test.go @@ -76,7 +76,7 @@ func makeChain(n int, seed byte, parent *types.Block) ([]common.Hash, map[common // fetcherTester is a test simulator for mocking out local block chain. type fetcherTester struct { - fetcher *Fetcher + fetcher *BlockFetcher hashes []common.Hash // Hash chain belonging to the tester blocks map[common.Hash]*types.Block // Blocks belonging to the tester @@ -92,7 +92,7 @@ func newTester() *fetcherTester { blocks: map[common.Hash]*types.Block{genesis.Hash(): genesis}, drops: make(map[string]bool), } - tester.fetcher = New(tester.getBlock, tester.verifyHeader, tester.broadcastBlock, tester.chainHeight, tester.insertChain, tester.dropPeer) + tester.fetcher = NewBlockFetcher(tester.getBlock, tester.verifyHeader, tester.broadcastBlock, tester.chainHeight, tester.insertChain, tester.dropPeer) tester.fetcher.Start() return tester diff --git a/eth/fetcher/metrics.go b/eth/fetcher/metrics.go index d68d12f000..b75889938d 100644 --- a/eth/fetcher/metrics.go +++ b/eth/fetcher/metrics.go @@ -23,15 +23,15 @@ import ( ) var ( - propAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/announces/in", nil) - propAnnounceOutTimer = metrics.NewRegisteredTimer("eth/fetcher/prop/announces/out", nil) - propAnnounceDropMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/announces/drop", nil) - propAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/announces/dos", nil) + blockAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/announces/in", nil) + blockAnnounceOutTimer = metrics.NewRegisteredTimer("eth/fetcher/prop/block/announces/out", nil) + blockAnnounceDropMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/announces/drop", nil) + blockAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/announces/dos", nil) - propBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/broadcasts/in", nil) - propBroadcastOutTimer = metrics.NewRegisteredTimer("eth/fetcher/prop/broadcasts/out", nil) - propBroadcastDropMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/broadcasts/drop", nil) - propBroadcastDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/broadcasts/dos", nil) + blockBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/broadcasts/in", nil) + blockBroadcastOutTimer = metrics.NewRegisteredTimer("eth/fetcher/prop/block/broadcasts/out", nil) + blockBroadcastDropMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/broadcasts/drop", nil) + blockBroadcastDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/broadcasts/dos", nil) headerFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/headers", nil) bodyFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/bodies", nil) @@ -40,4 +40,15 @@ var ( headerFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/headers/out", nil) bodyFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/bodies/in", nil) bodyFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/bodies/out", nil) + + txAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/transaction/announces/in", nil) + txAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/transaction/announces/dos", nil) + txAnnounceSkipMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/transaction/announces/skip", nil) + txAnnounceUnderpriceMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/transaction/announces/underprice", nil) + txBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/transaction/broadcasts/in", nil) + txFetchOutMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/transaction/out", nil) + txFetchSuccessMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/transaction/success", nil) + txFetchTimeoutMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/transaction/timeout", nil) + txFetchInvalidMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/transaction/invalid", nil) + txFetchDurationTimer = metrics.NewRegisteredTimer("eth/fetcher/fetch/transaction/duration", nil) ) diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go new file mode 100644 index 0000000000..1dabb0819f --- /dev/null +++ b/eth/fetcher/tx_fetcher.go @@ -0,0 +1,319 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package fetcher + +import ( + "math/rand" + "time" + + mapset "github.com/deckarep/golang-set" + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/log" +) + +var ( + // txAnnounceLimit is the maximum number of unique transaction a peer + // can announce in a short time. + txAnnounceLimit = 4096 + + // txFetchTimeout is the maximum allotted time to return an explicitly + // requested transaction. + txFetchTimeout = 5 * time.Second + + // MaxTransactionFetch is the maximum transaction number can be fetched + // in one request. The rationale to pick this value is: + // In eth protocol, the softResponseLimit is 2MB. Nowdays according to + // Etherscan the average transaction size is around 200B, so in theory + // we can include lots of transaction in a single protocol packet. However + // the maximum size of a single transaction is raised to 128KB, so pick + // a middle value here to ensure we can maximize the efficiency of the + // retrieval and response size overflow won't happen in most cases. + MaxTransactionFetch = 256 + + // underpriceSetSize is the size of underprice set which used for maintaining + // the set of underprice transactions. + underpriceSetSize = 4096 +) + +// txAnnounce is the notification of the availability of a single +// new transaction in the network. +type txAnnounce struct { + origin string // Identifier of the peer originating the notification + time time.Time // Timestamp of the announcement + fetchTxs func([]common.Hash) // Callback for retrieving transaction from specified peer +} + +// txsAnnounce is the notification of the availability of a batch +// of new transactions in the network. +type txsAnnounce struct { + hashes []common.Hash // Batch of transaction hashes being announced + origin string // Identifier of the peer originating the notification + time time.Time // Timestamp of the announcement + fetchTxs func([]common.Hash) // Callback for retrieving transaction from specified peer +} + +// TxFetcher is responsible for retrieving new transaction based +// on the announcement. +type TxFetcher struct { + notify chan *txsAnnounce + cleanup chan []common.Hash + quit chan struct{} + + // Announce states + announces map[string]int // Per peer transaction announce counts to prevent memory exhaustion + announced map[common.Hash][]*txAnnounce // Announced transactions, scheduled for fetching + fetching map[common.Hash]*txAnnounce // Announced transactions, currently fetching + underpriced mapset.Set // Transaction set whose price is too low for accepting + + // Callbacks + hasTx func(common.Hash) bool // Retrieves a tx from the local txpool + addTxs func([]*types.Transaction) []error // Insert a batch of transactions into local txpool + dropPeer func(string) // Drop the specified peer + + // Hooks + announceHook func([]common.Hash) // Hook which is called when a batch transactions are announced + importTxsHook func([]*types.Transaction) // Hook which is called when a batch of transactions are imported. + dropHook func(string) // Hook which is called when a peer is dropped + cleanupHook func([]common.Hash) // Hook which is called when internal status is cleaned + rejectUnderprice func(common.Hash) // Hook which is called when underprice transaction is rejected +} + +// NewTxFetcher creates a transaction fetcher to retrieve transaction +// based on hash announcements. +func NewTxFetcher(hasTx func(common.Hash) bool, addTxs func([]*types.Transaction) []error, dropPeer func(string)) *TxFetcher { + return &TxFetcher{ + notify: make(chan *txsAnnounce), + cleanup: make(chan []common.Hash), + quit: make(chan struct{}), + announces: make(map[string]int), + announced: make(map[common.Hash][]*txAnnounce), + fetching: make(map[common.Hash]*txAnnounce), + underpriced: mapset.NewSet(), + hasTx: hasTx, + addTxs: addTxs, + dropPeer: dropPeer, + } +} + +// Notify announces the fetcher of the potential availability of a +// new transaction in the network. +func (f *TxFetcher) Notify(peer string, hashes []common.Hash, time time.Time, fetchTxs func([]common.Hash)) error { + announce := &txsAnnounce{ + hashes: hashes, + time: time, + origin: peer, + fetchTxs: fetchTxs, + } + select { + case f.notify <- announce: + return nil + case <-f.quit: + return errTerminated + } +} + +// EnqueueTxs imports a batch of received transaction into fetcher. +func (f *TxFetcher) EnqueueTxs(peer string, txs []*types.Transaction) error { + var ( + drop bool + hashes []common.Hash + ) + errs := f.addTxs(txs) + for i, err := range errs { + if err != nil { + // Drop peer if the received transaction isn't signed properly. + drop = (drop || err == core.ErrInvalidSender) + txFetchInvalidMeter.Mark(1) + + // Track the transaction hash if the price is too low for us. + // Avoid re-request this transaction when we receive another + // announcement. + if err == core.ErrUnderpriced { + for f.underpriced.Cardinality() >= underpriceSetSize { + f.underpriced.Pop() + } + f.underpriced.Add(txs[i].Hash()) + } + } + hashes = append(hashes, txs[i].Hash()) + } + if f.importTxsHook != nil { + f.importTxsHook(txs) + } + // Drop the peer if some transaction failed signature verification. + // We can regard this peer is trying to DOS us by feeding lots of + // random hashes. + if drop { + f.dropPeer(peer) + if f.dropHook != nil { + f.dropHook(peer) + } + } + select { + case f.cleanup <- hashes: + return nil + case <-f.quit: + return errTerminated + } +} + +// Start boots up the announcement based synchroniser, accepting and processing +// hash notifications and block fetches until termination requested. +func (f *TxFetcher) Start() { + go f.loop() +} + +// Stop terminates the announcement based synchroniser, canceling all pending +// operations. +func (f *TxFetcher) Stop() { + close(f.quit) +} + +func (f *TxFetcher) loop() { + fetchTimer := time.NewTimer(0) + + for { + // Clean up any expired transaction fetches. + // There are many cases can lead to it: + // * We send the request to busy peer which can reply immediately + // * We send the request to malicious peer which doesn't reply deliberately + // * We send the request to normal peer for a batch of transaction, but some + // transactions have been included into blocks. According to EIP these txs + // won't be included. + // But it's fine to delete the fetching record and reschedule fetching iff we + // receive the annoucement again. + for hash, announce := range f.fetching { + if time.Since(announce.time) > txFetchTimeout { + delete(f.fetching, hash) + txFetchTimeoutMeter.Mark(1) + } + } + select { + case anno := <-f.notify: + txAnnounceInMeter.Mark(int64(len(anno.hashes))) + + // Drop the new announce if there are too many accumulated. + count := f.announces[anno.origin] + len(anno.hashes) + if count > txAnnounceLimit { + txAnnounceDOSMeter.Mark(int64(count - txAnnounceLimit)) + break + } + f.announces[anno.origin] = count + + // All is well, schedule the announce if transaction is not yet downloading + empty := len(f.announced) == 0 + for _, hash := range anno.hashes { + if _, ok := f.fetching[hash]; ok { + continue + } + if f.underpriced.Contains(hash) { + txAnnounceUnderpriceMeter.Mark(1) + if f.rejectUnderprice != nil { + f.rejectUnderprice(hash) + } + continue + } + f.announced[hash] = append(f.announced[hash], &txAnnounce{ + origin: anno.origin, + time: anno.time, + fetchTxs: anno.fetchTxs, + }) + } + if empty && len(f.announced) > 0 { + f.reschedule(fetchTimer) + } + if f.announceHook != nil { + f.announceHook(anno.hashes) + } + case <-fetchTimer.C: + // At least one tx's timer ran out, check for needing retrieval + request := make(map[string][]common.Hash) + + for hash, announces := range f.announced { + if time.Since(announces[0].time) > arriveTimeout-gatherSlack { + // Pick a random peer to retrieve from, reset all others + announce := announces[rand.Intn(len(announces))] + f.forgetHash(hash) + + // Skip fetching if we already receive the transaction. + if f.hasTx(hash) { + txAnnounceSkipMeter.Mark(1) + continue + } + // If the transaction still didn't arrive, queue for fetching + request[announce.origin] = append(request[announce.origin], hash) + f.fetching[hash] = announce + } + } + // Send out all block header requests + for peer, hashes := range request { + log.Trace("Fetching scheduled transactions", "peer", peer, "txs", hashes) + fetchTxs := f.fetching[hashes[0]].fetchTxs + fetchTxs(hashes) + txFetchOutMeter.Mark(int64(len(hashes))) + } + // Schedule the next fetch if blocks are still pending + f.reschedule(fetchTimer) + case hashes := <-f.cleanup: + for _, hash := range hashes { + f.forgetHash(hash) + anno, exist := f.fetching[hash] + if !exist { + txBroadcastInMeter.Mark(1) // Directly transaction propagation + continue + } + txFetchDurationTimer.UpdateSince(anno.time) + txFetchSuccessMeter.Mark(1) + delete(f.fetching, hash) + } + if f.cleanupHook != nil { + f.cleanupHook(hashes) + } + case <-f.quit: + return + } + } +} + +// rescheduleFetch resets the specified fetch timer to the next blockAnnounce timeout. +func (f *TxFetcher) reschedule(fetch *time.Timer) { + // Short circuit if no transactions are announced + if len(f.announced) == 0 { + return + } + // Otherwise find the earliest expiring announcement + earliest := time.Now() + for _, announces := range f.announced { + if earliest.After(announces[0].time) { + earliest = announces[0].time + } + } + fetch.Reset(arriveTimeout - time.Since(earliest)) +} + +func (f *TxFetcher) forgetHash(hash common.Hash) { + // Remove all pending announces and decrement DOS counters + for _, announce := range f.announced[hash] { + f.announces[announce.origin]-- + if f.announces[announce.origin] <= 0 { + delete(f.announces, announce.origin) + } + } + delete(f.announced, hash) +} diff --git a/eth/fetcher/tx_fetcher_test.go b/eth/fetcher/tx_fetcher_test.go new file mode 100644 index 0000000000..26f24f3f38 --- /dev/null +++ b/eth/fetcher/tx_fetcher_test.go @@ -0,0 +1,318 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package fetcher + +import ( + "crypto/ecdsa" + "math/big" + "math/rand" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/core" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/crypto" +) + +func init() { + rand.Seed(int64(time.Now().Nanosecond())) + + txAnnounceLimit = 64 + MaxTransactionFetch = 16 +} + +func makeTransactions(key *ecdsa.PrivateKey, target int) []*types.Transaction { + var txs []*types.Transaction + + for i := 0; i < target; i++ { + random := rand.Uint32() + tx := types.NewTransaction(uint64(random), common.Address{0x1, 0x2, 0x3}, big.NewInt(int64(random)), 100, big.NewInt(int64(random)), nil) + tx, _ = types.SignTx(tx, types.NewEIP155Signer(big.NewInt(1)), key) + txs = append(txs, tx) + } + return txs +} + +func makeUnsignedTransactions(key *ecdsa.PrivateKey, target int) []*types.Transaction { + var txs []*types.Transaction + + for i := 0; i < target; i++ { + random := rand.Uint32() + tx := types.NewTransaction(uint64(random), common.Address{0x1, 0x2, 0x3}, big.NewInt(int64(random)), 100, big.NewInt(int64(random)), nil) + txs = append(txs, tx) + } + return txs +} + +type txfetcherTester struct { + fetcher *TxFetcher + + priceLimit *big.Int + sender *ecdsa.PrivateKey + senderAddr common.Address + signer types.Signer + txs map[common.Hash]*types.Transaction + dropped map[string]struct{} + lock sync.RWMutex +} + +func newTxFetcherTester() *txfetcherTester { + key, _ := crypto.GenerateKey() + addr := crypto.PubkeyToAddress(key.PublicKey) + t := &txfetcherTester{ + sender: key, + senderAddr: addr, + signer: types.NewEIP155Signer(big.NewInt(1)), + txs: make(map[common.Hash]*types.Transaction), + dropped: make(map[string]struct{}), + } + t.fetcher = NewTxFetcher(t.hasTx, t.addTxs, t.dropPeer) + t.fetcher.Start() + return t +} + +func (t *txfetcherTester) hasTx(hash common.Hash) bool { + t.lock.RLock() + defer t.lock.RUnlock() + + return t.txs[hash] != nil +} + +func (t *txfetcherTester) addTxs(txs []*types.Transaction) []error { + t.lock.Lock() + defer t.lock.Unlock() + + var errors []error + for _, tx := range txs { + // Make sure the transaction is signed properly + _, err := types.Sender(t.signer, tx) + if err != nil { + errors = append(errors, core.ErrInvalidSender) + continue + } + // Make sure the price is high enough to accpet + if t.priceLimit != nil && tx.GasPrice().Cmp(t.priceLimit) < 0 { + errors = append(errors, core.ErrUnderpriced) + continue + } + t.txs[tx.Hash()] = tx + errors = append(errors, nil) + } + return errors +} + +func (t *txfetcherTester) dropPeer(id string) { + t.lock.Lock() + defer t.lock.Unlock() + + t.dropped[id] = struct{}{} +} + +// makeTxFetcher retrieves a batch of transaction associated with a simulated peer. +func (t *txfetcherTester) makeTxFetcher(peer string, txs []*types.Transaction) func(hashes []common.Hash) { + closure := make(map[common.Hash]*types.Transaction) + for _, tx := range txs { + closure[tx.Hash()] = tx + } + return func(hashes []common.Hash) { + var txs []*types.Transaction + for _, hash := range hashes { + tx := closure[hash] + if tx == nil { + continue + } + txs = append(txs, tx) + } + // Return on a new thread + go t.fetcher.EnqueueTxs(peer, txs) + } +} + +func TestSequentialTxAnnouncements(t *testing.T) { + tester := newTxFetcherTester() + txs := makeTransactions(tester.sender, txAnnounceLimit) + + retrieveTxs := tester.makeTxFetcher("peer", txs) + + newTxsCh := make(chan struct{}) + tester.fetcher.importTxsHook = func(transactions []*types.Transaction) { + newTxsCh <- struct{}{} + } + for _, tx := range txs { + tester.fetcher.Notify("peer", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout), retrieveTxs) + select { + case <-newTxsCh: + case <-time.NewTimer(time.Second).C: + t.Fatalf("timeout") + } + } + if len(tester.txs) != len(txs) { + t.Fatalf("Imported transaction number mismatch, want %d, got %d", len(txs), len(tester.txs)) + } +} + +func TestConcurrentAnnouncements(t *testing.T) { + tester := newTxFetcherTester() + txs := makeTransactions(tester.sender, txAnnounceLimit) + + txFetcherFn1 := tester.makeTxFetcher("peer1", txs) + txFetcherFn2 := tester.makeTxFetcher("peer2", txs) + + var ( + count uint32 + done = make(chan struct{}) + ) + tester.fetcher.importTxsHook = func(transactions []*types.Transaction) { + atomic.AddUint32(&count, uint32(len(transactions))) + if atomic.LoadUint32(&count) >= uint32(txAnnounceLimit) { + done <- struct{}{} + } + } + for _, tx := range txs { + tester.fetcher.Notify("peer1", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout), txFetcherFn1) + tester.fetcher.Notify("peer2", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout+time.Millisecond), txFetcherFn2) + tester.fetcher.Notify("peer2", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout-time.Millisecond), txFetcherFn2) + } + select { + case <-done: + case <-time.NewTimer(time.Second).C: + t.Fatalf("timeout") + } +} + +func TestBatchAnnouncements(t *testing.T) { + tester := newTxFetcherTester() + txs := makeTransactions(tester.sender, txAnnounceLimit) + + retrieveTxs := tester.makeTxFetcher("peer", txs) + + var count uint32 + var done = make(chan struct{}) + tester.fetcher.importTxsHook = func(txs []*types.Transaction) { + atomic.AddUint32(&count, uint32(len(txs))) + + if atomic.LoadUint32(&count) >= uint32(txAnnounceLimit) { + done <- struct{}{} + } + } + // Send all announces which exceeds the limit. + var hashes []common.Hash + for _, tx := range txs { + hashes = append(hashes, tx.Hash()) + } + tester.fetcher.Notify("peer", hashes, time.Now(), retrieveTxs) + + select { + case <-done: + case <-time.NewTimer(time.Second).C: + t.Fatalf("timeout") + } +} + +func TestPropagationAfterAnnounce(t *testing.T) { + tester := newTxFetcherTester() + txs := makeTransactions(tester.sender, txAnnounceLimit) + + var cleaned = make(chan struct{}) + tester.fetcher.cleanupHook = func(hashes []common.Hash) { + cleaned <- struct{}{} + } + retrieveTxs := tester.makeTxFetcher("peer", txs) + for _, tx := range txs { + tester.fetcher.Notify("peer", []common.Hash{tx.Hash()}, time.Now(), retrieveTxs) + tester.fetcher.EnqueueTxs("peer", []*types.Transaction{tx}) + + // It's ok to read the map directly since no write + // will happen in the same time. + <-cleaned + if len(tester.fetcher.announced) != 0 { + t.Fatalf("Announcement should be cleaned, got %d", len(tester.fetcher.announced)) + } + } +} + +func TestEnqueueTransactions(t *testing.T) { + tester := newTxFetcherTester() + txs := makeTransactions(tester.sender, txAnnounceLimit) + + done := make(chan struct{}) + tester.fetcher.importTxsHook = func(transactions []*types.Transaction) { + if len(transactions) == txAnnounceLimit { + done <- struct{}{} + } + } + go tester.fetcher.EnqueueTxs("peer", txs) + select { + case <-done: + case <-time.NewTimer(time.Second).C: + t.Fatalf("timeout") + } +} + +func TestInvalidTxAnnounces(t *testing.T) { + tester := newTxFetcherTester() + + var txs []*types.Transaction + txs = append(txs, makeUnsignedTransactions(tester.sender, 1)...) + txs = append(txs, makeTransactions(tester.sender, 1)...) + + txFetcherFn := tester.makeTxFetcher("peer", txs) + + dropped := make(chan string, 1) + tester.fetcher.dropHook = func(s string) { dropped <- s } + + for _, tx := range txs { + tester.fetcher.Notify("peer", []common.Hash{tx.Hash()}, time.Now(), txFetcherFn) + } + select { + case s := <-dropped: + if s != "peer" { + t.Fatalf("invalid dropped peer") + } + case <-time.NewTimer(time.Second).C: + t.Fatalf("timeout") + } +} + +func TestRejectUnderpriced(t *testing.T) { + tester := newTxFetcherTester() + tester.priceLimit = big.NewInt(10000) + + done := make(chan struct{}) + tester.fetcher.importTxsHook = func([]*types.Transaction) { done <- struct{}{} } + reject := make(chan struct{}) + tester.fetcher.rejectUnderprice = func(common.Hash) { reject <- struct{}{} } + + tx := types.NewTransaction(0, common.Address{0x1, 0x2, 0x3}, big.NewInt(int64(100)), 100, big.NewInt(int64(100)), nil) + tx, _ = types.SignTx(tx, types.NewEIP155Signer(big.NewInt(1)), tester.sender) + txFetcherFn := tester.makeTxFetcher("peer", []*types.Transaction{tx}) + + // Send the announcement first time + tester.fetcher.Notify("peer", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout), txFetcherFn) + <-done + + // Resend the announcement, shouldn't schedule fetching this time + tester.fetcher.Notify("peer", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout), txFetcherFn) + select { + case <-reject: + case <-time.NewTimer(time.Second).C: + t.Fatalf("timeout") + } +} diff --git a/eth/handler.go b/eth/handler.go index ae2b764cf6..d527b15d13 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -77,9 +77,10 @@ type ProtocolManager struct { blockchain *core.BlockChain maxPeers int - downloader *downloader.Downloader - fetcher *fetcher.Fetcher - peers *peerSet + downloader *downloader.Downloader + blockFetcher *fetcher.BlockFetcher + txFetcher *fetcher.TxFetcher + peers *peerSet eventMux *event.TypeMux txsCh chan core.NewTxsEvent @@ -97,6 +98,9 @@ type ProtocolManager struct { // wait group is used for graceful shutdowns during downloading // and processing wg sync.WaitGroup + + // Test fields or hooks + broadcastTxAnnouncesOnly bool // Testing field, disable transaction propagation } // NewProtocolManager returns a new Ethereum sub protocol manager. The Ethereum sub protocol manages peers capable @@ -187,7 +191,8 @@ func NewProtocolManager(config *params.ChainConfig, checkpoint *params.TrustedCh } return n, err } - manager.fetcher = fetcher.New(blockchain.GetBlockByHash, validator, manager.BroadcastBlock, heighter, inserter, manager.removePeer) + manager.blockFetcher = fetcher.NewBlockFetcher(blockchain.GetBlockByHash, validator, manager.BroadcastBlock, heighter, inserter, manager.removePeer) + manager.txFetcher = fetcher.NewTxFetcher(txpool.Has, txpool.AddRemotes, manager.removePeer) return manager, nil } @@ -203,7 +208,7 @@ func (pm *ProtocolManager) makeProtocol(version uint) p2p.Protocol { Version: version, Length: length, Run: func(p *p2p.Peer, rw p2p.MsgReadWriter) error { - peer := pm.newPeer(int(version), p, rw) + peer := pm.newPeer(int(version), p, rw, pm.txpool.Get) select { case pm.newPeerCh <- peer: pm.wg.Add(1) @@ -286,8 +291,8 @@ func (pm *ProtocolManager) Stop() { log.Info("Ethereum protocol stopped") } -func (pm *ProtocolManager) newPeer(pv int, p *p2p.Peer, rw p2p.MsgReadWriter) *peer { - return newPeer(pv, p, newMeteredMsgWriter(rw)) +func (pm *ProtocolManager) newPeer(pv int, p *p2p.Peer, rw p2p.MsgReadWriter, getPooledTx func(hash common.Hash) *types.Transaction) *peer { + return newPeer(pv, p, newMeteredMsgWriter(rw), getPooledTx) } // handle is the callback invoked to manage the life cycle of an eth peer. When @@ -514,7 +519,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { p.Log().Debug("Whitelist block verified", "number", headers[0].Number.Uint64(), "hash", want) } // Irrelevant of the fork checks, send the header to the fetcher just in case - headers = pm.fetcher.FilterHeaders(p.id, headers, time.Now()) + headers = pm.blockFetcher.FilterHeaders(p.id, headers, time.Now()) } if len(headers) > 0 || !filter { err := pm.downloader.DeliverHeaders(p.id, headers) @@ -567,7 +572,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // Filter out any explicitly requested bodies, deliver the rest to the downloader filter := len(transactions) > 0 || len(uncles) > 0 if filter { - transactions, uncles = pm.fetcher.FilterBodies(p.id, transactions, uncles, time.Now()) + transactions, uncles = pm.blockFetcher.FilterBodies(p.id, transactions, uncles, time.Now()) } if len(transactions) > 0 || len(uncles) > 0 || !filter { err := pm.downloader.DeliverBodies(p.id, transactions, uncles) @@ -678,7 +683,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } } for _, block := range unknown { - pm.fetcher.Notify(p.id, block.Hash, block.Number, time.Now(), p.RequestOneHeader, p.RequestBodies) + pm.blockFetcher.Notify(p.id, block.Hash, block.Number, time.Now(), p.RequestOneHeader, p.RequestBodies) } case msg.Code == NewBlockMsg: @@ -703,7 +708,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // Mark the peer as owning the block and schedule it for import p.MarkBlock(request.Block.Hash()) - pm.fetcher.Enqueue(p.id, request.Block) + pm.blockFetcher.Enqueue(p.id, request.Block) // Assuming the block is importable by the peer, but possibly not yet done so, // calculate the head hash and TD that the peer truly must have. @@ -724,6 +729,66 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } } + case msg.Code == NewPooledTransactionHashesMsg && p.version >= eth65: + // New transaction announcement arrived, make sure we have + // a valid and fresh chain to handle them + if atomic.LoadUint32(&pm.acceptTxs) == 0 { + break + } + var hashes []common.Hash + if err := msg.Decode(&hashes); err != nil { + return errResp(ErrDecode, "msg %v: %v", msg, err) + } + // Schedule all the unknown hashes for retrieval + var unknown []common.Hash + for _, hash := range hashes { + // Mark the hashes as present at the remote node + p.MarkTransaction(hash) + + // Filter duplicated transaction announcement. + // Notably we only dedupliate announcement in txpool, check the rationale + // behind in EIP https://github.com/ethereum/EIPs/pull/2464. + if pm.txpool.Has(hash) { + continue + } + unknown = append(unknown, hash) + } + pm.txFetcher.Notify(p.id, unknown, time.Now(), p.AsyncRequestTxs) + + case msg.Code == GetPooledTransactionsMsg && p.version >= eth65: + // Decode the retrieval message + msgStream := rlp.NewStream(msg.Payload, uint64(msg.Size)) + if _, err := msgStream.List(); err != nil { + return err + } + // Gather transactions until the fetch or network limits is reached + var ( + hash common.Hash + bytes int + txs []rlp.RawValue + ) + for bytes < softResponseLimit { + // Retrieve the hash of the next block + if err := msgStream.Decode(&hash); err == rlp.EOL { + break + } else if err != nil { + return errResp(ErrDecode, "msg %v: %v", msg, err) + } + // Retrieve the requested transaction, skipping if unknown to us + tx := pm.txpool.Get(hash) + if tx == nil { + continue + } + // If known, encode and queue for response packet + if encoded, err := rlp.EncodeToBytes(tx); err != nil { + log.Error("Failed to encode transaction", "err", err) + } else { + txs = append(txs, encoded) + bytes += len(encoded) + } + } + return p.SendTransactionRLP(txs) + case msg.Code == TxMsg: // Transactions arrived, make sure we have a valid and fresh chain to handle them if atomic.LoadUint32(&pm.acceptTxs) == 0 { @@ -741,7 +806,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } p.MarkTransaction(tx.Hash()) } - pm.txpool.AddRemotes(txs) + pm.txFetcher.EnqueueTxs(p.id, txs) default: return errResp(ErrInvalidMsgCode, "%v", msg.Code) @@ -791,20 +856,48 @@ func (pm *ProtocolManager) BroadcastBlock(block *types.Block, propagate bool) { // BroadcastTxs will propagate a batch of transactions to all peers which are not known to // already have the given transaction. -func (pm *ProtocolManager) BroadcastTxs(txs types.Transactions) { - var txset = make(map[*peer]types.Transactions) - +func (pm *ProtocolManager) BroadcastTxs(txs types.Transactions, propagate bool) { + var ( + txset = make(map[*peer][]common.Hash) + annos = make(map[*peer][]common.Hash) + ) // Broadcast transactions to a batch of peers not knowing about it + if propagate { + for _, tx := range txs { + peers := pm.peers.PeersWithoutTx(tx.Hash()) + + // Send the block to a subset of our peers + transferLen := int(math.Sqrt(float64(len(peers)))) + if transferLen < minBroadcastPeers { + transferLen = minBroadcastPeers + } + if transferLen > len(peers) { + transferLen = len(peers) + } + transfer := peers[:transferLen] + for _, peer := range transfer { + txset[peer] = append(txset[peer], tx.Hash()) + } + log.Trace("Broadcast transaction", "hash", tx.Hash(), "recipients", len(peers)) + } + for peer, hashes := range txset { + peer.AsyncSendTransactions(hashes) + } + return + } + // Otherwise only broadcast the announcement to peers for _, tx := range txs { peers := pm.peers.PeersWithoutTx(tx.Hash()) for _, peer := range peers { - txset[peer] = append(txset[peer], tx) + annos[peer] = append(annos[peer], tx.Hash()) } - log.Trace("Broadcast transaction", "hash", tx.Hash(), "recipients", len(peers)) } - // FIXME include this again: peers = peers[:int(math.Sqrt(float64(len(peers))))] - for peer, txs := range txset { - peer.AsyncSendTransactions(txs) + for peer, hashes := range annos { + if peer.version >= eth65 { + peer.AsyncSendTransactionHashes(hashes) + } else { + peer.AsyncSendTransactions(hashes) + } } } @@ -823,7 +916,13 @@ func (pm *ProtocolManager) txBroadcastLoop() { for { select { case event := <-pm.txsCh: - pm.BroadcastTxs(event.Txs) + // For testing purpose only, disable propagation + if pm.broadcastTxAnnouncesOnly { + pm.BroadcastTxs(event.Txs, false) + continue + } + pm.BroadcastTxs(event.Txs, true) // First propagate transactions to peers + pm.BroadcastTxs(event.Txs, false) // Only then announce to the rest // Err() channel will be closed when unsubscribing. case <-pm.txsSub.Err(): diff --git a/eth/handler_test.go b/eth/handler_test.go index 354cbc068c..97613a9834 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -495,7 +495,7 @@ func testCheckpointChallenge(t *testing.T, syncmode downloader.SyncMode, checkpo if err != nil { t.Fatalf("failed to create new blockchain: %v", err) } - pm, err := NewProtocolManager(config, cht, syncmode, DefaultConfig.NetworkId, new(event.TypeMux), new(testTxPool), ethash.NewFaker(), blockchain, db, 1, nil) + pm, err := NewProtocolManager(config, cht, syncmode, DefaultConfig.NetworkId, new(event.TypeMux), &testTxPool{pool: make(map[common.Hash]*types.Transaction)}, ethash.NewFaker(), blockchain, db, 1, nil) if err != nil { t.Fatalf("failed to start test protocol manager: %v", err) } @@ -582,7 +582,7 @@ func testBroadcastBlock(t *testing.T, totalPeers, broadcastExpected int) { if err != nil { t.Fatalf("failed to create new blockchain: %v", err) } - pm, err := NewProtocolManager(config, nil, downloader.FullSync, DefaultConfig.NetworkId, evmux, new(testTxPool), pow, blockchain, db, 1, nil) + pm, err := NewProtocolManager(config, nil, downloader.FullSync, DefaultConfig.NetworkId, evmux, &testTxPool{pool: make(map[common.Hash]*types.Transaction)}, pow, blockchain, db, 1, nil) if err != nil { t.Fatalf("failed to start test protocol manager: %v", err) } diff --git a/eth/helper_test.go b/eth/helper_test.go index e66910334f..bec37e16cb 100644 --- a/eth/helper_test.go +++ b/eth/helper_test.go @@ -68,7 +68,7 @@ func newTestProtocolManager(mode downloader.SyncMode, blocks int, generator func if _, err := blockchain.InsertChain(chain); err != nil { panic(err) } - pm, err := NewProtocolManager(gspec.Config, nil, mode, DefaultConfig.NetworkId, evmux, &testTxPool{added: newtx}, engine, blockchain, db, 1, nil) + pm, err := NewProtocolManager(gspec.Config, nil, mode, DefaultConfig.NetworkId, evmux, &testTxPool{added: newtx, pool: make(map[common.Hash]*types.Transaction)}, engine, blockchain, db, 1, nil) if err != nil { return nil, nil, err } @@ -91,22 +91,43 @@ func newTestProtocolManagerMust(t *testing.T, mode downloader.SyncMode, blocks i // testTxPool is a fake, helper transaction pool for testing purposes type testTxPool struct { txFeed event.Feed - pool []*types.Transaction // Collection of all transactions - added chan<- []*types.Transaction // Notification channel for new transactions + pool map[common.Hash]*types.Transaction // Hash map of collected transactions + added chan<- []*types.Transaction // Notification channel for new transactions lock sync.RWMutex // Protects the transaction pool } +// Has returns an indicator whether txpool has a transaction +// cached with the given hash. +func (p *testTxPool) Has(hash common.Hash) bool { + p.lock.Lock() + defer p.lock.Unlock() + + return p.pool[hash] != nil +} + +// Get retrieves the transaction from local txpool with given +// tx hash. +func (p *testTxPool) Get(hash common.Hash) *types.Transaction { + p.lock.Lock() + defer p.lock.Unlock() + + return p.pool[hash] +} + // AddRemotes appends a batch of transactions to the pool, and notifies any // listeners if the addition channel is non nil func (p *testTxPool) AddRemotes(txs []*types.Transaction) []error { p.lock.Lock() defer p.lock.Unlock() - p.pool = append(p.pool, txs...) + for _, tx := range txs { + p.pool[tx.Hash()] = tx + } if p.added != nil { p.added <- txs } + p.txFeed.Send(core.NewTxsEvent{Txs: txs}) return make([]error, len(txs)) } @@ -153,7 +174,7 @@ func newTestPeer(name string, version int, pm *ProtocolManager, shake bool) (*te var id enode.ID rand.Read(id[:]) - peer := pm.newPeer(version, p2p.NewPeer(id, name, nil), net) + peer := pm.newPeer(version, p2p.NewPeer(id, name, nil), net, pm.txpool.Get) // Start the peer on a new thread errc := make(chan error, 1) @@ -191,7 +212,7 @@ func (p *testPeer) handshake(t *testing.T, td *big.Int, head common.Hash, genesi CurrentBlock: head, GenesisBlock: genesis, } - case p.version == eth64: + case p.version >= eth64: msg = &statusData{ ProtocolVersion: uint32(p.version), NetworkID: DefaultConfig.NetworkId, diff --git a/eth/peer.go b/eth/peer.go index 0beec1d844..f4b939b71c 100644 --- a/eth/peer.go +++ b/eth/peer.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/forkid" "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/fetcher" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/rlp" ) @@ -41,24 +42,39 @@ const ( maxKnownTxs = 32768 // Maximum transactions hashes to keep in the known list (prevent DOS) maxKnownBlocks = 1024 // Maximum block hashes to keep in the known list (prevent DOS) - // maxQueuedTxs is the maximum number of transaction lists to queue up before - // dropping broadcasts. This is a sensitive number as a transaction list might - // contain a single transaction, or thousands. - maxQueuedTxs = 128 + // maxQueuedTxs is the maximum number of transactions to queue up before dropping + // broadcasts. + maxQueuedTxs = 4096 - // maxQueuedProps is the maximum number of block propagations to queue up before + // maxQueuedTxAnns is the maximum number of transaction announcements to queue up + // before dropping broadcasts. + maxQueuedTxAnns = 4096 + + // maxQueuedTxRetrieval is the maximum number of tx retrieval requests to queue up + // before dropping requests. + maxQueuedTxRetrieval = 4096 + + // maxQueuedBlocks is the maximum number of block propagations to queue up before // dropping broadcasts. There's not much point in queueing stale blocks, so a few // that might cover uncles should be enough. - maxQueuedProps = 4 + maxQueuedBlocks = 4 - // maxQueuedAnns is the maximum number of block announcements to queue up before + // maxQueuedBlockAnns is the maximum number of block announcements to queue up before // dropping broadcasts. Similarly to block propagations, there's no point to queue // above some healthy uncle limit, so use that. - maxQueuedAnns = 4 + maxQueuedBlockAnns = 4 handshakeTimeout = 5 * time.Second ) +// max is a helper function which returns the larger of the two given integers. +func max(a, b int) int { + if a > b { + return a + } + return b +} + // PeerInfo represents a short summary of the Ethereum sub-protocol metadata known // about a connected peer. type PeerInfo struct { @@ -86,48 +102,48 @@ type peer struct { td *big.Int lock sync.RWMutex - knownTxs mapset.Set // Set of transaction hashes known to be known by this peer - knownBlocks mapset.Set // Set of block hashes known to be known by this peer - queuedTxs chan []*types.Transaction // Queue of transactions to broadcast to the peer - queuedProps chan *propEvent // Queue of blocks to broadcast to the peer - queuedAnns chan *types.Block // Queue of blocks to announce to the peer - term chan struct{} // Termination channel to stop the broadcaster + knownTxs mapset.Set // Set of transaction hashes known to be known by this peer + knownBlocks mapset.Set // Set of block hashes known to be known by this peer + queuedBlocks chan *propEvent // Queue of blocks to broadcast to the peer + queuedBlockAnns chan *types.Block // Queue of blocks to announce to the peer + txPropagation chan []common.Hash // Channel used to queue transaction propagation requests + txAnnounce chan []common.Hash // Channel used to queue transaction announcement requests + txRetrieval chan []common.Hash // Channel used to queue transaction retrieval requests + getPooledTx func(common.Hash) *types.Transaction // Callback used to retrieve transaction from txpool + term chan struct{} // Termination channel to stop the broadcaster } -func newPeer(version int, p *p2p.Peer, rw p2p.MsgReadWriter) *peer { +func newPeer(version int, p *p2p.Peer, rw p2p.MsgReadWriter, getPooledTx func(hash common.Hash) *types.Transaction) *peer { return &peer{ - Peer: p, - rw: rw, - version: version, - id: fmt.Sprintf("%x", p.ID().Bytes()[:8]), - knownTxs: mapset.NewSet(), - knownBlocks: mapset.NewSet(), - queuedTxs: make(chan []*types.Transaction, maxQueuedTxs), - queuedProps: make(chan *propEvent, maxQueuedProps), - queuedAnns: make(chan *types.Block, maxQueuedAnns), - term: make(chan struct{}), - } -} - -// broadcast is a write loop that multiplexes block propagations, announcements -// and transaction broadcasts into the remote peer. The goal is to have an async -// writer that does not lock up node internals. -func (p *peer) broadcast() { + Peer: p, + rw: rw, + version: version, + id: fmt.Sprintf("%x", p.ID().Bytes()[:8]), + knownTxs: mapset.NewSet(), + knownBlocks: mapset.NewSet(), + queuedBlocks: make(chan *propEvent, maxQueuedBlocks), + queuedBlockAnns: make(chan *types.Block, maxQueuedBlockAnns), + txPropagation: make(chan []common.Hash), + txAnnounce: make(chan []common.Hash), + txRetrieval: make(chan []common.Hash), + getPooledTx: getPooledTx, + term: make(chan struct{}), + } +} + +// broadcastBlocks is a write loop that multiplexes block propagations, +// announcements into the remote peer. The goal is to have an async writer +// that does not lock up node internals. +func (p *peer) broadcastBlocks() { for { select { - case txs := <-p.queuedTxs: - if err := p.SendTransactions(txs); err != nil { - return - } - p.Log().Trace("Broadcast transactions", "count", len(txs)) - - case prop := <-p.queuedProps: + case prop := <-p.queuedBlocks: if err := p.SendNewBlock(prop.block, prop.td); err != nil { return } p.Log().Trace("Propagated block", "number", prop.block.Number(), "hash", prop.block.Hash(), "td", prop.td) - case block := <-p.queuedAnns: + case block := <-p.queuedBlockAnns: if err := p.SendNewBlockHashes([]common.Hash{block.Hash()}, []uint64{block.NumberU64()}); err != nil { return } @@ -139,6 +155,175 @@ func (p *peer) broadcast() { } } +// broadcastTxs is a write loop that multiplexes transaction propagations, +// announcements into the remote peer. The goal is to have an async writer +// that does not lock up node internals. +func (p *peer) broadcastTxs() { + var ( + txProps []common.Hash // Queue of transaction propagations to the peer + txAnnos []common.Hash // Queue of transaction announcements to the peer + done chan struct{} // Non-nil if background network sender routine is active. + errch = make(chan error) // Channel used to receive network error + ) + scheduleTask := func() { + // Short circuit if there already has a inflight task. + if done != nil { + return + } + // Spin up transaction propagation task if there is any + // queued hashes. + if len(txProps) > 0 { + var ( + hashes []common.Hash + txs []*types.Transaction + size common.StorageSize + ) + for i := 0; i < len(txProps) && size < txsyncPackSize; i++ { + if tx := p.getPooledTx(txProps[i]); tx != nil { + txs = append(txs, tx) + size += tx.Size() + } + hashes = append(hashes, txProps[i]) + } + txProps = txProps[:copy(txProps, txProps[len(hashes):])] + if len(txs) > 0 { + done = make(chan struct{}) + go func() { + if err := p.SendNewTransactions(txs); err != nil { + errch <- err + return + } + close(done) + p.Log().Trace("Sent transactions", "count", len(txs)) + }() + return + } + } + // Spin up transaction announcement task if there is any + // queued hashes. + if len(txAnnos) > 0 { + var ( + hashes []common.Hash + pending []common.Hash + size common.StorageSize + ) + for i := 0; i < len(txAnnos) && size < txsyncPackSize; i++ { + if tx := p.getPooledTx(txAnnos[i]); tx != nil { + pending = append(pending, txAnnos[i]) + size += common.HashLength + } + hashes = append(hashes, txAnnos[i]) + } + txAnnos = txAnnos[:copy(txAnnos, txAnnos[len(hashes):])] + if len(pending) > 0 { + done = make(chan struct{}) + go func() { + if err := p.SendNewTransactionHashes(pending); err != nil { + errch <- err + return + } + close(done) + p.Log().Trace("Sent transaction announcements", "count", len(pending)) + }() + } + } + } + + for { + scheduleTask() + select { + case hashes := <-p.txPropagation: + if len(txProps) == maxQueuedTxs { + continue + } + if len(txProps)+len(hashes) > maxQueuedTxs { + hashes = hashes[:maxQueuedTxs-len(txProps)] + } + txProps = append(txProps, hashes...) + + case hashes := <-p.txAnnounce: + if len(txAnnos) == maxQueuedTxAnns { + continue + } + if len(txAnnos)+len(hashes) > maxQueuedTxAnns { + hashes = hashes[:maxQueuedTxAnns-len(txAnnos)] + } + txAnnos = append(txAnnos, hashes...) + + case <-done: + done = nil + + case <-errch: + return + + case <-p.term: + return + } + } +} + +// retrievalTxs is a write loop which is responsible for retrieving transaction +// from the remote peer. The goal is to have an async writer that does not lock +// up node internals. If there are too many requests queued, then new arrival +// requests will be dropped silently so that we can ensure the memory assumption +// is fixed for each peer. +func (p *peer) retrievalTxs() { + var ( + requests []common.Hash // Queue of transaction requests to the peer + done chan struct{} // Non-nil if background network sender routine is active. + errch = make(chan error) // Channel used to receive network error + ) + // pick chooses a reasonble number of transaction hashes for retrieval. + pick := func() []common.Hash { + var ret []common.Hash + if len(requests) > fetcher.MaxTransactionFetch { + ret = requests[:fetcher.MaxTransactionFetch] + } else { + ret = requests[:] + } + requests = requests[:copy(requests, requests[len(ret):])] + return ret + } + // send sends transactions retrieval request. + send := func(hashes []common.Hash, done chan struct{}) { + if err := p.RequestTxs(hashes); err != nil { + errch <- err + return + } + close(done) + p.Log().Trace("Sent transaction retrieval request", "count", len(hashes)) + } + for { + select { + case hashes := <-p.txRetrieval: + if len(requests) == maxQueuedTxRetrieval { + continue + } + if len(requests)+len(hashes) > maxQueuedTxRetrieval { + hashes = hashes[:maxQueuedTxRetrieval-len(requests)] + } + requests = append(requests, hashes...) + if done == nil { + done = make(chan struct{}) + go send(pick(), done) + } + + case <-done: + done = nil + if pending := pick(); len(pending) > 0 { + done = make(chan struct{}) + go send(pending, done) + } + + case <- errch: + return + + case <-p.term: + return + } + } +} + // close signals the broadcast goroutine to terminate. func (p *peer) close() { close(p.term) @@ -194,33 +379,67 @@ func (p *peer) MarkTransaction(hash common.Hash) { p.knownTxs.Add(hash) } -// SendTransactions sends transactions to the peer and includes the hashes +// SendNewTransactionHashes sends a batch of transaction hashes to the peer and +// includes the hashes in its transaction hash set for future reference. +func (p *peer) SendNewTransactionHashes(hashes []common.Hash) error { + // Mark all the transactions as known, but ensure we don't overflow our limits + for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { + p.knownTxs.Pop() + } + for _, hash := range hashes { + p.knownTxs.Add(hash) + } + return p2p.Send(p.rw, NewPooledTransactionHashesMsg, hashes) +} + +// SendNewTransactions sends transactions to the peer and includes the hashes // in its transaction hash set for future reference. -func (p *peer) SendTransactions(txs types.Transactions) error { +func (p *peer) SendNewTransactions(txs types.Transactions) error { // Mark all the transactions as known, but ensure we don't overflow our limits + for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(txs)) { + p.knownTxs.Pop() + } for _, tx := range txs { p.knownTxs.Add(tx.Hash()) } - for p.knownTxs.Cardinality() >= maxKnownTxs { - p.knownTxs.Pop() - } + return p2p.Send(p.rw, TxMsg, txs) +} + +func (p *peer) SendTransactionRLP(txs []rlp.RawValue) error { return p2p.Send(p.rw, TxMsg, txs) } // AsyncSendTransactions queues list of transactions propagation to a remote // peer. If the peer's broadcast queue is full, the event is silently dropped. -func (p *peer) AsyncSendTransactions(txs []*types.Transaction) { +func (p *peer) AsyncSendTransactions(hashes []common.Hash) { select { - case p.queuedTxs <- txs: + case p.txPropagation <- hashes: // Mark all the transactions as known, but ensure we don't overflow our limits - for _, tx := range txs { - p.knownTxs.Add(tx.Hash()) + for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { + p.knownTxs.Pop() + } + for _, hash := range hashes { + p.knownTxs.Add(hash) } - for p.knownTxs.Cardinality() >= maxKnownTxs { + case <-p.term: + p.Log().Debug("Dropping transaction propagation", "count", len(hashes)) + } +} + +// AsyncSendTransactions queues list of transactions propagation to a remote +// peer. If the peer's broadcast queue is full, the event is silently dropped. +func (p *peer) AsyncSendTransactionHashes(hashes []common.Hash) { + select { + case p.txAnnounce <- hashes: + // Mark all the transactions as known, but ensure we don't overflow our limits + for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { p.knownTxs.Pop() } - default: - p.Log().Debug("Dropping transaction propagation", "count", len(txs)) + for _, hash := range hashes { + p.knownTxs.Add(hash) + } + case <-p.term: + p.Log().Debug("Dropping transaction announcement", "count", len(hashes)) } } @@ -228,12 +447,12 @@ func (p *peer) AsyncSendTransactions(txs []*types.Transaction) { // a hash notification. func (p *peer) SendNewBlockHashes(hashes []common.Hash, numbers []uint64) error { // Mark all the block hashes as known, but ensure we don't overflow our limits + for p.knownBlocks.Cardinality() > max(0, maxKnownBlocks-len(hashes)) { + p.knownBlocks.Pop() + } for _, hash := range hashes { p.knownBlocks.Add(hash) } - for p.knownBlocks.Cardinality() >= maxKnownBlocks { - p.knownBlocks.Pop() - } request := make(newBlockHashesData, len(hashes)) for i := 0; i < len(hashes); i++ { request[i].Hash = hashes[i] @@ -247,12 +466,12 @@ func (p *peer) SendNewBlockHashes(hashes []common.Hash, numbers []uint64) error // dropped. func (p *peer) AsyncSendNewBlockHash(block *types.Block) { select { - case p.queuedAnns <- block: + case p.queuedBlockAnns <- block: // Mark all the block hash as known, but ensure we don't overflow our limits - p.knownBlocks.Add(block.Hash()) for p.knownBlocks.Cardinality() >= maxKnownBlocks { p.knownBlocks.Pop() } + p.knownBlocks.Add(block.Hash()) default: p.Log().Debug("Dropping block announcement", "number", block.NumberU64(), "hash", block.Hash()) } @@ -261,10 +480,10 @@ func (p *peer) AsyncSendNewBlockHash(block *types.Block) { // SendNewBlock propagates an entire block to a remote peer. func (p *peer) SendNewBlock(block *types.Block, td *big.Int) error { // Mark all the block hash as known, but ensure we don't overflow our limits - p.knownBlocks.Add(block.Hash()) for p.knownBlocks.Cardinality() >= maxKnownBlocks { p.knownBlocks.Pop() } + p.knownBlocks.Add(block.Hash()) return p2p.Send(p.rw, NewBlockMsg, []interface{}{block, td}) } @@ -272,12 +491,12 @@ func (p *peer) SendNewBlock(block *types.Block, td *big.Int) error { // the peer's broadcast queue is full, the event is silently dropped. func (p *peer) AsyncSendNewBlock(block *types.Block, td *big.Int) { select { - case p.queuedProps <- &propEvent{block: block, td: td}: + case p.queuedBlocks <- &propEvent{block: block, td: td}: // Mark all the block hash as known, but ensure we don't overflow our limits - p.knownBlocks.Add(block.Hash()) for p.knownBlocks.Cardinality() >= maxKnownBlocks { p.knownBlocks.Pop() } + p.knownBlocks.Add(block.Hash()) default: p.Log().Debug("Dropping block propagation", "number", block.NumberU64(), "hash", block.Hash()) } @@ -352,6 +571,22 @@ func (p *peer) RequestReceipts(hashes []common.Hash) error { return p2p.Send(p.rw, GetReceiptsMsg, hashes) } +// RequestTxs fetches a batch of transactions from a remote node. +func (p *peer) RequestTxs(hashes []common.Hash) error { + p.Log().Debug("Fetching batch of transactions", "count", len(hashes)) + return p2p.Send(p.rw, GetPooledTransactionsMsg, hashes) +} + +// AsyncRequestTxs queues a tx retrieval request to a remote peer. If +// the peer's retrieval queue is full, the event is silently dropped. +func (p *peer) AsyncRequestTxs(hashes []common.Hash) { + select { + case p.txRetrieval <- hashes: + case <-p.term: + p.Log().Debug("Dropping transaction retrieval request", "count", len(hashes)) + } +} + // Handshake executes the eth protocol handshake, negotiating version number, // network IDs, difficulties, head and genesis blocks. func (p *peer) Handshake(network uint64, td *big.Int, head common.Hash, genesis common.Hash, forkID forkid.ID, forkFilter forkid.Filter) error { @@ -372,7 +607,7 @@ func (p *peer) Handshake(network uint64, td *big.Int, head common.Hash, genesis CurrentBlock: head, GenesisBlock: genesis, }) - case p.version == eth64: + case p.version >= eth64: errc <- p2p.Send(p.rw, StatusMsg, &statusData{ ProtocolVersion: uint32(p.version), NetworkID: network, @@ -389,7 +624,7 @@ func (p *peer) Handshake(network uint64, td *big.Int, head common.Hash, genesis switch { case p.version == eth63: errc <- p.readStatusLegacy(network, &status63, genesis) - case p.version == eth64: + case p.version >= eth64: errc <- p.readStatus(network, &status, genesis, forkFilter) default: panic(fmt.Sprintf("unsupported eth protocol version: %d", p.version)) @@ -410,7 +645,7 @@ func (p *peer) Handshake(network uint64, td *big.Int, head common.Hash, genesis switch { case p.version == eth63: p.td, p.head = status63.TD, status63.CurrentBlock - case p.version == eth64: + case p.version >= eth64: p.td, p.head = status.TD, status.Head default: panic(fmt.Sprintf("unsupported eth protocol version: %d", p.version)) @@ -511,7 +746,9 @@ func (ps *peerSet) Register(p *peer) error { return errAlreadyRegistered } ps.peers[p.id] = p - go p.broadcast() + go p.broadcastBlocks() + go p.broadcastTxs() + go p.retrievalTxs() return nil } diff --git a/eth/protocol.go b/eth/protocol.go index 62e4d13d14..1cef66adb2 100644 --- a/eth/protocol.go +++ b/eth/protocol.go @@ -33,16 +33,17 @@ import ( const ( eth63 = 63 eth64 = 64 + eth65 = 65 ) // protocolName is the official short name of the protocol used during capability negotiation. const protocolName = "eth" // ProtocolVersions are the supported versions of the eth protocol (first is primary). -var ProtocolVersions = []uint{eth64, eth63} +var ProtocolVersions = []uint{eth65, eth64, eth63} // protocolLengths are the number of implemented message corresponding to different protocol versions. -var protocolLengths = map[uint]uint64{eth64: 17, eth63: 17} +var protocolLengths = map[uint]uint64{eth65: 17, eth64: 17, eth63: 17} const protocolMaxMsgSize = 10 * 1024 * 1024 // Maximum cap on the size of a protocol message @@ -60,6 +61,13 @@ const ( NodeDataMsg = 0x0e GetReceiptsMsg = 0x0f ReceiptsMsg = 0x10 + + // New protocol message codes introduced in eth65 + // + // Previously these message ids(0x08, 0x09) were used by some + // legacy and unsupported eth protocols, reown them here. + NewPooledTransactionHashesMsg = 0x08 + GetPooledTransactionsMsg = 0x09 ) type errCode int @@ -94,6 +102,14 @@ var errorToString = map[int]string{ } type txPool interface { + // Has returns an indicator whether txpool has a transaction + // cached with the given hash. + Has(hash common.Hash) bool + + // Get retrieves the transaction from local txpool with given + // tx hash. + Get(hash common.Hash) *types.Transaction + // AddRemotes should add the given transactions to the pool. AddRemotes([]*types.Transaction) []error diff --git a/eth/protocol_test.go b/eth/protocol_test.go index ca418942bb..e9a1a511ef 100644 --- a/eth/protocol_test.go +++ b/eth/protocol_test.go @@ -20,6 +20,7 @@ import ( "fmt" "math/big" "sync" + "sync/atomic" "testing" "time" @@ -180,16 +181,16 @@ func TestForkIDSplit(t *testing.T) { blocksNoFork, _ = core.GenerateChain(configNoFork, genesisNoFork, engine, dbNoFork, 2, nil) blocksProFork, _ = core.GenerateChain(configProFork, genesisProFork, engine, dbProFork, 2, nil) - ethNoFork, _ = NewProtocolManager(configNoFork, nil, downloader.FullSync, 1, new(event.TypeMux), new(testTxPool), engine, chainNoFork, dbNoFork, 1, nil) - ethProFork, _ = NewProtocolManager(configProFork, nil, downloader.FullSync, 1, new(event.TypeMux), new(testTxPool), engine, chainProFork, dbProFork, 1, nil) + ethNoFork, _ = NewProtocolManager(configNoFork, nil, downloader.FullSync, 1, new(event.TypeMux), &testTxPool{pool: make(map[common.Hash]*types.Transaction)}, engine, chainNoFork, dbNoFork, 1, nil) + ethProFork, _ = NewProtocolManager(configProFork, nil, downloader.FullSync, 1, new(event.TypeMux), &testTxPool{pool: make(map[common.Hash]*types.Transaction)}, engine, chainProFork, dbProFork, 1, nil) ) ethNoFork.Start(1000) ethProFork.Start(1000) // Both nodes should allow the other to connect (same genesis, next fork is the same) p2pNoFork, p2pProFork := p2p.MsgPipe() - peerNoFork := newPeer(64, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork) - peerProFork := newPeer(64, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork) + peerNoFork := newPeer(64, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork, nil) + peerProFork := newPeer(64, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork, nil) errc := make(chan error, 2) go func() { errc <- ethNoFork.handle(peerProFork) }() @@ -207,8 +208,8 @@ func TestForkIDSplit(t *testing.T) { chainProFork.InsertChain(blocksProFork[:1]) p2pNoFork, p2pProFork = p2p.MsgPipe() - peerNoFork = newPeer(64, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork) - peerProFork = newPeer(64, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork) + peerNoFork = newPeer(64, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork, nil) + peerProFork = newPeer(64, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork, nil) errc = make(chan error, 2) go func() { errc <- ethNoFork.handle(peerProFork) }() @@ -226,8 +227,8 @@ func TestForkIDSplit(t *testing.T) { chainProFork.InsertChain(blocksProFork[1:2]) p2pNoFork, p2pProFork = p2p.MsgPipe() - peerNoFork = newPeer(64, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork) - peerProFork = newPeer(64, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork) + peerNoFork = newPeer(64, p2p.NewPeer(enode.ID{1}, "", nil), p2pNoFork, nil) + peerProFork = newPeer(64, p2p.NewPeer(enode.ID{2}, "", nil), p2pProFork, nil) errc = make(chan error, 2) go func() { errc <- ethNoFork.handle(peerProFork) }() @@ -246,6 +247,7 @@ func TestForkIDSplit(t *testing.T) { // This test checks that received transactions are added to the local pool. func TestRecvTransactions63(t *testing.T) { testRecvTransactions(t, 63) } func TestRecvTransactions64(t *testing.T) { testRecvTransactions(t, 64) } +func TestRecvTransactions65(t *testing.T) { testRecvTransactions(t, 65) } func testRecvTransactions(t *testing.T, protocol int) { txAdded := make(chan []*types.Transaction) @@ -274,6 +276,7 @@ func testRecvTransactions(t *testing.T, protocol int) { // This test checks that pending transactions are sent. func TestSendTransactions63(t *testing.T) { testSendTransactions(t, 63) } func TestSendTransactions64(t *testing.T) { testSendTransactions(t, 64) } +func TestSendTransactions65(t *testing.T) { testSendTransactions(t, 65) } func testSendTransactions(t *testing.T, protocol int) { pm, _ := newTestProtocolManagerMust(t, downloader.FullSync, 0, nil, nil) @@ -298,17 +301,43 @@ func testSendTransactions(t *testing.T, protocol int) { } for n := 0; n < len(alltxs) && !t.Failed(); { var txs []*types.Transaction - msg, err := p.app.ReadMsg() - if err != nil { - t.Errorf("%v: read error: %v", p.Peer, err) - } else if msg.Code != TxMsg { - t.Errorf("%v: got code %d, want TxMsg", p.Peer, msg.Code) - } - if err := msg.Decode(&txs); err != nil { - t.Errorf("%v: %v", p.Peer, err) + var hashes []common.Hash + var forAllHashes func(callback func(hash common.Hash)) + switch protocol { + case 63: + fallthrough + case 64: + msg, err := p.app.ReadMsg() + if err != nil { + t.Errorf("%v: read error: %v", p.Peer, err) + } else if msg.Code != TxMsg { + t.Errorf("%v: got code %d, want TxMsg", p.Peer, msg.Code) + } + if err := msg.Decode(&txs); err != nil { + t.Errorf("%v: %v", p.Peer, err) + } + forAllHashes = func(callback func(hash common.Hash)) { + for _, tx := range txs { + callback(tx.Hash()) + } + } + case 65: + msg, err := p.app.ReadMsg() + if err != nil { + t.Errorf("%v: read error: %v", p.Peer, err) + } else if msg.Code != NewPooledTransactionHashesMsg { + t.Errorf("%v: got code %d, want TxMsg", p.Peer, msg.Code) + } + if err := msg.Decode(&hashes); err != nil { + t.Errorf("%v: %v", p.Peer, err) + } + forAllHashes = func(callback func(hash common.Hash)) { + for _, h := range hashes { + callback(h) + } + } } - for _, tx := range txs { - hash := tx.Hash() + forAllHashes(func(hash common.Hash) { seentx, want := seen[hash] if seentx { t.Errorf("%v: got tx more than once: %x", p.Peer, hash) @@ -318,7 +347,7 @@ func testSendTransactions(t *testing.T, protocol int) { } seen[hash] = true n++ - } + }) } } for i := 0; i < 3; i++ { @@ -329,6 +358,53 @@ func testSendTransactions(t *testing.T, protocol int) { wg.Wait() } +func TestTransactionPropagation(t *testing.T) { testSyncTransaction(t, true) } +func TestTransactionAnnouncement(t *testing.T) { testSyncTransaction(t, false) } + +func testSyncTransaction(t *testing.T, propagtion bool) { + // Create a protocol manager for transaction fetcher and sender + pmFetcher, _ := newTestProtocolManagerMust(t, downloader.FastSync, 0, nil, nil) + defer pmFetcher.Stop() + pmSender, _ := newTestProtocolManagerMust(t, downloader.FastSync, 1024, nil, nil) + pmSender.broadcastTxAnnouncesOnly = !propagtion + defer pmSender.Stop() + + // Sync up the two peers + io1, io2 := p2p.MsgPipe() + + go pmSender.handle(pmSender.newPeer(65, p2p.NewPeer(enode.ID{}, "sender", nil), io2, pmSender.txpool.Get)) + go pmFetcher.handle(pmFetcher.newPeer(65, p2p.NewPeer(enode.ID{}, "fetcher", nil), io1, pmFetcher.txpool.Get)) + + time.Sleep(250 * time.Millisecond) + pmFetcher.synchronise(pmFetcher.peers.BestPeer()) + atomic.StoreUint32(&pmFetcher.acceptTxs, 1) + + newTxs := make(chan core.NewTxsEvent, 1024) + sub := pmFetcher.txpool.SubscribeNewTxsEvent(newTxs) + defer sub.Unsubscribe() + + // Fill the pool with new transactions + alltxs := make([]*types.Transaction, 1024) + for nonce := range alltxs { + alltxs[nonce] = newTestTransaction(testAccount, uint64(nonce), 0) + } + pmSender.txpool.AddRemotes(alltxs) + + var got int +loop: + for { + select { + case ev := <-newTxs: + got += len(ev.Txs) + if got == 1024 { + break loop + } + case <-time.NewTimer(time.Second).C: + t.Fatal("Failed to retrieve all transaction") + } + } +} + // Tests that the custom union field encoder and decoder works correctly. func TestGetBlockHeadersDataEncodeDecode(t *testing.T) { // Create a "random" hash for testing diff --git a/eth/sync.go b/eth/sync.go index 9e180ee200..93b2dd2ecf 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -38,8 +38,9 @@ const ( ) type txsync struct { - p *peer - txs []*types.Transaction + p *peer + hashes []common.Hash + txs []*types.Transaction } // syncTransactions starts sending all currently pending transactions to the given peer. @@ -53,7 +54,7 @@ func (pm *ProtocolManager) syncTransactions(p *peer) { return } select { - case pm.txsyncCh <- &txsync{p, txs}: + case pm.txsyncCh <- &txsync{p: p, txs: txs}: case <-pm.quitSync: } } @@ -69,26 +70,46 @@ func (pm *ProtocolManager) txsyncLoop() { pack = new(txsync) // the pack that is being sent done = make(chan error, 1) // result of the send ) - // send starts a sending a pack of transactions from the sync. send := func(s *txsync) { // Fill pack with transactions up to the target size. size := common.StorageSize(0) pack.p = s.p + pack.hashes = pack.hashes[:0] pack.txs = pack.txs[:0] - for i := 0; i < len(s.txs) && size < txsyncPackSize; i++ { - pack.txs = append(pack.txs, s.txs[i]) - size += s.txs[i].Size() - } - // Remove the transactions that will be sent. - s.txs = s.txs[:copy(s.txs, s.txs[len(pack.txs):])] - if len(s.txs) == 0 { - delete(pending, s.p.ID()) + if s.p.version >= eth65 { + // Eth65 introduces transaction announcement https://github.com/ethereum/EIPs/pull/2464, + // only txhashes are transferred here. + for i := 0; i < len(s.txs) && size < txsyncPackSize; i++ { + pack.hashes = append(pack.hashes, s.txs[i].Hash()) + size += common.HashLength + } + // Remove the transactions that will be sent. + s.txs = s.txs[:copy(s.txs, s.txs[len(pack.hashes):])] + if len(s.txs) == 0 { + delete(pending, s.p.ID()) + } + // Send the pack in the background. + s.p.Log().Trace("Sending batch of transaction announcements", "count", len(pack.hashes), "bytes", size) + sending = true + go func() { done <- pack.p.SendNewTransactionHashes(pack.hashes) }() + } else { + // Legacy eth protocol doesn't have transaction announcement protocol + // message, transfer the whole pending transaction slice. + for i := 0; i < len(s.txs) && size < txsyncPackSize; i++ { + pack.txs = append(pack.txs, s.txs[i]) + size += s.txs[i].Size() + } + // Remove the transactions that will be sent. + s.txs = s.txs[:copy(s.txs, s.txs[len(pack.txs):])] + if len(s.txs) == 0 { + delete(pending, s.p.ID()) + } + // Send the pack in the background. + s.p.Log().Trace("Sending batch of transactions", "count", len(pack.txs), "bytes", size) + sending = true + go func() { done <- pack.p.SendNewTransactions(pack.txs) }() } - // Send the pack in the background. - s.p.Log().Trace("Sending batch of transactions", "count", len(pack.txs), "bytes", size) - sending = true - go func() { done <- pack.p.SendTransactions(pack.txs) }() } // pick chooses the next pending sync. @@ -133,8 +154,10 @@ func (pm *ProtocolManager) txsyncLoop() { // downloading hashes and blocks as well as handling the announcement handler. func (pm *ProtocolManager) syncer() { // Start and ensure cleanup of sync mechanisms - pm.fetcher.Start() - defer pm.fetcher.Stop() + pm.blockFetcher.Start() + pm.txFetcher.Start() + defer pm.blockFetcher.Stop() + defer pm.txFetcher.Stop() defer pm.downloader.Terminate() // Wait for different events to fire synchronisation operations diff --git a/eth/sync_test.go b/eth/sync_test.go index e4c99ff587..d02bc57108 100644 --- a/eth/sync_test.go +++ b/eth/sync_test.go @@ -26,9 +26,13 @@ import ( "github.com/ethereum/go-ethereum/p2p/enode" ) +func TestFastSyncDisabling63(t *testing.T) { testFastSyncDisabling(t, 63) } +func TestFastSyncDisabling64(t *testing.T) { testFastSyncDisabling(t, 64) } +func TestFastSyncDisabling65(t *testing.T) { testFastSyncDisabling(t, 65) } + // Tests that fast sync gets disabled as soon as a real block is successfully // imported into the blockchain. -func TestFastSyncDisabling(t *testing.T) { +func testFastSyncDisabling(t *testing.T, protocol int) { // Create a pristine protocol manager, check that fast sync is left enabled pmEmpty, _ := newTestProtocolManagerMust(t, downloader.FastSync, 0, nil, nil) if atomic.LoadUint32(&pmEmpty.fastSync) == 0 { @@ -42,8 +46,8 @@ func TestFastSyncDisabling(t *testing.T) { // Sync up the two peers io1, io2 := p2p.MsgPipe() - go pmFull.handle(pmFull.newPeer(63, p2p.NewPeer(enode.ID{}, "empty", nil), io2)) - go pmEmpty.handle(pmEmpty.newPeer(63, p2p.NewPeer(enode.ID{}, "full", nil), io1)) + go pmFull.handle(pmFull.newPeer(protocol, p2p.NewPeer(enode.ID{}, "empty", nil), io2, pmFull.txpool.Get)) + go pmEmpty.handle(pmEmpty.newPeer(protocol, p2p.NewPeer(enode.ID{}, "full", nil), io1, pmEmpty.txpool.Get)) time.Sleep(250 * time.Millisecond) pmEmpty.synchronise(pmEmpty.peers.BestPeer()) From c22fdec3c7c48bc5076cb85b92a55a68951db167 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 11 Feb 2020 16:36:49 +0100 Subject: [PATCH 100/128] common/mclock: add NewTimer and Timer.Reset (#20634) These methods can be helpful when migrating existing timer code. --- common/mclock/mclock.go | 65 +++++++++++++-- common/mclock/simclock.go | 143 ++++++++++++++++++++++----------- common/mclock/simclock_test.go | 63 +++++++++++++-- 3 files changed, 207 insertions(+), 64 deletions(-) diff --git a/common/mclock/mclock.go b/common/mclock/mclock.go index d0e0cd78be..3aca257cb3 100644 --- a/common/mclock/mclock.go +++ b/common/mclock/mclock.go @@ -31,44 +31,93 @@ func Now() AbsTime { return AbsTime(monotime.Now()) } -// Add returns t + d. +// Add returns t + d as absolute time. func (t AbsTime) Add(d time.Duration) AbsTime { return t + AbsTime(d) } +// Sub returns t - t2 as a duration. +func (t AbsTime) Sub(t2 AbsTime) time.Duration { + return time.Duration(t - t2) +} + // The Clock interface makes it possible to replace the monotonic system clock with // a simulated clock. type Clock interface { Now() AbsTime Sleep(time.Duration) - After(time.Duration) <-chan time.Time + NewTimer(time.Duration) ChanTimer + After(time.Duration) <-chan AbsTime AfterFunc(d time.Duration, f func()) Timer } -// Timer represents a cancellable event returned by AfterFunc +// Timer is a cancellable event created by AfterFunc. type Timer interface { + // Stop cancels the timer. It returns false if the timer has already + // expired or been stopped. Stop() bool } +// ChanTimer is a cancellable event created by NewTimer. +type ChanTimer interface { + Timer + + // The channel returned by C receives a value when the timer expires. + C() <-chan AbsTime + // Reset reschedules the timer with a new timeout. + // It should be invoked only on stopped or expired timers with drained channels. + Reset(time.Duration) +} + // System implements Clock using the system clock. type System struct{} // Now returns the current monotonic time. -func (System) Now() AbsTime { +func (c System) Now() AbsTime { return AbsTime(monotime.Now()) } // Sleep blocks for the given duration. -func (System) Sleep(d time.Duration) { +func (c System) Sleep(d time.Duration) { time.Sleep(d) } +// NewTimer creates a timer which can be rescheduled. +func (c System) NewTimer(d time.Duration) ChanTimer { + ch := make(chan AbsTime, 1) + t := time.AfterFunc(d, func() { + // This send is non-blocking because that's how time.Timer + // behaves. It doesn't matter in the happy case, but does + // when Reset is misused. + select { + case ch <- c.Now(): + default: + } + }) + return &systemTimer{t, ch} +} + // After returns a channel which receives the current time after d has elapsed. -func (System) After(d time.Duration) <-chan time.Time { - return time.After(d) +func (c System) After(d time.Duration) <-chan AbsTime { + ch := make(chan AbsTime, 1) + time.AfterFunc(d, func() { ch <- c.Now() }) + return ch } // AfterFunc runs f on a new goroutine after the duration has elapsed. -func (System) AfterFunc(d time.Duration, f func()) Timer { +func (c System) AfterFunc(d time.Duration, f func()) Timer { return time.AfterFunc(d, f) } + +type systemTimer struct { + *time.Timer + ch <-chan AbsTime +} + +func (st *systemTimer) Reset(d time.Duration) { + st.Timer.Reset(d) +} + +func (st *systemTimer) C() <-chan AbsTime { + return st.ch +} diff --git a/common/mclock/simclock.go b/common/mclock/simclock.go index 4d351252ff..766ca0f873 100644 --- a/common/mclock/simclock.go +++ b/common/mclock/simclock.go @@ -17,6 +17,7 @@ package mclock import ( + "container/heap" "sync" "time" ) @@ -32,18 +33,24 @@ import ( // the timeout using a channel or semaphore. type Simulated struct { now AbsTime - scheduled []*simTimer + scheduled simTimerHeap mu sync.RWMutex cond *sync.Cond - lastId uint64 } -// simTimer implements Timer on the virtual clock. +// simTimer implements ChanTimer on the virtual clock. type simTimer struct { - do func() - at AbsTime - id uint64 - s *Simulated + at AbsTime + index int // position in s.scheduled + s *Simulated + do func() + ch <-chan AbsTime +} + +func (s *Simulated) init() { + if s.cond == nil { + s.cond = sync.NewCond(&s.mu) + } } // Run moves the clock by the given duration, executing all timers before that duration. @@ -53,14 +60,9 @@ func (s *Simulated) Run(d time.Duration) { end := s.now + AbsTime(d) var do []func() - for len(s.scheduled) > 0 { - ev := s.scheduled[0] - if ev.at > end { - break - } - s.now = ev.at + for len(s.scheduled) > 0 && s.scheduled[0].at <= end { + ev := heap.Pop(&s.scheduled).(*simTimer) do = append(do, ev.do) - s.scheduled = s.scheduled[1:] } s.now = end s.mu.Unlock() @@ -102,14 +104,22 @@ func (s *Simulated) Sleep(d time.Duration) { <-s.After(d) } +// NewTimer creates a timer which fires when the clock has advanced by d. +func (s *Simulated) NewTimer(d time.Duration) ChanTimer { + s.mu.Lock() + defer s.mu.Unlock() + + ch := make(chan AbsTime, 1) + var timer *simTimer + timer = s.schedule(d, func() { ch <- timer.at }) + timer.ch = ch + return timer +} + // After returns a channel which receives the current time after the clock // has advanced by d. -func (s *Simulated) After(d time.Duration) <-chan time.Time { - after := make(chan time.Time, 1) - s.AfterFunc(d, func() { - after <- (time.Time{}).Add(time.Duration(s.now)) - }) - return after +func (s *Simulated) After(d time.Duration) <-chan AbsTime { + return s.NewTimer(d).C() } // AfterFunc runs fn after the clock has advanced by d. Unlike with the system @@ -117,46 +127,83 @@ func (s *Simulated) After(d time.Duration) <-chan time.Time { func (s *Simulated) AfterFunc(d time.Duration, fn func()) Timer { s.mu.Lock() defer s.mu.Unlock() + + return s.schedule(d, fn) +} + +func (s *Simulated) schedule(d time.Duration, fn func()) *simTimer { s.init() at := s.now + AbsTime(d) - s.lastId++ - id := s.lastId - l, h := 0, len(s.scheduled) - ll := h - for l != h { - m := (l + h) / 2 - if (at < s.scheduled[m].at) || ((at == s.scheduled[m].at) && (id < s.scheduled[m].id)) { - h = m - } else { - l = m + 1 - } - } ev := &simTimer{do: fn, at: at, s: s} - s.scheduled = append(s.scheduled, nil) - copy(s.scheduled[l+1:], s.scheduled[l:ll]) - s.scheduled[l] = ev + heap.Push(&s.scheduled, ev) s.cond.Broadcast() return ev } func (ev *simTimer) Stop() bool { - s := ev.s - s.mu.Lock() - defer s.mu.Unlock() + ev.s.mu.Lock() + defer ev.s.mu.Unlock() - for i := 0; i < len(s.scheduled); i++ { - if s.scheduled[i] == ev { - s.scheduled = append(s.scheduled[:i], s.scheduled[i+1:]...) - s.cond.Broadcast() - return true - } + if ev.index < 0 { + return false } - return false + heap.Remove(&ev.s.scheduled, ev.index) + ev.s.cond.Broadcast() + ev.index = -1 + return true } -func (s *Simulated) init() { - if s.cond == nil { - s.cond = sync.NewCond(&s.mu) +func (ev *simTimer) Reset(d time.Duration) { + if ev.ch == nil { + panic("mclock: Reset() on timer created by AfterFunc") } + + ev.s.mu.Lock() + defer ev.s.mu.Unlock() + ev.at = ev.s.now.Add(d) + if ev.index < 0 { + heap.Push(&ev.s.scheduled, ev) // already expired + } else { + heap.Fix(&ev.s.scheduled, ev.index) // hasn't fired yet, reschedule + } + ev.s.cond.Broadcast() +} + +func (ev *simTimer) C() <-chan AbsTime { + if ev.ch == nil { + panic("mclock: C() on timer created by AfterFunc") + } + return ev.ch +} + +type simTimerHeap []*simTimer + +func (h *simTimerHeap) Len() int { + return len(*h) +} + +func (h *simTimerHeap) Less(i, j int) bool { + return (*h)[i].at < (*h)[j].at +} + +func (h *simTimerHeap) Swap(i, j int) { + (*h)[i], (*h)[j] = (*h)[j], (*h)[i] + (*h)[i].index = i + (*h)[j].index = j +} + +func (h *simTimerHeap) Push(x interface{}) { + t := x.(*simTimer) + t.index = len(*h) + *h = append(*h, t) +} + +func (h *simTimerHeap) Pop() interface{} { + end := len(*h) - 1 + t := (*h)[end] + t.index = -1 + (*h)[end] = nil + *h = (*h)[:end] + return t } diff --git a/common/mclock/simclock_test.go b/common/mclock/simclock_test.go index 09e4391c1c..94aa4f2b39 100644 --- a/common/mclock/simclock_test.go +++ b/common/mclock/simclock_test.go @@ -25,14 +25,16 @@ var _ Clock = System{} var _ Clock = new(Simulated) func TestSimulatedAfter(t *testing.T) { - const timeout = 30 * time.Minute - const adv = time.Minute - var ( - c Simulated - end = c.Now().Add(timeout) - ch = c.After(timeout) + timeout = 30 * time.Minute + offset = 99 * time.Hour + adv = 11 * time.Minute + c Simulated ) + c.Run(offset) + + end := c.Now().Add(timeout) + ch := c.After(timeout) for c.Now() < end.Add(-adv) { c.Run(adv) select { @@ -45,8 +47,8 @@ func TestSimulatedAfter(t *testing.T) { c.Run(adv) select { case stamp := <-ch: - want := time.Time{}.Add(timeout) - if !stamp.Equal(want) { + want := AbsTime(0).Add(offset).Add(timeout) + if stamp != want { t.Errorf("Wrong time sent on timer channel: got %v, want %v", stamp, want) } default: @@ -113,3 +115,48 @@ func TestSimulatedSleep(t *testing.T) { t.Fatal("Sleep didn't return in time") } } + +func TestSimulatedTimerReset(t *testing.T) { + var ( + c Simulated + timeout = 1 * time.Hour + ) + timer := c.NewTimer(timeout) + c.Run(2 * timeout) + select { + case ftime := <-timer.C(): + if ftime != AbsTime(timeout) { + t.Fatalf("wrong time %v sent on timer channel, want %v", ftime, AbsTime(timeout)) + } + default: + t.Fatal("timer didn't fire") + } + + timer.Reset(timeout) + c.Run(2 * timeout) + select { + case ftime := <-timer.C(): + if ftime != AbsTime(3*timeout) { + t.Fatalf("wrong time %v sent on timer channel, want %v", ftime, AbsTime(3*timeout)) + } + default: + t.Fatal("timer didn't fire again") + } +} + +func TestSimulatedTimerStop(t *testing.T) { + var ( + c Simulated + timeout = 1 * time.Hour + ) + timer := c.NewTimer(timeout) + c.Run(2 * timeout) + if timer.Stop() { + t.Errorf("Stop returned true for fired timer") + } + select { + case <-timer.C(): + default: + t.Fatal("timer didn't fire") + } +} From 8045504abf64a865be4b1dbc780b796a9f5d11cc Mon Sep 17 00:00:00 2001 From: Adam Schmideg Date: Tue, 11 Feb 2020 16:46:32 +0100 Subject: [PATCH 101/128] les: log disconnect reason when light server is not synced (#20643) Co-authored-by: ligi --- les/server_handler.go | 1 + 1 file changed, 1 insertion(+) diff --git a/les/server_handler.go b/les/server_handler.go index 38aeac4e88..98e9294b07 100644 --- a/les/server_handler.go +++ b/les/server_handler.go @@ -128,6 +128,7 @@ func (h *serverHandler) handle(p *peer) error { } // Reject light clients if server is not synced. if !h.synced() { + p.Log().Debug("Light server not synced, rejecting peer") return p2p.DiscRequested } defer p.fcClient.Disconnect() From 18213281627b310ac68d22bca7c401ab5f67e1da Mon Sep 17 00:00:00 2001 From: Boqin Qin Date: Wed, 12 Feb 2020 17:33:31 +0800 Subject: [PATCH 102/128] event: add missing unlock before panic (#20653) --- event/feed.go | 1 + 1 file changed, 1 insertion(+) diff --git a/event/feed.go b/event/feed.go index 02f3ca6875..33dafe5886 100644 --- a/event/feed.go +++ b/event/feed.go @@ -138,6 +138,7 @@ func (f *Feed) Send(value interface{}) (nsent int) { if !f.typecheck(rvalue.Type()) { f.sendLock <- struct{}{} + f.mu.Unlock() panic(feedTypeError{op: "Send", got: rvalue.Type(), want: f.etype}) } f.mu.Unlock() From 46c4b699c8b5b537416d395852a3f44fd7a45f49 Mon Sep 17 00:00:00 2001 From: Marius van der Wijden Date: Wed, 12 Feb 2020 11:33:17 +0100 Subject: [PATCH 103/128] accounts/abi/bind/backends: add support for historical state (#20644) --- accounts/abi/bind/backends/simulated.go | 40 +++++++++++++++++-------- 1 file changed, 28 insertions(+), 12 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index 0b2555c994..f7f3dec839 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -127,15 +127,28 @@ func (b *SimulatedBackend) rollback() { b.pendingState, _ = state.New(b.pendingBlock.Root(), statedb.Database()) } +// stateByBlockNumber retrieves a state by a given blocknumber. +func (b *SimulatedBackend) stateByBlockNumber(ctx context.Context, blockNumber *big.Int) (*state.StateDB, error) { + if blockNumber == nil || blockNumber.Cmp(b.blockchain.CurrentBlock().Number()) == 0 { + return b.blockchain.State() + } + block, err := b.BlockByNumber(ctx, blockNumber) + if err != nil { + return nil, err + } + return b.blockchain.StateAt(block.Hash()) +} + // CodeAt returns the code associated with a certain account in the blockchain. func (b *SimulatedBackend) CodeAt(ctx context.Context, contract common.Address, blockNumber *big.Int) ([]byte, error) { b.mu.Lock() defer b.mu.Unlock() - if blockNumber != nil && blockNumber.Cmp(b.blockchain.CurrentBlock().Number()) != 0 { - return nil, errBlockNumberUnsupported + statedb, err := b.stateByBlockNumber(ctx, blockNumber) + if err != nil { + return nil, err } - statedb, _ := b.blockchain.State() + return statedb.GetCode(contract), nil } @@ -144,10 +157,11 @@ func (b *SimulatedBackend) BalanceAt(ctx context.Context, contract common.Addres b.mu.Lock() defer b.mu.Unlock() - if blockNumber != nil && blockNumber.Cmp(b.blockchain.CurrentBlock().Number()) != 0 { - return nil, errBlockNumberUnsupported + statedb, err := b.stateByBlockNumber(ctx, blockNumber) + if err != nil { + return nil, err } - statedb, _ := b.blockchain.State() + return statedb.GetBalance(contract), nil } @@ -156,10 +170,11 @@ func (b *SimulatedBackend) NonceAt(ctx context.Context, contract common.Address, b.mu.Lock() defer b.mu.Unlock() - if blockNumber != nil && blockNumber.Cmp(b.blockchain.CurrentBlock().Number()) != 0 { - return 0, errBlockNumberUnsupported + statedb, err := b.stateByBlockNumber(ctx, blockNumber) + if err != nil { + return 0, err } - statedb, _ := b.blockchain.State() + return statedb.GetNonce(contract), nil } @@ -168,10 +183,11 @@ func (b *SimulatedBackend) StorageAt(ctx context.Context, contract common.Addres b.mu.Lock() defer b.mu.Unlock() - if blockNumber != nil && blockNumber.Cmp(b.blockchain.CurrentBlock().Number()) != 0 { - return nil, errBlockNumberUnsupported + statedb, err := b.stateByBlockNumber(ctx, blockNumber) + if err != nil { + return nil, err } - statedb, _ := b.blockchain.State() + val := statedb.GetState(contract, key) return val[:], nil } From a9614c3c9167d2da3b5c72e773e64984ab504814 Mon Sep 17 00:00:00 2001 From: Boqin Qin Date: Wed, 12 Feb 2020 22:19:47 +0800 Subject: [PATCH 104/128] event, p2p/simulations/adapters: fix rare goroutine leaks (#20657) Co-authored-by: Felix Lange --- event/subscription.go | 17 ++++++++--------- event/subscription_test.go | 2 +- p2p/simulations/adapters/exec.go | 2 +- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/event/subscription.go b/event/subscription.go index d03f465075..c80d171f3a 100644 --- a/event/subscription.go +++ b/event/subscription.go @@ -145,7 +145,6 @@ func (s *resubscribeSub) loop() { func (s *resubscribeSub) subscribe() Subscription { subscribed := make(chan error) var sub Subscription -retry: for { s.lastTry = mclock.Now() ctx, cancel := context.WithCancel(context.Background()) @@ -157,19 +156,19 @@ retry: select { case err := <-subscribed: cancel() - if err != nil { - // Subscribing failed, wait before launching the next try. - if s.backoffWait() { - return nil + if err == nil { + if sub == nil { + panic("event: ResubscribeFunc returned nil subscription and no error") } - continue retry + return sub } - if sub == nil { - panic("event: ResubscribeFunc returned nil subscription and no error") + // Subscribing failed, wait before launching the next try. + if s.backoffWait() { + return nil // unsubscribed during wait } - return sub case <-s.unsub: cancel() + <-subscribed // avoid leaking the s.fn goroutine. return nil } } diff --git a/event/subscription_test.go b/event/subscription_test.go index 5b8a2c8ede..c48be3aa30 100644 --- a/event/subscription_test.go +++ b/event/subscription_test.go @@ -102,7 +102,7 @@ func TestResubscribe(t *testing.T) { func TestResubscribeAbort(t *testing.T) { t.Parallel() - done := make(chan error) + done := make(chan error, 1) sub := Resubscribe(0, func(ctx context.Context) (Subscription, error) { select { case <-ctx.Done(): diff --git a/p2p/simulations/adapters/exec.go b/p2p/simulations/adapters/exec.go index 7c6ec94621..18ec9c69b8 100644 --- a/p2p/simulations/adapters/exec.go +++ b/p2p/simulations/adapters/exec.go @@ -287,7 +287,7 @@ func (n *ExecNode) Stop() error { if err := n.Cmd.Process.Signal(syscall.SIGTERM); err != nil { return n.Cmd.Process.Kill() } - waitErr := make(chan error) + waitErr := make(chan error, 1) go func() { waitErr <- n.Cmd.Wait() }() From 5f2002bbcc1ad21818d9b08badea84acac6d0481 Mon Sep 17 00:00:00 2001 From: Boqin Qin Date: Wed, 12 Feb 2020 22:20:50 +0800 Subject: [PATCH 105/128] accounts: add walletsNoLock to avoid double read lock (#20655) --- accounts/manager.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/accounts/manager.go b/accounts/manager.go index 731d12ea30..acf41ed8e9 100644 --- a/accounts/manager.go +++ b/accounts/manager.go @@ -141,6 +141,11 @@ func (am *Manager) Wallets() []Wallet { am.lock.RLock() defer am.lock.RUnlock() + return am.walletsNoLock() +} + +// walletsNoLock returns all registered wallets. Callers must hold am.lock. +func (am *Manager) walletsNoLock() []Wallet { cpy := make([]Wallet, len(am.wallets)) copy(cpy, am.wallets) return cpy @@ -155,7 +160,7 @@ func (am *Manager) Wallet(url string) (Wallet, error) { if err != nil { return nil, err } - for _, wallet := range am.Wallets() { + for _, wallet := range am.walletsNoLock() { if wallet.URL() == parsed { return wallet, nil } From 90caa2cabbd46c3e712eb7216bd668dd02bb15a8 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 13 Feb 2020 11:10:03 +0100 Subject: [PATCH 106/128] p2p: new dial scheduler (#20592) * p2p: new dial scheduler This change replaces the peer-to-peer dial scheduler with a new and improved implementation. The new code is better than the previous implementation in two key aspects: - The time between discovery of a node and dialing that node is significantly lower in the new version. The old dialState kept a buffer of nodes and launched a task to refill it whenever the buffer became empty. This worked well with the discovery interface we used to have, but doesn't really work with the new iterator-based discovery API. - Selection of static dial candidates (created by Server.AddPeer or through static-nodes.json) performs much better for large amounts of static peers. Connections to static nodes are now limited like dynanic dials and can no longer overstep MaxPeers or the dial ratio. * p2p/simulations/adapters: adapt to new NodeDialer interface * p2p: re-add check for self in checkDial * p2p: remove peersetCh * p2p: allow static dials when discovery is disabled * p2p: add test for dialScheduler.removeStatic * p2p: remove blank line * p2p: fix documentation of maxDialPeers * p2p: change "ok" to "added" in static node log * p2p: improve dialTask docs Also increase log level for "Can't resolve node" * p2p: ensure dial resolver is truly nil without discovery * p2p: add "looking for peers" log message * p2p: clean up Server.run comments * p2p: fix maxDialedConns for maxpeers < dialRatio Always allocate at least one dial slot unless dialing is disabled using NoDial or MaxPeers == 0. Most importantly, this fixes MaxPeers == 1 to dedicate the sole slot to dialing instead of listening. * p2p: fix RemovePeer to disconnect the peer again Also make RemovePeer synchronous and add a test. * p2p: remove "Connection set up" log message * p2p: clean up connection logging We previously logged outgoing connection failures up to three times. - in SetupConn() as "Setting up connection failed addr=..." - in setupConn() with an error-specific message and "id=... addr=..." - in dial() as "Dial error task=..." This commit ensures a single log message is emitted per failure and adds "id=... addr=... conn=..." everywhere (id= omitted when the ID isn't known yet). Also avoid printing a log message when a static dial fails but can't be resolved because discv4 is disabled. The light client hit this case all the time, increasing the message count to four lines per failed connection. * p2p: document that RemovePeer blocks --- p2p/dial.go | 591 ++++++++++----- p2p/dial_test.go | 1084 +++++++++++++++------------- p2p/peer_test.go | 44 +- p2p/server.go | 320 ++++---- p2p/server_test.go | 218 ++---- p2p/simulations/adapters/inproc.go | 3 +- p2p/util.go | 20 +- p2p/util_test.go | 12 +- 8 files changed, 1244 insertions(+), 1048 deletions(-) diff --git a/p2p/dial.go b/p2p/dial.go index 3975b488bf..d190e866af 100644 --- a/p2p/dial.go +++ b/p2p/dial.go @@ -17,11 +17,17 @@ package p2p import ( + "context" + crand "crypto/rand" + "encoding/binary" "errors" "fmt" + mrand "math/rand" "net" + "sync" "time" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/netutil" @@ -33,8 +39,9 @@ const ( // private networks. dialHistoryExpiration = inboundThrottleTime + 5*time.Second - // If no peers are found for this amount of time, the initial bootnodes are dialed. - fallbackInterval = 20 * time.Second + // Config for the "Looking for peers" message. + dialStatsLogInterval = 10 * time.Second // printed at most this often + dialStatsPeerLimit = 3 // but not if more than this many dialed peers // Endpoint resolution is throttled with bounded backoff. initialResolveDelay = 60 * time.Second @@ -42,219 +49,443 @@ const ( ) // NodeDialer is used to connect to nodes in the network, typically by using -// an underlying net.Dialer but also using net.Pipe in tests +// an underlying net.Dialer but also using net.Pipe in tests. type NodeDialer interface { - Dial(*enode.Node) (net.Conn, error) + Dial(context.Context, *enode.Node) (net.Conn, error) } type nodeResolver interface { Resolve(*enode.Node) *enode.Node } -// TCPDialer implements the NodeDialer interface by using a net.Dialer to -// create TCP connections to nodes in the network -type TCPDialer struct { - *net.Dialer +// tcpDialer implements NodeDialer using real TCP connections. +type tcpDialer struct { + d *net.Dialer } -// Dial creates a TCP connection to the node -func (t TCPDialer) Dial(dest *enode.Node) (net.Conn, error) { - addr := &net.TCPAddr{IP: dest.IP(), Port: dest.TCP()} - return t.Dialer.Dial("tcp", addr.String()) +func (t tcpDialer) Dial(ctx context.Context, dest *enode.Node) (net.Conn, error) { + return t.d.DialContext(ctx, "tcp", nodeAddr(dest).String()) } -// dialstate schedules dials and discovery lookups. -// It gets a chance to compute new tasks on every iteration -// of the main loop in Server.run. -type dialstate struct { - maxDynDials int - netrestrict *netutil.Netlist - self enode.ID - bootnodes []*enode.Node // default dials when there are no peers - log log.Logger +func nodeAddr(n *enode.Node) net.Addr { + return &net.TCPAddr{IP: n.IP(), Port: n.TCP()} +} + +// checkDial errors: +var ( + errSelf = errors.New("is self") + errAlreadyDialing = errors.New("already dialing") + errAlreadyConnected = errors.New("already connected") + errRecentlyDialed = errors.New("recently dialed") + errNotWhitelisted = errors.New("not contained in netrestrict whitelist") +) + +// dialer creates outbound connections and submits them into Server. +// Two types of peer connections can be created: +// +// - static dials are pre-configured connections. The dialer attempts +// keep these nodes connected at all times. +// +// - dynamic dials are created from node discovery results. The dialer +// continuously reads candidate nodes from its input iterator and attempts +// to create peer connections to nodes arriving through the iterator. +// +type dialScheduler struct { + dialConfig + setupFunc dialSetupFunc + wg sync.WaitGroup + cancel context.CancelFunc + ctx context.Context + nodesIn chan *enode.Node + doneCh chan *dialTask + addStaticCh chan *enode.Node + remStaticCh chan *enode.Node + addPeerCh chan *conn + remPeerCh chan *conn + + // Everything below here belongs to loop and + // should only be accessed by code on the loop goroutine. + dialing map[enode.ID]*dialTask // active tasks + peers map[enode.ID]connFlag // all connected peers + dialPeers int // current number of dialed peers + + // The static map tracks all static dial tasks. The subset of usable static dial tasks + // (i.e. those passing checkDial) is kept in staticPool. The scheduler prefers + // launching random static tasks from the pool over launching dynamic dials from the + // iterator. + static map[enode.ID]*dialTask + staticPool []*dialTask + + // The dial history keeps recently dialed nodes. Members of history are not dialed. + history expHeap + historyTimer mclock.Timer + historyTimerTime mclock.AbsTime + + // for logStats + lastStatsLog mclock.AbsTime + doneSinceLastLog int +} - start time.Time // time when the dialer was first used - lookupRunning bool - dialing map[enode.ID]connFlag - lookupBuf []*enode.Node // current discovery lookup results - static map[enode.ID]*dialTask - hist expHeap +type dialSetupFunc func(net.Conn, connFlag, *enode.Node) error + +type dialConfig struct { + self enode.ID // our own ID + maxDialPeers int // maximum number of dialed peers + maxActiveDials int // maximum number of active dials + netRestrict *netutil.Netlist // IP whitelist, disabled if nil + resolver nodeResolver + dialer NodeDialer + log log.Logger + clock mclock.Clock + rand *mrand.Rand } -type task interface { - Do(*Server) +func (cfg dialConfig) withDefaults() dialConfig { + if cfg.maxActiveDials == 0 { + cfg.maxActiveDials = defaultMaxPendingPeers + } + if cfg.log == nil { + cfg.log = log.Root() + } + if cfg.clock == nil { + cfg.clock = mclock.System{} + } + if cfg.rand == nil { + seedb := make([]byte, 8) + crand.Read(seedb) + seed := int64(binary.BigEndian.Uint64(seedb)) + cfg.rand = mrand.New(mrand.NewSource(seed)) + } + return cfg } -func newDialState(self enode.ID, maxdyn int, cfg *Config) *dialstate { - s := &dialstate{ - maxDynDials: maxdyn, - self: self, - netrestrict: cfg.NetRestrict, - log: cfg.Logger, +func newDialScheduler(config dialConfig, it enode.Iterator, setupFunc dialSetupFunc) *dialScheduler { + d := &dialScheduler{ + dialConfig: config.withDefaults(), + setupFunc: setupFunc, + dialing: make(map[enode.ID]*dialTask), static: make(map[enode.ID]*dialTask), - dialing: make(map[enode.ID]connFlag), - bootnodes: make([]*enode.Node, len(cfg.BootstrapNodes)), + peers: make(map[enode.ID]connFlag), + doneCh: make(chan *dialTask), + nodesIn: make(chan *enode.Node), + addStaticCh: make(chan *enode.Node), + remStaticCh: make(chan *enode.Node), + addPeerCh: make(chan *conn), + remPeerCh: make(chan *conn), } - copy(s.bootnodes, cfg.BootstrapNodes) - if s.log == nil { - s.log = log.Root() + d.lastStatsLog = d.clock.Now() + d.ctx, d.cancel = context.WithCancel(context.Background()) + d.wg.Add(2) + go d.readNodes(it) + go d.loop(it) + return d +} + +// stop shuts down the dialer, canceling all current dial tasks. +func (d *dialScheduler) stop() { + d.cancel() + d.wg.Wait() +} + +// addStatic adds a static dial candidate. +func (d *dialScheduler) addStatic(n *enode.Node) { + select { + case d.addStaticCh <- n: + case <-d.ctx.Done(): } - for _, n := range cfg.StaticNodes { - s.addStatic(n) +} + +// removeStatic removes a static dial candidate. +func (d *dialScheduler) removeStatic(n *enode.Node) { + select { + case d.remStaticCh <- n: + case <-d.ctx.Done(): } - return s } -func (s *dialstate) addStatic(n *enode.Node) { - // This overwrites the task instead of updating an existing - // entry, giving users the opportunity to force a resolve operation. - s.static[n.ID()] = &dialTask{flags: staticDialedConn, dest: n} +// peerAdded updates the peer set. +func (d *dialScheduler) peerAdded(c *conn) { + select { + case d.addPeerCh <- c: + case <-d.ctx.Done(): + } } -func (s *dialstate) removeStatic(n *enode.Node) { - // This removes a task so future attempts to connect will not be made. - delete(s.static, n.ID()) +// peerRemoved updates the peer set. +func (d *dialScheduler) peerRemoved(c *conn) { + select { + case d.remPeerCh <- c: + case <-d.ctx.Done(): + } } -func (s *dialstate) newTasks(nRunning int, peers map[enode.ID]*Peer, now time.Time) []task { - var newtasks []task - addDial := func(flag connFlag, n *enode.Node) bool { - if err := s.checkDial(n, peers); err != nil { - s.log.Trace("Skipping dial candidate", "id", n.ID(), "addr", &net.TCPAddr{IP: n.IP(), Port: n.TCP()}, "err", err) - return false - } - s.dialing[n.ID()] = flag - newtasks = append(newtasks, &dialTask{flags: flag, dest: n}) - return true - } - - if s.start.IsZero() { - s.start = now - } - s.hist.expire(now) - - // Create dials for static nodes if they are not connected. - for id, t := range s.static { - err := s.checkDial(t.dest, peers) - switch err { - case errNotWhitelisted, errSelf: - s.log.Warn("Removing static dial candidate", "id", t.dest.ID, "addr", &net.TCPAddr{IP: t.dest.IP(), Port: t.dest.TCP()}, "err", err) - delete(s.static, t.dest.ID()) - case nil: - s.dialing[id] = t.flags - newtasks = append(newtasks, t) +// loop is the main loop of the dialer. +func (d *dialScheduler) loop(it enode.Iterator) { + var ( + nodesCh chan *enode.Node + historyExp = make(chan struct{}, 1) + ) + +loop: + for { + // Launch new dials if slots are available. + slots := d.freeDialSlots() + slots -= d.startStaticDials(slots) + if slots > 0 { + nodesCh = d.nodesIn + } else { + nodesCh = nil } - } + d.rearmHistoryTimer(historyExp) + d.logStats() + + select { + case node := <-nodesCh: + if err := d.checkDial(node); err != nil { + d.log.Trace("Discarding dial candidate", "id", node.ID(), "ip", node.IP(), "reason", err) + } else { + d.startDial(newDialTask(node, dynDialedConn)) + } + + case task := <-d.doneCh: + id := task.dest.ID() + delete(d.dialing, id) + d.updateStaticPool(id) + d.doneSinceLastLog++ + + case c := <-d.addPeerCh: + if c.is(dynDialedConn) || c.is(staticDialedConn) { + d.dialPeers++ + } + id := c.node.ID() + d.peers[id] = c.flags + // Remove from static pool because the node is now connected. + task := d.static[id] + if task != nil && task.staticPoolIndex >= 0 { + d.removeFromStaticPool(task.staticPoolIndex) + } + // TODO: cancel dials to connected peers + + case c := <-d.remPeerCh: + if c.is(dynDialedConn) || c.is(staticDialedConn) { + d.dialPeers-- + } + delete(d.peers, c.node.ID()) + d.updateStaticPool(c.node.ID()) + + case node := <-d.addStaticCh: + id := node.ID() + _, exists := d.static[id] + d.log.Trace("Adding static node", "id", id, "ip", node.IP(), "added", !exists) + if exists { + continue loop + } + task := newDialTask(node, staticDialedConn) + d.static[id] = task + if d.checkDial(node) == nil { + d.addToStaticPool(task) + } + + case node := <-d.remStaticCh: + id := node.ID() + task := d.static[id] + d.log.Trace("Removing static node", "id", id, "ok", task != nil) + if task != nil { + delete(d.static, id) + if task.staticPoolIndex >= 0 { + d.removeFromStaticPool(task.staticPoolIndex) + } + } - // Compute number of dynamic dials needed. - needDynDials := s.maxDynDials - for _, p := range peers { - if p.rw.is(dynDialedConn) { - needDynDials-- + case <-historyExp: + d.expireHistory() + + case <-d.ctx.Done(): + it.Close() + break loop } } - for _, flag := range s.dialing { - if flag&dynDialedConn != 0 { - needDynDials-- - } + + d.stopHistoryTimer(historyExp) + for range d.dialing { + <-d.doneCh } + d.wg.Done() +} + +// readNodes runs in its own goroutine and delivers nodes from +// the input iterator to the nodesIn channel. +func (d *dialScheduler) readNodes(it enode.Iterator) { + defer d.wg.Done() - // If we don't have any peers whatsoever, try to dial a random bootnode. This - // scenario is useful for the testnet (and private networks) where the discovery - // table might be full of mostly bad peers, making it hard to find good ones. - if len(peers) == 0 && len(s.bootnodes) > 0 && needDynDials > 0 && now.Sub(s.start) > fallbackInterval { - bootnode := s.bootnodes[0] - s.bootnodes = append(s.bootnodes[:0], s.bootnodes[1:]...) - s.bootnodes = append(s.bootnodes, bootnode) - if addDial(dynDialedConn, bootnode) { - needDynDials-- + for it.Next() { + select { + case d.nodesIn <- it.Node(): + case <-d.ctx.Done(): } } +} - // Create dynamic dials from discovery results. - i := 0 - for ; i < len(s.lookupBuf) && needDynDials > 0; i++ { - if addDial(dynDialedConn, s.lookupBuf[i]) { - needDynDials-- - } +// logStats prints dialer statistics to the log. The message is suppressed when enough +// peers are connected because users should only see it while their client is starting up +// or comes back online. +func (d *dialScheduler) logStats() { + now := d.clock.Now() + if d.lastStatsLog.Add(dialStatsLogInterval) > now { + return + } + if d.dialPeers < dialStatsPeerLimit && d.dialPeers < d.maxDialPeers { + d.log.Info("Looking for peers", "peercount", len(d.peers), "tried", d.doneSinceLastLog, "static", len(d.static)) } - s.lookupBuf = s.lookupBuf[:copy(s.lookupBuf, s.lookupBuf[i:])] + d.doneSinceLastLog = 0 + d.lastStatsLog = now +} - // Launch a discovery lookup if more candidates are needed. - if len(s.lookupBuf) < needDynDials && !s.lookupRunning { - s.lookupRunning = true - newtasks = append(newtasks, &discoverTask{want: needDynDials - len(s.lookupBuf)}) +// rearmHistoryTimer configures d.historyTimer to fire when the +// next item in d.history expires. +func (d *dialScheduler) rearmHistoryTimer(ch chan struct{}) { + if len(d.history) == 0 || d.historyTimerTime == d.history.nextExpiry() { + return } + d.stopHistoryTimer(ch) + d.historyTimerTime = d.history.nextExpiry() + timeout := time.Duration(d.historyTimerTime - d.clock.Now()) + d.historyTimer = d.clock.AfterFunc(timeout, func() { ch <- struct{}{} }) +} - // Launch a timer to wait for the next node to expire if all - // candidates have been tried and no task is currently active. - // This should prevent cases where the dialer logic is not ticked - // because there are no pending events. - if nRunning == 0 && len(newtasks) == 0 && s.hist.Len() > 0 { - t := &waitExpireTask{s.hist.nextExpiry().Sub(now)} - newtasks = append(newtasks, t) +// stopHistoryTimer stops the timer and drains the channel it sends on. +func (d *dialScheduler) stopHistoryTimer(ch chan struct{}) { + if d.historyTimer != nil && !d.historyTimer.Stop() { + <-ch } - return newtasks } -var ( - errSelf = errors.New("is self") - errAlreadyDialing = errors.New("already dialing") - errAlreadyConnected = errors.New("already connected") - errRecentlyDialed = errors.New("recently dialed") - errNotWhitelisted = errors.New("not contained in netrestrict whitelist") -) +// expireHistory removes expired items from d.history. +func (d *dialScheduler) expireHistory() { + d.historyTimer.Stop() + d.historyTimer = nil + d.historyTimerTime = 0 + d.history.expire(d.clock.Now(), func(hkey string) { + var id enode.ID + copy(id[:], hkey) + d.updateStaticPool(id) + }) +} + +// freeDialSlots returns the number of free dial slots. The result can be negative +// when peers are connected while their task is still running. +func (d *dialScheduler) freeDialSlots() int { + slots := (d.maxDialPeers - d.dialPeers) * 2 + if slots > d.maxActiveDials { + slots = d.maxActiveDials + } + free := slots - len(d.dialing) + return free +} -func (s *dialstate) checkDial(n *enode.Node, peers map[enode.ID]*Peer) error { - _, dialing := s.dialing[n.ID()] - switch { - case dialing: +// checkDial returns an error if node n should not be dialed. +func (d *dialScheduler) checkDial(n *enode.Node) error { + if n.ID() == d.self { + return errSelf + } + if _, ok := d.dialing[n.ID()]; ok { return errAlreadyDialing - case peers[n.ID()] != nil: + } + if _, ok := d.peers[n.ID()]; ok { return errAlreadyConnected - case n.ID() == s.self: - return errSelf - case s.netrestrict != nil && !s.netrestrict.Contains(n.IP()): + } + if d.netRestrict != nil && !d.netRestrict.Contains(n.IP()) { return errNotWhitelisted - case s.hist.contains(string(n.ID().Bytes())): + } + if d.history.contains(string(n.ID().Bytes())) { return errRecentlyDialed } return nil } -func (s *dialstate) taskDone(t task, now time.Time) { - switch t := t.(type) { - case *dialTask: - s.hist.add(string(t.dest.ID().Bytes()), now.Add(dialHistoryExpiration)) - delete(s.dialing, t.dest.ID()) - case *discoverTask: - s.lookupRunning = false - s.lookupBuf = append(s.lookupBuf, t.results...) +// startStaticDials starts n static dial tasks. +func (d *dialScheduler) startStaticDials(n int) (started int) { + for started = 0; started < n && len(d.staticPool) > 0; started++ { + idx := d.rand.Intn(len(d.staticPool)) + task := d.staticPool[idx] + d.startDial(task) + d.removeFromStaticPool(idx) + } + return started +} + +// updateStaticPool attempts to move the given static dial back into staticPool. +func (d *dialScheduler) updateStaticPool(id enode.ID) { + task, ok := d.static[id] + if ok && task.staticPoolIndex < 0 && d.checkDial(task.dest) == nil { + d.addToStaticPool(task) + } +} + +func (d *dialScheduler) addToStaticPool(task *dialTask) { + if task.staticPoolIndex >= 0 { + panic("attempt to add task to staticPool twice") } + d.staticPool = append(d.staticPool, task) + task.staticPoolIndex = len(d.staticPool) - 1 } -// A dialTask is generated for each node that is dialed. Its -// fields cannot be accessed while the task is running. +// removeFromStaticPool removes the task at idx from staticPool. It does that by moving the +// current last element of the pool to idx and then shortening the pool by one. +func (d *dialScheduler) removeFromStaticPool(idx int) { + task := d.staticPool[idx] + end := len(d.staticPool) - 1 + d.staticPool[idx] = d.staticPool[end] + d.staticPool[idx].staticPoolIndex = idx + d.staticPool[end] = nil + d.staticPool = d.staticPool[:end] + task.staticPoolIndex = -1 +} + +// startDial runs the given dial task in a separate goroutine. +func (d *dialScheduler) startDial(task *dialTask) { + d.log.Trace("Starting p2p dial", "id", task.dest.ID(), "ip", task.dest.IP(), "flag", task.flags) + hkey := string(task.dest.ID().Bytes()) + d.history.add(hkey, d.clock.Now().Add(dialHistoryExpiration)) + d.dialing[task.dest.ID()] = task + go func() { + task.run(d) + d.doneCh <- task + }() +} + +// A dialTask generated for each node that is dialed. type dialTask struct { - flags connFlag + staticPoolIndex int + flags connFlag + // These fields are private to the task and should not be + // accessed by dialScheduler while the task is running. dest *enode.Node - lastResolved time.Time + lastResolved mclock.AbsTime resolveDelay time.Duration } -func (t *dialTask) Do(srv *Server) { +func newDialTask(dest *enode.Node, flags connFlag) *dialTask { + return &dialTask{dest: dest, flags: flags, staticPoolIndex: -1} +} + +type dialError struct { + error +} + +func (t *dialTask) run(d *dialScheduler) { if t.dest.Incomplete() { - if !t.resolve(srv) { + if !t.resolve(d) { return } } - err := t.dial(srv, t.dest) + + err := t.dial(d, t.dest) if err != nil { - srv.log.Trace("Dial error", "task", t, "err", err) // Try resolving the ID of static nodes if dialing failed. if _, ok := err.(*dialError); ok && t.flags&staticDialedConn != 0 { - if t.resolve(srv) { - t.dial(srv, t.dest) + if t.resolve(d) { + t.dial(d, t.dest) } } } @@ -266,46 +497,42 @@ func (t *dialTask) Do(srv *Server) { // Resolve operations are throttled with backoff to avoid flooding the // discovery network with useless queries for nodes that don't exist. // The backoff delay resets when the node is found. -func (t *dialTask) resolve(srv *Server) bool { - if srv.staticNodeResolver == nil { - srv.log.Debug("Can't resolve node", "id", t.dest.ID(), "err", "discovery is disabled") +func (t *dialTask) resolve(d *dialScheduler) bool { + if d.resolver == nil { return false } if t.resolveDelay == 0 { t.resolveDelay = initialResolveDelay } - if time.Since(t.lastResolved) < t.resolveDelay { + if t.lastResolved > 0 && time.Duration(d.clock.Now()-t.lastResolved) < t.resolveDelay { return false } - resolved := srv.staticNodeResolver.Resolve(t.dest) - t.lastResolved = time.Now() + resolved := d.resolver.Resolve(t.dest) + t.lastResolved = d.clock.Now() if resolved == nil { t.resolveDelay *= 2 if t.resolveDelay > maxResolveDelay { t.resolveDelay = maxResolveDelay } - srv.log.Debug("Resolving node failed", "id", t.dest.ID(), "newdelay", t.resolveDelay) + d.log.Debug("Resolving node failed", "id", t.dest.ID(), "newdelay", t.resolveDelay) return false } // The node was found. t.resolveDelay = initialResolveDelay t.dest = resolved - srv.log.Debug("Resolved node", "id", t.dest.ID(), "addr", &net.TCPAddr{IP: t.dest.IP(), Port: t.dest.TCP()}) + d.log.Debug("Resolved node", "id", t.dest.ID(), "addr", &net.TCPAddr{IP: t.dest.IP(), Port: t.dest.TCP()}) return true } -type dialError struct { - error -} - // dial performs the actual connection attempt. -func (t *dialTask) dial(srv *Server, dest *enode.Node) error { - fd, err := srv.Dialer.Dial(dest) +func (t *dialTask) dial(d *dialScheduler, dest *enode.Node) error { + fd, err := d.dialer.Dial(d.ctx, t.dest) if err != nil { + d.log.Trace("Dial error", "id", t.dest.ID(), "addr", nodeAddr(t.dest), "conn", t.flags, "err", cleanupDialErr(err)) return &dialError{err} } mfd := newMeteredConn(fd, false, &net.TCPAddr{IP: dest.IP(), Port: dest.TCP()}) - return srv.SetupConn(mfd, t.flags, dest) + return d.setupFunc(mfd, t.flags, dest) } func (t *dialTask) String() string { @@ -313,37 +540,9 @@ func (t *dialTask) String() string { return fmt.Sprintf("%v %x %v:%d", t.flags, id[:8], t.dest.IP(), t.dest.TCP()) } -// discoverTask runs discovery table operations. -// Only one discoverTask is active at any time. -// discoverTask.Do performs a random lookup. -type discoverTask struct { - want int - results []*enode.Node -} - -func (t *discoverTask) Do(srv *Server) { - t.results = enode.ReadNodes(srv.discmix, t.want) -} - -func (t *discoverTask) String() string { - s := "discovery query" - if len(t.results) > 0 { - s += fmt.Sprintf(" (%d results)", len(t.results)) - } else { - s += fmt.Sprintf(" (want %d)", t.want) +func cleanupDialErr(err error) error { + if netErr, ok := err.(*net.OpError); ok && netErr.Op == "dial" { + return netErr.Err } - return s -} - -// A waitExpireTask is generated if there are no other tasks -// to keep the loop in Server.run ticking. -type waitExpireTask struct { - time.Duration -} - -func (t waitExpireTask) Do(*Server) { - time.Sleep(t.Duration) -} -func (t waitExpireTask) String() string { - return fmt.Sprintf("wait for dial hist expire (%v)", t.Duration) + return err } diff --git a/p2p/dial_test.go b/p2p/dial_test.go index 6189ec4d0b..cd8dedff1c 100644 --- a/p2p/dial_test.go +++ b/p2p/dial_test.go @@ -17,574 +17,656 @@ package p2p import ( - "encoding/binary" + "context" + "errors" + "fmt" + "math/rand" "net" "reflect" - "strings" + "sync" "testing" "time" - "github.com/davecgh/go-spew/spew" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/internal/testlog" "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/p2p/enode" - "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/p2p/netutil" ) -func init() { - spew.Config.Indent = "\t" -} - -type dialtest struct { - init *dialstate // state before and after the test. - rounds []round -} - -type round struct { - peers []*Peer // current peer set - done []task // tasks that got done this round - new []task // the result must match this one -} +// This test checks that dynamic dials are launched from discovery results. +func TestDialSchedDynDial(t *testing.T) { + t.Parallel() -func runDialTest(t *testing.T, test dialtest) { - var ( - vtime time.Time - running int - ) - pm := func(ps []*Peer) map[enode.ID]*Peer { - m := make(map[enode.ID]*Peer) - for _, p := range ps { - m[p.ID()] = p - } - return m + config := dialConfig{ + maxActiveDials: 5, + maxDialPeers: 4, } - for i, round := range test.rounds { - for _, task := range round.done { - running-- - if running < 0 { - panic("running task counter underflow") - } - test.init.taskDone(task, vtime) - } + runDialTest(t, config, []dialTestRound{ + // 3 out of 4 peers are connected, leaving 2 dial slots. + // 9 nodes are discovered, but only 2 are dialed. + { + peersAdded: []*conn{ + {flags: staticDialedConn, node: newNode(uintID(0x00), "")}, + {flags: dynDialedConn, node: newNode(uintID(0x01), "")}, + {flags: dynDialedConn, node: newNode(uintID(0x02), "")}, + }, + discovered: []*enode.Node{ + newNode(uintID(0x00), "127.0.0.1:30303"), // not dialed because already connected as static peer + newNode(uintID(0x02), "127.0.0.1:30303"), // ... + newNode(uintID(0x03), "127.0.0.1:30303"), + newNode(uintID(0x04), "127.0.0.1:30303"), + newNode(uintID(0x05), "127.0.0.1:30303"), // not dialed because there are only two slots + newNode(uintID(0x06), "127.0.0.1:30303"), // ... + newNode(uintID(0x07), "127.0.0.1:30303"), // ... + newNode(uintID(0x08), "127.0.0.1:30303"), // ... + }, + wantNewDials: []*enode.Node{ + newNode(uintID(0x03), "127.0.0.1:30303"), + newNode(uintID(0x04), "127.0.0.1:30303"), + }, + }, - new := test.init.newTasks(running, pm(round.peers), vtime) - if !sametasks(new, round.new) { - t.Errorf("ERROR round %d: got %v\nwant %v\nstate: %v\nrunning: %v", - i, spew.Sdump(new), spew.Sdump(round.new), spew.Sdump(test.init), spew.Sdump(running)) - } - t.Logf("round %d (running %d) new tasks: %s", i, running, strings.TrimSpace(spew.Sdump(new))) + // One dial completes, freeing one dial slot. + { + failed: []enode.ID{ + uintID(0x04), + }, + wantNewDials: []*enode.Node{ + newNode(uintID(0x05), "127.0.0.1:30303"), + }, + }, - // Time advances by 16 seconds on every round. - vtime = vtime.Add(16 * time.Second) - running += len(new) - } -} + // Dial to 0x03 completes, filling the last remaining peer slot. + { + succeeded: []enode.ID{ + uintID(0x03), + }, + failed: []enode.ID{ + uintID(0x05), + }, + discovered: []*enode.Node{ + newNode(uintID(0x09), "127.0.0.1:30303"), // not dialed because there are no free slots + }, + }, -// This test checks that dynamic dials are launched from discovery results. -func TestDialStateDynDial(t *testing.T) { - config := &Config{Logger: testlog.Logger(t, log.LvlTrace)} - runDialTest(t, dialtest{ - init: newDialState(enode.ID{}, 5, config), - rounds: []round{ - // A discovery query is launched. - { - peers: []*Peer{ - {rw: &conn{flags: staticDialedConn, node: newNode(uintID(0), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(1), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(2), nil)}}, - }, - new: []task{ - &discoverTask{want: 3}, - }, - }, - // Dynamic dials are launched when it completes. - { - peers: []*Peer{ - {rw: &conn{flags: staticDialedConn, node: newNode(uintID(0), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(1), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(2), nil)}}, - }, - done: []task{ - &discoverTask{results: []*enode.Node{ - newNode(uintID(2), nil), // this one is already connected and not dialed. - newNode(uintID(3), nil), - newNode(uintID(4), nil), - newNode(uintID(5), nil), - newNode(uintID(6), nil), // these are not tried because max dyn dials is 5 - newNode(uintID(7), nil), // ... - }}, - }, - new: []task{ - &dialTask{flags: dynDialedConn, dest: newNode(uintID(3), nil)}, - &dialTask{flags: dynDialedConn, dest: newNode(uintID(4), nil)}, - &dialTask{flags: dynDialedConn, dest: newNode(uintID(5), nil)}, - }, - }, - // Some of the dials complete but no new ones are launched yet because - // the sum of active dial count and dynamic peer count is == maxDynDials. - { - peers: []*Peer{ - {rw: &conn{flags: staticDialedConn, node: newNode(uintID(0), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(1), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(2), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(3), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(4), nil)}}, - }, - done: []task{ - &dialTask{flags: dynDialedConn, dest: newNode(uintID(3), nil)}, - &dialTask{flags: dynDialedConn, dest: newNode(uintID(4), nil)}, - }, - }, - // No new dial tasks are launched in the this round because - // maxDynDials has been reached. - { - peers: []*Peer{ - {rw: &conn{flags: staticDialedConn, node: newNode(uintID(0), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(1), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(2), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(3), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(4), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(5), nil)}}, - }, - done: []task{ - &dialTask{flags: dynDialedConn, dest: newNode(uintID(5), nil)}, - }, - new: []task{ - &waitExpireTask{Duration: 19 * time.Second}, - }, - }, - // In this round, the peer with id 2 drops off. The query - // results from last discovery lookup are reused. - { - peers: []*Peer{ - {rw: &conn{flags: staticDialedConn, node: newNode(uintID(0), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(1), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(3), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(4), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(5), nil)}}, - }, - new: []task{ - &dialTask{flags: dynDialedConn, dest: newNode(uintID(6), nil)}, - }, - }, - // More peers (3,4) drop off and dial for ID 6 completes. - // The last query result from the discovery lookup is reused - // and a new one is spawned because more candidates are needed. - { - peers: []*Peer{ - {rw: &conn{flags: staticDialedConn, node: newNode(uintID(0), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(1), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(5), nil)}}, - }, - done: []task{ - &dialTask{flags: dynDialedConn, dest: newNode(uintID(6), nil)}, - }, - new: []task{ - &dialTask{flags: dynDialedConn, dest: newNode(uintID(7), nil)}, - &discoverTask{want: 2}, - }, - }, - // Peer 7 is connected, but there still aren't enough dynamic peers - // (4 out of 5). However, a discovery is already running, so ensure - // no new is started. - { - peers: []*Peer{ - {rw: &conn{flags: staticDialedConn, node: newNode(uintID(0), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(1), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(5), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(7), nil)}}, - }, - done: []task{ - &dialTask{flags: dynDialedConn, dest: newNode(uintID(7), nil)}, - }, - }, - // Finish the running node discovery with an empty set. A new lookup - // should be immediately requested. - { - peers: []*Peer{ - {rw: &conn{flags: staticDialedConn, node: newNode(uintID(0), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(1), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(5), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(7), nil)}}, - }, - done: []task{ - &discoverTask{}, - }, - new: []task{ - &discoverTask{want: 2}, - }, + // 3 peers drop off, creating 6 dial slots. Check that 5 of those slots + // (i.e. up to maxActiveDialTasks) are used. + { + peersRemoved: []enode.ID{ + uintID(0x00), + uintID(0x01), + uintID(0x02), + }, + discovered: []*enode.Node{ + newNode(uintID(0x0a), "127.0.0.1:30303"), + newNode(uintID(0x0b), "127.0.0.1:30303"), + newNode(uintID(0x0c), "127.0.0.1:30303"), + newNode(uintID(0x0d), "127.0.0.1:30303"), + newNode(uintID(0x0f), "127.0.0.1:30303"), + }, + wantNewDials: []*enode.Node{ + newNode(uintID(0x06), "127.0.0.1:30303"), + newNode(uintID(0x07), "127.0.0.1:30303"), + newNode(uintID(0x08), "127.0.0.1:30303"), + newNode(uintID(0x09), "127.0.0.1:30303"), + newNode(uintID(0x0a), "127.0.0.1:30303"), }, }, }) } -// Tests that bootnodes are dialed if no peers are connectd, but not otherwise. -func TestDialStateDynDialBootnode(t *testing.T) { - config := &Config{ - BootstrapNodes: []*enode.Node{ - newNode(uintID(1), nil), - newNode(uintID(2), nil), - newNode(uintID(3), nil), - }, - Logger: testlog.Logger(t, log.LvlTrace), +// This test checks that candidates that do not match the netrestrict list are not dialed. +func TestDialSchedNetRestrict(t *testing.T) { + t.Parallel() + + nodes := []*enode.Node{ + newNode(uintID(0x01), "127.0.0.1:30303"), + newNode(uintID(0x02), "127.0.0.2:30303"), + newNode(uintID(0x03), "127.0.0.3:30303"), + newNode(uintID(0x04), "127.0.0.4:30303"), + newNode(uintID(0x05), "127.0.2.5:30303"), + newNode(uintID(0x06), "127.0.2.6:30303"), + newNode(uintID(0x07), "127.0.2.7:30303"), + newNode(uintID(0x08), "127.0.2.8:30303"), + } + config := dialConfig{ + netRestrict: new(netutil.Netlist), + maxActiveDials: 10, + maxDialPeers: 10, } - runDialTest(t, dialtest{ - init: newDialState(enode.ID{}, 5, config), - rounds: []round{ - { - new: []task{ - &discoverTask{want: 5}, - }, - }, - { - done: []task{ - &discoverTask{ - results: []*enode.Node{ - newNode(uintID(4), nil), - newNode(uintID(5), nil), - }, - }, - }, - new: []task{ - &dialTask{flags: dynDialedConn, dest: newNode(uintID(4), nil)}, - &dialTask{flags: dynDialedConn, dest: newNode(uintID(5), nil)}, - &discoverTask{want: 3}, - }, - }, - // No dials succeed, bootnodes still pending fallback interval - {}, - // 1 bootnode attempted as fallback interval was reached - { - done: []task{ - &dialTask{flags: dynDialedConn, dest: newNode(uintID(4), nil)}, - &dialTask{flags: dynDialedConn, dest: newNode(uintID(5), nil)}, - }, - new: []task{ - &dialTask{flags: dynDialedConn, dest: newNode(uintID(1), nil)}, - }, - }, - // No dials succeed, 2nd bootnode is attempted - { - done: []task{ - &dialTask{flags: dynDialedConn, dest: newNode(uintID(1), nil)}, - }, - new: []task{ - &dialTask{flags: dynDialedConn, dest: newNode(uintID(2), nil)}, - }, - }, - // No dials succeed, 3rd bootnode is attempted - { - done: []task{ - &dialTask{flags: dynDialedConn, dest: newNode(uintID(2), nil)}, - }, - new: []task{ - &dialTask{flags: dynDialedConn, dest: newNode(uintID(3), nil)}, - }, - }, - // No dials succeed, 1st bootnode is attempted again, expired random nodes retried - { - done: []task{ - &dialTask{flags: dynDialedConn, dest: newNode(uintID(3), nil)}, - &discoverTask{results: []*enode.Node{ - newNode(uintID(6), nil), - }}, - }, - new: []task{ - &dialTask{flags: dynDialedConn, dest: newNode(uintID(6), nil)}, - &discoverTask{want: 4}, - }, - }, - // Random dial succeeds, no more bootnodes are attempted - { - peers: []*Peer{ - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(6), nil)}}, - }, + config.netRestrict.Add("127.0.2.0/24") + runDialTest(t, config, []dialTestRound{ + { + discovered: nodes, + wantNewDials: nodes[4:8], + }, + { + succeeded: []enode.ID{ + nodes[4].ID(), + nodes[5].ID(), + nodes[6].ID(), + nodes[7].ID(), }, }, }) } -func newNode(id enode.ID, ip net.IP) *enode.Node { - var r enr.Record - if ip != nil { - r.Set(enr.IP(ip)) +// This test checks that static dials work and obey the limits. +func TestDialSchedStaticDial(t *testing.T) { + t.Parallel() + + config := dialConfig{ + maxActiveDials: 5, + maxDialPeers: 4, } - return enode.SignNull(&r, id) + runDialTest(t, config, []dialTestRound{ + // Static dials are launched for the nodes that + // aren't yet connected. + { + peersAdded: []*conn{ + {flags: dynDialedConn, node: newNode(uintID(0x01), "127.0.0.1:30303")}, + {flags: dynDialedConn, node: newNode(uintID(0x02), "127.0.0.2:30303")}, + }, + update: func(d *dialScheduler) { + // These two are not dialed because they're already connected + // as dynamic peers. + d.addStatic(newNode(uintID(0x01), "127.0.0.1:30303")) + d.addStatic(newNode(uintID(0x02), "127.0.0.2:30303")) + // These nodes will be dialed: + d.addStatic(newNode(uintID(0x03), "127.0.0.3:30303")) + d.addStatic(newNode(uintID(0x04), "127.0.0.4:30303")) + d.addStatic(newNode(uintID(0x05), "127.0.0.5:30303")) + d.addStatic(newNode(uintID(0x06), "127.0.0.6:30303")) + d.addStatic(newNode(uintID(0x07), "127.0.0.7:30303")) + d.addStatic(newNode(uintID(0x08), "127.0.0.8:30303")) + d.addStatic(newNode(uintID(0x09), "127.0.0.9:30303")) + }, + wantNewDials: []*enode.Node{ + newNode(uintID(0x03), "127.0.0.3:30303"), + newNode(uintID(0x04), "127.0.0.4:30303"), + newNode(uintID(0x05), "127.0.0.5:30303"), + newNode(uintID(0x06), "127.0.0.6:30303"), + }, + }, + // Dial to 0x03 completes, filling a peer slot. One slot remains, + // two dials are launched to attempt to fill it. + { + succeeded: []enode.ID{ + uintID(0x03), + }, + failed: []enode.ID{ + uintID(0x04), + uintID(0x05), + uintID(0x06), + }, + wantResolves: map[enode.ID]*enode.Node{ + uintID(0x04): nil, + uintID(0x05): nil, + uintID(0x06): nil, + }, + wantNewDials: []*enode.Node{ + newNode(uintID(0x08), "127.0.0.8:30303"), + newNode(uintID(0x09), "127.0.0.9:30303"), + }, + }, + // Peer 0x01 drops and 0x07 connects as inbound peer. + // Only 0x01 is dialed. + { + peersAdded: []*conn{ + {flags: inboundConn, node: newNode(uintID(0x07), "127.0.0.7:30303")}, + }, + peersRemoved: []enode.ID{ + uintID(0x01), + }, + wantNewDials: []*enode.Node{ + newNode(uintID(0x01), "127.0.0.1:30303"), + }, + }, + }) } -// // This test checks that candidates that do not match the netrestrict list are not dialed. -func TestDialStateNetRestrict(t *testing.T) { - // This table always returns the same random nodes - // in the order given below. - nodes := []*enode.Node{ - newNode(uintID(1), net.ParseIP("127.0.0.1")), - newNode(uintID(2), net.ParseIP("127.0.0.2")), - newNode(uintID(3), net.ParseIP("127.0.0.3")), - newNode(uintID(4), net.ParseIP("127.0.0.4")), - newNode(uintID(5), net.ParseIP("127.0.2.5")), - newNode(uintID(6), net.ParseIP("127.0.2.6")), - newNode(uintID(7), net.ParseIP("127.0.2.7")), - newNode(uintID(8), net.ParseIP("127.0.2.8")), +// This test checks that removing static nodes stops connecting to them. +func TestDialSchedRemoveStatic(t *testing.T) { + t.Parallel() + + config := dialConfig{ + maxActiveDials: 1, + maxDialPeers: 1, } - restrict := new(netutil.Netlist) - restrict.Add("127.0.2.0/24") - - runDialTest(t, dialtest{ - init: newDialState(enode.ID{}, 10, &Config{NetRestrict: restrict}), - rounds: []round{ - { - new: []task{ - &discoverTask{want: 10}, - }, - }, - { - done: []task{ - &discoverTask{results: nodes}, - }, - new: []task{ - &dialTask{flags: dynDialedConn, dest: nodes[4]}, - &dialTask{flags: dynDialedConn, dest: nodes[5]}, - &dialTask{flags: dynDialedConn, dest: nodes[6]}, - &dialTask{flags: dynDialedConn, dest: nodes[7]}, - &discoverTask{want: 6}, - }, + runDialTest(t, config, []dialTestRound{ + // Add static nodes. + { + update: func(d *dialScheduler) { + d.addStatic(newNode(uintID(0x01), "127.0.0.1:30303")) + d.addStatic(newNode(uintID(0x02), "127.0.0.2:30303")) + d.addStatic(newNode(uintID(0x03), "127.0.0.3:30303")) + }, + wantNewDials: []*enode.Node{ + newNode(uintID(0x01), "127.0.0.1:30303"), + }, + }, + // Dial to 0x01 fails. + { + failed: []enode.ID{ + uintID(0x01), + }, + wantResolves: map[enode.ID]*enode.Node{ + uintID(0x01): nil, + }, + wantNewDials: []*enode.Node{ + newNode(uintID(0x02), "127.0.0.2:30303"), + }, + }, + // All static nodes are removed. 0x01 is in history, 0x02 is being + // dialed, 0x03 is in staticPool. + { + update: func(d *dialScheduler) { + d.removeStatic(newNode(uintID(0x01), "127.0.0.1:30303")) + d.removeStatic(newNode(uintID(0x02), "127.0.0.2:30303")) + d.removeStatic(newNode(uintID(0x03), "127.0.0.3:30303")) + }, + failed: []enode.ID{ + uintID(0x02), + }, + wantResolves: map[enode.ID]*enode.Node{ + uintID(0x02): nil, }, }, + // Since all static nodes are removed, they should not be dialed again. + {}, {}, {}, }) } -// This test checks that static dials are launched. -func TestDialStateStaticDial(t *testing.T) { - config := &Config{ - StaticNodes: []*enode.Node{ - newNode(uintID(1), nil), - newNode(uintID(2), nil), - newNode(uintID(3), nil), - newNode(uintID(4), nil), - newNode(uintID(5), nil), +// This test checks that static dials are selected at random. +func TestDialSchedManyStaticNodes(t *testing.T) { + t.Parallel() + + config := dialConfig{maxDialPeers: 2} + runDialTest(t, config, []dialTestRound{ + { + peersAdded: []*conn{ + {flags: dynDialedConn, node: newNode(uintID(0xFFFE), "")}, + {flags: dynDialedConn, node: newNode(uintID(0xFFFF), "")}, + }, + update: func(d *dialScheduler) { + for id := uint16(0); id < 2000; id++ { + n := newNode(uintID(id), "127.0.0.1:30303") + d.addStatic(n) + } + }, }, - Logger: testlog.Logger(t, log.LvlTrace), - } - runDialTest(t, dialtest{ - init: newDialState(enode.ID{}, 0, config), - rounds: []round{ - // Static dials are launched for the nodes that - // aren't yet connected. - { - peers: []*Peer{ - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(1), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(2), nil)}}, - }, - new: []task{ - &dialTask{flags: staticDialedConn, dest: newNode(uintID(3), nil)}, - &dialTask{flags: staticDialedConn, dest: newNode(uintID(4), nil)}, - &dialTask{flags: staticDialedConn, dest: newNode(uintID(5), nil)}, - }, - }, - // No new tasks are launched in this round because all static - // nodes are either connected or still being dialed. - { - peers: []*Peer{ - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(1), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(2), nil)}}, - {rw: &conn{flags: staticDialedConn, node: newNode(uintID(3), nil)}}, - }, - done: []task{ - &dialTask{flags: staticDialedConn, dest: newNode(uintID(3), nil)}, - }, - }, - // No new dial tasks are launched because all static - // nodes are now connected. - { - peers: []*Peer{ - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(1), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(2), nil)}}, - {rw: &conn{flags: staticDialedConn, node: newNode(uintID(3), nil)}}, - {rw: &conn{flags: staticDialedConn, node: newNode(uintID(4), nil)}}, - {rw: &conn{flags: staticDialedConn, node: newNode(uintID(5), nil)}}, - }, - done: []task{ - &dialTask{flags: staticDialedConn, dest: newNode(uintID(4), nil)}, - &dialTask{flags: staticDialedConn, dest: newNode(uintID(5), nil)}, - }, - new: []task{ - &waitExpireTask{Duration: 19 * time.Second}, - }, - }, - // Wait a round for dial history to expire, no new tasks should spawn. - { - peers: []*Peer{ - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(1), nil)}}, - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(2), nil)}}, - {rw: &conn{flags: staticDialedConn, node: newNode(uintID(3), nil)}}, - {rw: &conn{flags: staticDialedConn, node: newNode(uintID(4), nil)}}, - {rw: &conn{flags: staticDialedConn, node: newNode(uintID(5), nil)}}, - }, - }, - // If a static node is dropped, it should be immediately redialed, - // irrespective whether it was originally static or dynamic. - { - done: []task{ - &waitExpireTask{Duration: 19 * time.Second}, - }, - peers: []*Peer{ - {rw: &conn{flags: dynDialedConn, node: newNode(uintID(1), nil)}}, - {rw: &conn{flags: staticDialedConn, node: newNode(uintID(3), nil)}}, - {rw: &conn{flags: staticDialedConn, node: newNode(uintID(5), nil)}}, - }, - new: []task{ - &dialTask{flags: staticDialedConn, dest: newNode(uintID(2), nil)}, - }, + { + peersRemoved: []enode.ID{ + uintID(0xFFFE), + uintID(0xFFFF), + }, + wantNewDials: []*enode.Node{ + newNode(uintID(0x0085), "127.0.0.1:30303"), + newNode(uintID(0x02dc), "127.0.0.1:30303"), + newNode(uintID(0x0285), "127.0.0.1:30303"), + newNode(uintID(0x00cb), "127.0.0.1:30303"), }, }, }) } // This test checks that past dials are not retried for some time. -func TestDialStateCache(t *testing.T) { - config := &Config{ - StaticNodes: []*enode.Node{ - newNode(uintID(1), nil), - newNode(uintID(2), nil), - newNode(uintID(3), nil), +func TestDialSchedHistory(t *testing.T) { + t.Parallel() + + config := dialConfig{ + maxActiveDials: 3, + maxDialPeers: 3, + } + runDialTest(t, config, []dialTestRound{ + { + update: func(d *dialScheduler) { + d.addStatic(newNode(uintID(0x01), "127.0.0.1:30303")) + d.addStatic(newNode(uintID(0x02), "127.0.0.2:30303")) + d.addStatic(newNode(uintID(0x03), "127.0.0.3:30303")) + }, + wantNewDials: []*enode.Node{ + newNode(uintID(0x01), "127.0.0.1:30303"), + newNode(uintID(0x02), "127.0.0.2:30303"), + newNode(uintID(0x03), "127.0.0.3:30303"), + }, }, - Logger: testlog.Logger(t, log.LvlTrace), + // No new tasks are launched in this round because all static + // nodes are either connected or still being dialed. + { + succeeded: []enode.ID{ + uintID(0x01), + uintID(0x02), + }, + failed: []enode.ID{ + uintID(0x03), + }, + wantResolves: map[enode.ID]*enode.Node{ + uintID(0x03): nil, + }, + }, + // Nothing happens in this round because we're waiting for + // node 0x3's history entry to expire. + {}, + // The cache entry for node 0x03 has expired and is retried. + { + wantNewDials: []*enode.Node{ + newNode(uintID(0x03), "127.0.0.3:30303"), + }, + }, + }) +} + +func TestDialSchedResolve(t *testing.T) { + t.Parallel() + + config := dialConfig{ + maxActiveDials: 1, + maxDialPeers: 1, } - runDialTest(t, dialtest{ - init: newDialState(enode.ID{}, 0, config), - rounds: []round{ - // Static dials are launched for the nodes that - // aren't yet connected. - { - peers: nil, - new: []task{ - &dialTask{flags: staticDialedConn, dest: newNode(uintID(1), nil)}, - &dialTask{flags: staticDialedConn, dest: newNode(uintID(2), nil)}, - &dialTask{flags: staticDialedConn, dest: newNode(uintID(3), nil)}, - }, - }, - // No new tasks are launched in this round because all static - // nodes are either connected or still being dialed. - { - peers: []*Peer{ - {rw: &conn{flags: staticDialedConn, node: newNode(uintID(1), nil)}}, - {rw: &conn{flags: staticDialedConn, node: newNode(uintID(2), nil)}}, - }, - done: []task{ - &dialTask{flags: staticDialedConn, dest: newNode(uintID(1), nil)}, - &dialTask{flags: staticDialedConn, dest: newNode(uintID(2), nil)}, - }, - }, - // A salvage task is launched to wait for node 3's history - // entry to expire. - { - peers: []*Peer{ - {rw: &conn{flags: staticDialedConn, node: newNode(uintID(1), nil)}}, - {rw: &conn{flags: staticDialedConn, node: newNode(uintID(2), nil)}}, - }, - done: []task{ - &dialTask{flags: staticDialedConn, dest: newNode(uintID(3), nil)}, - }, - new: []task{ - &waitExpireTask{Duration: 19 * time.Second}, - }, - }, - // Still waiting for node 3's entry to expire in the cache. - { - peers: []*Peer{ - {rw: &conn{flags: staticDialedConn, node: newNode(uintID(1), nil)}}, - {rw: &conn{flags: staticDialedConn, node: newNode(uintID(2), nil)}}, - }, - }, - { - peers: []*Peer{ - {rw: &conn{flags: staticDialedConn, node: newNode(uintID(1), nil)}}, - {rw: &conn{flags: staticDialedConn, node: newNode(uintID(2), nil)}}, - }, - }, - // The cache entry for node 3 has expired and is retried. - { - done: []task{ - &waitExpireTask{Duration: 19 * time.Second}, - }, - peers: []*Peer{ - {rw: &conn{flags: staticDialedConn, node: newNode(uintID(1), nil)}}, - {rw: &conn{flags: staticDialedConn, node: newNode(uintID(2), nil)}}, - }, - new: []task{ - &dialTask{flags: staticDialedConn, dest: newNode(uintID(3), nil)}, - }, + node := newNode(uintID(0x01), "") + resolved := newNode(uintID(0x01), "127.0.0.1:30303") + resolved2 := newNode(uintID(0x01), "127.0.0.55:30303") + runDialTest(t, config, []dialTestRound{ + { + update: func(d *dialScheduler) { + d.addStatic(node) + }, + wantResolves: map[enode.ID]*enode.Node{ + uintID(0x01): resolved, + }, + wantNewDials: []*enode.Node{ + resolved, + }, + }, + { + failed: []enode.ID{ + uintID(0x01), + }, + wantResolves: map[enode.ID]*enode.Node{ + uintID(0x01): resolved2, + }, + wantNewDials: []*enode.Node{ + resolved2, }, }, }) } -func TestDialResolve(t *testing.T) { - config := &Config{ - Logger: testlog.Logger(t, log.LvlTrace), - Dialer: TCPDialer{&net.Dialer{Deadline: time.Now().Add(-5 * time.Minute)}}, +// ------- +// Code below here is the framework for the tests above. + +type dialTestRound struct { + peersAdded []*conn + peersRemoved []enode.ID + update func(*dialScheduler) // called at beginning of round + discovered []*enode.Node // newly discovered nodes + succeeded []enode.ID // dials which succeed this round + failed []enode.ID // dials which fail this round + wantResolves map[enode.ID]*enode.Node + wantNewDials []*enode.Node // dials that should be launched in this round +} + +func runDialTest(t *testing.T, config dialConfig, rounds []dialTestRound) { + var ( + clock = new(mclock.Simulated) + iterator = newDialTestIterator() + dialer = newDialTestDialer() + resolver = new(dialTestResolver) + peers = make(map[enode.ID]*conn) + setupCh = make(chan *conn) + ) + + // Override config. + config.clock = clock + config.dialer = dialer + config.resolver = resolver + config.log = testlog.Logger(t, log.LvlTrace) + config.rand = rand.New(rand.NewSource(0x1111)) + + // Set up the dialer. The setup function below runs on the dialTask + // goroutine and adds the peer. + var dialsched *dialScheduler + setup := func(fd net.Conn, f connFlag, node *enode.Node) error { + conn := &conn{flags: f, node: node} + dialsched.peerAdded(conn) + setupCh <- conn + return nil } - resolved := newNode(uintID(1), net.IP{127, 0, 55, 234}) - resolver := &resolveMock{answer: resolved} - state := newDialState(enode.ID{}, 0, config) - - // Check that the task is generated with an incomplete ID. - dest := newNode(uintID(1), nil) - state.addStatic(dest) - tasks := state.newTasks(0, nil, time.Time{}) - if !reflect.DeepEqual(tasks, []task{&dialTask{flags: staticDialedConn, dest: dest}}) { - t.Fatalf("expected dial task, got %#v", tasks) + dialsched = newDialScheduler(config, iterator, setup) + defer dialsched.stop() + + for i, round := range rounds { + // Apply peer set updates. + for _, c := range round.peersAdded { + if peers[c.node.ID()] != nil { + t.Fatalf("round %d: peer %v already connected", i, c.node.ID()) + } + dialsched.peerAdded(c) + peers[c.node.ID()] = c + } + for _, id := range round.peersRemoved { + c := peers[id] + if c == nil { + t.Fatalf("round %d: can't remove non-existent peer %v", i, id) + } + dialsched.peerRemoved(c) + } + + // Init round. + t.Logf("round %d (%d peers)", i, len(peers)) + resolver.setAnswers(round.wantResolves) + if round.update != nil { + round.update(dialsched) + } + iterator.addNodes(round.discovered) + + // Unblock dialTask goroutines. + if err := dialer.completeDials(round.succeeded, nil); err != nil { + t.Fatalf("round %d: %v", i, err) + } + for range round.succeeded { + conn := <-setupCh + peers[conn.node.ID()] = conn + } + if err := dialer.completeDials(round.failed, errors.New("oops")); err != nil { + t.Fatalf("round %d: %v", i, err) + } + + // Wait for new tasks. + if err := dialer.waitForDials(round.wantNewDials); err != nil { + t.Fatalf("round %d: %v", i, err) + } + if !resolver.checkCalls() { + t.Fatalf("unexpected calls to Resolve: %v", resolver.calls) + } + + clock.Run(16 * time.Second) } +} - // Now run the task, it should resolve the ID once. - srv := &Server{ - Config: *config, - log: config.Logger, - staticNodeResolver: resolver, +// dialTestIterator is the input iterator for dialer tests. This works a bit like a channel +// with infinite buffer: nodes are added to the buffer with addNodes, which unblocks Next +// and returns them from the iterator. +type dialTestIterator struct { + cur *enode.Node + + mu sync.Mutex + buf []*enode.Node + cond *sync.Cond + closed bool +} + +func newDialTestIterator() *dialTestIterator { + it := &dialTestIterator{} + it.cond = sync.NewCond(&it.mu) + return it +} + +// addNodes adds nodes to the iterator buffer and unblocks Next. +func (it *dialTestIterator) addNodes(nodes []*enode.Node) { + it.mu.Lock() + defer it.mu.Unlock() + + it.buf = append(it.buf, nodes...) + it.cond.Signal() +} + +// Node returns the current node. +func (it *dialTestIterator) Node() *enode.Node { + return it.cur +} + +// Next moves to the next node. +func (it *dialTestIterator) Next() bool { + it.mu.Lock() + defer it.mu.Unlock() + + it.cur = nil + for len(it.buf) == 0 && !it.closed { + it.cond.Wait() } - tasks[0].Do(srv) - if !reflect.DeepEqual(resolver.calls, []*enode.Node{dest}) { - t.Fatalf("wrong resolve calls, got %v", resolver.calls) + if it.closed { + return false } + it.cur = it.buf[0] + copy(it.buf[:], it.buf[1:]) + it.buf = it.buf[:len(it.buf)-1] + return true +} + +// Close ends the iterator, unblocking Next. +func (it *dialTestIterator) Close() { + it.mu.Lock() + defer it.mu.Unlock() + + it.closed = true + it.buf = nil + it.cond.Signal() +} + +// dialTestDialer is the NodeDialer used by runDialTest. +type dialTestDialer struct { + init chan *dialTestReq + blocked map[enode.ID]*dialTestReq +} - // Report it as done to the dialer, which should update the static node record. - state.taskDone(tasks[0], time.Now()) - if state.static[uintID(1)].dest != resolved { - t.Fatalf("state.dest not updated") +type dialTestReq struct { + n *enode.Node + unblock chan error +} + +func newDialTestDialer() *dialTestDialer { + return &dialTestDialer{ + init: make(chan *dialTestReq), + blocked: make(map[enode.ID]*dialTestReq), } } -// compares task lists but doesn't care about the order. -func sametasks(a, b []task) bool { - if len(a) != len(b) { - return false +// Dial implements NodeDialer. +func (d *dialTestDialer) Dial(ctx context.Context, n *enode.Node) (net.Conn, error) { + req := &dialTestReq{n: n, unblock: make(chan error, 1)} + select { + case d.init <- req: + select { + case err := <-req.unblock: + pipe, _ := net.Pipe() + return pipe, err + case <-ctx.Done(): + return nil, ctx.Err() + } + case <-ctx.Done(): + return nil, ctx.Err() } -next: - for _, ta := range a { - for _, tb := range b { - if reflect.DeepEqual(ta, tb) { - continue next +} + +// waitForDials waits for calls to Dial with the given nodes as argument. +// Those calls will be held blocking until completeDials is called with the same nodes. +func (d *dialTestDialer) waitForDials(nodes []*enode.Node) error { + waitset := make(map[enode.ID]*enode.Node) + for _, n := range nodes { + waitset[n.ID()] = n + } + timeout := time.NewTimer(1 * time.Second) + defer timeout.Stop() + + for len(waitset) > 0 { + select { + case req := <-d.init: + want, ok := waitset[req.n.ID()] + if !ok { + return fmt.Errorf("attempt to dial unexpected node %v", req.n.ID()) + } + if !reflect.DeepEqual(req.n, want) { + return fmt.Errorf("ENR of dialed node %v does not match test", req.n.ID()) } + delete(waitset, req.n.ID()) + d.blocked[req.n.ID()] = req + case <-timeout.C: + var waitlist []enode.ID + for id := range waitset { + waitlist = append(waitlist, id) + } + return fmt.Errorf("timed out waiting for dials to %v", waitlist) } - return false } - return true + + return d.checkUnexpectedDial() +} + +func (d *dialTestDialer) checkUnexpectedDial() error { + select { + case req := <-d.init: + return fmt.Errorf("attempt to dial unexpected node %v", req.n.ID()) + case <-time.After(150 * time.Millisecond): + return nil + } } -func uintID(i uint32) enode.ID { - var id enode.ID - binary.BigEndian.PutUint32(id[:], i) - return id +// completeDials unblocks calls to Dial for the given nodes. +func (d *dialTestDialer) completeDials(ids []enode.ID, err error) error { + for _, id := range ids { + req := d.blocked[id] + if req == nil { + return fmt.Errorf("can't complete dial to %v", id) + } + req.unblock <- err + } + return nil } -// for TestDialResolve -type resolveMock struct { - calls []*enode.Node - answer *enode.Node +// dialTestResolver tracks calls to resolve. +type dialTestResolver struct { + mu sync.Mutex + calls []enode.ID + answers map[enode.ID]*enode.Node } -func (t *resolveMock) Resolve(n *enode.Node) *enode.Node { - t.calls = append(t.calls, n) - return t.answer +func (t *dialTestResolver) setAnswers(m map[enode.ID]*enode.Node) { + t.mu.Lock() + defer t.mu.Unlock() + + t.answers = m + t.calls = nil +} + +func (t *dialTestResolver) checkCalls() bool { + t.mu.Lock() + defer t.mu.Unlock() + + for _, id := range t.calls { + if _, ok := t.answers[id]; !ok { + return false + } + } + return true +} + +func (t *dialTestResolver) Resolve(n *enode.Node) *enode.Node { + t.mu.Lock() + defer t.mu.Unlock() + + t.calls = append(t.calls, n.ID()) + return t.answers[n.ID()] } diff --git a/p2p/peer_test.go b/p2p/peer_test.go index a2393ba854..e40deb98f0 100644 --- a/p2p/peer_test.go +++ b/p2p/peer_test.go @@ -17,15 +17,20 @@ package p2p import ( + "encoding/binary" "errors" "fmt" "math/rand" "net" "reflect" + "strconv" + "strings" "testing" "time" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/p2p/enode" + "github.com/ethereum/go-ethereum/p2p/enr" ) var discard = Protocol{ @@ -45,10 +50,45 @@ var discard = Protocol{ }, } +// uintID encodes i into a node ID. +func uintID(i uint16) enode.ID { + var id enode.ID + binary.BigEndian.PutUint16(id[:], i) + return id +} + +// newNode creates a node record with the given address. +func newNode(id enode.ID, addr string) *enode.Node { + var r enr.Record + if addr != "" { + // Set the port if present. + if strings.Contains(addr, ":") { + hs, ps, err := net.SplitHostPort(addr) + if err != nil { + panic(fmt.Errorf("invalid address %q", addr)) + } + port, err := strconv.Atoi(ps) + if err != nil { + panic(fmt.Errorf("invalid port in %q", addr)) + } + r.Set(enr.TCP(port)) + r.Set(enr.UDP(port)) + addr = hs + } + // Set the IP. + ip := net.ParseIP(addr) + if ip == nil { + panic(fmt.Errorf("invalid IP %q", addr)) + } + r.Set(enr.IP(ip)) + } + return enode.SignNull(&r, id) +} + func testPeer(protos []Protocol) (func(), *conn, *Peer, <-chan error) { fd1, fd2 := net.Pipe() - c1 := &conn{fd: fd1, node: newNode(randomID(), nil), transport: newTestTransport(&newkey().PublicKey, fd1)} - c2 := &conn{fd: fd2, node: newNode(randomID(), nil), transport: newTestTransport(&newkey().PublicKey, fd2)} + c1 := &conn{fd: fd1, node: newNode(randomID(), ""), transport: newTestTransport(&newkey().PublicKey, fd1)} + c2 := &conn{fd: fd2, node: newNode(randomID(), ""), transport: newTestTransport(&newkey().PublicKey, fd2)} for _, p := range protos { c1.caps = append(c1.caps, p.cap()) c2.caps = append(c2.caps, p.cap()) diff --git a/p2p/server.go b/p2p/server.go index fda72e3793..1ed1be2ac6 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -51,7 +51,6 @@ const ( discmixTimeout = 5 * time.Second // Connectivity defaults. - maxActiveDialTasks = 16 defaultMaxPendingPeers = 50 defaultDialRatio = 3 @@ -156,6 +155,8 @@ type Config struct { // Logger is a custom logger to use with the p2p.Server. Logger log.Logger `toml:",omitempty"` + + clock mclock.Clock } // Server manages all peer connections. @@ -183,13 +184,10 @@ type Server struct { ntab *discover.UDPv4 DiscV5 *discv5.Network discmix *enode.FairMix - - staticNodeResolver nodeResolver + dialsched *dialScheduler // Channels into the run loop. quit chan struct{} - addstatic chan *enode.Node - removestatic chan *enode.Node addtrusted chan *enode.Node removetrusted chan *enode.Node peerOp chan peerOpFunc @@ -302,47 +300,57 @@ func (srv *Server) LocalNode() *enode.LocalNode { // Peers returns all connected peers. func (srv *Server) Peers() []*Peer { var ps []*Peer - select { - // Note: We'd love to put this function into a variable but - // that seems to cause a weird compiler error in some - // environments. - case srv.peerOp <- func(peers map[enode.ID]*Peer) { + srv.doPeerOp(func(peers map[enode.ID]*Peer) { for _, p := range peers { ps = append(ps, p) } - }: - <-srv.peerOpDone - case <-srv.quit: - } + }) return ps } // PeerCount returns the number of connected peers. func (srv *Server) PeerCount() int { var count int - select { - case srv.peerOp <- func(ps map[enode.ID]*Peer) { count = len(ps) }: - <-srv.peerOpDone - case <-srv.quit: - } + srv.doPeerOp(func(ps map[enode.ID]*Peer) { + count = len(ps) + }) return count } -// AddPeer connects to the given node and maintains the connection until the -// server is shut down. If the connection fails for any reason, the server will -// attempt to reconnect the peer. +// AddPeer adds the given node to the static node set. When there is room in the peer set, +// the server will connect to the node. If the connection fails for any reason, the server +// will attempt to reconnect the peer. func (srv *Server) AddPeer(node *enode.Node) { - select { - case srv.addstatic <- node: - case <-srv.quit: - } + srv.dialsched.addStatic(node) } -// RemovePeer disconnects from the given node +// RemovePeer removes a node from the static node set. It also disconnects from the given +// node if it is currently connected as a peer. +// +// This method blocks until all protocols have exited and the peer is removed. Do not use +// RemovePeer in protocol implementations, call Disconnect on the Peer instead. func (srv *Server) RemovePeer(node *enode.Node) { - select { - case srv.removestatic <- node: - case <-srv.quit: + var ( + ch chan *PeerEvent + sub event.Subscription + ) + // Disconnect the peer on the main loop. + srv.doPeerOp(func(peers map[enode.ID]*Peer) { + srv.dialsched.removeStatic(node) + if peer := peers[node.ID()]; peer != nil { + ch = make(chan *PeerEvent, 1) + sub = srv.peerFeed.Subscribe(ch) + peer.Disconnect(DiscRequested) + } + }) + // Wait for the peer connection to end. + if ch != nil { + defer sub.Unsubscribe() + for ev := range ch { + if ev.Peer == node.ID() && ev.Type == PeerEventTypeDrop { + return + } + } } } @@ -437,6 +445,9 @@ func (srv *Server) Start() (err error) { if srv.log == nil { srv.log = log.Root() } + if srv.clock == nil { + srv.clock = mclock.System{} + } if srv.NoDial && srv.ListenAddr == "" { srv.log.Warn("P2P server will be useless, neither dialing nor listening") } @@ -451,15 +462,10 @@ func (srv *Server) Start() (err error) { if srv.listenFunc == nil { srv.listenFunc = net.Listen } - if srv.Dialer == nil { - srv.Dialer = TCPDialer{&net.Dialer{Timeout: defaultDialTimeout}} - } srv.quit = make(chan struct{}) srv.delpeer = make(chan peerDrop) srv.checkpointPostHandshake = make(chan *conn) srv.checkpointAddPeer = make(chan *conn) - srv.addstatic = make(chan *enode.Node) - srv.removestatic = make(chan *enode.Node) srv.addtrusted = make(chan *enode.Node) srv.removetrusted = make(chan *enode.Node) srv.peerOp = make(chan peerOpFunc) @@ -476,11 +482,10 @@ func (srv *Server) Start() (err error) { if err := srv.setupDiscovery(); err != nil { return err } + srv.setupDialScheduler() - dynPeers := srv.maxDialedConns() - dialer := newDialState(srv.localnode.ID(), dynPeers, &srv.Config) srv.loopWG.Add(1) - go srv.run(dialer) + go srv.run() return nil } @@ -583,7 +588,6 @@ func (srv *Server) setupDiscovery() error { } srv.ntab = ntab srv.discmix.AddSource(ntab.RandomNodes()) - srv.staticNodeResolver = ntab } // Discovery V5 @@ -606,6 +610,47 @@ func (srv *Server) setupDiscovery() error { return nil } +func (srv *Server) setupDialScheduler() { + config := dialConfig{ + self: srv.localnode.ID(), + maxDialPeers: srv.maxDialedConns(), + maxActiveDials: srv.MaxPendingPeers, + log: srv.Logger, + netRestrict: srv.NetRestrict, + dialer: srv.Dialer, + clock: srv.clock, + } + if srv.ntab != nil { + config.resolver = srv.ntab + } + if config.dialer == nil { + config.dialer = tcpDialer{&net.Dialer{Timeout: defaultDialTimeout}} + } + srv.dialsched = newDialScheduler(config, srv.discmix, srv.SetupConn) + for _, n := range srv.StaticNodes { + srv.dialsched.addStatic(n) + } +} + +func (srv *Server) maxInboundConns() int { + return srv.MaxPeers - srv.maxDialedConns() +} + +func (srv *Server) maxDialedConns() (limit int) { + if srv.NoDial || srv.MaxPeers == 0 { + return 0 + } + if srv.DialRatio == 0 { + limit = srv.MaxPeers / defaultDialRatio + } else { + limit = srv.MaxPeers / srv.DialRatio + } + if limit == 0 { + limit = 1 + } + return limit +} + func (srv *Server) setupListening() error { // Launch the listener. listener, err := srv.listenFunc("tcp", srv.ListenAddr) @@ -632,112 +677,55 @@ func (srv *Server) setupListening() error { return nil } -type dialer interface { - newTasks(running int, peers map[enode.ID]*Peer, now time.Time) []task - taskDone(task, time.Time) - addStatic(*enode.Node) - removeStatic(*enode.Node) +// doPeerOp runs fn on the main loop. +func (srv *Server) doPeerOp(fn peerOpFunc) { + select { + case srv.peerOp <- fn: + <-srv.peerOpDone + case <-srv.quit: + } } -func (srv *Server) run(dialstate dialer) { +// run is the main loop of the server. +func (srv *Server) run() { srv.log.Info("Started P2P networking", "self", srv.localnode.Node().URLv4()) defer srv.loopWG.Done() defer srv.nodedb.Close() defer srv.discmix.Close() + defer srv.dialsched.stop() var ( peers = make(map[enode.ID]*Peer) inboundCount = 0 trusted = make(map[enode.ID]bool, len(srv.TrustedNodes)) - taskdone = make(chan task, maxActiveDialTasks) - tick = time.NewTicker(30 * time.Second) - runningTasks []task - queuedTasks []task // tasks that can't run yet ) - defer tick.Stop() - // Put trusted nodes into a map to speed up checks. // Trusted peers are loaded on startup or added via AddTrustedPeer RPC. for _, n := range srv.TrustedNodes { trusted[n.ID()] = true } - // removes t from runningTasks - delTask := func(t task) { - for i := range runningTasks { - if runningTasks[i] == t { - runningTasks = append(runningTasks[:i], runningTasks[i+1:]...) - break - } - } - } - // starts until max number of active tasks is satisfied - startTasks := func(ts []task) (rest []task) { - i := 0 - for ; len(runningTasks) < maxActiveDialTasks && i < len(ts); i++ { - t := ts[i] - srv.log.Trace("New dial task", "task", t) - go func() { t.Do(srv); taskdone <- t }() - runningTasks = append(runningTasks, t) - } - return ts[i:] - } - scheduleTasks := func() { - // Start from queue first. - queuedTasks = append(queuedTasks[:0], startTasks(queuedTasks)...) - // Query dialer for new tasks and start as many as possible now. - if len(runningTasks) < maxActiveDialTasks { - nt := dialstate.newTasks(len(runningTasks)+len(queuedTasks), peers, time.Now()) - queuedTasks = append(queuedTasks, startTasks(nt)...) - } - } - running: for { - scheduleTasks() - select { - case <-tick.C: - // This is just here to ensure the dial scheduler runs occasionally. - case <-srv.quit: // The server was stopped. Run the cleanup logic. break running - case n := <-srv.addstatic: - // This channel is used by AddPeer to add to the - // ephemeral static peer list. Add it to the dialer, - // it will keep the node connected. - srv.log.Trace("Adding static node", "node", n) - dialstate.addStatic(n) - - case n := <-srv.removestatic: - // This channel is used by RemovePeer to send a - // disconnect request to a peer and begin the - // stop keeping the node connected. - srv.log.Trace("Removing static node", "node", n) - dialstate.removeStatic(n) - if p, ok := peers[n.ID()]; ok { - p.Disconnect(DiscRequested) - } - case n := <-srv.addtrusted: - // This channel is used by AddTrustedPeer to add an enode + // This channel is used by AddTrustedPeer to add a node // to the trusted node set. srv.log.Trace("Adding trusted node", "node", n) trusted[n.ID()] = true - // Mark any already-connected peer as trusted if p, ok := peers[n.ID()]; ok { p.rw.set(trustedConn, true) } case n := <-srv.removetrusted: - // This channel is used by RemoveTrustedPeer to remove an enode + // This channel is used by RemoveTrustedPeer to remove a node // from the trusted node set. srv.log.Trace("Removing trusted node", "node", n) delete(trusted, n.ID()) - - // Unmark any already-connected peer as trusted if p, ok := peers[n.ID()]; ok { p.rw.set(trustedConn, false) } @@ -747,14 +735,6 @@ running: op(peers) srv.peerOpDone <- struct{}{} - case t := <-taskdone: - // A task got done. Tell dialstate about it so it - // can update its state and remove it from the active - // tasks list. - srv.log.Trace("Dial task done", "task", t) - dialstate.taskDone(t, time.Now()) - delTask(t) - case c := <-srv.checkpointPostHandshake: // A connection has passed the encryption handshake so // the remote identity is known (but hasn't been verified yet). @@ -771,33 +751,25 @@ running: err := srv.addPeerChecks(peers, inboundCount, c) if err == nil { // The handshakes are done and it passed all checks. - p := newPeer(srv.log, c, srv.Protocols) - // If message events are enabled, pass the peerFeed - // to the peer - if srv.EnableMsgEvents { - p.events = &srv.peerFeed - } - name := truncateName(c.name) - p.log.Debug("Adding p2p peer", "addr", p.RemoteAddr(), "peers", len(peers)+1, "name", name) - go srv.runPeer(p) + p := srv.launchPeer(c) peers[c.node.ID()] = p - if p.Inbound() { - inboundCount++ - } + srv.log.Debug("Adding p2p peer", "peercount", len(peers), "id", p.ID(), "conn", c.flags, "addr", p.RemoteAddr(), "name", truncateName(c.name)) + srv.dialsched.peerAdded(c) if conn, ok := c.fd.(*meteredConn); ok { conn.handshakeDone(p) } + if p.Inbound() { + inboundCount++ + } } - // The dialer logic relies on the assumption that - // dial tasks complete after the peer has been added or - // discarded. Unblock the task last. c.cont <- err case pd := <-srv.delpeer: // A peer disconnected. d := common.PrettyDuration(mclock.Now() - pd.created) - pd.log.Debug("Removing p2p peer", "addr", pd.RemoteAddr(), "peers", len(peers)-1, "duration", d, "req", pd.requested, "err", pd.err) delete(peers, pd.ID()) + srv.log.Debug("Removing p2p peer", "peercount", len(peers), "id", pd.ID(), "duration", d, "req", pd.requested, "err", pd.err) + srv.dialsched.peerRemoved(pd.rw) if pd.Inbound() { inboundCount-- } @@ -822,14 +794,14 @@ running: // is closed. for len(peers) > 0 { p := <-srv.delpeer - p.log.Trace("<-delpeer (spindown)", "remainingTasks", len(runningTasks)) + p.log.Trace("<-delpeer (spindown)") delete(peers, p.ID()) } } func (srv *Server) postHandshakeChecks(peers map[enode.ID]*Peer, inboundCount int, c *conn) error { switch { - case !c.is(trustedConn|staticDialedConn) && len(peers) >= srv.MaxPeers: + case !c.is(trustedConn) && len(peers) >= srv.MaxPeers: return DiscTooManyPeers case !c.is(trustedConn) && c.is(inboundConn) && inboundCount >= srv.maxInboundConns(): return DiscTooManyPeers @@ -852,21 +824,6 @@ func (srv *Server) addPeerChecks(peers map[enode.ID]*Peer, inboundCount int, c * return srv.postHandshakeChecks(peers, inboundCount, c) } -func (srv *Server) maxInboundConns() int { - return srv.MaxPeers - srv.maxDialedConns() -} - -func (srv *Server) maxDialedConns() int { - if srv.NoDiscovery || srv.NoDial { - return 0 - } - r := srv.DialRatio - if r == 0 { - r = defaultDialRatio - } - return srv.MaxPeers / r -} - // listenLoop runs in its own goroutine and accepts // inbound connections. func (srv *Server) listenLoop() { @@ -935,18 +892,20 @@ func (srv *Server) listenLoop() { } func (srv *Server) checkInboundConn(fd net.Conn, remoteIP net.IP) error { - if remoteIP != nil { - // Reject connections that do not match NetRestrict. - if srv.NetRestrict != nil && !srv.NetRestrict.Contains(remoteIP) { - return fmt.Errorf("not whitelisted in NetRestrict") - } - // Reject Internet peers that try too often. - srv.inboundHistory.expire(time.Now()) - if !netutil.IsLAN(remoteIP) && srv.inboundHistory.contains(remoteIP.String()) { - return fmt.Errorf("too many attempts") - } - srv.inboundHistory.add(remoteIP.String(), time.Now().Add(inboundThrottleTime)) + if remoteIP == nil { + return nil + } + // Reject connections that do not match NetRestrict. + if srv.NetRestrict != nil && !srv.NetRestrict.Contains(remoteIP) { + return fmt.Errorf("not whitelisted in NetRestrict") } + // Reject Internet peers that try too often. + now := srv.clock.Now() + srv.inboundHistory.expire(now, nil) + if !netutil.IsLAN(remoteIP) && srv.inboundHistory.contains(remoteIP.String()) { + return fmt.Errorf("too many attempts") + } + srv.inboundHistory.add(remoteIP.String(), now.Add(inboundThrottleTime)) return nil } @@ -958,7 +917,6 @@ func (srv *Server) SetupConn(fd net.Conn, flags connFlag, dialDest *enode.Node) err := srv.setupConn(c, flags, dialDest) if err != nil { c.close(err) - srv.log.Trace("Setting up connection failed", "addr", fd.RemoteAddr(), "err", err) } return err } @@ -977,7 +935,9 @@ func (srv *Server) setupConn(c *conn, flags connFlag, dialDest *enode.Node) erro if dialDest != nil { dialPubkey = new(ecdsa.PublicKey) if err := dialDest.Load((*enode.Secp256k1)(dialPubkey)); err != nil { - return errors.New("dial destination doesn't have a secp256k1 public key") + err = errors.New("dial destination doesn't have a secp256k1 public key") + srv.log.Trace("Setting up connection failed", "addr", c.fd.RemoteAddr(), "conn", c.flags, "err", err) + return err } } @@ -1006,7 +966,7 @@ func (srv *Server) setupConn(c *conn, flags connFlag, dialDest *enode.Node) erro // Run the capability negotiation handshake. phs, err := c.doProtoHandshake(srv.ourHandshake) if err != nil { - clog.Trace("Failed proto handshake", "err", err) + clog.Trace("Failed p2p handshake", "err", err) return err } if id := c.node.ID(); !bytes.Equal(crypto.Keccak256(phs.ID), id[:]) { @@ -1020,9 +980,6 @@ func (srv *Server) setupConn(c *conn, flags connFlag, dialDest *enode.Node) erro return err } - // If the checks completed successfully, the connection has been added as a peer and - // runPeer has been launched. - clog.Trace("Connection set up", "inbound", dialDest == nil) return nil } @@ -1054,15 +1011,22 @@ func (srv *Server) checkpoint(c *conn, stage chan<- *conn) error { return <-c.cont } +func (srv *Server) launchPeer(c *conn) *Peer { + p := newPeer(srv.log, c, srv.Protocols) + if srv.EnableMsgEvents { + // If message events are enabled, pass the peerFeed + // to the peer. + p.events = &srv.peerFeed + } + go srv.runPeer(p) + return p +} + // runPeer runs in its own goroutine for each peer. -// it waits until the Peer logic returns and removes -// the peer. func (srv *Server) runPeer(p *Peer) { if srv.newPeerHook != nil { srv.newPeerHook(p) } - - // broadcast peer add srv.peerFeed.Send(&PeerEvent{ Type: PeerEventTypeAdd, Peer: p.ID(), @@ -1070,10 +1034,18 @@ func (srv *Server) runPeer(p *Peer) { LocalAddress: p.LocalAddr().String(), }) - // run the protocol + // Run the per-peer main loop. remoteRequested, err := p.run() - // broadcast peer drop + // Announce disconnect on the main loop to update the peer set. + // The main loop waits for existing peers to be sent on srv.delpeer + // before returning, so this send should not select on srv.quit. + srv.delpeer <- peerDrop{p, err, remoteRequested} + + // Broadcast peer drop to external subscribers. This needs to be + // after the send to delpeer so subscribers have a consistent view of + // the peer set (i.e. Server.Peers() doesn't include the peer when the + // event is received. srv.peerFeed.Send(&PeerEvent{ Type: PeerEventTypeDrop, Peer: p.ID(), @@ -1081,10 +1053,6 @@ func (srv *Server) runPeer(p *Peer) { RemoteAddress: p.RemoteAddr().String(), LocalAddress: p.LocalAddr().String(), }) - - // Note: run waits for existing peers to be sent on srv.delpeer - // before returning, so this send should not select on srv.quit. - srv.delpeer <- peerDrop{p, err, remoteRequested} } // NodeInfo represents a short summary of the information known about the host. diff --git a/p2p/server_test.go b/p2p/server_test.go index 383445c833..958eb29129 100644 --- a/p2p/server_test.go +++ b/p2p/server_test.go @@ -34,10 +34,6 @@ import ( "golang.org/x/crypto/sha3" ) -// func init() { -// log.Root().SetHandler(log.LvlFilterHandler(log.LvlTrace, log.StreamHandler(os.Stderr, log.TerminalFormat(false)))) -// } - type testTransport struct { rpub *ecdsa.PublicKey *rlpx @@ -72,11 +68,12 @@ func (c *testTransport) close(err error) { func startTestServer(t *testing.T, remoteKey *ecdsa.PublicKey, pf func(*Peer)) *Server { config := Config{ - Name: "test", - MaxPeers: 10, - ListenAddr: "127.0.0.1:0", - PrivateKey: newkey(), - Logger: testlog.Logger(t, log.LvlTrace), + Name: "test", + MaxPeers: 10, + ListenAddr: "127.0.0.1:0", + NoDiscovery: true, + PrivateKey: newkey(), + Logger: testlog.Logger(t, log.LvlTrace), } server := &Server{ Config: config, @@ -131,11 +128,10 @@ func TestServerDial(t *testing.T) { t.Fatalf("could not setup listener: %v", err) } defer listener.Close() - accepted := make(chan net.Conn) + accepted := make(chan net.Conn, 1) go func() { conn, err := listener.Accept() if err != nil { - t.Error("accept error:", err) return } accepted <- conn @@ -205,155 +201,38 @@ func TestServerDial(t *testing.T) { } } -// This test checks that tasks generated by dialstate are -// actually executed and taskdone is called for them. -func TestServerTaskScheduling(t *testing.T) { - var ( - done = make(chan *testTask) - quit, returned = make(chan struct{}), make(chan struct{}) - tc = 0 - tg = taskgen{ - newFunc: func(running int, peers map[enode.ID]*Peer) []task { - tc++ - return []task{&testTask{index: tc - 1}} - }, - doneFunc: func(t task) { - select { - case done <- t.(*testTask): - case <-quit: - } - }, - } - ) +// This test checks that RemovePeer disconnects the peer if it is connected. +func TestServerRemovePeerDisconnect(t *testing.T) { + srv1 := &Server{Config: Config{ + PrivateKey: newkey(), + MaxPeers: 1, + NoDiscovery: true, + Logger: testlog.Logger(t, log.LvlTrace).New("server", "1"), + }} + srv2 := &Server{Config: Config{ + PrivateKey: newkey(), + MaxPeers: 1, + NoDiscovery: true, + NoDial: true, + ListenAddr: "127.0.0.1:0", + Logger: testlog.Logger(t, log.LvlTrace).New("server", "2"), + }} + srv1.Start() + defer srv1.Stop() + srv2.Start() + defer srv2.Stop() - // The Server in this test isn't actually running - // because we're only interested in what run does. - db, _ := enode.OpenDB("") - srv := &Server{ - Config: Config{MaxPeers: 10}, - localnode: enode.NewLocalNode(db, newkey()), - nodedb: db, - discmix: enode.NewFairMix(0), - quit: make(chan struct{}), - running: true, - log: log.New(), - } - srv.loopWG.Add(1) - go func() { - srv.run(tg) - close(returned) - }() - - var gotdone []*testTask - for i := 0; i < 100; i++ { - gotdone = append(gotdone, <-done) + if !syncAddPeer(srv1, srv2.Self()) { + t.Fatal("peer not connected") } - for i, task := range gotdone { - if task.index != i { - t.Errorf("task %d has wrong index, got %d", i, task.index) - break - } - if !task.called { - t.Errorf("task %d was not called", i) - break - } - } - - close(quit) - srv.Stop() - select { - case <-returned: - case <-time.After(500 * time.Millisecond): - t.Error("Server.run did not return within 500ms") + srv1.RemovePeer(srv2.Self()) + if srv1.PeerCount() > 0 { + t.Fatal("removed peer still connected") } } -// This test checks that Server doesn't drop tasks, -// even if newTasks returns more than the maximum number of tasks. -func TestServerManyTasks(t *testing.T) { - alltasks := make([]task, 300) - for i := range alltasks { - alltasks[i] = &testTask{index: i} - } - - var ( - db, _ = enode.OpenDB("") - srv = &Server{ - quit: make(chan struct{}), - localnode: enode.NewLocalNode(db, newkey()), - nodedb: db, - running: true, - log: log.New(), - discmix: enode.NewFairMix(0), - } - done = make(chan *testTask) - start, end = 0, 0 - ) - defer srv.Stop() - srv.loopWG.Add(1) - go srv.run(taskgen{ - newFunc: func(running int, peers map[enode.ID]*Peer) []task { - start, end = end, end+maxActiveDialTasks+10 - if end > len(alltasks) { - end = len(alltasks) - } - return alltasks[start:end] - }, - doneFunc: func(tt task) { - done <- tt.(*testTask) - }, - }) - - doneset := make(map[int]bool) - timeout := time.After(2 * time.Second) - for len(doneset) < len(alltasks) { - select { - case tt := <-done: - if doneset[tt.index] { - t.Errorf("task %d got done more than once", tt.index) - } else { - doneset[tt.index] = true - } - case <-timeout: - t.Errorf("%d of %d tasks got done within 2s", len(doneset), len(alltasks)) - for i := 0; i < len(alltasks); i++ { - if !doneset[i] { - t.Logf("task %d not done", i) - } - } - return - } - } -} - -type taskgen struct { - newFunc func(running int, peers map[enode.ID]*Peer) []task - doneFunc func(task) -} - -func (tg taskgen) newTasks(running int, peers map[enode.ID]*Peer, now time.Time) []task { - return tg.newFunc(running, peers) -} -func (tg taskgen) taskDone(t task, now time.Time) { - tg.doneFunc(t) -} -func (tg taskgen) addStatic(*enode.Node) { -} -func (tg taskgen) removeStatic(*enode.Node) { -} - -type testTask struct { - index int - called bool -} - -func (t *testTask) Do(srv *Server) { - t.called = true -} - -// This test checks that connections are disconnected -// just after the encryption handshake when the server is -// at capacity. Trusted connections should still be accepted. +// This test checks that connections are disconnected just after the encryption handshake +// when the server is at capacity. Trusted connections should still be accepted. func TestServerAtCap(t *testing.T) { trustedNode := newkey() trustedID := enode.PubkeyToIDV4(&trustedNode.PublicKey) @@ -363,7 +242,8 @@ func TestServerAtCap(t *testing.T) { MaxPeers: 10, NoDial: true, NoDiscovery: true, - TrustedNodes: []*enode.Node{newNode(trustedID, nil)}, + TrustedNodes: []*enode.Node{newNode(trustedID, "")}, + Logger: testlog.Logger(t, log.LvlTrace), }, } if err := srv.Start(); err != nil { @@ -401,14 +281,14 @@ func TestServerAtCap(t *testing.T) { } // Remove from trusted set and try again - srv.RemoveTrustedPeer(newNode(trustedID, nil)) + srv.RemoveTrustedPeer(newNode(trustedID, "")) c = newconn(trustedID) if err := srv.checkpoint(c, srv.checkpointPostHandshake); err != DiscTooManyPeers { t.Error("wrong error for insert:", err) } // Add anotherID to trusted set and try again - srv.AddTrustedPeer(newNode(anotherID, nil)) + srv.AddTrustedPeer(newNode(anotherID, "")) c = newconn(anotherID) if err := srv.checkpoint(c, srv.checkpointPostHandshake); err != nil { t.Error("unexpected error for trusted conn @posthandshake:", err) @@ -439,9 +319,9 @@ func TestServerPeerLimits(t *testing.T) { NoDial: true, NoDiscovery: true, Protocols: []Protocol{discard}, + Logger: testlog.Logger(t, log.LvlTrace), }, newTransport: func(fd net.Conn) transport { return tp }, - log: log.New(), } if err := srv.Start(); err != nil { t.Fatalf("couldn't start server: %v", err) @@ -724,3 +604,23 @@ func (l *fakeAddrListener) Accept() (net.Conn, error) { func (c *fakeAddrConn) RemoteAddr() net.Addr { return c.remoteAddr } + +func syncAddPeer(srv *Server, node *enode.Node) bool { + var ( + ch = make(chan *PeerEvent) + sub = srv.SubscribeEvents(ch) + timeout = time.After(2 * time.Second) + ) + defer sub.Unsubscribe() + srv.AddPeer(node) + for { + select { + case ev := <-ch: + if ev.Type == PeerEventTypeAdd && ev.Peer == node.ID() { + return true + } + case <-timeout: + return false + } + } +} diff --git a/p2p/simulations/adapters/inproc.go b/p2p/simulations/adapters/inproc.go index 9787082e18..651d9546ae 100644 --- a/p2p/simulations/adapters/inproc.go +++ b/p2p/simulations/adapters/inproc.go @@ -17,6 +17,7 @@ package adapters import ( + "context" "errors" "fmt" "math" @@ -126,7 +127,7 @@ func (s *SimAdapter) NewNode(config *NodeConfig) (Node, error) { // Dial implements the p2p.NodeDialer interface by connecting to the node using // an in-memory net.Pipe -func (s *SimAdapter) Dial(dest *enode.Node) (conn net.Conn, err error) { +func (s *SimAdapter) Dial(ctx context.Context, dest *enode.Node) (conn net.Conn, err error) { node, ok := s.GetNode(dest.ID()) if !ok { return nil, fmt.Errorf("unknown node: %s", dest.ID()) diff --git a/p2p/util.go b/p2p/util.go index 018cc40e98..3c5f6b8508 100644 --- a/p2p/util.go +++ b/p2p/util.go @@ -18,7 +18,8 @@ package p2p import ( "container/heap" - "time" + + "github.com/ethereum/go-ethereum/common/mclock" ) // expHeap tracks strings and their expiry time. @@ -27,16 +28,16 @@ type expHeap []expItem // expItem is an entry in addrHistory. type expItem struct { item string - exp time.Time + exp mclock.AbsTime } // nextExpiry returns the next expiry time. -func (h *expHeap) nextExpiry() time.Time { +func (h *expHeap) nextExpiry() mclock.AbsTime { return (*h)[0].exp } // add adds an item and sets its expiry time. -func (h *expHeap) add(item string, exp time.Time) { +func (h *expHeap) add(item string, exp mclock.AbsTime) { heap.Push(h, expItem{item, exp}) } @@ -51,15 +52,18 @@ func (h expHeap) contains(item string) bool { } // expire removes items with expiry time before 'now'. -func (h *expHeap) expire(now time.Time) { - for h.Len() > 0 && h.nextExpiry().Before(now) { - heap.Pop(h) +func (h *expHeap) expire(now mclock.AbsTime, onExp func(string)) { + for h.Len() > 0 && h.nextExpiry() < now { + item := heap.Pop(h) + if onExp != nil { + onExp(item.(expItem).item) + } } } // heap.Interface boilerplate func (h expHeap) Len() int { return len(h) } -func (h expHeap) Less(i, j int) bool { return h[i].exp.Before(h[j].exp) } +func (h expHeap) Less(i, j int) bool { return h[i].exp < h[j].exp } func (h expHeap) Swap(i, j int) { h[i], h[j] = h[j], h[i] } func (h *expHeap) Push(x interface{}) { *h = append(*h, x.(expItem)) } func (h *expHeap) Pop() interface{} { diff --git a/p2p/util_test.go b/p2p/util_test.go index c9f2648dc9..cc0d2b215f 100644 --- a/p2p/util_test.go +++ b/p2p/util_test.go @@ -19,30 +19,32 @@ package p2p import ( "testing" "time" + + "github.com/ethereum/go-ethereum/common/mclock" ) func TestExpHeap(t *testing.T) { var h expHeap var ( - basetime = time.Unix(4000, 0) + basetime = mclock.AbsTime(10) exptimeA = basetime.Add(2 * time.Second) exptimeB = basetime.Add(3 * time.Second) exptimeC = basetime.Add(4 * time.Second) ) - h.add("a", exptimeA) h.add("b", exptimeB) + h.add("a", exptimeA) h.add("c", exptimeC) - if !h.nextExpiry().Equal(exptimeA) { + if h.nextExpiry() != exptimeA { t.Fatal("wrong nextExpiry") } if !h.contains("a") || !h.contains("b") || !h.contains("c") { t.Fatal("heap doesn't contain all live items") } - h.expire(exptimeA.Add(1)) - if !h.nextExpiry().Equal(exptimeB) { + h.expire(exptimeA.Add(1), nil) + if h.nextExpiry() != exptimeB { t.Fatal("wrong nextExpiry") } if h.contains("a") { From 9938d954c8391682947682543cf9b52196507a88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Wed, 22 Jan 2020 16:39:43 +0200 Subject: [PATCH 107/128] eth: rework tx fetcher to use O(1) ops + manage network requests --- core/tx_pool.go | 9 +- eth/fetcher/block_fetcher.go | 21 + eth/fetcher/metrics.go | 54 - eth/fetcher/tx_fetcher.go | 931 +++++++-- eth/fetcher/tx_fetcher_test.go | 1672 ++++++++++++++--- eth/handler.go | 59 +- eth/metrics.go | 139 -- eth/peer.go | 316 ++-- eth/protocol.go | 7 +- eth/protocol_test.go | 25 +- eth/sync.go | 75 +- fuzzbuzz.yaml | 8 + ...0151ee1d0db4c74d3bcdfa4f7396a4c8538748c9-2 | 1 + ...020dd7b492a6eb34ff0b7d8ee46189422c37e4a7-6 | Bin 0 -> 14 bytes ...021d1144e359233c496e22c3250609b11b213e9f-4 | 12 + ...0d28327b1fb52c1ba02a6eb96675c31633921bb2-2 | 15 + ...0fcd827b57ded58e91f7ba2ac2b7ea4d25ebedca-7 | 1 + ...109bc9b8fd4fef63493e104c703c79bc4a5e8d34-6 | Bin 0 -> 470 bytes ...163785ab002746452619f31e8dfcb4549e6f8b6e-6 | Bin 0 -> 30 bytes ...1adfa6b9ddf5766220c8ff7ede2926ca241bb947-3 | 11 + ...1b9a02e9a48fea1d2fc3fb77946ada278e152079-4 | Bin 0 -> 17 bytes ...1e14c7ea1faef92890988061b5abe96db7190f98-7 | 1 + ...1e7d05f00e99cbf3ff0ef1cd7ea8dd07ad6dff23-6 | Bin 0 -> 19 bytes ...1ec95e347fd522e6385b5091aa81aa2485be4891-4 | Bin 0 -> 833 bytes ...1fbfa5d214060d2a0905846a589fd6f78d411451-4 | Bin 0 -> 22 bytes ...1fd84ee194e791783a7f18f0a6deab8efe05fc04-2 | 1 + ...21e76b9fca21d94d97f860c1c82f40697a83471b-8 | 3 + ...220a87fed0c92474923054094eb7aff14289cf5e-4 | Bin 0 -> 4 bytes ...23ddcd66aa92fe3d78b7f5b6e7cddb1b55c5f5df-3 | 12 + ...2441d249faf9a859e38c49f6e305b394280c6ea5-1 | Bin 0 -> 55 bytes ...2da1f0635e11283b1927974f418aadd8837ad31e-7 | Bin 0 -> 15 bytes ...2e1853fbf8efe40098b1583224fe3b5f335e7037-6 | Bin 0 -> 26 bytes ...2f25490dc49c103d653843ed47324b310ee7105e-7 | Bin 0 -> 23 bytes ...30494b85bb60ad7f099fa49d427007a761620d8f-5 | 10 + ...316024ca3aaf09c1de5258733ff5fe3d799648d3-4 | 15 + ...32a089e2c439a91f4c1b67a13d52429bcded0dd9-7 | Bin 0 -> 1665 bytes ...33ec1dc0bfeb93d16edee3c07125fec6ac1aa17d-2 | 1 + ...37a0d207700b52caa005ec8aeb344dcb13150ed2-5 | Bin 0 -> 55 bytes ...382f59c66d0ddb6747d3177263279789ca15c2db-5 | Bin 0 -> 14 bytes ...3a010483a4ad8d7215447ce27e0fac3791235c99-4 | 7 + ...3a3b717fcfe7ffb000b906e5a76f32248a576bf7-6 | Bin 0 -> 1141 bytes ...3c37f6d58b8029971935f127f53e6aaeba558445-6 | 2 + ...3c73b63bafa9f535c882ec17189adaf02b58f432-6 | 1 + ...3d11500c4f66b20c73bbdfb1a7bddd7bbf92b29c-5 | Bin 0 -> 453 bytes ...3d8b5bf36c80d6f65802280039f85421f32b5055-6 | Bin 0 -> 23 bytes ...3f99c546a3962256176d566c19e3fffb62072078-1 | 1 + ...408ec46539af27acd82b3d01e863597030882458-8 | Bin 0 -> 1884 bytes ...436154e5bb6487673f6642e6d2a582c01b083c08-8 | 1 + ...45f565cd14b8de1ba2e925047ce776c2682b4b8d-3 | Bin 0 -> 25 bytes ...4a0a12f5b033c8c160cc3b5133692ea1e92c6cdf-7 | 3 + ...550f15ef65230cc4dcfab7fea67de212d9212ff8-8 | Bin 0 -> 44 bytes ...5552213d659fef900a194c52718ffeffdc72d043-3 | Bin 0 -> 11 bytes ...5570ef82893a9b9b9158572d43a7de7537121d2d-1 | 1 + ...5e10f734f8af4116fbd164d96eec67aa53e6228c-5 | Bin 0 -> 40 bytes ...608200b402488b3989ec8ec5f4190ccb537b8ea4-4 | Bin 0 -> 24 bytes ...61e89c3fbdf9eff74bd250ea73cc2e61f8ca0d97-5 | 1 + ...62817a48c78fbf2c12fcdc5ca58e2ca60c43543a-7 | Bin 0 -> 27 bytes ...6782da8f1a432a77306d60d2ac2470c35b98004f-3 | 1 + ...68fb55290cb9d6da5b259017c34bcecf96c944aa-5 | Bin 0 -> 49 bytes ...6a5059bc86872526241d21ab5dae9f0afd3b9ae1-3 | 1 + ...717928e0e2d478c680c6409b173552ca98469ba5-6 | 1 + ...71d22f25419543e437f249ca437823b87ac926b1-6 | Bin 0 -> 27 bytes ...7312a0f31ae5d773ed4fd74abc7521eb14754683-8 | 2 + ...76e413a50dc8861e3756e556f796f1737bec2675-4 | Bin 0 -> 24 bytes ...78480977d5c07386b06e9b37f5c82f5ed86c2f09-3 | 14 + ...7a113cd3c178934cdb64353af86d51462d7080a4-5 | 10 + ...7ea9f71020f3eb783f743f744eba8d8ca4b2582f-3 | 9 + ...84f8c275f3ffbaf8c32c21782af13de10e7de28b-3 | 1 + ...85dfe7ddee0e52aa19115c0ebb9ed28a14e488c6-5 | Bin 0 -> 11 bytes ...87bba5b1e3da38fed8cb5a9bc5c8baa819e83d05-5 | Bin 0 -> 1137 bytes ...8a9ebedfbfec584d8b22761e6121dc1ca0248548-4 | Bin 0 -> 722 bytes ...8ff3bd49f93079e5e1c7f8f2461ba7ee612900c3-5 | Bin 0 -> 40 bytes ...9034aaf45143996a2b14465c352ab0c6fa26b221-2 | 1 + ...92cefdc6251d04896349a464b29be03d6bb04c3d-2 | 1 + ...9613e580ccb69df7c9074f0e2f6886ac6b34ca55-5 | Bin 0 -> 446 bytes ...98afc8970a680fdc4aee0b5d48784f650c566b75-6 | Bin 0 -> 31 bytes ...9dfc92f4ca2ece0167096fca6751ff314765f08b-8 | Bin 0 -> 23 bytes ...9ebcbbfdaf0e98c87652e57226a4d8a35170c67d-4 | 5 + ...9ff520eb8b8319a5fdafbe4d1cbb02a75058d93b-7 | 2 + ...a0b57a12e25ac5adcedb2a5c45915f0f62aee869-4 | Bin 0 -> 242 bytes ...a2684adccf16e036b051c12f283734fa803746e8-6 | Bin 0 -> 25 bytes ...a37305974cf477ecfe65fa92f37b1f51dea25910-4 | Bin 0 -> 15 bytes ...a7eb43926bd14b1f62a66a33107776e487434d32-7 | Bin 0 -> 516 bytes ...a8f7c254eb64a40fd2a77b79979c7bbdac6a760c-4 | 2 + ...a9a8f287d6af24e47d8db468e8f967aa44fb5a1f-7 | Bin 0 -> 106 bytes ...aa7444d8e326158046862590a0db993c07aef372-7 | 1 + ...ae4593626d8796e079a358c2395a4f6c9ddd6a44-6 | 8 + ...b2942d4413a66939cda7db93020dee79eb17788c-9 | Bin 0 -> 18 bytes ...b4614117cdfd147d38f4e8a4d85f5a2bb99a6a4f-5 | Bin 0 -> 838 bytes ...b631ef3291fa405cd6517d11f4d1b9b6d02912d4-2 | 1 + ...b7a91e338cc11f50ebdb2c414610efc4d5be3137-4 | Bin 0 -> 34 bytes ...b858cb282617fb0956d960215c8e84d1ccf909c6-2 | 1 + ...bc9d570aacf3acd39600feda8e72a293a4667da4-1 | 1 + ...be7eed35b245b5d5d2adcdb4c67f07794eb86b24-3 | 2 + ...c010b0cd70c7edbc5bd332fc9e2e91c6a1cbcdc4-5 | 4 + ...c1690698607eb0f4c4244e9f9629968be4beb6bc-8 | 2 + ...c1f435e4f53a9a17578d9e8c4789860f962a1379-6 | Bin 0 -> 1889 bytes ...c298a75334c3acf04bd129a8867447a25c8bacf8-7 | Bin 0 -> 27 bytes ...c42287c7d225e530e822f23bbbba6819a9e48f38-6 | Bin 0 -> 10 bytes ...4cdbb891f3ee76476b7375d5ed51691fed95421-10 | Bin 0 -> 16 bytes ...cc9572d72dfa2937074b1766dcbcff9cc58d1137-4 | Bin 0 -> 1522 bytes ...cd1d73b4e101bc7b979e3f6f135cb12d4594d348-5 | 1 + ...d0acdc8fca32bbd58d368eeac3bd9eaa46f59d27-5 | Bin 0 -> 9 bytes ...d0e43b715fd00953f7bdd6dfad95811985e81396-4 | Bin 0 -> 10 bytes ...d925fbd22c8bc0de34d6a9d1258ce3d2928d0927-8 | Bin 0 -> 22 bytes ...d9ba78cb7425724185d5fa300cd5c03aec2683bb-7 | Bin 0 -> 510 bytes .../da39a3ee5e6b4b0d3255bfef95601890afd80709 | 0 ...dcdb7758b87648b5d766b1b341a65834420cf621-7 | Bin 0 -> 22 bytes ...dd441bd24581332c9ce19e008260a69287aa3cbc-6 | 2 + ...def879fe0fd637a745c00c8f1da340518db8688c-2 | 1 + ...df6c30a9781b93bd6d2f5e97e5592d5945210003-7 | Bin 0 -> 1881 bytes ...dfc1c3a2e3ccdaf6f88c515fd00e8ad08421e431-6 | Bin 0 -> 10 bytes ...e1dcc4e7ead6dfd1139ece7bf57d776cb9dac72d-7 | Bin 0 -> 20 bytes ...e39c2de2c8937d2cbd4339b13d6a0ce94d94f8d2-8 | Bin 0 -> 1275 bytes ...e72f76b9579c792e545d02fe405d9186f0d6c39b-6 | Bin 0 -> 47 bytes ...eb70814d6355a4498b8f301ba8dbc34f895a9947-5 | Bin 0 -> 57 bytes ...ebdc17efe343e412634dca57cecd5a0e1ce1c1c7-5 | Bin 0 -> 54 bytes ...ec0a25eba8966b8f628d821b3cfbdf2dfd4bbb4c-3 | 13 + ...eebe3b76aeba6deed965d17d2b024f7eae1a43f1-5 | Bin 0 -> 10 bytes ...ef8741a9faf030794d98ff113f556c68a24719a5-6 | Bin 0 -> 31 bytes ...efb7410d02418befeba25a43d676cc6124129125-4 | 1 + ...f6f97d781a5a749903790e07db8619866cb7c3a1-6 | Bin 0 -> 22 bytes ...f7a3cd00fa0e57742e7dbbb8283dcaea067eaf7b-5 | 2 + ...f94d60a6c556ce485ab60088291760b8be25776c-6 | 2 + ...f9e627b2cb82ffa1ea5e0c6d7f2802f3000b18a8-6 | Bin 0 -> 11 bytes ...fb3775aa24e5667e658920c05ba4b7b19ff256fb-5 | 1 + ...fd6386548e119a50db96b2fa406e54924c45a2d5-6 | Bin 0 -> 28 bytes tests/fuzzers/txfetcher/txfetcher_fuzzer.go | 199 ++ 128 files changed, 2852 insertions(+), 856 deletions(-) delete mode 100644 eth/fetcher/metrics.go delete mode 100644 eth/metrics.go create mode 100644 tests/fuzzers/txfetcher/corpus/0151ee1d0db4c74d3bcdfa4f7396a4c8538748c9-2 create mode 100644 tests/fuzzers/txfetcher/corpus/020dd7b492a6eb34ff0b7d8ee46189422c37e4a7-6 create mode 100644 tests/fuzzers/txfetcher/corpus/021d1144e359233c496e22c3250609b11b213e9f-4 create mode 100644 tests/fuzzers/txfetcher/corpus/0d28327b1fb52c1ba02a6eb96675c31633921bb2-2 create mode 100644 tests/fuzzers/txfetcher/corpus/0fcd827b57ded58e91f7ba2ac2b7ea4d25ebedca-7 create mode 100644 tests/fuzzers/txfetcher/corpus/109bc9b8fd4fef63493e104c703c79bc4a5e8d34-6 create mode 100644 tests/fuzzers/txfetcher/corpus/163785ab002746452619f31e8dfcb4549e6f8b6e-6 create mode 100644 tests/fuzzers/txfetcher/corpus/1adfa6b9ddf5766220c8ff7ede2926ca241bb947-3 create mode 100644 tests/fuzzers/txfetcher/corpus/1b9a02e9a48fea1d2fc3fb77946ada278e152079-4 create mode 100644 tests/fuzzers/txfetcher/corpus/1e14c7ea1faef92890988061b5abe96db7190f98-7 create mode 100644 tests/fuzzers/txfetcher/corpus/1e7d05f00e99cbf3ff0ef1cd7ea8dd07ad6dff23-6 create mode 100644 tests/fuzzers/txfetcher/corpus/1ec95e347fd522e6385b5091aa81aa2485be4891-4 create mode 100644 tests/fuzzers/txfetcher/corpus/1fbfa5d214060d2a0905846a589fd6f78d411451-4 create mode 100644 tests/fuzzers/txfetcher/corpus/1fd84ee194e791783a7f18f0a6deab8efe05fc04-2 create mode 100644 tests/fuzzers/txfetcher/corpus/21e76b9fca21d94d97f860c1c82f40697a83471b-8 create mode 100644 tests/fuzzers/txfetcher/corpus/220a87fed0c92474923054094eb7aff14289cf5e-4 create mode 100644 tests/fuzzers/txfetcher/corpus/23ddcd66aa92fe3d78b7f5b6e7cddb1b55c5f5df-3 create mode 100644 tests/fuzzers/txfetcher/corpus/2441d249faf9a859e38c49f6e305b394280c6ea5-1 create mode 100644 tests/fuzzers/txfetcher/corpus/2da1f0635e11283b1927974f418aadd8837ad31e-7 create mode 100644 tests/fuzzers/txfetcher/corpus/2e1853fbf8efe40098b1583224fe3b5f335e7037-6 create mode 100644 tests/fuzzers/txfetcher/corpus/2f25490dc49c103d653843ed47324b310ee7105e-7 create mode 100644 tests/fuzzers/txfetcher/corpus/30494b85bb60ad7f099fa49d427007a761620d8f-5 create mode 100644 tests/fuzzers/txfetcher/corpus/316024ca3aaf09c1de5258733ff5fe3d799648d3-4 create mode 100644 tests/fuzzers/txfetcher/corpus/32a089e2c439a91f4c1b67a13d52429bcded0dd9-7 create mode 100644 tests/fuzzers/txfetcher/corpus/33ec1dc0bfeb93d16edee3c07125fec6ac1aa17d-2 create mode 100644 tests/fuzzers/txfetcher/corpus/37a0d207700b52caa005ec8aeb344dcb13150ed2-5 create mode 100644 tests/fuzzers/txfetcher/corpus/382f59c66d0ddb6747d3177263279789ca15c2db-5 create mode 100644 tests/fuzzers/txfetcher/corpus/3a010483a4ad8d7215447ce27e0fac3791235c99-4 create mode 100644 tests/fuzzers/txfetcher/corpus/3a3b717fcfe7ffb000b906e5a76f32248a576bf7-6 create mode 100644 tests/fuzzers/txfetcher/corpus/3c37f6d58b8029971935f127f53e6aaeba558445-6 create mode 100644 tests/fuzzers/txfetcher/corpus/3c73b63bafa9f535c882ec17189adaf02b58f432-6 create mode 100644 tests/fuzzers/txfetcher/corpus/3d11500c4f66b20c73bbdfb1a7bddd7bbf92b29c-5 create mode 100644 tests/fuzzers/txfetcher/corpus/3d8b5bf36c80d6f65802280039f85421f32b5055-6 create mode 100644 tests/fuzzers/txfetcher/corpus/3f99c546a3962256176d566c19e3fffb62072078-1 create mode 100644 tests/fuzzers/txfetcher/corpus/408ec46539af27acd82b3d01e863597030882458-8 create mode 100644 tests/fuzzers/txfetcher/corpus/436154e5bb6487673f6642e6d2a582c01b083c08-8 create mode 100644 tests/fuzzers/txfetcher/corpus/45f565cd14b8de1ba2e925047ce776c2682b4b8d-3 create mode 100644 tests/fuzzers/txfetcher/corpus/4a0a12f5b033c8c160cc3b5133692ea1e92c6cdf-7 create mode 100644 tests/fuzzers/txfetcher/corpus/550f15ef65230cc4dcfab7fea67de212d9212ff8-8 create mode 100644 tests/fuzzers/txfetcher/corpus/5552213d659fef900a194c52718ffeffdc72d043-3 create mode 100644 tests/fuzzers/txfetcher/corpus/5570ef82893a9b9b9158572d43a7de7537121d2d-1 create mode 100644 tests/fuzzers/txfetcher/corpus/5e10f734f8af4116fbd164d96eec67aa53e6228c-5 create mode 100644 tests/fuzzers/txfetcher/corpus/608200b402488b3989ec8ec5f4190ccb537b8ea4-4 create mode 100644 tests/fuzzers/txfetcher/corpus/61e89c3fbdf9eff74bd250ea73cc2e61f8ca0d97-5 create mode 100644 tests/fuzzers/txfetcher/corpus/62817a48c78fbf2c12fcdc5ca58e2ca60c43543a-7 create mode 100644 tests/fuzzers/txfetcher/corpus/6782da8f1a432a77306d60d2ac2470c35b98004f-3 create mode 100644 tests/fuzzers/txfetcher/corpus/68fb55290cb9d6da5b259017c34bcecf96c944aa-5 create mode 100644 tests/fuzzers/txfetcher/corpus/6a5059bc86872526241d21ab5dae9f0afd3b9ae1-3 create mode 100644 tests/fuzzers/txfetcher/corpus/717928e0e2d478c680c6409b173552ca98469ba5-6 create mode 100644 tests/fuzzers/txfetcher/corpus/71d22f25419543e437f249ca437823b87ac926b1-6 create mode 100644 tests/fuzzers/txfetcher/corpus/7312a0f31ae5d773ed4fd74abc7521eb14754683-8 create mode 100644 tests/fuzzers/txfetcher/corpus/76e413a50dc8861e3756e556f796f1737bec2675-4 create mode 100644 tests/fuzzers/txfetcher/corpus/78480977d5c07386b06e9b37f5c82f5ed86c2f09-3 create mode 100644 tests/fuzzers/txfetcher/corpus/7a113cd3c178934cdb64353af86d51462d7080a4-5 create mode 100644 tests/fuzzers/txfetcher/corpus/7ea9f71020f3eb783f743f744eba8d8ca4b2582f-3 create mode 100644 tests/fuzzers/txfetcher/corpus/84f8c275f3ffbaf8c32c21782af13de10e7de28b-3 create mode 100644 tests/fuzzers/txfetcher/corpus/85dfe7ddee0e52aa19115c0ebb9ed28a14e488c6-5 create mode 100644 tests/fuzzers/txfetcher/corpus/87bba5b1e3da38fed8cb5a9bc5c8baa819e83d05-5 create mode 100644 tests/fuzzers/txfetcher/corpus/8a9ebedfbfec584d8b22761e6121dc1ca0248548-4 create mode 100644 tests/fuzzers/txfetcher/corpus/8ff3bd49f93079e5e1c7f8f2461ba7ee612900c3-5 create mode 100644 tests/fuzzers/txfetcher/corpus/9034aaf45143996a2b14465c352ab0c6fa26b221-2 create mode 100644 tests/fuzzers/txfetcher/corpus/92cefdc6251d04896349a464b29be03d6bb04c3d-2 create mode 100644 tests/fuzzers/txfetcher/corpus/9613e580ccb69df7c9074f0e2f6886ac6b34ca55-5 create mode 100644 tests/fuzzers/txfetcher/corpus/98afc8970a680fdc4aee0b5d48784f650c566b75-6 create mode 100644 tests/fuzzers/txfetcher/corpus/9dfc92f4ca2ece0167096fca6751ff314765f08b-8 create mode 100644 tests/fuzzers/txfetcher/corpus/9ebcbbfdaf0e98c87652e57226a4d8a35170c67d-4 create mode 100644 tests/fuzzers/txfetcher/corpus/9ff520eb8b8319a5fdafbe4d1cbb02a75058d93b-7 create mode 100644 tests/fuzzers/txfetcher/corpus/a0b57a12e25ac5adcedb2a5c45915f0f62aee869-4 create mode 100644 tests/fuzzers/txfetcher/corpus/a2684adccf16e036b051c12f283734fa803746e8-6 create mode 100644 tests/fuzzers/txfetcher/corpus/a37305974cf477ecfe65fa92f37b1f51dea25910-4 create mode 100644 tests/fuzzers/txfetcher/corpus/a7eb43926bd14b1f62a66a33107776e487434d32-7 create mode 100644 tests/fuzzers/txfetcher/corpus/a8f7c254eb64a40fd2a77b79979c7bbdac6a760c-4 create mode 100644 tests/fuzzers/txfetcher/corpus/a9a8f287d6af24e47d8db468e8f967aa44fb5a1f-7 create mode 100644 tests/fuzzers/txfetcher/corpus/aa7444d8e326158046862590a0db993c07aef372-7 create mode 100644 tests/fuzzers/txfetcher/corpus/ae4593626d8796e079a358c2395a4f6c9ddd6a44-6 create mode 100644 tests/fuzzers/txfetcher/corpus/b2942d4413a66939cda7db93020dee79eb17788c-9 create mode 100644 tests/fuzzers/txfetcher/corpus/b4614117cdfd147d38f4e8a4d85f5a2bb99a6a4f-5 create mode 100644 tests/fuzzers/txfetcher/corpus/b631ef3291fa405cd6517d11f4d1b9b6d02912d4-2 create mode 100644 tests/fuzzers/txfetcher/corpus/b7a91e338cc11f50ebdb2c414610efc4d5be3137-4 create mode 100644 tests/fuzzers/txfetcher/corpus/b858cb282617fb0956d960215c8e84d1ccf909c6-2 create mode 100644 tests/fuzzers/txfetcher/corpus/bc9d570aacf3acd39600feda8e72a293a4667da4-1 create mode 100644 tests/fuzzers/txfetcher/corpus/be7eed35b245b5d5d2adcdb4c67f07794eb86b24-3 create mode 100644 tests/fuzzers/txfetcher/corpus/c010b0cd70c7edbc5bd332fc9e2e91c6a1cbcdc4-5 create mode 100644 tests/fuzzers/txfetcher/corpus/c1690698607eb0f4c4244e9f9629968be4beb6bc-8 create mode 100644 tests/fuzzers/txfetcher/corpus/c1f435e4f53a9a17578d9e8c4789860f962a1379-6 create mode 100644 tests/fuzzers/txfetcher/corpus/c298a75334c3acf04bd129a8867447a25c8bacf8-7 create mode 100644 tests/fuzzers/txfetcher/corpus/c42287c7d225e530e822f23bbbba6819a9e48f38-6 create mode 100644 tests/fuzzers/txfetcher/corpus/c4cdbb891f3ee76476b7375d5ed51691fed95421-10 create mode 100644 tests/fuzzers/txfetcher/corpus/cc9572d72dfa2937074b1766dcbcff9cc58d1137-4 create mode 100644 tests/fuzzers/txfetcher/corpus/cd1d73b4e101bc7b979e3f6f135cb12d4594d348-5 create mode 100644 tests/fuzzers/txfetcher/corpus/d0acdc8fca32bbd58d368eeac3bd9eaa46f59d27-5 create mode 100644 tests/fuzzers/txfetcher/corpus/d0e43b715fd00953f7bdd6dfad95811985e81396-4 create mode 100644 tests/fuzzers/txfetcher/corpus/d925fbd22c8bc0de34d6a9d1258ce3d2928d0927-8 create mode 100644 tests/fuzzers/txfetcher/corpus/d9ba78cb7425724185d5fa300cd5c03aec2683bb-7 create mode 100644 tests/fuzzers/txfetcher/corpus/da39a3ee5e6b4b0d3255bfef95601890afd80709 create mode 100644 tests/fuzzers/txfetcher/corpus/dcdb7758b87648b5d766b1b341a65834420cf621-7 create mode 100644 tests/fuzzers/txfetcher/corpus/dd441bd24581332c9ce19e008260a69287aa3cbc-6 create mode 100644 tests/fuzzers/txfetcher/corpus/def879fe0fd637a745c00c8f1da340518db8688c-2 create mode 100644 tests/fuzzers/txfetcher/corpus/df6c30a9781b93bd6d2f5e97e5592d5945210003-7 create mode 100644 tests/fuzzers/txfetcher/corpus/dfc1c3a2e3ccdaf6f88c515fd00e8ad08421e431-6 create mode 100644 tests/fuzzers/txfetcher/corpus/e1dcc4e7ead6dfd1139ece7bf57d776cb9dac72d-7 create mode 100644 tests/fuzzers/txfetcher/corpus/e39c2de2c8937d2cbd4339b13d6a0ce94d94f8d2-8 create mode 100644 tests/fuzzers/txfetcher/corpus/e72f76b9579c792e545d02fe405d9186f0d6c39b-6 create mode 100644 tests/fuzzers/txfetcher/corpus/eb70814d6355a4498b8f301ba8dbc34f895a9947-5 create mode 100644 tests/fuzzers/txfetcher/corpus/ebdc17efe343e412634dca57cecd5a0e1ce1c1c7-5 create mode 100644 tests/fuzzers/txfetcher/corpus/ec0a25eba8966b8f628d821b3cfbdf2dfd4bbb4c-3 create mode 100644 tests/fuzzers/txfetcher/corpus/eebe3b76aeba6deed965d17d2b024f7eae1a43f1-5 create mode 100644 tests/fuzzers/txfetcher/corpus/ef8741a9faf030794d98ff113f556c68a24719a5-6 create mode 100644 tests/fuzzers/txfetcher/corpus/efb7410d02418befeba25a43d676cc6124129125-4 create mode 100644 tests/fuzzers/txfetcher/corpus/f6f97d781a5a749903790e07db8619866cb7c3a1-6 create mode 100644 tests/fuzzers/txfetcher/corpus/f7a3cd00fa0e57742e7dbbb8283dcaea067eaf7b-5 create mode 100644 tests/fuzzers/txfetcher/corpus/f94d60a6c556ce485ab60088291760b8be25776c-6 create mode 100644 tests/fuzzers/txfetcher/corpus/f9e627b2cb82ffa1ea5e0c6d7f2802f3000b18a8-6 create mode 100644 tests/fuzzers/txfetcher/corpus/fb3775aa24e5667e658920c05ba4b7b19ff256fb-5 create mode 100644 tests/fuzzers/txfetcher/corpus/fd6386548e119a50db96b2fa406e54924c45a2d5-6 create mode 100644 tests/fuzzers/txfetcher/txfetcher_fuzzer.go diff --git a/core/tx_pool.go b/core/tx_pool.go index 16d80d644f..e2137ca4f4 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -18,7 +18,6 @@ package core import ( "errors" - "fmt" "math" "math/big" "sort" @@ -53,6 +52,10 @@ const ( ) var ( + // ErrAlreadyKnown is returned if the transactions is already contained + // within the pool. + ErrAlreadyKnown = errors.New("already known") + // ErrInvalidSender is returned if the transaction contains an invalid signature. ErrInvalidSender = errors.New("invalid sender") @@ -579,7 +582,7 @@ func (pool *TxPool) add(tx *types.Transaction, local bool) (replaced bool, err e if pool.all.Get(hash) != nil { log.Trace("Discarding already known transaction", "hash", hash) knownTxMeter.Mark(1) - return false, fmt.Errorf("known transaction: %x", hash) + return false, ErrAlreadyKnown } // If the transaction fails basic validation, discard it if err := pool.validateTx(tx, local); err != nil { @@ -786,7 +789,7 @@ func (pool *TxPool) addTxs(txs []*types.Transaction, local, sync bool) []error { for i, tx := range txs { // If the transaction is known, pre-set the error slot if pool.all.Get(tx.Hash()) != nil { - errs[i] = fmt.Errorf("known transaction: %x", tx.Hash()) + errs[i] = ErrAlreadyKnown knownTxMeter.Mark(1) continue } diff --git a/eth/fetcher/block_fetcher.go b/eth/fetcher/block_fetcher.go index 7395ec83de..b6cab05deb 100644 --- a/eth/fetcher/block_fetcher.go +++ b/eth/fetcher/block_fetcher.go @@ -27,6 +27,7 @@ import ( "github.com/ethereum/go-ethereum/consensus" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" ) const ( @@ -42,6 +43,26 @@ const ( blockLimit = 64 // Maximum number of unique blocks a peer may have delivered ) +var ( + blockAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/announces/in", nil) + blockAnnounceOutTimer = metrics.NewRegisteredTimer("eth/fetcher/block/announces/out", nil) + blockAnnounceDropMeter = metrics.NewRegisteredMeter("eth/fetcher/block/announces/drop", nil) + blockAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/block/announces/dos", nil) + + blockBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/broadcasts/in", nil) + blockBroadcastOutTimer = metrics.NewRegisteredTimer("eth/fetcher/block/broadcasts/out", nil) + blockBroadcastDropMeter = metrics.NewRegisteredMeter("eth/fetcher/block/broadcasts/drop", nil) + blockBroadcastDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/block/broadcasts/dos", nil) + + headerFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/block/headers", nil) + bodyFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/block/bodies", nil) + + headerFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/headers/in", nil) + headerFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/headers/out", nil) + bodyFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/bodies/in", nil) + bodyFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/block/filter/bodies/out", nil) +) + var ( errTerminated = errors.New("terminated") ) diff --git a/eth/fetcher/metrics.go b/eth/fetcher/metrics.go deleted file mode 100644 index b75889938d..0000000000 --- a/eth/fetcher/metrics.go +++ /dev/null @@ -1,54 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -// Contains the metrics collected by the fetcher. - -package fetcher - -import ( - "github.com/ethereum/go-ethereum/metrics" -) - -var ( - blockAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/announces/in", nil) - blockAnnounceOutTimer = metrics.NewRegisteredTimer("eth/fetcher/prop/block/announces/out", nil) - blockAnnounceDropMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/announces/drop", nil) - blockAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/announces/dos", nil) - - blockBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/broadcasts/in", nil) - blockBroadcastOutTimer = metrics.NewRegisteredTimer("eth/fetcher/prop/block/broadcasts/out", nil) - blockBroadcastDropMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/broadcasts/drop", nil) - blockBroadcastDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/block/broadcasts/dos", nil) - - headerFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/headers", nil) - bodyFetchMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/bodies", nil) - - headerFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/headers/in", nil) - headerFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/headers/out", nil) - bodyFilterInMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/bodies/in", nil) - bodyFilterOutMeter = metrics.NewRegisteredMeter("eth/fetcher/filter/bodies/out", nil) - - txAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/transaction/announces/in", nil) - txAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/transaction/announces/dos", nil) - txAnnounceSkipMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/transaction/announces/skip", nil) - txAnnounceUnderpriceMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/transaction/announces/underprice", nil) - txBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/prop/transaction/broadcasts/in", nil) - txFetchOutMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/transaction/out", nil) - txFetchSuccessMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/transaction/success", nil) - txFetchTimeoutMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/transaction/timeout", nil) - txFetchInvalidMeter = metrics.NewRegisteredMeter("eth/fetcher/fetch/transaction/invalid", nil) - txFetchDurationTimer = metrics.NewRegisteredTimer("eth/fetcher/fetch/transaction/duration", nil) -) diff --git a/eth/fetcher/tx_fetcher.go b/eth/fetcher/tx_fetcher.go index 1dabb0819f..c497cebb45 100644 --- a/eth/fetcher/tx_fetcher.go +++ b/eth/fetcher/tx_fetcher.go @@ -17,108 +17,236 @@ package fetcher import ( - "math/rand" + "bytes" + "fmt" + mrand "math/rand" + "sort" "time" mapset "github.com/deckarep/golang-set" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" "github.com/ethereum/go-ethereum/log" + "github.com/ethereum/go-ethereum/metrics" ) -var ( - // txAnnounceLimit is the maximum number of unique transaction a peer +const ( + // maxTxAnnounces is the maximum number of unique transaction a peer // can announce in a short time. - txAnnounceLimit = 4096 + maxTxAnnounces = 4096 + + // maxTxRetrievals is the maximum transaction number can be fetched in one + // request. The rationale to pick 256 is: + // - In eth protocol, the softResponseLimit is 2MB. Nowadays according to + // Etherscan the average transaction size is around 200B, so in theory + // we can include lots of transaction in a single protocol packet. + // - However the maximum size of a single transaction is raised to 128KB, + // so pick a middle value here to ensure we can maximize the efficiency + // of the retrieval and response size overflow won't happen in most cases. + maxTxRetrievals = 256 + + // maxTxUnderpricedSetSize is the size of the underpriced transaction set that + // is used to track recent transactions that have been dropped so we don't + // re-request them. + maxTxUnderpricedSetSize = 32768 + + // txArriveTimeout is the time allowance before an announced transaction is + // explicitly requested. + txArriveTimeout = 500 * time.Millisecond + + // txGatherSlack is the interval used to collate almost-expired announces + // with network fetches. + txGatherSlack = 100 * time.Millisecond +) +var ( // txFetchTimeout is the maximum allotted time to return an explicitly // requested transaction. txFetchTimeout = 5 * time.Second +) + +var ( + txAnnounceInMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/announces/in", nil) + txAnnounceKnownMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/announces/known", nil) + txAnnounceUnderpricedMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/announces/underpriced", nil) + txAnnounceDOSMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/announces/dos", nil) + + txBroadcastInMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/broadcasts/in", nil) + txBroadcastKnownMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/broadcasts/known", nil) + txBroadcastUnderpricedMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/broadcasts/underpriced", nil) + txBroadcastOtherRejectMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/broadcasts/otherreject", nil) - // MaxTransactionFetch is the maximum transaction number can be fetched - // in one request. The rationale to pick this value is: - // In eth protocol, the softResponseLimit is 2MB. Nowdays according to - // Etherscan the average transaction size is around 200B, so in theory - // we can include lots of transaction in a single protocol packet. However - // the maximum size of a single transaction is raised to 128KB, so pick - // a middle value here to ensure we can maximize the efficiency of the - // retrieval and response size overflow won't happen in most cases. - MaxTransactionFetch = 256 - - // underpriceSetSize is the size of underprice set which used for maintaining - // the set of underprice transactions. - underpriceSetSize = 4096 + txRequestOutMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/request/out", nil) + txRequestFailMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/request/fail", nil) + txRequestDoneMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/request/done", nil) + txRequestTimeoutMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/request/timeout", nil) + + txReplyInMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/replies/in", nil) + txReplyKnownMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/replies/known", nil) + txReplyUnderpricedMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/replies/underpriced", nil) + txReplyOtherRejectMeter = metrics.NewRegisteredMeter("eth/fetcher/transaction/replies/otherreject", nil) + + txFetcherWaitingPeers = metrics.NewRegisteredGauge("eth/fetcher/transaction/waiting/peers", nil) + txFetcherWaitingHashes = metrics.NewRegisteredGauge("eth/fetcher/transaction/waiting/hashes", nil) + txFetcherQueueingPeers = metrics.NewRegisteredGauge("eth/fetcher/transaction/queueing/peers", nil) + txFetcherQueueingHashes = metrics.NewRegisteredGauge("eth/fetcher/transaction/queueing/hashes", nil) + txFetcherFetchingPeers = metrics.NewRegisteredGauge("eth/fetcher/transaction/fetching/peers", nil) + txFetcherFetchingHashes = metrics.NewRegisteredGauge("eth/fetcher/transaction/fetching/hashes", nil) ) -// txAnnounce is the notification of the availability of a single -// new transaction in the network. +// txAnnounce is the notification of the availability of a batch +// of new transactions in the network. type txAnnounce struct { - origin string // Identifier of the peer originating the notification - time time.Time // Timestamp of the announcement - fetchTxs func([]common.Hash) // Callback for retrieving transaction from specified peer + origin string // Identifier of the peer originating the notification + hashes []common.Hash // Batch of transaction hashes being announced } -// txsAnnounce is the notification of the availability of a batch -// of new transactions in the network. -type txsAnnounce struct { - hashes []common.Hash // Batch of transaction hashes being announced - origin string // Identifier of the peer originating the notification - time time.Time // Timestamp of the announcement - fetchTxs func([]common.Hash) // Callback for retrieving transaction from specified peer +// txRequest represents an in-flight transaction retrieval request destined to +// a specific peers. +type txRequest struct { + hashes []common.Hash // Transactions having been requested + stolen map[common.Hash]struct{} // Deliveries by someone else (don't re-request) + time mclock.AbsTime // Timestamp of the request +} + +// txDelivery is the notification that a batch of transactions have been added +// to the pool and should be untracked. +type txDelivery struct { + origin string // Identifier of the peer originating the notification + hashes []common.Hash // Batch of transaction hashes having been delivered + direct bool // Whether this is a direct reply or a broadcast +} + +// txDrop is the notiication that a peer has disconnected. +type txDrop struct { + peer string } -// TxFetcher is responsible for retrieving new transaction based -// on the announcement. +// TxFetcher is responsible for retrieving new transaction based on announcements. +// +// The fetcher operates in 3 stages: +// - Transactions that are newly discovered are moved into a wait list. +// - After ~500ms passes, transactions from the wait list that have not been +// broadcast to us in whole are moved into a queueing area. +// - When a connected peer doesn't have in-flight retrieval requests, any +// transaction queued up (and announced by the peer) are allocated to the +// peer and moved into a fetching status until it's fulfilled or fails. +// +// The invariants of the fetcher are: +// - Each tracked transaction (hash) must only be present in one of the +// three stages. This ensures that the fetcher operates akin to a finite +// state automata and there's do data leak. +// - Each peer that announced transactions may be scheduled retrievals, but +// only ever one concurrently. This ensures we can immediately know what is +// missing from a reply and reschedule it. type TxFetcher struct { - notify chan *txsAnnounce - cleanup chan []common.Hash + notify chan *txAnnounce + cleanup chan *txDelivery + drop chan *txDrop quit chan struct{} - // Announce states - announces map[string]int // Per peer transaction announce counts to prevent memory exhaustion - announced map[common.Hash][]*txAnnounce // Announced transactions, scheduled for fetching - fetching map[common.Hash]*txAnnounce // Announced transactions, currently fetching - underpriced mapset.Set // Transaction set whose price is too low for accepting + underpriced mapset.Set // Transactions discarded as too cheap (don't re-fetch) + + // Stage 1: Waiting lists for newly discovered transactions that might be + // broadcast without needing explicit request/reply round trips. + waitlist map[common.Hash]map[string]struct{} // Transactions waiting for an potential broadcast + waittime map[common.Hash]mclock.AbsTime // Timestamps when transactions were added to the waitlist + waitslots map[string]map[common.Hash]struct{} // Waiting announcement sgroupped by peer (DoS protection) + + // Stage 2: Queue of transactions that waiting to be allocated to some peer + // to be retrieved directly. + announces map[string]map[common.Hash]struct{} // Set of announced transactions, grouped by origin peer + announced map[common.Hash]map[string]struct{} // Set of download locations, grouped by transaction hash + + // Stage 3: Set of transactions currently being retrieved, some which may be + // fulfilled and some rescheduled. Note, this step shares 'announces' from the + // previous stage to avoid having to duplicate (need it for DoS checks). + fetching map[common.Hash]string // Transaction set currently being retrieved + requests map[string]*txRequest // In-flight transaction retrievals + alternates map[common.Hash]map[string]struct{} // In-flight transaction alternate origins if retrieval fails // Callbacks hasTx func(common.Hash) bool // Retrieves a tx from the local txpool addTxs func([]*types.Transaction) []error // Insert a batch of transactions into local txpool - dropPeer func(string) // Drop the specified peer - - // Hooks - announceHook func([]common.Hash) // Hook which is called when a batch transactions are announced - importTxsHook func([]*types.Transaction) // Hook which is called when a batch of transactions are imported. - dropHook func(string) // Hook which is called when a peer is dropped - cleanupHook func([]common.Hash) // Hook which is called when internal status is cleaned - rejectUnderprice func(common.Hash) // Hook which is called when underprice transaction is rejected + fetchTxs func(string, []common.Hash) error // Retrieves a set of txs from a remote peer + + step chan struct{} // Notification channel when the fetcher loop iterates + clock mclock.Clock // Time wrapper to simulate in tests + rand *mrand.Rand // Randomizer to use in tests instead of map range loops (soft-random) } // NewTxFetcher creates a transaction fetcher to retrieve transaction // based on hash announcements. -func NewTxFetcher(hasTx func(common.Hash) bool, addTxs func([]*types.Transaction) []error, dropPeer func(string)) *TxFetcher { +func NewTxFetcher(hasTx func(common.Hash) bool, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error) *TxFetcher { + return NewTxFetcherForTests(hasTx, addTxs, fetchTxs, mclock.System{}, nil) +} + +// NewTxFetcherForTests is a testing method to mock out the realtime clock with +// a simulated version and the internal randomness with a deterministic one. +func NewTxFetcherForTests( + hasTx func(common.Hash) bool, addTxs func([]*types.Transaction) []error, fetchTxs func(string, []common.Hash) error, + clock mclock.Clock, rand *mrand.Rand) *TxFetcher { return &TxFetcher{ - notify: make(chan *txsAnnounce), - cleanup: make(chan []common.Hash), + notify: make(chan *txAnnounce), + cleanup: make(chan *txDelivery), + drop: make(chan *txDrop), quit: make(chan struct{}), - announces: make(map[string]int), - announced: make(map[common.Hash][]*txAnnounce), - fetching: make(map[common.Hash]*txAnnounce), + waitlist: make(map[common.Hash]map[string]struct{}), + waittime: make(map[common.Hash]mclock.AbsTime), + waitslots: make(map[string]map[common.Hash]struct{}), + announces: make(map[string]map[common.Hash]struct{}), + announced: make(map[common.Hash]map[string]struct{}), + fetching: make(map[common.Hash]string), + requests: make(map[string]*txRequest), + alternates: make(map[common.Hash]map[string]struct{}), underpriced: mapset.NewSet(), hasTx: hasTx, addTxs: addTxs, - dropPeer: dropPeer, + fetchTxs: fetchTxs, + clock: clock, + rand: rand, } } -// Notify announces the fetcher of the potential availability of a -// new transaction in the network. -func (f *TxFetcher) Notify(peer string, hashes []common.Hash, time time.Time, fetchTxs func([]common.Hash)) error { - announce := &txsAnnounce{ - hashes: hashes, - time: time, - origin: peer, - fetchTxs: fetchTxs, +// Notify announces the fetcher of the potential availability of a new batch of +// transactions in the network. +func (f *TxFetcher) Notify(peer string, hashes []common.Hash) error { + // Keep track of all the announced transactions + txAnnounceInMeter.Mark(int64(len(hashes))) + + // Skip any transaction announcements that we already know of, or that we've + // previously marked as cheap and discarded. This check is of course racey, + // because multiple concurrent notifies will still manage to pass it, but it's + // still valuable to check here because it runs concurrent to the internal + // loop, so anything caught here is time saved internally. + var ( + unknowns = make([]common.Hash, 0, len(hashes)) + duplicate, underpriced int64 + ) + for _, hash := range hashes { + switch { + case f.hasTx(hash): + duplicate++ + + case f.underpriced.Contains(hash): + underpriced++ + + default: + unknowns = append(unknowns, hash) + } + } + txAnnounceKnownMeter.Mark(duplicate) + txAnnounceUnderpricedMeter.Mark(underpriced) + + // If anything's left to announce, push it into the internal loop + if len(unknowns) == 0 { + return nil + } + announce := &txAnnounce{ + origin: peer, + hashes: unknowns, } select { case f.notify <- announce: @@ -128,45 +256,75 @@ func (f *TxFetcher) Notify(peer string, hashes []common.Hash, time time.Time, fe } } -// EnqueueTxs imports a batch of received transaction into fetcher. -func (f *TxFetcher) EnqueueTxs(peer string, txs []*types.Transaction) error { +// Enqueue imports a batch of received transaction into the transaction pool +// and the fetcher. This method may be called by both transaction broadcasts and +// direct request replies. The differentiation is important so the fetcher can +// re-shedule missing transactions as soon as possible. +func (f *TxFetcher) Enqueue(peer string, txs []*types.Transaction, direct bool) error { + // Keep track of all the propagated transactions + if direct { + txReplyInMeter.Mark(int64(len(txs))) + } else { + txBroadcastInMeter.Mark(int64(len(txs))) + } + // Push all the transactions into the pool, tracking underpriced ones to avoid + // re-requesting them and dropping the peer in case of malicious transfers. var ( - drop bool - hashes []common.Hash + added = make([]common.Hash, 0, len(txs)) + duplicate int64 + underpriced int64 + otherreject int64 ) errs := f.addTxs(txs) for i, err := range errs { if err != nil { - // Drop peer if the received transaction isn't signed properly. - drop = (drop || err == core.ErrInvalidSender) - txFetchInvalidMeter.Mark(1) - // Track the transaction hash if the price is too low for us. // Avoid re-request this transaction when we receive another // announcement. - if err == core.ErrUnderpriced { - for f.underpriced.Cardinality() >= underpriceSetSize { + if err == core.ErrUnderpriced || err == core.ErrReplaceUnderpriced { + for f.underpriced.Cardinality() >= maxTxUnderpricedSetSize { f.underpriced.Pop() } f.underpriced.Add(txs[i].Hash()) } + // Track a few interesting failure types + switch err { + case nil: // Noop, but need to handle to not count these + + case core.ErrAlreadyKnown: + duplicate++ + + case core.ErrUnderpriced, core.ErrReplaceUnderpriced: + underpriced++ + + default: + otherreject++ + } } - hashes = append(hashes, txs[i].Hash()) + added = append(added, txs[i].Hash()) } - if f.importTxsHook != nil { - f.importTxsHook(txs) + if direct { + txReplyKnownMeter.Mark(duplicate) + txReplyUnderpricedMeter.Mark(underpriced) + txReplyOtherRejectMeter.Mark(otherreject) + } else { + txBroadcastKnownMeter.Mark(duplicate) + txBroadcastUnderpricedMeter.Mark(underpriced) + txBroadcastOtherRejectMeter.Mark(otherreject) } - // Drop the peer if some transaction failed signature verification. - // We can regard this peer is trying to DOS us by feeding lots of - // random hashes. - if drop { - f.dropPeer(peer) - if f.dropHook != nil { - f.dropHook(peer) - } + select { + case f.cleanup <- &txDelivery{origin: peer, hashes: added, direct: direct}: + return nil + case <-f.quit: + return errTerminated } +} + +// Drop should be called when a peer disconnects. It cleans up all the internal +// data structures of the given node. +func (f *TxFetcher) Drop(peer string) error { select { - case f.cleanup <- hashes: + case f.drop <- &txDrop{peer: peer}: return nil case <-f.quit: return errTerminated @@ -186,134 +344,551 @@ func (f *TxFetcher) Stop() { } func (f *TxFetcher) loop() { - fetchTimer := time.NewTimer(0) + var ( + waitTimer = new(mclock.Timer) + timeoutTimer = new(mclock.Timer) + waitTrigger = make(chan struct{}, 1) + timeoutTrigger = make(chan struct{}, 1) + ) for { - // Clean up any expired transaction fetches. - // There are many cases can lead to it: - // * We send the request to busy peer which can reply immediately - // * We send the request to malicious peer which doesn't reply deliberately - // * We send the request to normal peer for a batch of transaction, but some - // transactions have been included into blocks. According to EIP these txs - // won't be included. - // But it's fine to delete the fetching record and reschedule fetching iff we - // receive the annoucement again. - for hash, announce := range f.fetching { - if time.Since(announce.time) > txFetchTimeout { - delete(f.fetching, hash) - txFetchTimeoutMeter.Mark(1) - } - } select { - case anno := <-f.notify: - txAnnounceInMeter.Mark(int64(len(anno.hashes))) - - // Drop the new announce if there are too many accumulated. - count := f.announces[anno.origin] + len(anno.hashes) - if count > txAnnounceLimit { - txAnnounceDOSMeter.Mark(int64(count - txAnnounceLimit)) + case ann := <-f.notify: + // Drop part of the new announcements if there are too many accumulated. + // Note, we could but do not filter already known transactions here as + // the probability of something arriving between this call and the pre- + // filter outside is essentially zero. + used := len(f.waitslots[ann.origin]) + len(f.announces[ann.origin]) + if used >= maxTxAnnounces { + // This can happen if a set of transactions are requested but not + // all fulfilled, so the remainder are rescheduled without the cap + // check. Should be fine as the limit is in the thousands and the + // request size in the hundreds. + txAnnounceDOSMeter.Mark(int64(len(ann.hashes))) break } - f.announces[anno.origin] = count + want := used + len(ann.hashes) + if want > maxTxAnnounces { + txAnnounceDOSMeter.Mark(int64(want - maxTxAnnounces)) + ann.hashes = ann.hashes[:want-maxTxAnnounces] + } + // All is well, schedule the remainder of the transactions + idleWait := len(f.waittime) == 0 + _, oldPeer := f.announces[ann.origin] + + for _, hash := range ann.hashes { + // If the transaction is already downloading, add it to the list + // of possible alternates (in case the current retrieval fails) and + // also account it for the peer. + if f.alternates[hash] != nil { + f.alternates[hash][ann.origin] = struct{}{} - // All is well, schedule the announce if transaction is not yet downloading - empty := len(f.announced) == 0 - for _, hash := range anno.hashes { - if _, ok := f.fetching[hash]; ok { + // Stage 2 and 3 share the set of origins per tx + if announces := f.announces[ann.origin]; announces != nil { + announces[hash] = struct{}{} + } else { + f.announces[ann.origin] = map[common.Hash]struct{}{hash: struct{}{}} + } continue } - if f.underpriced.Contains(hash) { - txAnnounceUnderpriceMeter.Mark(1) - if f.rejectUnderprice != nil { - f.rejectUnderprice(hash) + // If the transaction is not downloading, but is already queued + // from a different peer, track it for the new peer too. + if f.announced[hash] != nil { + f.announced[hash][ann.origin] = struct{}{} + + // Stage 2 and 3 share the set of origins per tx + if announces := f.announces[ann.origin]; announces != nil { + announces[hash] = struct{}{} + } else { + f.announces[ann.origin] = map[common.Hash]struct{}{hash: struct{}{}} } continue } - f.announced[hash] = append(f.announced[hash], &txAnnounce{ - origin: anno.origin, - time: anno.time, - fetchTxs: anno.fetchTxs, - }) + // If the transaction is already known to the fetcher, but not + // yet downloading, add the peer as an alternate origin in the + // waiting list. + if f.waitlist[hash] != nil { + f.waitlist[hash][ann.origin] = struct{}{} + + if waitslots := f.waitslots[ann.origin]; waitslots != nil { + waitslots[hash] = struct{}{} + } else { + f.waitslots[ann.origin] = map[common.Hash]struct{}{hash: struct{}{}} + } + continue + } + // Transaction unknown to the fetcher, insert it into the waiting list + f.waitlist[hash] = map[string]struct{}{ann.origin: struct{}{}} + f.waittime[hash] = f.clock.Now() + + if waitslots := f.waitslots[ann.origin]; waitslots != nil { + waitslots[hash] = struct{}{} + } else { + f.waitslots[ann.origin] = map[common.Hash]struct{}{hash: struct{}{}} + } } - if empty && len(f.announced) > 0 { - f.reschedule(fetchTimer) + // If a new item was added to the waitlist, schedule it into the fetcher + if idleWait && len(f.waittime) > 0 { + f.rescheduleWait(waitTimer, waitTrigger) } - if f.announceHook != nil { - f.announceHook(anno.hashes) + // If this peer is new and announced something already queued, maybe + // request transactions from them + if !oldPeer && len(f.announces[ann.origin]) > 0 { + f.scheduleFetches(timeoutTimer, timeoutTrigger, map[string]struct{}{ann.origin: struct{}{}}) } - case <-fetchTimer.C: - // At least one tx's timer ran out, check for needing retrieval - request := make(map[string][]common.Hash) - - for hash, announces := range f.announced { - if time.Since(announces[0].time) > arriveTimeout-gatherSlack { - // Pick a random peer to retrieve from, reset all others - announce := announces[rand.Intn(len(announces))] - f.forgetHash(hash) - - // Skip fetching if we already receive the transaction. - if f.hasTx(hash) { - txAnnounceSkipMeter.Mark(1) - continue + + case <-waitTrigger: + // At least one transaction's waiting time ran out, push all expired + // ones into the retrieval queues + actives := make(map[string]struct{}) + for hash, instance := range f.waittime { + if time.Duration(f.clock.Now()-instance)+txGatherSlack > txArriveTimeout { + // Transaction expired without propagation, schedule for retrieval + if f.announced[hash] != nil { + panic("announce tracker already contains waitlist item") + } + f.announced[hash] = f.waitlist[hash] + for peer := range f.waitlist[hash] { + if announces := f.announces[peer]; announces != nil { + announces[hash] = struct{}{} + } else { + f.announces[peer] = map[common.Hash]struct{}{hash: struct{}{}} + } + delete(f.waitslots[peer], hash) + if len(f.waitslots[peer]) == 0 { + delete(f.waitslots, peer) + } + actives[peer] = struct{}{} } - // If the transaction still didn't arrive, queue for fetching - request[announce.origin] = append(request[announce.origin], hash) - f.fetching[hash] = announce + delete(f.waittime, hash) + delete(f.waitlist, hash) } } - // Send out all block header requests - for peer, hashes := range request { - log.Trace("Fetching scheduled transactions", "peer", peer, "txs", hashes) - fetchTxs := f.fetching[hashes[0]].fetchTxs - fetchTxs(hashes) - txFetchOutMeter.Mark(int64(len(hashes))) + // If transactions are still waiting for propagation, reschedule the wait timer + if len(f.waittime) > 0 { + f.rescheduleWait(waitTimer, waitTrigger) } - // Schedule the next fetch if blocks are still pending - f.reschedule(fetchTimer) - case hashes := <-f.cleanup: - for _, hash := range hashes { - f.forgetHash(hash) - anno, exist := f.fetching[hash] - if !exist { - txBroadcastInMeter.Mark(1) // Directly transaction propagation - continue + // If any peers became active and are idle, request transactions from them + if len(actives) > 0 { + f.scheduleFetches(timeoutTimer, timeoutTrigger, actives) + } + + case <-timeoutTrigger: + // Clean up any expired retrievals and avoid re-requesting them from the + // same peer (either overloaded or malicious, useless in both cases). We + // could also penalize (Drop), but there's nothing to gain, and if could + // possibly further increase the load on it. + for peer, req := range f.requests { + if time.Duration(f.clock.Now()-req.time)+txGatherSlack > txFetchTimeout { + txRequestTimeoutMeter.Mark(int64(len(req.hashes))) + + // Reschedule all the not-yet-delivered fetches to alternate peers + for _, hash := range req.hashes { + // Skip rescheduling hashes already delivered by someone else + if req.stolen != nil { + if _, ok := req.stolen[hash]; ok { + continue + } + } + // Move the delivery back from fetching to queued + if _, ok := f.announced[hash]; ok { + panic("announced tracker already contains alternate item") + } + if f.alternates[hash] != nil { // nil if tx was broadcast during fetch + f.announced[hash] = f.alternates[hash] + } + delete(f.announced[hash], peer) + if len(f.announced[hash]) == 0 { + delete(f.announced, hash) + } + delete(f.announces[peer], hash) + delete(f.alternates, hash) + delete(f.fetching, hash) + } + if len(f.announces[peer]) == 0 { + delete(f.announces, peer) + } + // Keep track of the request as dangling, but never expire + f.requests[peer].hashes = nil + } + } + // Schedule a new transaction retrieval + f.scheduleFetches(timeoutTimer, timeoutTrigger, nil) + + // No idea if we sheduled something or not, trigger the timer if needed + // TODO(karalabe): this is kind of lame, can't we dump it into scheduleFetches somehow? + f.rescheduleTimeout(timeoutTimer, timeoutTrigger) + + case delivery := <-f.cleanup: + // Independent if the delivery was direct or broadcast, remove all + // traces of the hash from internal trackers + for _, hash := range delivery.hashes { + if _, ok := f.waitlist[hash]; ok { + for peer, txset := range f.waitslots { + delete(txset, hash) + if len(txset) == 0 { + delete(f.waitslots, peer) + } + } + delete(f.waitlist, hash) + delete(f.waittime, hash) + } else { + for peer, txset := range f.announces { + delete(txset, hash) + if len(txset) == 0 { + delete(f.announces, peer) + } + } + delete(f.announced, hash) + delete(f.alternates, hash) + + // If a transaction currently being fetched from a different + // origin was delivered (delivery stolen), mark it so the + // actual delivery won't double schedule it. + if origin, ok := f.fetching[hash]; ok && (origin != delivery.origin || !delivery.direct) { + stolen := f.requests[origin].stolen + if stolen == nil { + f.requests[origin].stolen = make(map[common.Hash]struct{}) + stolen = f.requests[origin].stolen + } + stolen[hash] = struct{}{} + } + delete(f.fetching, hash) + } + } + // In case of a direct delivery, also reschedule anything missing + // from the original query + if delivery.direct { + // Mark the reqesting successful (independent of individual status) + txRequestDoneMeter.Mark(int64(len(delivery.hashes))) + + // Make sure something was pending, nuke it + req := f.requests[delivery.origin] + if req == nil { + log.Warn("Unexpected transaction delivery", "peer", delivery.origin) + break + } + delete(f.requests, delivery.origin) + + // Anything not delivered should be re-scheduled (with or without + // this peer, depending on the response cutoff) + delivered := make(map[common.Hash]struct{}) + for _, hash := range delivery.hashes { + delivered[hash] = struct{}{} + } + cutoff := len(req.hashes) // If nothing is delivered, assume everything is missing, don't retry!!! + for i, hash := range req.hashes { + if _, ok := delivered[hash]; ok { + cutoff = i + } + } + // Reschedule missing hashes from alternates, not-fulfilled from alt+self + for i, hash := range req.hashes { + // Skip rescheduling hashes already delivered by someone else + if req.stolen != nil { + if _, ok := req.stolen[hash]; ok { + continue + } + } + if _, ok := delivered[hash]; !ok { + if i < cutoff { + delete(f.alternates[hash], delivery.origin) + delete(f.announces[delivery.origin], hash) + if len(f.announces[delivery.origin]) == 0 { + delete(f.announces, delivery.origin) + } + } + if len(f.alternates[hash]) > 0 { + if _, ok := f.announced[hash]; ok { + panic(fmt.Sprintf("announced tracker already contains alternate item: %v", f.announced[hash])) + } + f.announced[hash] = f.alternates[hash] + } + } + delete(f.alternates, hash) + delete(f.fetching, hash) + } + // Something was delivered, try to rechedule requests + f.scheduleFetches(timeoutTimer, timeoutTrigger, nil) // Partial delivery may enable others to deliver too + } + + case drop := <-f.drop: + // A peer was dropped, remove all traces of it + if _, ok := f.waitslots[drop.peer]; ok { + for hash := range f.waitslots[drop.peer] { + delete(f.waitlist[hash], drop.peer) + if len(f.waitlist[hash]) == 0 { + delete(f.waitlist, hash) + delete(f.waittime, hash) + } + } + delete(f.waitslots, drop.peer) + if len(f.waitlist) > 0 { + f.rescheduleWait(waitTimer, waitTrigger) + } + } + // Clean up any active requests + var request *txRequest + if request = f.requests[drop.peer]; request != nil { + for _, hash := range request.hashes { + // Skip rescheduling hashes already delivered by someone else + if request.stolen != nil { + if _, ok := request.stolen[hash]; ok { + continue + } + } + // Undelivered hash, reschedule if there's an alternative origin available + delete(f.alternates[hash], drop.peer) + if len(f.alternates[hash]) == 0 { + delete(f.alternates, hash) + } else { + f.announced[hash] = f.alternates[hash] + delete(f.alternates, hash) + } + delete(f.fetching, hash) + } + delete(f.requests, drop.peer) + } + // Clean up general announcement tracking + if _, ok := f.announces[drop.peer]; ok { + for hash := range f.announces[drop.peer] { + delete(f.announced[hash], drop.peer) + if len(f.announced[hash]) == 0 { + delete(f.announced, hash) + } } - txFetchDurationTimer.UpdateSince(anno.time) - txFetchSuccessMeter.Mark(1) - delete(f.fetching, hash) + delete(f.announces, drop.peer) } - if f.cleanupHook != nil { - f.cleanupHook(hashes) + // If a request was cancelled, check if anything needs to be rescheduled + if request != nil { + f.scheduleFetches(timeoutTimer, timeoutTrigger, nil) + f.rescheduleTimeout(timeoutTimer, timeoutTrigger) } + case <-f.quit: return } + // No idea what happened, but bump some sanity metrics + txFetcherWaitingPeers.Update(int64(len(f.waitslots))) + txFetcherWaitingHashes.Update(int64(len(f.waitlist))) + txFetcherQueueingPeers.Update(int64(len(f.announces) - len(f.requests))) + txFetcherQueueingHashes.Update(int64(len(f.announced))) + txFetcherFetchingPeers.Update(int64(len(f.requests))) + txFetcherFetchingHashes.Update(int64(len(f.fetching))) + + // Loop did something, ping the step notifier if needed (tests) + if f.step != nil { + f.step <- struct{}{} + } + } +} + +// rescheduleWait iterates over all the transactions currently in the waitlist +// and schedules the movement into the fetcher for the earliest. +// +// The method has a granularity of 'gatherSlack', since there's not much point in +// spinning over all the transactions just to maybe find one that should trigger +// a few ms earlier. +func (f *TxFetcher) rescheduleWait(timer *mclock.Timer, trigger chan struct{}) { + if *timer != nil { + (*timer).Stop() + } + now := f.clock.Now() + + earliest := now + for _, instance := range f.waittime { + if earliest > instance { + earliest = instance + if txArriveTimeout-time.Duration(now-earliest) < gatherSlack { + break + } + } + } + *timer = f.clock.AfterFunc(txArriveTimeout-time.Duration(now-earliest), func() { + trigger <- struct{}{} + }) +} + +// rescheduleTimeout iterates over all the transactions currently in flight and +// schedules a cleanup run when the first would trigger. +// +// The method has a granularity of 'gatherSlack', since there's not much point in +// spinning over all the transactions just to maybe find one that should trigger +// a few ms earlier. +// +// This method is a bit "flaky" "by design". In theory the timeout timer only ever +// should be rescheduled if some request is pending. In practice, a timeout will +// cause the timer to be rescheduled every 5 secs (until the peer comes through or +// disconnects). This is a limitation of the fetcher code because we don't trac +// pending requests and timed out requests separatey. Without double tracking, if +// we simply didn't reschedule the timer on all-timeout then the timer would never +// be set again since len(request) > 0 => something's running. +func (f *TxFetcher) rescheduleTimeout(timer *mclock.Timer, trigger chan struct{}) { + if *timer != nil { + (*timer).Stop() + } + now := f.clock.Now() + + earliest := now + for _, req := range f.requests { + // If this request already timed out, skip it altogether + if req.hashes == nil { + continue + } + if earliest > req.time { + earliest = req.time + if txFetchTimeout-time.Duration(now-earliest) < gatherSlack { + break + } + } } + *timer = f.clock.AfterFunc(txFetchTimeout-time.Duration(now-earliest), func() { + trigger <- struct{}{} + }) } -// rescheduleFetch resets the specified fetch timer to the next blockAnnounce timeout. -func (f *TxFetcher) reschedule(fetch *time.Timer) { - // Short circuit if no transactions are announced - if len(f.announced) == 0 { +// scheduleFetches starts a batch of retrievals for all available idle peers. +func (f *TxFetcher) scheduleFetches(timer *mclock.Timer, timeout chan struct{}, whitelist map[string]struct{}) { + // Gather the set of peers we want to retrieve from (default to all) + actives := whitelist + if actives == nil { + actives = make(map[string]struct{}) + for peer := range f.announces { + actives[peer] = struct{}{} + } + } + if len(actives) == 0 { return } - // Otherwise find the earliest expiring announcement - earliest := time.Now() - for _, announces := range f.announced { - if earliest.After(announces[0].time) { - earliest = announces[0].time + // For each active peer, try to schedule some transaction fetches + idle := len(f.requests) == 0 + + f.forEachPeer(actives, func(peer string) { + if f.requests[peer] != nil { + return // continue in the for-each } + if len(f.announces[peer]) == 0 { + return // continue in the for-each + } + hashes := make([]common.Hash, 0, maxTxRetrievals) + f.forEachHash(f.announces[peer], func(hash common.Hash) bool { + if _, ok := f.fetching[hash]; !ok { + // Mark the hash as fetching and stash away possible alternates + f.fetching[hash] = peer + + if _, ok := f.alternates[hash]; ok { + panic(fmt.Sprintf("alternate tracker already contains fetching item: %v", f.alternates[hash])) + } + f.alternates[hash] = f.announced[hash] + delete(f.announced, hash) + + // Accumulate the hash and stop if the limit was reached + hashes = append(hashes, hash) + if len(hashes) >= maxTxRetrievals { + return false // break in the for-each + } + } + return true // continue in the for-each + }) + // If any hashes were allocated, request them from the peer + if len(hashes) > 0 { + f.requests[peer] = &txRequest{hashes: hashes, time: f.clock.Now()} + txRequestOutMeter.Mark(int64(len(hashes))) + + go func(peer string, hashes []common.Hash) { + // Try to fetch the transactions, but in case of a request + // failure (e.g. peer disconnected), reschedule the hashes. + if err := f.fetchTxs(peer, hashes); err != nil { + txRequestFailMeter.Mark(int64(len(hashes))) + f.Drop(peer) + } + }(peer, hashes) + } + }) + // If a new request was fired, schedule a timeout timer + if idle && len(f.requests) > 0 { + f.rescheduleTimeout(timer, timeout) } - fetch.Reset(arriveTimeout - time.Since(earliest)) } -func (f *TxFetcher) forgetHash(hash common.Hash) { - // Remove all pending announces and decrement DOS counters - for _, announce := range f.announced[hash] { - f.announces[announce.origin]-- - if f.announces[announce.origin] <= 0 { - delete(f.announces, announce.origin) +// forEachPeer does a range loop over a map of peers in production, but during +// testing it does a deterministic sorted random to allow reproducing issues. +func (f *TxFetcher) forEachPeer(peers map[string]struct{}, do func(peer string)) { + // If we're running production, use whatever Go's map gives us + if f.rand == nil { + for peer := range peers { + do(peer) } + return + } + // We're running the test suite, make iteration deterministic + list := make([]string, 0, len(peers)) + for peer := range peers { + list = append(list, peer) + } + sort.Strings(list) + rotateStrings(list, f.rand.Intn(len(list))) + for _, peer := range list { + do(peer) + } +} + +// forEachHash does a range loop over a map of hashes in production, but during +// testing it does a deterministic sorted random to allow reproducing issues. +func (f *TxFetcher) forEachHash(hashes map[common.Hash]struct{}, do func(hash common.Hash) bool) { + // If we're running production, use whatever Go's map gives us + if f.rand == nil { + for hash := range hashes { + if !do(hash) { + return + } + } + return + } + // We're running the test suite, make iteration deterministic + list := make([]common.Hash, 0, len(hashes)) + for hash := range hashes { + list = append(list, hash) + } + sortHashes(list) + rotateHashes(list, f.rand.Intn(len(list))) + for _, hash := range list { + if !do(hash) { + return + } + } +} + +// rotateStrings rotates the contents of a slice by n steps. This method is only +// used in tests to simulate random map iteration but keep it deterministic. +func rotateStrings(slice []string, n int) { + orig := make([]string, len(slice)) + copy(orig, slice) + + for i := 0; i < len(orig); i++ { + slice[i] = orig[(i+n)%len(orig)] + } +} + +// sortHashes sorts a slice of hashes. This method is only used in tests in order +// to simulate random map iteration but keep it deterministic. +func sortHashes(slice []common.Hash) { + for i := 0; i < len(slice); i++ { + for j := i + 1; j < len(slice); j++ { + if bytes.Compare(slice[i][:], slice[j][:]) > 0 { + slice[i], slice[j] = slice[j], slice[i] + } + } + } +} + +// rotateHashes rotates the contents of a slice by n steps. This method is only +// used in tests to simulate random map iteration but keep it deterministic. +func rotateHashes(slice []common.Hash, n int) { + orig := make([]common.Hash, len(slice)) + copy(orig, slice) + + for i := 0; i < len(orig); i++ { + slice[i] = orig[(i+n)%len(orig)] } - delete(f.announced, hash) } diff --git a/eth/fetcher/tx_fetcher_test.go b/eth/fetcher/tx_fetcher_test.go index 26f24f3f38..c5c198da88 100644 --- a/eth/fetcher/tx_fetcher_test.go +++ b/eth/fetcher/tx_fetcher_test.go @@ -17,302 +17,1512 @@ package fetcher import ( - "crypto/ecdsa" + "errors" "math/big" "math/rand" - "sync" - "sync/atomic" "testing" "time" "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/mclock" "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" ) -func init() { - rand.Seed(int64(time.Now().Nanosecond())) +var ( + // testTxs is a set of transactions to use during testing that have meaninful hashes. + testTxs = []*types.Transaction{ + types.NewTransaction(rand.Uint64(), common.Address{byte(rand.Intn(256))}, new(big.Int), 0, new(big.Int), nil), + types.NewTransaction(rand.Uint64(), common.Address{byte(rand.Intn(256))}, new(big.Int), 0, new(big.Int), nil), + types.NewTransaction(rand.Uint64(), common.Address{byte(rand.Intn(256))}, new(big.Int), 0, new(big.Int), nil), + types.NewTransaction(rand.Uint64(), common.Address{byte(rand.Intn(256))}, new(big.Int), 0, new(big.Int), nil), + } + // testTxsHashes is the hashes of the test transactions above + testTxsHashes = []common.Hash{testTxs[0].Hash(), testTxs[1].Hash(), testTxs[2].Hash(), testTxs[3].Hash()} +) - txAnnounceLimit = 64 - MaxTransactionFetch = 16 +type doTxNotify struct { + peer string + hashes []common.Hash +} +type doTxEnqueue struct { + peer string + txs []*types.Transaction + direct bool +} +type doWait struct { + time time.Duration + step bool } +type doDrop string +type doFunc func() -func makeTransactions(key *ecdsa.PrivateKey, target int) []*types.Transaction { - var txs []*types.Transaction +type isWaiting map[string][]common.Hash +type isScheduled struct { + tracking map[string][]common.Hash + fetching map[string][]common.Hash + dangling map[string][]common.Hash +} +type isUnderpriced int - for i := 0; i < target; i++ { - random := rand.Uint32() - tx := types.NewTransaction(uint64(random), common.Address{0x1, 0x2, 0x3}, big.NewInt(int64(random)), 100, big.NewInt(int64(random)), nil) - tx, _ = types.SignTx(tx, types.NewEIP155Signer(big.NewInt(1)), key) - txs = append(txs, tx) - } - return txs +// txFetcherTest represents a test scenario that can be executed by the test +// runner. +type txFetcherTest struct { + init func() *TxFetcher + steps []interface{} } -func makeUnsignedTransactions(key *ecdsa.PrivateKey, target int) []*types.Transaction { - var txs []*types.Transaction +// Tests that transaction announcements are added to a waitlist, and none +// of them are scheduled for retrieval until the wait expires. +func TestTransactionFetcherWaiting(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + nil, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Initial announcement to get something into the waitlist + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}}, + isWaiting(map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }), + // Announce from a new peer to check that no overwrite happens + doTxNotify{peer: "B", hashes: []common.Hash{{0x03}, {0x04}}}, + isWaiting(map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + "B": {{0x03}, {0x04}}, + }), + // Announce clashing hashes but unique new peer + doTxNotify{peer: "C", hashes: []common.Hash{{0x01}, {0x04}}}, + isWaiting(map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + "B": {{0x03}, {0x04}}, + "C": {{0x01}, {0x04}}, + }), + // Announce existing and clashing hashes from existing peer + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x03}, {0x05}}}, + isWaiting(map[string][]common.Hash{ + "A": {{0x01}, {0x02}, {0x03}, {0x05}}, + "B": {{0x03}, {0x04}}, + "C": {{0x01}, {0x04}}, + }), + isScheduled{tracking: nil, fetching: nil}, - for i := 0; i < target; i++ { - random := rand.Uint32() - tx := types.NewTransaction(uint64(random), common.Address{0x1, 0x2, 0x3}, big.NewInt(int64(random)), 100, big.NewInt(int64(random)), nil) - txs = append(txs, tx) - } - return txs + // Wait for the arrival timeout which should move all expired items + // from the wait list to the scheduler + doWait{time: txArriveTimeout, step: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}, {0x02}, {0x03}, {0x05}}, + "B": {{0x03}, {0x04}}, + "C": {{0x01}, {0x04}}, + }, + fetching: map[string][]common.Hash{ // Depends on deterministic test randomizer + "A": {{0x02}, {0x03}, {0x05}}, + "C": {{0x01}, {0x04}}, + }, + }, + // Queue up a non-fetchable transaction and then trigger it with a new + // peer (weird case to test 1 line in the fetcher) + doTxNotify{peer: "C", hashes: []common.Hash{{0x06}, {0x07}}}, + isWaiting(map[string][]common.Hash{ + "C": {{0x06}, {0x07}}, + }), + doWait{time: txArriveTimeout, step: true}, + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}, {0x02}, {0x03}, {0x05}}, + "B": {{0x03}, {0x04}}, + "C": {{0x01}, {0x04}, {0x06}, {0x07}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x02}, {0x03}, {0x05}}, + "C": {{0x01}, {0x04}}, + }, + }, + doTxNotify{peer: "D", hashes: []common.Hash{{0x06}, {0x07}}}, + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}, {0x02}, {0x03}, {0x05}}, + "B": {{0x03}, {0x04}}, + "C": {{0x01}, {0x04}, {0x06}, {0x07}}, + "D": {{0x06}, {0x07}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x02}, {0x03}, {0x05}}, + "C": {{0x01}, {0x04}}, + "D": {{0x06}, {0x07}}, + }, + }, + }, + }) } -type txfetcherTester struct { - fetcher *TxFetcher +// Tests that transaction announcements skip the waiting list if they are +// already scheduled. +func TestTransactionFetcherSkipWaiting(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + nil, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Push an initial announcement through to the scheduled stage + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}}, + isWaiting(map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }), + isScheduled{tracking: nil, fetching: nil}, - priceLimit *big.Int - sender *ecdsa.PrivateKey - senderAddr common.Address - signer types.Signer - txs map[common.Hash]*types.Transaction - dropped map[string]struct{} - lock sync.RWMutex + doWait{time: txArriveTimeout, step: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + }, + // Announce overlaps from the same peer, ensure the new ones end up + // in stage one, and clashing ones don't get double tracked + doTxNotify{peer: "A", hashes: []common.Hash{{0x02}, {0x03}}}, + isWaiting(map[string][]common.Hash{ + "A": {{0x03}}, + }), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + }, + // Announce overlaps from a new peer, ensure new transactions end up + // in stage one and clashing ones get tracked for the new peer + doTxNotify{peer: "B", hashes: []common.Hash{{0x02}, {0x03}, {0x04}}}, + isWaiting(map[string][]common.Hash{ + "A": {{0x03}}, + "B": {{0x03}, {0x04}}, + }), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + "B": {{0x02}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + }, + }, + }) } -func newTxFetcherTester() *txfetcherTester { - key, _ := crypto.GenerateKey() - addr := crypto.PubkeyToAddress(key.PublicKey) - t := &txfetcherTester{ - sender: key, - senderAddr: addr, - signer: types.NewEIP155Signer(big.NewInt(1)), - txs: make(map[common.Hash]*types.Transaction), - dropped: make(map[string]struct{}), - } - t.fetcher = NewTxFetcher(t.hasTx, t.addTxs, t.dropPeer) - t.fetcher.Start() - return t +// Tests that only a single transaction request gets scheduled to a peer +// and subsequent announces block or get allotted to someone else. +func TestTransactionFetcherSingletonRequesting(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + nil, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Push an initial announcement through to the scheduled stage + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}}, + isWaiting(map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }), + isScheduled{tracking: nil, fetching: nil}, + + doWait{time: txArriveTimeout, step: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + }, + // Announce a new set of transactions from the same peer and ensure + // they do not start fetching since the peer is already busy + doTxNotify{peer: "A", hashes: []common.Hash{{0x03}, {0x04}}}, + isWaiting(map[string][]common.Hash{ + "A": {{0x03}, {0x04}}, + }), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + }, + doWait{time: txArriveTimeout, step: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}, {0x02}, {0x03}, {0x04}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + }, + // Announce a duplicate set of transactions from a new peer and ensure + // uniquely new ones start downloading, even if clashing. + doTxNotify{peer: "B", hashes: []common.Hash{{0x02}, {0x03}, {0x05}, {0x06}}}, + isWaiting(map[string][]common.Hash{ + "B": {{0x05}, {0x06}}, + }), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}, {0x02}, {0x03}, {0x04}}, + "B": {{0x02}, {0x03}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + "B": {{0x03}}, + }, + }, + }, + }) } -func (t *txfetcherTester) hasTx(hash common.Hash) bool { - t.lock.RLock() - defer t.lock.RUnlock() +// Tests that if a transaction retrieval fails, all the transactions get +// instantly schedule back to someone else or the announcements dropped +// if no alternate source is available. +func TestTransactionFetcherFailedRescheduling(t *testing.T) { + // Create a channel to control when tx requests can fail + proceed := make(chan struct{}) - return t.txs[hash] != nil + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + nil, + func(origin string, hashes []common.Hash) error { + <-proceed + return errors.New("peer disconnected") + }, + ) + }, + steps: []interface{}{ + // Push an initial announcement through to the scheduled stage + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}, {0x02}}}, + isWaiting(map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }), + isScheduled{tracking: nil, fetching: nil}, + + doWait{time: txArriveTimeout, step: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + }, + // While the original peer is stuck in the request, push in an second + // data source. + doTxNotify{peer: "B", hashes: []common.Hash{{0x02}}}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + "B": {{0x02}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + }, + // Wait until the original request fails and check that transactions + // are either rescheduled or dropped + doFunc(func() { + proceed <- struct{}{} // Allow peer A to return the failure + }), + doWait{time: 0, step: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "B": {{0x02}}, + }, + fetching: map[string][]common.Hash{ + "B": {{0x02}}, + }, + }, + doFunc(func() { + proceed <- struct{}{} // Allow peer B to return the failure + }), + doWait{time: 0, step: true}, + isWaiting(nil), + isScheduled{nil, nil, nil}, + }, + }) } -func (t *txfetcherTester) addTxs(txs []*types.Transaction) []error { - t.lock.Lock() - defer t.lock.Unlock() - - var errors []error - for _, tx := range txs { - // Make sure the transaction is signed properly - _, err := types.Sender(t.signer, tx) - if err != nil { - errors = append(errors, core.ErrInvalidSender) - continue - } - // Make sure the price is high enough to accpet - if t.priceLimit != nil && tx.GasPrice().Cmp(t.priceLimit) < 0 { - errors = append(errors, core.ErrUnderpriced) - continue - } - t.txs[tx.Hash()] = tx - errors = append(errors, nil) - } - return errors +// Tests that if a transaction retrieval succeeds, all alternate origins +// are cleaned up. +func TestTransactionFetcherCleanup(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Push an initial announcement through to the scheduled stage + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + isWaiting(map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }), + isScheduled{tracking: nil, fetching: nil}, + + doWait{time: txArriveTimeout, step: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }, + fetching: map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }, + }, + // Request should be delivered + doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}, direct: true}, + isScheduled{nil, nil, nil}, + }, + }) } -func (t *txfetcherTester) dropPeer(id string) { - t.lock.Lock() - defer t.lock.Unlock() +// Tests that if a transaction retrieval succeeds, but the response is empty (no +// transactions available, then all are nuked instead of being rescheduled (yes, +// this was a bug)). +func TestTransactionFetcherCleanupEmpty(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Push an initial announcement through to the scheduled stage + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + isWaiting(map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }), + isScheduled{tracking: nil, fetching: nil}, - t.dropped[id] = struct{}{} + doWait{time: txArriveTimeout, step: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }, + fetching: map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }, + }, + // Deliver an empty response and ensure the transaction is cleared, not rescheduled + doTxEnqueue{peer: "A", txs: []*types.Transaction{}, direct: true}, + isScheduled{nil, nil, nil}, + }, + }) } -// makeTxFetcher retrieves a batch of transaction associated with a simulated peer. -func (t *txfetcherTester) makeTxFetcher(peer string, txs []*types.Transaction) func(hashes []common.Hash) { - closure := make(map[common.Hash]*types.Transaction) - for _, tx := range txs { - closure[tx.Hash()] = tx - } - return func(hashes []common.Hash) { - var txs []*types.Transaction - for _, hash := range hashes { - tx := closure[hash] - if tx == nil { - continue - } - txs = append(txs, tx) - } - // Return on a new thread - go t.fetcher.EnqueueTxs(peer, txs) - } +// Tests that non-returned transactions are either re-sheduled from a +// different peer, or self if they are after the cutoff point. +func TestTransactionFetcherMissingRescheduling(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Push an initial announcement through to the scheduled stage + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]}}, + isWaiting(map[string][]common.Hash{ + "A": {testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]}, + }), + isScheduled{tracking: nil, fetching: nil}, + + doWait{time: txArriveTimeout, step: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]}, + }, + fetching: map[string][]common.Hash{ + "A": {testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]}, + }, + }, + // Deliver the middle transaction requested, the one before which + // should be dropped and the one after re-requested. + doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}, direct: true}, // This depends on the deterministic random + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {testTxsHashes[2]}, + }, + fetching: map[string][]common.Hash{ + "A": {testTxsHashes[2]}, + }, + }, + }, + }) } -func TestSequentialTxAnnouncements(t *testing.T) { - tester := newTxFetcherTester() - txs := makeTransactions(tester.sender, txAnnounceLimit) +// Tests that out of two transactions, if one is missing and the last is +// delivered, the peer gets properly cleaned out from the internal state. +func TestTransactionFetcherMissingCleanup(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Push an initial announcement through to the scheduled stage + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1]}}, + isWaiting(map[string][]common.Hash{ + "A": {testTxsHashes[0], testTxsHashes[1]}, + }), + isScheduled{tracking: nil, fetching: nil}, - retrieveTxs := tester.makeTxFetcher("peer", txs) + doWait{time: txArriveTimeout, step: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {testTxsHashes[0], testTxsHashes[1]}, + }, + fetching: map[string][]common.Hash{ + "A": {testTxsHashes[0], testTxsHashes[1]}, + }, + }, + // Deliver the middle transaction requested, the one before which + // should be dropped and the one after re-requested. + doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[1]}, direct: true}, // This depends on the deterministic random + isScheduled{nil, nil, nil}, + }, + }) +} + +// Tests that transaction broadcasts properly clean up announcements. +func TestTransactionFetcherBroadcasts(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Set up three transactions to be in different stats, waiting, queued and fetching + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + doWait{time: txArriveTimeout, step: true}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[1]}}, + doWait{time: txArriveTimeout, step: true}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[2]}}, + + isWaiting(map[string][]common.Hash{ + "A": {testTxsHashes[2]}, + }), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {testTxsHashes[0], testTxsHashes[1]}, + }, + fetching: map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }, + }, + // Broadcast all the transactions and ensure everything gets cleaned + // up, but the dangling request is left alone to avoid doing multiple + // concurrent requests. + doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0], testTxs[1], testTxs[2]}, direct: false}, + isWaiting(nil), + isScheduled{ + tracking: nil, + fetching: nil, + dangling: map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }, + }, + // Deliver the requested hashes + doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0], testTxs[1], testTxs[2]}, direct: true}, + isScheduled{nil, nil, nil}, + }, + }) +} - newTxsCh := make(chan struct{}) - tester.fetcher.importTxsHook = func(transactions []*types.Transaction) { - newTxsCh <- struct{}{} +// Tests that the waiting list timers properly reset and reschedule. +func TestTransactionFetcherWaitTimerResets(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + nil, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}}, + isWaiting(map[string][]common.Hash{ + "A": {{0x01}}, + }), + isScheduled{nil, nil, nil}, + doWait{time: txArriveTimeout / 2, step: false}, + isWaiting(map[string][]common.Hash{ + "A": {{0x01}}, + }), + isScheduled{nil, nil, nil}, + + doTxNotify{peer: "A", hashes: []common.Hash{{0x02}}}, + isWaiting(map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }), + isScheduled{nil, nil, nil}, + doWait{time: txArriveTimeout / 2, step: true}, + isWaiting(map[string][]common.Hash{ + "A": {{0x02}}, + }), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x01}}, + }, + }, + + doWait{time: txArriveTimeout / 2, step: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x01}}, + }, + }, + }, + }) +} + +// Tests that if a transaction request is not replied to, it will time +// out and be re-scheduled for someone else. +func TestTransactionFetcherTimeoutRescheduling(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Push an initial announcement through to the scheduled stage + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + isWaiting(map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }), + isScheduled{tracking: nil, fetching: nil}, + + doWait{time: txArriveTimeout, step: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }, + fetching: map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }, + }, + // Wait until the delivery times out, everything should be cleaned up + doWait{time: txFetchTimeout, step: true}, + isWaiting(nil), + isScheduled{ + tracking: nil, + fetching: nil, + dangling: map[string][]common.Hash{ + "A": {}, + }, + }, + // Ensure that followup announcements don't get scheduled + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[1]}}, + doWait{time: txArriveTimeout, step: true}, + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {testTxsHashes[1]}, + }, + fetching: nil, + dangling: map[string][]common.Hash{ + "A": {}, + }, + }, + // If the dangling request arrives a bit later, do not choke + doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}, direct: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {testTxsHashes[1]}, + }, + fetching: map[string][]common.Hash{ + "A": {testTxsHashes[1]}, + }, + }, + }, + }) +} + +// Tests that the fetching timeout timers properly reset and reschedule. +func TestTransactionFetcherTimeoutTimerResets(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + nil, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}}, + doWait{time: txArriveTimeout, step: true}, + doTxNotify{peer: "B", hashes: []common.Hash{{0x02}}}, + doWait{time: txArriveTimeout, step: true}, + + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}}, + "B": {{0x02}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x01}}, + "B": {{0x02}}, + }, + }, + doWait{time: txFetchTimeout - txArriveTimeout, step: true}, + isScheduled{ + tracking: map[string][]common.Hash{ + "B": {{0x02}}, + }, + fetching: map[string][]common.Hash{ + "B": {{0x02}}, + }, + dangling: map[string][]common.Hash{ + "A": {}, + }, + }, + doWait{time: txArriveTimeout, step: true}, + isScheduled{ + tracking: nil, + fetching: nil, + dangling: map[string][]common.Hash{ + "A": {}, + "B": {}, + }, + }, + }, + }) +} + +// Tests that if thousands of transactions are announces, only a small +// number of them will be requested at a time. +func TestTransactionFetcherRateLimiting(t *testing.T) { + // Create a slew of transactions and to announce them + var hashes []common.Hash + for i := 0; i < maxTxAnnounces; i++ { + hashes = append(hashes, common.Hash{byte(i / 256), byte(i % 256)}) } - for _, tx := range txs { - tester.fetcher.Notify("peer", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout), retrieveTxs) - select { - case <-newTxsCh: - case <-time.NewTimer(time.Second).C: - t.Fatalf("timeout") - } + + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + nil, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Announce all the transactions, wait a bit and ensure only a small + // percentage gets requested + doTxNotify{peer: "A", hashes: hashes}, + doWait{time: txArriveTimeout, step: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": hashes, + }, + fetching: map[string][]common.Hash{ + "A": hashes[1643 : 1643+maxTxRetrievals], + }, + }, + }, + }) +} + +// Tests that then number of transactions a peer is allowed to announce and/or +// request at the same time is hard capped. +func TestTransactionFetcherDoSProtection(t *testing.T) { + // Create a slew of transactions and to announce them + var hashesA []common.Hash + for i := 0; i < maxTxAnnounces+1; i++ { + hashesA = append(hashesA, common.Hash{0x01, byte(i / 256), byte(i % 256)}) } - if len(tester.txs) != len(txs) { - t.Fatalf("Imported transaction number mismatch, want %d, got %d", len(txs), len(tester.txs)) + var hashesB []common.Hash + for i := 0; i < maxTxAnnounces+1; i++ { + hashesB = append(hashesB, common.Hash{0x02, byte(i / 256), byte(i % 256)}) } + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + nil, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Announce half of the transaction and wait for them to be scheduled + doTxNotify{peer: "A", hashes: hashesA[:maxTxAnnounces/2]}, + doTxNotify{peer: "B", hashes: hashesB[:maxTxAnnounces/2-1]}, + doWait{time: txArriveTimeout, step: true}, + + // Announce the second half and keep them in the wait list + doTxNotify{peer: "A", hashes: hashesA[maxTxAnnounces/2 : maxTxAnnounces]}, + doTxNotify{peer: "B", hashes: hashesB[maxTxAnnounces/2-1 : maxTxAnnounces-1]}, + + // Ensure the hashes are split half and half + isWaiting(map[string][]common.Hash{ + "A": hashesA[maxTxAnnounces/2 : maxTxAnnounces], + "B": hashesB[maxTxAnnounces/2-1 : maxTxAnnounces-1], + }), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": hashesA[:maxTxAnnounces/2], + "B": hashesB[:maxTxAnnounces/2-1], + }, + fetching: map[string][]common.Hash{ + "A": hashesA[1643 : 1643+maxTxRetrievals], + "B": append(append([]common.Hash{}, hashesB[maxTxAnnounces/2-3:maxTxAnnounces/2-1]...), hashesB[:maxTxRetrievals-2]...), + }, + }, + // Ensure that adding even one more hash results in dropping the hash + doTxNotify{peer: "A", hashes: []common.Hash{hashesA[maxTxAnnounces]}}, + doTxNotify{peer: "B", hashes: hashesB[maxTxAnnounces-1 : maxTxAnnounces+1]}, + + isWaiting(map[string][]common.Hash{ + "A": hashesA[maxTxAnnounces/2 : maxTxAnnounces], + "B": hashesB[maxTxAnnounces/2-1 : maxTxAnnounces], + }), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": hashesA[:maxTxAnnounces/2], + "B": hashesB[:maxTxAnnounces/2-1], + }, + fetching: map[string][]common.Hash{ + "A": hashesA[1643 : 1643+maxTxRetrievals], + "B": append(append([]common.Hash{}, hashesB[maxTxAnnounces/2-3:maxTxAnnounces/2-1]...), hashesB[:maxTxRetrievals-2]...), + }, + }, + }, + }) } -func TestConcurrentAnnouncements(t *testing.T) { - tester := newTxFetcherTester() - txs := makeTransactions(tester.sender, txAnnounceLimit) - - txFetcherFn1 := tester.makeTxFetcher("peer1", txs) - txFetcherFn2 := tester.makeTxFetcher("peer2", txs) - - var ( - count uint32 - done = make(chan struct{}) - ) - tester.fetcher.importTxsHook = func(transactions []*types.Transaction) { - atomic.AddUint32(&count, uint32(len(transactions))) - if atomic.LoadUint32(&count) >= uint32(txAnnounceLimit) { - done <- struct{}{} - } +// Tests that underpriced transactions don't get rescheduled after being rejected. +func TestTransactionFetcherUnderpricedDedup(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + errs := make([]error, len(txs)) + for i := 0; i < len(errs); i++ { + if i%2 == 0 { + errs[i] = core.ErrUnderpriced + } else { + errs[i] = core.ErrReplaceUnderpriced + } + } + return errs + }, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Deliver a transaction through the fetcher, but reject as underpriced + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1]}}, + doWait{time: txArriveTimeout, step: true}, + doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0], testTxs[1]}, direct: true}, + isScheduled{nil, nil, nil}, + + // Try to announce the transaction again, ensure it's not scheduled back + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1], testTxsHashes[2]}}, // [2] is needed to force a step in the fetcher + isWaiting(map[string][]common.Hash{ + "A": {testTxsHashes[2]}, + }), + isScheduled{nil, nil, nil}, + }, + }) +} + +// Tests that underpriced transactions don't get rescheduled after being rejected, +// but at the same time there's a hard cap on the number of transactions that are +// tracked. +func TestTransactionFetcherUnderpricedDoSProtection(t *testing.T) { + // Temporarily disable fetch timeouts as they massively mess up the simulated clock + defer func(timeout time.Duration) { txFetchTimeout = timeout }(txFetchTimeout) + txFetchTimeout = 24 * time.Hour + + // Create a slew of transactions to max out the underpriced set + var txs []*types.Transaction + for i := 0; i < maxTxUnderpricedSetSize+1; i++ { + txs = append(txs, types.NewTransaction(rand.Uint64(), common.Address{byte(rand.Intn(256))}, new(big.Int), 0, new(big.Int), nil)) } - for _, tx := range txs { - tester.fetcher.Notify("peer1", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout), txFetcherFn1) - tester.fetcher.Notify("peer2", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout+time.Millisecond), txFetcherFn2) - tester.fetcher.Notify("peer2", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout-time.Millisecond), txFetcherFn2) + hashes := make([]common.Hash, len(txs)) + for i, tx := range txs { + hashes[i] = tx.Hash() } - select { - case <-done: - case <-time.NewTimer(time.Second).C: - t.Fatalf("timeout") + // Generate a set of steps to announce and deliver the entire set of transactions + var steps []interface{} + for i := 0; i < maxTxUnderpricedSetSize/maxTxRetrievals; i++ { + steps = append(steps, doTxNotify{peer: "A", hashes: hashes[i*maxTxRetrievals : (i+1)*maxTxRetrievals]}) + steps = append(steps, isWaiting(map[string][]common.Hash{ + "A": hashes[i*maxTxRetrievals : (i+1)*maxTxRetrievals], + })) + steps = append(steps, doWait{time: txArriveTimeout, step: true}) + steps = append(steps, isScheduled{ + tracking: map[string][]common.Hash{ + "A": hashes[i*maxTxRetrievals : (i+1)*maxTxRetrievals], + }, + fetching: map[string][]common.Hash{ + "A": hashes[i*maxTxRetrievals : (i+1)*maxTxRetrievals], + }, + }) + steps = append(steps, doTxEnqueue{peer: "A", txs: txs[i*maxTxRetrievals : (i+1)*maxTxRetrievals], direct: true}) + steps = append(steps, isWaiting(nil)) + steps = append(steps, isScheduled{nil, nil, nil}) + steps = append(steps, isUnderpriced((i+1)*maxTxRetrievals)) } + testTransactionFetcher(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + errs := make([]error, len(txs)) + for i := 0; i < len(errs); i++ { + errs[i] = core.ErrUnderpriced + } + return errs + }, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: append(steps, []interface{}{ + // The preparation of the test has already been done in `steps`, add the last check + doTxNotify{peer: "A", hashes: []common.Hash{hashes[maxTxUnderpricedSetSize]}}, + doWait{time: txArriveTimeout, step: true}, + doTxEnqueue{peer: "A", txs: []*types.Transaction{txs[maxTxUnderpricedSetSize]}, direct: true}, + isUnderpriced(maxTxUnderpricedSetSize), + }...), + }) } -func TestBatchAnnouncements(t *testing.T) { - tester := newTxFetcherTester() - txs := makeTransactions(tester.sender, txAnnounceLimit) +// Tests that unexpected deliveries don't corrupt the internal state. +func TestTransactionFetcherOutOfBoundDeliveries(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Deliver something out of the blue + isWaiting(nil), + isScheduled{nil, nil, nil}, + doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}, direct: false}, + isWaiting(nil), + isScheduled{nil, nil, nil}, - retrieveTxs := tester.makeTxFetcher("peer", txs) + // Set up a few hashes into various stages + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + doWait{time: txArriveTimeout, step: true}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[1]}}, + doWait{time: txArriveTimeout, step: true}, + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[2]}}, - var count uint32 - var done = make(chan struct{}) - tester.fetcher.importTxsHook = func(txs []*types.Transaction) { - atomic.AddUint32(&count, uint32(len(txs))) + isWaiting(map[string][]common.Hash{ + "A": {testTxsHashes[2]}, + }), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {testTxsHashes[0], testTxsHashes[1]}, + }, + fetching: map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }, + }, + // Deliver everything and more out of the blue + doTxEnqueue{peer: "B", txs: []*types.Transaction{testTxs[0], testTxs[1], testTxs[2], testTxs[3]}, direct: true}, + isWaiting(nil), + isScheduled{ + tracking: nil, + fetching: nil, + dangling: map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }, + }, + }, + }) +} - if atomic.LoadUint32(&count) >= uint32(txAnnounceLimit) { - done <- struct{}{} - } - } - // Send all announces which exceeds the limit. - var hashes []common.Hash - for _, tx := range txs { - hashes = append(hashes, tx.Hash()) - } - tester.fetcher.Notify("peer", hashes, time.Now(), retrieveTxs) +// Tests that dropping a peer cleans out all internal data structures in all the +// live or danglng stages. +func TestTransactionFetcherDrop(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Set up a few hashes into various stages + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}}, + doWait{time: txArriveTimeout, step: true}, + doTxNotify{peer: "A", hashes: []common.Hash{{0x02}}}, + doWait{time: txArriveTimeout, step: true}, + doTxNotify{peer: "A", hashes: []common.Hash{{0x03}}}, - select { - case <-done: - case <-time.NewTimer(time.Second).C: - t.Fatalf("timeout") - } + isWaiting(map[string][]common.Hash{ + "A": {{0x03}}, + }), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}, {0x02}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x01}}, + }, + }, + // Drop the peer and ensure everything's cleaned out + doDrop("A"), + isWaiting(nil), + isScheduled{nil, nil, nil}, + + // Push the node into a dangling (timeout) state + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + doWait{time: txArriveTimeout, step: true}, + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }, + fetching: map[string][]common.Hash{ + "A": {testTxsHashes[0]}, + }, + }, + doWait{time: txFetchTimeout, step: true}, + isWaiting(nil), + isScheduled{ + tracking: nil, + fetching: nil, + dangling: map[string][]common.Hash{ + "A": {}, + }, + }, + // Drop the peer and ensure everything's cleaned out + doDrop("A"), + isWaiting(nil), + isScheduled{nil, nil, nil}, + }, + }) } -func TestPropagationAfterAnnounce(t *testing.T) { - tester := newTxFetcherTester() - txs := makeTransactions(tester.sender, txAnnounceLimit) +// Tests that dropping a peer instantly reschedules failed announcements to any +// available peer. +func TestTransactionFetcherDropRescheduling(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Set up a few hashes into various stages + doTxNotify{peer: "A", hashes: []common.Hash{{0x01}}}, + doWait{time: txArriveTimeout, step: true}, + doTxNotify{peer: "B", hashes: []common.Hash{{0x01}}}, - var cleaned = make(chan struct{}) - tester.fetcher.cleanupHook = func(hashes []common.Hash) { - cleaned <- struct{}{} - } - retrieveTxs := tester.makeTxFetcher("peer", txs) - for _, tx := range txs { - tester.fetcher.Notify("peer", []common.Hash{tx.Hash()}, time.Now(), retrieveTxs) - tester.fetcher.EnqueueTxs("peer", []*types.Transaction{tx}) - - // It's ok to read the map directly since no write - // will happen in the same time. - <-cleaned - if len(tester.fetcher.announced) != 0 { - t.Fatalf("Announcement should be cleaned, got %d", len(tester.fetcher.announced)) - } - } + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "A": {{0x01}}, + "B": {{0x01}}, + }, + fetching: map[string][]common.Hash{ + "A": {{0x01}}, + }, + }, + // Drop the peer and ensure everything's cleaned out + doDrop("A"), + isWaiting(nil), + isScheduled{ + tracking: map[string][]common.Hash{ + "B": {{0x01}}, + }, + fetching: map[string][]common.Hash{ + "B": {{0x01}}, + }, + }, + }, + }) } -func TestEnqueueTransactions(t *testing.T) { - tester := newTxFetcherTester() - txs := makeTransactions(tester.sender, txAnnounceLimit) +// This test reproduces a crash caught by the fuzzer. The root cause was a +// dangling transaction timing out and clashing on readd with a concurrently +// announced one. +func TestTransactionFetcherFuzzCrash01(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Get a transaction into fetching mode and make it dangling with a broadcast + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + doWait{time: txArriveTimeout, step: true}, + doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}}, - done := make(chan struct{}) - tester.fetcher.importTxsHook = func(transactions []*types.Transaction) { - if len(transactions) == txAnnounceLimit { - done <- struct{}{} - } - } - go tester.fetcher.EnqueueTxs("peer", txs) - select { - case <-done: - case <-time.NewTimer(time.Second).C: - t.Fatalf("timeout") - } + // Notify the dangling transaction once more and crash via a timeout + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + doWait{time: txFetchTimeout, step: true}, + }, + }) } -func TestInvalidTxAnnounces(t *testing.T) { - tester := newTxFetcherTester() +// This test reproduces a crash caught by the fuzzer. The root cause was a +// dangling transaction getting peer-dropped and clashing on readd with a +// concurrently announced one. +func TestTransactionFetcherFuzzCrash02(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Get a transaction into fetching mode and make it dangling with a broadcast + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + doWait{time: txArriveTimeout, step: true}, + doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}}, - var txs []*types.Transaction - txs = append(txs, makeUnsignedTransactions(tester.sender, 1)...) - txs = append(txs, makeTransactions(tester.sender, 1)...) + // Notify the dangling transaction once more, re-fetch, and crash via a drop and timeout + doTxNotify{peer: "B", hashes: []common.Hash{testTxsHashes[0]}}, + doWait{time: txArriveTimeout, step: true}, + doDrop("A"), + doWait{time: txFetchTimeout, step: true}, + }, + }) +} - txFetcherFn := tester.makeTxFetcher("peer", txs) +// This test reproduces a crash caught by the fuzzer. The root cause was a +// dangling transaction getting rescheduled via a partial delivery, clashing +// with a concurrent notify. +func TestTransactionFetcherFuzzCrash03(t *testing.T) { + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + ) + }, + steps: []interface{}{ + // Get a transaction into fetching mode and make it dangling with a broadcast + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0], testTxsHashes[1]}}, + doWait{time: txFetchTimeout, step: true}, + doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0], testTxs[1]}}, - dropped := make(chan string, 1) - tester.fetcher.dropHook = func(s string) { dropped <- s } + // Notify the dangling transaction once more, partially deliver, clash&crash with a timeout + doTxNotify{peer: "B", hashes: []common.Hash{testTxsHashes[0]}}, + doWait{time: txArriveTimeout, step: true}, - for _, tx := range txs { - tester.fetcher.Notify("peer", []common.Hash{tx.Hash()}, time.Now(), txFetcherFn) - } - select { - case s := <-dropped: - if s != "peer" { - t.Fatalf("invalid dropped peer") + doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[1]}, direct: true}, + doWait{time: txFetchTimeout, step: true}, + }, + }) +} + +// This test reproduces a crash caught by the fuzzer. The root cause was a +// dangling transaction getting rescheduled via a disconnect, clashing with +// a concurrent notify. +func TestTransactionFetcherFuzzCrash04(t *testing.T) { + // Create a channel to control when tx requests can fail + proceed := make(chan struct{}) + + testTransactionFetcherParallel(t, txFetcherTest{ + init: func() *TxFetcher { + return NewTxFetcher( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { + <-proceed + return errors.New("peer disconnected") + }, + ) + }, + steps: []interface{}{ + // Get a transaction into fetching mode and make it dangling with a broadcast + doTxNotify{peer: "A", hashes: []common.Hash{testTxsHashes[0]}}, + doWait{time: txArriveTimeout, step: true}, + doTxEnqueue{peer: "A", txs: []*types.Transaction{testTxs[0]}}, + + // Notify the dangling transaction once more, re-fetch, and crash via an in-flight disconnect + doTxNotify{peer: "B", hashes: []common.Hash{testTxsHashes[0]}}, + doWait{time: txArriveTimeout, step: true}, + doFunc(func() { + proceed <- struct{}{} // Allow peer A to return the failure + }), + doWait{time: 0, step: true}, + doWait{time: txFetchTimeout, step: true}, + }, + }) +} + +func testTransactionFetcherParallel(t *testing.T, tt txFetcherTest) { + t.Parallel() + testTransactionFetcher(t, tt) +} + +func testTransactionFetcher(t *testing.T, tt txFetcherTest) { + // Create a fetcher and hook into it's simulated fields + clock := new(mclock.Simulated) + wait := make(chan struct{}) + + fetcher := tt.init() + fetcher.clock = clock + fetcher.step = wait + fetcher.rand = rand.New(rand.NewSource(0x3a29)) + + fetcher.Start() + defer fetcher.Stop() + + // Crunch through all the test steps and execute them + for i, step := range tt.steps { + switch step := step.(type) { + case doTxNotify: + if err := fetcher.Notify(step.peer, step.hashes); err != nil { + t.Errorf("step %d: %v", i, err) + } + <-wait // Fetcher needs to process this, wait until it's done + select { + case <-wait: + panic("wtf") + case <-time.After(time.Millisecond): + } + + case doTxEnqueue: + if err := fetcher.Enqueue(step.peer, step.txs, step.direct); err != nil { + t.Errorf("step %d: %v", i, err) + } + <-wait // Fetcher needs to process this, wait until it's done + + case doWait: + clock.Run(step.time) + if step.step { + <-wait // Fetcher supposed to do something, wait until it's done + } + + case doDrop: + if err := fetcher.Drop(string(step)); err != nil { + t.Errorf("step %d: %v", i, err) + } + <-wait // Fetcher needs to process this, wait until it's done + + case doFunc: + step() + + case isWaiting: + // We need to check that the waiting list (stage 1) internals + // match with the expected set. Check the peer->hash mappings + // first. + for peer, hashes := range step { + waiting := fetcher.waitslots[peer] + if waiting == nil { + t.Errorf("step %d: peer %s missing from waitslots", i, peer) + continue + } + for _, hash := range hashes { + if _, ok := waiting[hash]; !ok { + t.Errorf("step %d, peer %s: hash %x missing from waitslots", i, peer, hash) + } + } + for hash := range waiting { + if !containsHash(hashes, hash) { + t.Errorf("step %d, peer %s: hash %x extra in waitslots", i, peer, hash) + } + } + } + for peer := range fetcher.waitslots { + if _, ok := step[peer]; !ok { + t.Errorf("step %d: peer %s extra in waitslots", i, peer) + } + } + // Peer->hash sets correct, check the hash->peer and timeout sets + for peer, hashes := range step { + for _, hash := range hashes { + if _, ok := fetcher.waitlist[hash][peer]; !ok { + t.Errorf("step %d, hash %x: peer %s missing from waitlist", i, hash, peer) + } + if _, ok := fetcher.waittime[hash]; !ok { + t.Errorf("step %d: hash %x missing from waittime", i, hash) + } + } + } + for hash, peers := range fetcher.waitlist { + if len(peers) == 0 { + t.Errorf("step %d, hash %x: empty peerset in waitlist", i, hash) + } + for peer := range peers { + if !containsHash(step[peer], hash) { + t.Errorf("step %d, hash %x: peer %s extra in waitlist", i, hash, peer) + } + } + } + for hash := range fetcher.waittime { + var found bool + for _, hashes := range step { + if containsHash(hashes, hash) { + found = true + break + } + } + if !found { + t.Errorf("step %d,: hash %x extra in waittime", i, hash) + } + } + + case isScheduled: + // Check that all scheduled announces are accounted for and no + // extra ones are present. + for peer, hashes := range step.tracking { + scheduled := fetcher.announces[peer] + if scheduled == nil { + t.Errorf("step %d: peer %s missing from announces", i, peer) + continue + } + for _, hash := range hashes { + if _, ok := scheduled[hash]; !ok { + t.Errorf("step %d, peer %s: hash %x missing from announces", i, peer, hash) + } + } + for hash := range scheduled { + if !containsHash(hashes, hash) { + t.Errorf("step %d, peer %s: hash %x extra in announces", i, peer, hash) + } + } + } + for peer := range fetcher.announces { + if _, ok := step.tracking[peer]; !ok { + t.Errorf("step %d: peer %s extra in announces", i, peer) + } + } + // Check that all announces required to be fetching are in the + // appropriate sets + for peer, hashes := range step.fetching { + request := fetcher.requests[peer] + if request == nil { + t.Errorf("step %d: peer %s missing from requests", i, peer) + continue + } + for _, hash := range hashes { + if !containsHash(request.hashes, hash) { + t.Errorf("step %d, peer %s: hash %x missing from requests", i, peer, hash) + } + } + for _, hash := range request.hashes { + if !containsHash(hashes, hash) { + t.Errorf("step %d, peer %s: hash %x extra in requests", i, peer, hash) + } + } + } + for peer := range fetcher.requests { + if _, ok := step.fetching[peer]; !ok { + if _, ok := step.dangling[peer]; !ok { + t.Errorf("step %d: peer %s extra in requests", i, peer) + } + } + } + for peer, hashes := range step.fetching { + for _, hash := range hashes { + if _, ok := fetcher.fetching[hash]; !ok { + t.Errorf("step %d, peer %s: hash %x missing from fetching", i, peer, hash) + } + } + } + for hash := range fetcher.fetching { + var found bool + for _, req := range fetcher.requests { + if containsHash(req.hashes, hash) { + found = true + break + } + } + if !found { + t.Errorf("step %d: hash %x extra in fetching", i, hash) + } + } + for _, hashes := range step.fetching { + for _, hash := range hashes { + alternates := fetcher.alternates[hash] + if alternates == nil { + t.Errorf("step %d: hash %x missing from alternates", i, hash) + continue + } + for peer := range alternates { + if _, ok := fetcher.announces[peer]; !ok { + t.Errorf("step %d: peer %s extra in alternates", i, peer) + continue + } + if _, ok := fetcher.announces[peer][hash]; !ok { + t.Errorf("step %d, peer %s: hash %x extra in alternates", i, hash, peer) + continue + } + } + for p := range fetcher.announced[hash] { + if _, ok := alternates[p]; !ok { + t.Errorf("step %d, hash %x: peer %s missing from alternates", i, hash, p) + continue + } + } + } + } + for peer, hashes := range step.dangling { + request := fetcher.requests[peer] + if request == nil { + t.Errorf("step %d: peer %s missing from requests", i, peer) + continue + } + for _, hash := range hashes { + if !containsHash(request.hashes, hash) { + t.Errorf("step %d, peer %s: hash %x missing from requests", i, peer, hash) + } + } + for _, hash := range request.hashes { + if !containsHash(hashes, hash) { + t.Errorf("step %d, peer %s: hash %x extra in requests", i, peer, hash) + } + } + } + // Check that all transaction announces that are scheduled for + // retrieval but not actively being downloaded are tracked only + // in the stage 2 `announced` map. + var queued []common.Hash + for _, hashes := range step.tracking { + for _, hash := range hashes { + var found bool + for _, hs := range step.fetching { + if containsHash(hs, hash) { + found = true + break + } + } + if !found { + queued = append(queued, hash) + } + } + } + for _, hash := range queued { + if _, ok := fetcher.announced[hash]; !ok { + t.Errorf("step %d: hash %x missing from announced", i, hash) + } + } + for hash := range fetcher.announced { + if !containsHash(queued, hash) { + t.Errorf("step %d: hash %x extra in announced", i, hash) + } + } + + case isUnderpriced: + if fetcher.underpriced.Cardinality() != int(step) { + t.Errorf("step %d: underpriced set size mismatch: have %d, want %d", i, fetcher.underpriced.Cardinality(), step) + } + + default: + t.Fatalf("step %d: unknown step type %T", i, step) + } + // After every step, cross validate the internal uniqueness invariants + // between stage one and stage two. + for hash := range fetcher.waittime { + if _, ok := fetcher.announced[hash]; ok { + t.Errorf("step %d: hash %s present in both stage 1 and 2", i, hash) + } } - case <-time.NewTimer(time.Second).C: - t.Fatalf("timeout") } } -func TestRejectUnderpriced(t *testing.T) { - tester := newTxFetcherTester() - tester.priceLimit = big.NewInt(10000) - - done := make(chan struct{}) - tester.fetcher.importTxsHook = func([]*types.Transaction) { done <- struct{}{} } - reject := make(chan struct{}) - tester.fetcher.rejectUnderprice = func(common.Hash) { reject <- struct{}{} } - - tx := types.NewTransaction(0, common.Address{0x1, 0x2, 0x3}, big.NewInt(int64(100)), 100, big.NewInt(int64(100)), nil) - tx, _ = types.SignTx(tx, types.NewEIP155Signer(big.NewInt(1)), tester.sender) - txFetcherFn := tester.makeTxFetcher("peer", []*types.Transaction{tx}) - - // Send the announcement first time - tester.fetcher.Notify("peer", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout), txFetcherFn) - <-done - - // Resend the announcement, shouldn't schedule fetching this time - tester.fetcher.Notify("peer", []common.Hash{tx.Hash()}, time.Now().Add(-arriveTimeout), txFetcherFn) - select { - case <-reject: - case <-time.NewTimer(time.Second).C: - t.Fatalf("timeout") +// containsHash returns whether a hash is contained within a hash slice. +func containsHash(slice []common.Hash, hash common.Hash) bool { + for _, have := range slice { + if have == hash { + return true + } } + return false } diff --git a/eth/handler.go b/eth/handler.go index d527b15d13..fc6c74cfe6 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -51,7 +51,7 @@ const ( // The number is referenced from the size of tx pool. txChanSize = 4096 - // minimim number of peers to broadcast new blocks to + // minimim number of peers to broadcast entire blocks and transactions too. minBroadcastPeers = 4 ) @@ -192,7 +192,15 @@ func NewProtocolManager(config *params.ChainConfig, checkpoint *params.TrustedCh return n, err } manager.blockFetcher = fetcher.NewBlockFetcher(blockchain.GetBlockByHash, validator, manager.BroadcastBlock, heighter, inserter, manager.removePeer) - manager.txFetcher = fetcher.NewTxFetcher(txpool.Has, txpool.AddRemotes, manager.removePeer) + + fetchTx := func(peer string, hashes []common.Hash) error { + p := manager.peers.Peer(peer) + if p == nil { + return errors.New("unknown peer") + } + return p.RequestTxs(hashes) + } + manager.txFetcher = fetcher.NewTxFetcher(txpool.Has, txpool.AddRemotes, fetchTx) return manager, nil } @@ -240,6 +248,8 @@ func (pm *ProtocolManager) removePeer(id string) { // Unregister the peer from the downloader and Ethereum peer set pm.downloader.UnregisterPeer(id) + pm.txFetcher.Drop(id) + if err := pm.peers.Unregister(id); err != nil { log.Error("Peer removal failed", "peer", id, "err", err) } @@ -263,7 +273,7 @@ func (pm *ProtocolManager) Start(maxPeers int) { // start sync handlers go pm.syncer() - go pm.txsyncLoop() + go pm.txsyncLoop64() // TODO(karalabe): Legacy initial tx echange, drop with eth/64. } func (pm *ProtocolManager) Stop() { @@ -292,7 +302,7 @@ func (pm *ProtocolManager) Stop() { } func (pm *ProtocolManager) newPeer(pv int, p *p2p.Peer, rw p2p.MsgReadWriter, getPooledTx func(hash common.Hash) *types.Transaction) *peer { - return newPeer(pv, p, newMeteredMsgWriter(rw), getPooledTx) + return newPeer(pv, p, rw, getPooledTx) } // handle is the callback invoked to manage the life cycle of an eth peer. When @@ -316,9 +326,6 @@ func (pm *ProtocolManager) handle(p *peer) error { p.Log().Debug("Ethereum handshake failed", "err", err) return err } - if rw, ok := p.rw.(*meteredMsgReadWriter); ok { - rw.Init(p.version) - } // Register the peer locally if err := pm.peers.Register(p); err != nil { p.Log().Error("Ethereum peer registration failed", "err", err) @@ -740,20 +747,10 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { return errResp(ErrDecode, "msg %v: %v", msg, err) } // Schedule all the unknown hashes for retrieval - var unknown []common.Hash for _, hash := range hashes { - // Mark the hashes as present at the remote node p.MarkTransaction(hash) - - // Filter duplicated transaction announcement. - // Notably we only dedupliate announcement in txpool, check the rationale - // behind in EIP https://github.com/ethereum/EIPs/pull/2464. - if pm.txpool.Has(hash) { - continue - } - unknown = append(unknown, hash) } - pm.txFetcher.Notify(p.id, unknown, time.Now(), p.AsyncRequestTxs) + pm.txFetcher.Notify(p.id, hashes) case msg.Code == GetPooledTransactionsMsg && p.version >= eth65: // Decode the retrieval message @@ -763,9 +760,10 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } // Gather transactions until the fetch or network limits is reached var ( - hash common.Hash - bytes int - txs []rlp.RawValue + hash common.Hash + bytes int + hashes []common.Hash + txs []rlp.RawValue ) for bytes < softResponseLimit { // Retrieve the hash of the next block @@ -783,13 +781,14 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { if encoded, err := rlp.EncodeToBytes(tx); err != nil { log.Error("Failed to encode transaction", "err", err) } else { + hashes = append(hashes, hash) txs = append(txs, encoded) bytes += len(encoded) } } - return p.SendTransactionRLP(txs) + return p.SendPooledTransactionsRLP(hashes, txs) - case msg.Code == TxMsg: + case msg.Code == TransactionMsg || (msg.Code == PooledTransactionsMsg && p.version >= eth65): // Transactions arrived, make sure we have a valid and fresh chain to handle them if atomic.LoadUint32(&pm.acceptTxs) == 0 { break @@ -806,7 +805,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { } p.MarkTransaction(tx.Hash()) } - pm.txFetcher.EnqueueTxs(p.id, txs) + pm.txFetcher.Enqueue(p.id, txs, msg.Code == PooledTransactionsMsg) default: return errResp(ErrInvalidMsgCode, "%v", msg.Code) @@ -854,9 +853,9 @@ func (pm *ProtocolManager) BroadcastBlock(block *types.Block, propagate bool) { } } -// BroadcastTxs will propagate a batch of transactions to all peers which are not known to +// BroadcastTransactions will propagate a batch of transactions to all peers which are not known to // already have the given transaction. -func (pm *ProtocolManager) BroadcastTxs(txs types.Transactions, propagate bool) { +func (pm *ProtocolManager) BroadcastTransactions(txs types.Transactions, propagate bool) { var ( txset = make(map[*peer][]common.Hash) annos = make(map[*peer][]common.Hash) @@ -894,7 +893,7 @@ func (pm *ProtocolManager) BroadcastTxs(txs types.Transactions, propagate bool) } for peer, hashes := range annos { if peer.version >= eth65 { - peer.AsyncSendTransactionHashes(hashes) + peer.AsyncSendPooledTransactionHashes(hashes) } else { peer.AsyncSendTransactions(hashes) } @@ -918,11 +917,11 @@ func (pm *ProtocolManager) txBroadcastLoop() { case event := <-pm.txsCh: // For testing purpose only, disable propagation if pm.broadcastTxAnnouncesOnly { - pm.BroadcastTxs(event.Txs, false) + pm.BroadcastTransactions(event.Txs, false) continue } - pm.BroadcastTxs(event.Txs, true) // First propagate transactions to peers - pm.BroadcastTxs(event.Txs, false) // Only then announce to the rest + pm.BroadcastTransactions(event.Txs, true) // First propagate transactions to peers + pm.BroadcastTransactions(event.Txs, false) // Only then announce to the rest // Err() channel will be closed when unsubscribing. case <-pm.txsSub.Err(): diff --git a/eth/metrics.go b/eth/metrics.go deleted file mode 100644 index 0533a2a875..0000000000 --- a/eth/metrics.go +++ /dev/null @@ -1,139 +0,0 @@ -// Copyright 2015 The go-ethereum Authors -// This file is part of the go-ethereum library. -// -// The go-ethereum library is free software: you can redistribute it and/or modify -// it under the terms of the GNU Lesser General Public License as published by -// the Free Software Foundation, either version 3 of the License, or -// (at your option) any later version. -// -// The go-ethereum library is distributed in the hope that it will be useful, -// but WITHOUT ANY WARRANTY; without even the implied warranty of -// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -// GNU Lesser General Public License for more details. -// -// You should have received a copy of the GNU Lesser General Public License -// along with the go-ethereum library. If not, see . - -package eth - -import ( - "github.com/ethereum/go-ethereum/metrics" - "github.com/ethereum/go-ethereum/p2p" -) - -var ( - propTxnInPacketsMeter = metrics.NewRegisteredMeter("eth/prop/txns/in/packets", nil) - propTxnInTrafficMeter = metrics.NewRegisteredMeter("eth/prop/txns/in/traffic", nil) - propTxnOutPacketsMeter = metrics.NewRegisteredMeter("eth/prop/txns/out/packets", nil) - propTxnOutTrafficMeter = metrics.NewRegisteredMeter("eth/prop/txns/out/traffic", nil) - propHashInPacketsMeter = metrics.NewRegisteredMeter("eth/prop/hashes/in/packets", nil) - propHashInTrafficMeter = metrics.NewRegisteredMeter("eth/prop/hashes/in/traffic", nil) - propHashOutPacketsMeter = metrics.NewRegisteredMeter("eth/prop/hashes/out/packets", nil) - propHashOutTrafficMeter = metrics.NewRegisteredMeter("eth/prop/hashes/out/traffic", nil) - propBlockInPacketsMeter = metrics.NewRegisteredMeter("eth/prop/blocks/in/packets", nil) - propBlockInTrafficMeter = metrics.NewRegisteredMeter("eth/prop/blocks/in/traffic", nil) - propBlockOutPacketsMeter = metrics.NewRegisteredMeter("eth/prop/blocks/out/packets", nil) - propBlockOutTrafficMeter = metrics.NewRegisteredMeter("eth/prop/blocks/out/traffic", nil) - reqHeaderInPacketsMeter = metrics.NewRegisteredMeter("eth/req/headers/in/packets", nil) - reqHeaderInTrafficMeter = metrics.NewRegisteredMeter("eth/req/headers/in/traffic", nil) - reqHeaderOutPacketsMeter = metrics.NewRegisteredMeter("eth/req/headers/out/packets", nil) - reqHeaderOutTrafficMeter = metrics.NewRegisteredMeter("eth/req/headers/out/traffic", nil) - reqBodyInPacketsMeter = metrics.NewRegisteredMeter("eth/req/bodies/in/packets", nil) - reqBodyInTrafficMeter = metrics.NewRegisteredMeter("eth/req/bodies/in/traffic", nil) - reqBodyOutPacketsMeter = metrics.NewRegisteredMeter("eth/req/bodies/out/packets", nil) - reqBodyOutTrafficMeter = metrics.NewRegisteredMeter("eth/req/bodies/out/traffic", nil) - reqStateInPacketsMeter = metrics.NewRegisteredMeter("eth/req/states/in/packets", nil) - reqStateInTrafficMeter = metrics.NewRegisteredMeter("eth/req/states/in/traffic", nil) - reqStateOutPacketsMeter = metrics.NewRegisteredMeter("eth/req/states/out/packets", nil) - reqStateOutTrafficMeter = metrics.NewRegisteredMeter("eth/req/states/out/traffic", nil) - reqReceiptInPacketsMeter = metrics.NewRegisteredMeter("eth/req/receipts/in/packets", nil) - reqReceiptInTrafficMeter = metrics.NewRegisteredMeter("eth/req/receipts/in/traffic", nil) - reqReceiptOutPacketsMeter = metrics.NewRegisteredMeter("eth/req/receipts/out/packets", nil) - reqReceiptOutTrafficMeter = metrics.NewRegisteredMeter("eth/req/receipts/out/traffic", nil) - miscInPacketsMeter = metrics.NewRegisteredMeter("eth/misc/in/packets", nil) - miscInTrafficMeter = metrics.NewRegisteredMeter("eth/misc/in/traffic", nil) - miscOutPacketsMeter = metrics.NewRegisteredMeter("eth/misc/out/packets", nil) - miscOutTrafficMeter = metrics.NewRegisteredMeter("eth/misc/out/traffic", nil) -) - -// meteredMsgReadWriter is a wrapper around a p2p.MsgReadWriter, capable of -// accumulating the above defined metrics based on the data stream contents. -type meteredMsgReadWriter struct { - p2p.MsgReadWriter // Wrapped message stream to meter - version int // Protocol version to select correct meters -} - -// newMeteredMsgWriter wraps a p2p MsgReadWriter with metering support. If the -// metrics system is disabled, this function returns the original object. -func newMeteredMsgWriter(rw p2p.MsgReadWriter) p2p.MsgReadWriter { - if !metrics.Enabled { - return rw - } - return &meteredMsgReadWriter{MsgReadWriter: rw} -} - -// Init sets the protocol version used by the stream to know which meters to -// increment in case of overlapping message ids between protocol versions. -func (rw *meteredMsgReadWriter) Init(version int) { - rw.version = version -} - -func (rw *meteredMsgReadWriter) ReadMsg() (p2p.Msg, error) { - // Read the message and short circuit in case of an error - msg, err := rw.MsgReadWriter.ReadMsg() - if err != nil { - return msg, err - } - // Account for the data traffic - packets, traffic := miscInPacketsMeter, miscInTrafficMeter - switch { - case msg.Code == BlockHeadersMsg: - packets, traffic = reqHeaderInPacketsMeter, reqHeaderInTrafficMeter - case msg.Code == BlockBodiesMsg: - packets, traffic = reqBodyInPacketsMeter, reqBodyInTrafficMeter - - case rw.version >= eth63 && msg.Code == NodeDataMsg: - packets, traffic = reqStateInPacketsMeter, reqStateInTrafficMeter - case rw.version >= eth63 && msg.Code == ReceiptsMsg: - packets, traffic = reqReceiptInPacketsMeter, reqReceiptInTrafficMeter - - case msg.Code == NewBlockHashesMsg: - packets, traffic = propHashInPacketsMeter, propHashInTrafficMeter - case msg.Code == NewBlockMsg: - packets, traffic = propBlockInPacketsMeter, propBlockInTrafficMeter - case msg.Code == TxMsg: - packets, traffic = propTxnInPacketsMeter, propTxnInTrafficMeter - } - packets.Mark(1) - traffic.Mark(int64(msg.Size)) - - return msg, err -} - -func (rw *meteredMsgReadWriter) WriteMsg(msg p2p.Msg) error { - // Account for the data traffic - packets, traffic := miscOutPacketsMeter, miscOutTrafficMeter - switch { - case msg.Code == BlockHeadersMsg: - packets, traffic = reqHeaderOutPacketsMeter, reqHeaderOutTrafficMeter - case msg.Code == BlockBodiesMsg: - packets, traffic = reqBodyOutPacketsMeter, reqBodyOutTrafficMeter - - case rw.version >= eth63 && msg.Code == NodeDataMsg: - packets, traffic = reqStateOutPacketsMeter, reqStateOutTrafficMeter - case rw.version >= eth63 && msg.Code == ReceiptsMsg: - packets, traffic = reqReceiptOutPacketsMeter, reqReceiptOutTrafficMeter - - case msg.Code == NewBlockHashesMsg: - packets, traffic = propHashOutPacketsMeter, propHashOutTrafficMeter - case msg.Code == NewBlockMsg: - packets, traffic = propBlockOutPacketsMeter, propBlockOutTrafficMeter - case msg.Code == TxMsg: - packets, traffic = propTxnOutPacketsMeter, propTxnOutTrafficMeter - } - packets.Mark(1) - traffic.Mark(int64(msg.Size)) - - // Send the packet to the p2p layer - return rw.MsgReadWriter.WriteMsg(msg) -} diff --git a/eth/peer.go b/eth/peer.go index f4b939b71c..2d22603467 100644 --- a/eth/peer.go +++ b/eth/peer.go @@ -27,7 +27,6 @@ import ( "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/core/forkid" "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/fetcher" "github.com/ethereum/go-ethereum/p2p" "github.com/ethereum/go-ethereum/rlp" ) @@ -43,17 +42,13 @@ const ( maxKnownBlocks = 1024 // Maximum block hashes to keep in the known list (prevent DOS) // maxQueuedTxs is the maximum number of transactions to queue up before dropping - // broadcasts. + // older broadcasts. maxQueuedTxs = 4096 // maxQueuedTxAnns is the maximum number of transaction announcements to queue up - // before dropping broadcasts. + // before dropping older announcements. maxQueuedTxAnns = 4096 - // maxQueuedTxRetrieval is the maximum number of tx retrieval requests to queue up - // before dropping requests. - maxQueuedTxRetrieval = 4096 - // maxQueuedBlocks is the maximum number of block propagations to queue up before // dropping broadcasts. There's not much point in queueing stale blocks, so a few // that might cover uncles should be enough. @@ -102,15 +97,16 @@ type peer struct { td *big.Int lock sync.RWMutex - knownTxs mapset.Set // Set of transaction hashes known to be known by this peer - knownBlocks mapset.Set // Set of block hashes known to be known by this peer - queuedBlocks chan *propEvent // Queue of blocks to broadcast to the peer - queuedBlockAnns chan *types.Block // Queue of blocks to announce to the peer - txPropagation chan []common.Hash // Channel used to queue transaction propagation requests - txAnnounce chan []common.Hash // Channel used to queue transaction announcement requests - txRetrieval chan []common.Hash // Channel used to queue transaction retrieval requests - getPooledTx func(common.Hash) *types.Transaction // Callback used to retrieve transaction from txpool - term chan struct{} // Termination channel to stop the broadcaster + knownBlocks mapset.Set // Set of block hashes known to be known by this peer + queuedBlocks chan *propEvent // Queue of blocks to broadcast to the peer + queuedBlockAnns chan *types.Block // Queue of blocks to announce to the peer + + knownTxs mapset.Set // Set of transaction hashes known to be known by this peer + txBroadcast chan []common.Hash // Channel used to queue transaction propagation requests + txAnnounce chan []common.Hash // Channel used to queue transaction announcement requests + getPooledTx func(common.Hash) *types.Transaction // Callback used to retrieve transaction from txpool + + term chan struct{} // Termination channel to stop the broadcaster } func newPeer(version int, p *p2p.Peer, rw p2p.MsgReadWriter, getPooledTx func(hash common.Hash) *types.Transaction) *peer { @@ -123,17 +119,16 @@ func newPeer(version int, p *p2p.Peer, rw p2p.MsgReadWriter, getPooledTx func(ha knownBlocks: mapset.NewSet(), queuedBlocks: make(chan *propEvent, maxQueuedBlocks), queuedBlockAnns: make(chan *types.Block, maxQueuedBlockAnns), - txPropagation: make(chan []common.Hash), + txBroadcast: make(chan []common.Hash), txAnnounce: make(chan []common.Hash), - txRetrieval: make(chan []common.Hash), getPooledTx: getPooledTx, term: make(chan struct{}), } } -// broadcastBlocks is a write loop that multiplexes block propagations, -// announcements into the remote peer. The goal is to have an async writer -// that does not lock up node internals. +// broadcastBlocks is a write loop that multiplexes blocks and block accouncements +// to the remote peer. The goal is to have an async writer that does not lock up +// node internals and at the same time rate limits queued data. func (p *peer) broadcastBlocks() { for { select { @@ -155,71 +150,101 @@ func (p *peer) broadcastBlocks() { } } -// broadcastTxs is a write loop that multiplexes transaction propagations, -// announcements into the remote peer. The goal is to have an async writer -// that does not lock up node internals. -func (p *peer) broadcastTxs() { +// broadcastTransactions is a write loop that schedules transaction broadcasts +// to the remote peer. The goal is to have an async writer that does not lock up +// node internals and at the same time rate limits queued data. +func (p *peer) broadcastTransactions() { var ( - txProps []common.Hash // Queue of transaction propagations to the peer - txAnnos []common.Hash // Queue of transaction announcements to the peer - done chan struct{} // Non-nil if background network sender routine is active. - errch = make(chan error) // Channel used to receive network error + queue []common.Hash // Queue of hashes to broadcast as full transactions + done chan struct{} // Non-nil if background broadcaster is running + fail = make(chan error) // Channel used to receive network error ) - scheduleTask := func() { - // Short circuit if there already has a inflight task. - if done != nil { - return - } - // Spin up transaction propagation task if there is any - // queued hashes. - if len(txProps) > 0 { + for { + // If there's no in-flight broadcast running, check if a new one is needed + if done == nil && len(queue) > 0 { + // Pile transaction until we reach our allowed network limit var ( hashes []common.Hash txs []*types.Transaction size common.StorageSize ) - for i := 0; i < len(txProps) && size < txsyncPackSize; i++ { - if tx := p.getPooledTx(txProps[i]); tx != nil { + for i := 0; i < len(queue) && size < txsyncPackSize; i++ { + if tx := p.getPooledTx(queue[i]); tx != nil { txs = append(txs, tx) size += tx.Size() } - hashes = append(hashes, txProps[i]) + hashes = append(hashes, queue[i]) } - txProps = txProps[:copy(txProps, txProps[len(hashes):])] + queue = queue[:copy(queue, queue[len(hashes):])] + + // If there's anything available to transfer, fire up an async writer if len(txs) > 0 { done = make(chan struct{}) go func() { - if err := p.SendNewTransactions(txs); err != nil { - errch <- err + if err := p.sendTransactions(txs); err != nil { + fail <- err return } close(done) p.Log().Trace("Sent transactions", "count", len(txs)) }() - return } } - // Spin up transaction announcement task if there is any - // queued hashes. - if len(txAnnos) > 0 { + // Transfer goroutine may or may not have been started, listen for events + select { + case hashes := <-p.txBroadcast: + // New batch of transactions to be broadcast, queue them (with cap) + queue = append(queue, hashes...) + if len(queue) > maxQueuedTxs { + // Fancy copy and resize to ensure buffer doesn't grow indefinitely + queue = queue[:copy(queue, queue[len(queue)-maxQueuedTxs:])] + } + + case <-done: + done = nil + + case <-fail: + return + + case <-p.term: + return + } + } +} + +// announceTransactions is a write loop that schedules transaction broadcasts +// to the remote peer. The goal is to have an async writer that does not lock up +// node internals and at the same time rate limits queued data. +func (p *peer) announceTransactions() { + var ( + queue []common.Hash // Queue of hashes to announce as transaction stubs + done chan struct{} // Non-nil if background announcer is running + fail = make(chan error) // Channel used to receive network error + ) + for { + // If there's no in-flight announce running, check if a new one is needed + if done == nil && len(queue) > 0 { + // Pile transaction hashes until we reach our allowed network limit var ( hashes []common.Hash pending []common.Hash size common.StorageSize ) - for i := 0; i < len(txAnnos) && size < txsyncPackSize; i++ { - if tx := p.getPooledTx(txAnnos[i]); tx != nil { - pending = append(pending, txAnnos[i]) + for i := 0; i < len(queue) && size < txsyncPackSize; i++ { + if p.getPooledTx(queue[i]) != nil { + pending = append(pending, queue[i]) size += common.HashLength } - hashes = append(hashes, txAnnos[i]) + hashes = append(hashes, queue[i]) } - txAnnos = txAnnos[:copy(txAnnos, txAnnos[len(hashes):])] + queue = queue[:copy(queue, queue[len(hashes):])] + + // If there's anything available to transfer, fire up an async writer if len(pending) > 0 { done = make(chan struct{}) go func() { - if err := p.SendNewTransactionHashes(pending); err != nil { - errch <- err + if err := p.sendPooledTransactionHashes(pending); err != nil { + fail <- err return } close(done) @@ -227,95 +252,20 @@ func (p *peer) broadcastTxs() { }() } } - } - - for { - scheduleTask() + // Transfer goroutine may or may not have been started, listen for events select { - case hashes := <-p.txPropagation: - if len(txProps) == maxQueuedTxs { - continue - } - if len(txProps)+len(hashes) > maxQueuedTxs { - hashes = hashes[:maxQueuedTxs-len(txProps)] - } - txProps = append(txProps, hashes...) - case hashes := <-p.txAnnounce: - if len(txAnnos) == maxQueuedTxAnns { - continue - } - if len(txAnnos)+len(hashes) > maxQueuedTxAnns { - hashes = hashes[:maxQueuedTxAnns-len(txAnnos)] - } - txAnnos = append(txAnnos, hashes...) - - case <-done: - done = nil - - case <-errch: - return - - case <-p.term: - return - } - } -} - -// retrievalTxs is a write loop which is responsible for retrieving transaction -// from the remote peer. The goal is to have an async writer that does not lock -// up node internals. If there are too many requests queued, then new arrival -// requests will be dropped silently so that we can ensure the memory assumption -// is fixed for each peer. -func (p *peer) retrievalTxs() { - var ( - requests []common.Hash // Queue of transaction requests to the peer - done chan struct{} // Non-nil if background network sender routine is active. - errch = make(chan error) // Channel used to receive network error - ) - // pick chooses a reasonble number of transaction hashes for retrieval. - pick := func() []common.Hash { - var ret []common.Hash - if len(requests) > fetcher.MaxTransactionFetch { - ret = requests[:fetcher.MaxTransactionFetch] - } else { - ret = requests[:] - } - requests = requests[:copy(requests, requests[len(ret):])] - return ret - } - // send sends transactions retrieval request. - send := func(hashes []common.Hash, done chan struct{}) { - if err := p.RequestTxs(hashes); err != nil { - errch <- err - return - } - close(done) - p.Log().Trace("Sent transaction retrieval request", "count", len(hashes)) - } - for { - select { - case hashes := <-p.txRetrieval: - if len(requests) == maxQueuedTxRetrieval { - continue - } - if len(requests)+len(hashes) > maxQueuedTxRetrieval { - hashes = hashes[:maxQueuedTxRetrieval-len(requests)] - } - requests = append(requests, hashes...) - if done == nil { - done = make(chan struct{}) - go send(pick(), done) + // New batch of transactions to be broadcast, queue them (with cap) + queue = append(queue, hashes...) + if len(queue) > maxQueuedTxAnns { + // Fancy copy and resize to ensure buffer doesn't grow indefinitely + queue = queue[:copy(queue, queue[len(queue)-maxQueuedTxs:])] } case <-done: done = nil - if pending := pick(); len(pending) > 0 { - done = make(chan struct{}) - go send(pending, done) - } - case <- errch: + case <-fail: return case <-p.term: @@ -379,22 +329,22 @@ func (p *peer) MarkTransaction(hash common.Hash) { p.knownTxs.Add(hash) } -// SendNewTransactionHashes sends a batch of transaction hashes to the peer and -// includes the hashes in its transaction hash set for future reference. -func (p *peer) SendNewTransactionHashes(hashes []common.Hash) error { - // Mark all the transactions as known, but ensure we don't overflow our limits - for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { - p.knownTxs.Pop() - } - for _, hash := range hashes { - p.knownTxs.Add(hash) - } - return p2p.Send(p.rw, NewPooledTransactionHashesMsg, hashes) +// SendTransactions64 sends transactions to the peer and includes the hashes +// in its transaction hash set for future reference. +// +// This method is legacy support for initial transaction exchange in eth/64 and +// prior. For eth/65 and higher use SendPooledTransactionHashes. +func (p *peer) SendTransactions64(txs types.Transactions) error { + return p.sendTransactions(txs) } -// SendNewTransactions sends transactions to the peer and includes the hashes +// sendTransactions sends transactions to the peer and includes the hashes // in its transaction hash set for future reference. -func (p *peer) SendNewTransactions(txs types.Transactions) error { +// +// This method is a helper used by the async transaction sender. Don't call it +// directly as the queueing (memory) and transmission (bandwidth) costs should +// not be managed directly. +func (p *peer) sendTransactions(txs types.Transactions) error { // Mark all the transactions as known, but ensure we don't overflow our limits for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(txs)) { p.knownTxs.Pop() @@ -402,18 +352,15 @@ func (p *peer) SendNewTransactions(txs types.Transactions) error { for _, tx := range txs { p.knownTxs.Add(tx.Hash()) } - return p2p.Send(p.rw, TxMsg, txs) -} - -func (p *peer) SendTransactionRLP(txs []rlp.RawValue) error { - return p2p.Send(p.rw, TxMsg, txs) + return p2p.Send(p.rw, TransactionMsg, txs) } -// AsyncSendTransactions queues list of transactions propagation to a remote -// peer. If the peer's broadcast queue is full, the event is silently dropped. +// AsyncSendTransactions queues a list of transactions (by hash) to eventually +// propagate to a remote peer. The number of pending sends are capped (new ones +// will force old sends to be dropped) func (p *peer) AsyncSendTransactions(hashes []common.Hash) { select { - case p.txPropagation <- hashes: + case p.txBroadcast <- hashes: // Mark all the transactions as known, but ensure we don't overflow our limits for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { p.knownTxs.Pop() @@ -426,9 +373,27 @@ func (p *peer) AsyncSendTransactions(hashes []common.Hash) { } } -// AsyncSendTransactions queues list of transactions propagation to a remote -// peer. If the peer's broadcast queue is full, the event is silently dropped. -func (p *peer) AsyncSendTransactionHashes(hashes []common.Hash) { +// sendPooledTransactionHashes sends transaction hashes to the peer and includes +// them in its transaction hash set for future reference. +// +// This method is a helper used by the async transaction announcer. Don't call it +// directly as the queueing (memory) and transmission (bandwidth) costs should +// not be managed directly. +func (p *peer) sendPooledTransactionHashes(hashes []common.Hash) error { + // Mark all the transactions as known, but ensure we don't overflow our limits + for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { + p.knownTxs.Pop() + } + for _, hash := range hashes { + p.knownTxs.Add(hash) + } + return p2p.Send(p.rw, NewPooledTransactionHashesMsg, hashes) +} + +// AsyncSendPooledTransactionHashes queues a list of transactions hashes to eventually +// announce to a remote peer. The number of pending sends are capped (new ones +// will force old sends to be dropped) +func (p *peer) AsyncSendPooledTransactionHashes(hashes []common.Hash) { select { case p.txAnnounce <- hashes: // Mark all the transactions as known, but ensure we don't overflow our limits @@ -443,6 +408,22 @@ func (p *peer) AsyncSendTransactionHashes(hashes []common.Hash) { } } +// SendPooledTransactionsRLP sends requested transactions to the peer and adds the +// hashes in its transaction hash set for future reference. +// +// Note, the method assumes the hashes are correct and correspond to the list of +// transactions being sent. +func (p *peer) SendPooledTransactionsRLP(hashes []common.Hash, txs []rlp.RawValue) error { + // Mark all the transactions as known, but ensure we don't overflow our limits + for p.knownTxs.Cardinality() > max(0, maxKnownTxs-len(hashes)) { + p.knownTxs.Pop() + } + for _, hash := range hashes { + p.knownTxs.Add(hash) + } + return p2p.Send(p.rw, PooledTransactionsMsg, txs) +} + // SendNewBlockHashes announces the availability of a number of blocks through // a hash notification. func (p *peer) SendNewBlockHashes(hashes []common.Hash, numbers []uint64) error { @@ -577,16 +558,6 @@ func (p *peer) RequestTxs(hashes []common.Hash) error { return p2p.Send(p.rw, GetPooledTransactionsMsg, hashes) } -// AsyncRequestTxs queues a tx retrieval request to a remote peer. If -// the peer's retrieval queue is full, the event is silently dropped. -func (p *peer) AsyncRequestTxs(hashes []common.Hash) { - select { - case p.txRetrieval <- hashes: - case <-p.term: - p.Log().Debug("Dropping transaction retrieval request", "count", len(hashes)) - } -} - // Handshake executes the eth protocol handshake, negotiating version number, // network IDs, difficulties, head and genesis blocks. func (p *peer) Handshake(network uint64, td *big.Int, head common.Hash, genesis common.Hash, forkID forkid.ID, forkFilter forkid.Filter) error { @@ -746,9 +717,10 @@ func (ps *peerSet) Register(p *peer) error { return errAlreadyRegistered } ps.peers[p.id] = p + go p.broadcastBlocks() - go p.broadcastTxs() - go p.retrievalTxs() + go p.broadcastTransactions() + go p.announceTransactions() return nil } diff --git a/eth/protocol.go b/eth/protocol.go index 1cef66adb2..dc75d6b31a 100644 --- a/eth/protocol.go +++ b/eth/protocol.go @@ -51,7 +51,7 @@ const protocolMaxMsgSize = 10 * 1024 * 1024 // Maximum cap on the size of a prot const ( StatusMsg = 0x00 NewBlockHashesMsg = 0x01 - TxMsg = 0x02 + TransactionMsg = 0x02 GetBlockHeadersMsg = 0x03 BlockHeadersMsg = 0x04 GetBlockBodiesMsg = 0x05 @@ -64,10 +64,11 @@ const ( // New protocol message codes introduced in eth65 // - // Previously these message ids(0x08, 0x09) were used by some - // legacy and unsupported eth protocols, reown them here. + // Previously these message ids were used by some legacy and unsupported + // eth protocols, reown them here. NewPooledTransactionHashesMsg = 0x08 GetPooledTransactionsMsg = 0x09 + PooledTransactionsMsg = 0x0a ) type errCode int diff --git a/eth/protocol_test.go b/eth/protocol_test.go index e9a1a511ef..4bbfe9bd3c 100644 --- a/eth/protocol_test.go +++ b/eth/protocol_test.go @@ -62,7 +62,7 @@ func TestStatusMsgErrors63(t *testing.T) { wantError error }{ { - code: TxMsg, data: []interface{}{}, + code: TransactionMsg, data: []interface{}{}, wantError: errResp(ErrNoStatusMsg, "first msg has code 2 (!= 0)"), }, { @@ -114,7 +114,7 @@ func TestStatusMsgErrors64(t *testing.T) { wantError error }{ { - code: TxMsg, data: []interface{}{}, + code: TransactionMsg, data: []interface{}{}, wantError: errResp(ErrNoStatusMsg, "first msg has code 2 (!= 0)"), }, { @@ -258,7 +258,7 @@ func testRecvTransactions(t *testing.T, protocol int) { defer p.close() tx := newTestTransaction(testAccount, 0, 0) - if err := p2p.Send(p.app, TxMsg, []interface{}{tx}); err != nil { + if err := p2p.Send(p.app, TransactionMsg, []interface{}{tx}); err != nil { t.Fatalf("send error: %v", err) } select { @@ -282,13 +282,16 @@ func testSendTransactions(t *testing.T, protocol int) { pm, _ := newTestProtocolManagerMust(t, downloader.FullSync, 0, nil, nil) defer pm.Stop() - // Fill the pool with big transactions. + // Fill the pool with big transactions (use a subscription to wait until all + // the transactions are announced to avoid spurious events causing extra + // broadcasts). const txsize = txsyncPackSize / 10 alltxs := make([]*types.Transaction, 100) for nonce := range alltxs { alltxs[nonce] = newTestTransaction(testAccount, uint64(nonce), txsize) } pm.txpool.AddRemotes(alltxs) + time.Sleep(100 * time.Millisecond) // Wait until new tx even gets out of the system (lame) // Connect several peers. They should all receive the pending transactions. var wg sync.WaitGroup @@ -300,8 +303,6 @@ func testSendTransactions(t *testing.T, protocol int) { seen[tx.Hash()] = false } for n := 0; n < len(alltxs) && !t.Failed(); { - var txs []*types.Transaction - var hashes []common.Hash var forAllHashes func(callback func(hash common.Hash)) switch protocol { case 63: @@ -310,11 +311,15 @@ func testSendTransactions(t *testing.T, protocol int) { msg, err := p.app.ReadMsg() if err != nil { t.Errorf("%v: read error: %v", p.Peer, err) - } else if msg.Code != TxMsg { + continue + } else if msg.Code != TransactionMsg { t.Errorf("%v: got code %d, want TxMsg", p.Peer, msg.Code) + continue } + var txs []*types.Transaction if err := msg.Decode(&txs); err != nil { t.Errorf("%v: %v", p.Peer, err) + continue } forAllHashes = func(callback func(hash common.Hash)) { for _, tx := range txs { @@ -325,11 +330,15 @@ func testSendTransactions(t *testing.T, protocol int) { msg, err := p.app.ReadMsg() if err != nil { t.Errorf("%v: read error: %v", p.Peer, err) + continue } else if msg.Code != NewPooledTransactionHashesMsg { - t.Errorf("%v: got code %d, want TxMsg", p.Peer, msg.Code) + t.Errorf("%v: got code %d, want NewPooledTransactionHashesMsg", p.Peer, msg.Code) + continue } + var hashes []common.Hash if err := msg.Decode(&hashes); err != nil { t.Errorf("%v: %v", p.Peer, err) + continue } forAllHashes = func(callback func(hash common.Hash)) { for _, h := range hashes { diff --git a/eth/sync.go b/eth/sync.go index 93b2dd2ecf..d5c678a74a 100644 --- a/eth/sync.go +++ b/eth/sync.go @@ -38,13 +38,18 @@ const ( ) type txsync struct { - p *peer - hashes []common.Hash - txs []*types.Transaction + p *peer + txs []*types.Transaction } // syncTransactions starts sending all currently pending transactions to the given peer. func (pm *ProtocolManager) syncTransactions(p *peer) { + // Assemble the set of transaction to broadcast or announce to the remote + // peer. Fun fact, this is quite an expensive operation as it needs to sort + // the transactions if the sorting is not cached yet. However, with a random + // order, insertions could overflow the non-executable queues and get dropped. + // + // TODO(karalabe): Figure out if we could get away with random order somehow var txs types.Transactions pending, _ := pm.txpool.Pending() for _, batch := range pending { @@ -53,17 +58,29 @@ func (pm *ProtocolManager) syncTransactions(p *peer) { if len(txs) == 0 { return } + // The eth/65 protocol introduces proper transaction announcements, so instead + // of dripping transactions across multiple peers, just send the entire list as + // an announcement and let the remote side decide what they need (likely nothing). + if p.version >= eth65 { + hashes := make([]common.Hash, len(txs)) + for i, tx := range txs { + hashes[i] = tx.Hash() + } + p.AsyncSendPooledTransactionHashes(hashes) + return + } + // Out of luck, peer is running legacy protocols, drop the txs over select { case pm.txsyncCh <- &txsync{p: p, txs: txs}: case <-pm.quitSync: } } -// txsyncLoop takes care of the initial transaction sync for each new +// txsyncLoop64 takes care of the initial transaction sync for each new // connection. When a new peer appears, we relay all currently pending // transactions. In order to minimise egress bandwidth usage, we send // the transactions in small packs to one peer at a time. -func (pm *ProtocolManager) txsyncLoop() { +func (pm *ProtocolManager) txsyncLoop64() { var ( pending = make(map[enode.ID]*txsync) sending = false // whether a send is active @@ -72,44 +89,26 @@ func (pm *ProtocolManager) txsyncLoop() { ) // send starts a sending a pack of transactions from the sync. send := func(s *txsync) { + if s.p.version >= eth65 { + panic("initial transaction syncer running on eth/65+") + } // Fill pack with transactions up to the target size. size := common.StorageSize(0) pack.p = s.p - pack.hashes = pack.hashes[:0] pack.txs = pack.txs[:0] - if s.p.version >= eth65 { - // Eth65 introduces transaction announcement https://github.com/ethereum/EIPs/pull/2464, - // only txhashes are transferred here. - for i := 0; i < len(s.txs) && size < txsyncPackSize; i++ { - pack.hashes = append(pack.hashes, s.txs[i].Hash()) - size += common.HashLength - } - // Remove the transactions that will be sent. - s.txs = s.txs[:copy(s.txs, s.txs[len(pack.hashes):])] - if len(s.txs) == 0 { - delete(pending, s.p.ID()) - } - // Send the pack in the background. - s.p.Log().Trace("Sending batch of transaction announcements", "count", len(pack.hashes), "bytes", size) - sending = true - go func() { done <- pack.p.SendNewTransactionHashes(pack.hashes) }() - } else { - // Legacy eth protocol doesn't have transaction announcement protocol - // message, transfer the whole pending transaction slice. - for i := 0; i < len(s.txs) && size < txsyncPackSize; i++ { - pack.txs = append(pack.txs, s.txs[i]) - size += s.txs[i].Size() - } - // Remove the transactions that will be sent. - s.txs = s.txs[:copy(s.txs, s.txs[len(pack.txs):])] - if len(s.txs) == 0 { - delete(pending, s.p.ID()) - } - // Send the pack in the background. - s.p.Log().Trace("Sending batch of transactions", "count", len(pack.txs), "bytes", size) - sending = true - go func() { done <- pack.p.SendNewTransactions(pack.txs) }() + for i := 0; i < len(s.txs) && size < txsyncPackSize; i++ { + pack.txs = append(pack.txs, s.txs[i]) + size += s.txs[i].Size() + } + // Remove the transactions that will be sent. + s.txs = s.txs[:copy(s.txs, s.txs[len(pack.txs):])] + if len(s.txs) == 0 { + delete(pending, s.p.ID()) } + // Send the pack in the background. + s.p.Log().Trace("Sending batch of transactions", "count", len(pack.txs), "bytes", size) + sending = true + go func() { done <- pack.p.SendTransactions64(pack.txs) }() } // pick chooses the next pending sync. diff --git a/fuzzbuzz.yaml b/fuzzbuzz.yaml index 72c3835ace..2a4f0c296f 100644 --- a/fuzzbuzz.yaml +++ b/fuzzbuzz.yaml @@ -26,6 +26,14 @@ targets: function: Fuzz package: github.com/ethereum/go-ethereum/tests/fuzzers/trie checkout: github.com/ethereum/go-ethereum/ + - name: txfetcher + language: go + version: "1.13" + corpus: ./fuzzers/txfetcher/corpus + harness: + function: Fuzz + package: github.com/ethereum/go-ethereum/tests/fuzzers/txfetcher + checkout: github.com/ethereum/go-ethereum/ - name: whisperv6 language: go version: "1.13" diff --git a/tests/fuzzers/txfetcher/corpus/0151ee1d0db4c74d3bcdfa4f7396a4c8538748c9-2 b/tests/fuzzers/txfetcher/corpus/0151ee1d0db4c74d3bcdfa4f7396a4c8538748c9-2 new file mode 100644 index 0000000000..2c75e9c7a7 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/0151ee1d0db4c74d3bcdfa4f7396a4c8538748c9-2 @@ -0,0 +1 @@ +�� \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/020dd7b492a6eb34ff0b7d8ee46189422c37e4a7-6 b/tests/fuzzers/txfetcher/corpus/020dd7b492a6eb34ff0b7d8ee46189422c37e4a7-6 new file mode 100644 index 0000000000000000000000000000000000000000..8d3b57789e79a7046916b2c7646f038f443671ad GIT binary patch literal 14 VcmZRuu&`iYU~sUour#;W4*(9%0>A(O literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/021d1144e359233c496e22c3250609b11b213e9f-4 b/tests/fuzzers/txfetcher/corpus/021d1144e359233c496e22c3250609b11b213e9f-4 new file mode 100644 index 0000000000..73731899d5 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/021d1144e359233c496e22c3250609b11b213e9f-4 @@ -0,0 +1,12 @@ + TESTING KEY----- +MIICXgIBAAKBgQDuLnQAI3mDgey3VBzWnB2L39JUU4txjeVE6myuDqkM/uGlfjb9 +SjY1bIw4iAJm2gsvvZhIrCHS3l6afab4pZB +l2+XsDlrKBxKKtDrGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTtqJQIDAQAB +AoGAGRzwwir7XvBOAy5tuV6ef6anZzus1s1Y1Clb6HbnWWF/wbZGOpet +3m4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKZTXtdZrh+k7hx0nTP8Jcb +uqFk541awmMogY/EfbWd6IOkp+4xqjlFBEDytgbIECQQDvH/6nk+hgN4H +qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz84SHEg1Ak/7KCxmD/sfgS5TeuNi8DoUBEmiSJwm7FX +ftxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su43sjXNueLKH8+ph2UfQuU9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xl/DoCzjA0CQQDU +y2pGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYiZ6PYj013sovGKUFfYAqVXVlxtIďż˝oďż˝X +qUn3Xh9ps8ZfjLZO7BAkEAlT4R5Yl6cGhaJQYZHOde3JMhNRcVFMO8dDaFo +f9Oeos0UotgiDktdQHxdNEwLjQlJBz+OtwwA=---E RATTIEY- \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/0d28327b1fb52c1ba02a6eb96675c31633921bb2-2 b/tests/fuzzers/txfetcher/corpus/0d28327b1fb52c1ba02a6eb96675c31633921bb2-2 new file mode 100644 index 0000000000..8cc3039cb8 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/0d28327b1fb52c1ba02a6eb96675c31633921bb2-2 @@ -0,0 +1,15 @@ +ďż˝&^��o�ȗ-----BEGIN RSA TESTING KEY----- +MIICXgIBAAKBgQDuLnQAI3mDgey3VBzWnB2L39JUU4txjeVE6myuDqkM/uGlfjb9 +SjY1bIw4iA5sBBZzHi3z0h1YV8QPuxEbi4nW91IJm2gsvvZhIrCHS3l6afab4pZB +l2+XsDulrKBxKKtD1rGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTvqJQIDAQAB +AoGAGRzwwir7XvBOAy5tM/uV6e+Zf6anZzus1s1Y1ClbjbE6HXbnWWF/wbZGOpet +3Zm4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKwZatTXtdZrhu+Jk7hx0nTPy8Jcb +uJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+hgN4H +qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz8vp +jy4SHEg1AkEA/v13/5M47K9vCxmb8QeD/asydfsgS5TeuNi8DoUBEmiSJwma7FXY +fFUtxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su4b3sjXNueLKH85Q+phy2U +fQtuUE9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xlp/DoCzjA0CQQDU +y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYiZ6PYj013sovGKUFfYAqVXVlxtIX +qyUBnu3X9ps8ZfjLZO7BAkEAlT4R5Yl6cGhaJQYZHOde3JEMhNRcVFMO8dJDaFeo +f9Oeos0UUothgiDktdQHxdNEwLjQf7lJJBzV+5OtwswCWA== +-----END RSA TESTING KEY-----Q_ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/0fcd827b57ded58e91f7ba2ac2b7ea4d25ebedca-7 b/tests/fuzzers/txfetcher/corpus/0fcd827b57ded58e91f7ba2ac2b7ea4d25ebedca-7 new file mode 100644 index 0000000000..8ceee16af1 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/0fcd827b57ded58e91f7ba2ac2b7ea4d25ebedca-7 @@ -0,0 +1 @@ +ďż˝ap����ďż˝���￿�����������������V�������#��&�� \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/109bc9b8fd4fef63493e104c703c79bc4a5e8d34-6 b/tests/fuzzers/txfetcher/corpus/109bc9b8fd4fef63493e104c703c79bc4a5e8d34-6 new file mode 100644 index 0000000000000000000000000000000000000000..df9b986af1005f1e472eaee9d42135411f3cad63 GIT binary patch literal 470 zcmXBPNsgjW0D$4DEvGQ)r7@r&!=e@sdB{8nD6kU{kfE3!lXvU%k@^a4rw;Ic$zQfx zYUxCchXTqX8J;qOlCW-BNUrGREK=HOVq~%x}s(2NsgIr{eR1PWA?-k_w>v}L;@%v7V}lB?YOX7S+`=lZ(i)wVud5Y zY!6BZa8jojm77j=WDEG3q(kN$-N$G=S;6CB{dAPMzB8g9V5#V#Wt=p|pNhFKjy%yC zFpBAMMa)X+HjvSN7$b5hd}DlQ5^HbU{NQ1!jYbc0Xo)I!+*2KCFqw4WsZh_wU=>#O zr1r)kwlZ8qgkQCI1Blc6)Wg<8J2*FT4y<|`;v!3_^cGx#XJ`clF*MUzO#{$fuhoKN zAM^Nc(r*X02+fQKC8*rXv7+O1epiA}A{Chb{qy_R=Z6uCy2&Pra2Fp<%9)R5AkasN o5@|^cIBRywREomC%wfKLePO%ZjwD%%fQGPfz{Yi4`Mgp80htP#Qvd(} literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/163785ab002746452619f31e8dfcb4549e6f8b6e-6 b/tests/fuzzers/txfetcher/corpus/163785ab002746452619f31e8dfcb4549e6f8b6e-6 new file mode 100644 index 0000000000000000000000000000000000000000..55467373d46141a3786f3d38a439ab7fa65d9b1c GIT binary patch literal 30 kcmZShPPx# literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/1adfa6b9ddf5766220c8ff7ede2926ca241bb947-3 b/tests/fuzzers/txfetcher/corpus/1adfa6b9ddf5766220c8ff7ede2926ca241bb947-3 new file mode 100644 index 0000000000..4a593aa28d --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/1adfa6b9ddf5766220c8ff7ede2926ca241bb947-3 @@ -0,0 +1,11 @@ +TAKBgDuLnQA3gey3VBznB39JUtxjeE6myuDkM/uGlfjb +S1w4iA5sBzzh8uxEbi4nW91IJm2gsvvZhICHS3l6ab4pZB +l2DulrKBxKKtD1rGxlG4LncabFn9vLZad2bSysqz/qTAUSTvqJQIDAQAB +AoGAGRzwwir7XvBOAy5tM/uV6e+Zf6anZzus1s1Y1ClbjbE6HXbnWWF/wbZGOpet +3Z4vMXc7jpTLryzTQIvVdfQbRc6+MUVeLKZatTXtdZrhu+Jk7hx0nTPy8Jcb +uJqFk54MogxEcfbWd6IOkp+4xqFLBEDtgbIECnk+hgN4H +qzzxxr397vWrjrIgbJpQvBv8QeeuNi8DoUBEmiSJwa7FXY +FUtxuvL7XvjwjN5B30pEbc6Iuyt7y4MQJBAIt21su4b3sjphy2tuUE9xblTu14qgHZ6+AiZovGKU--FfYAqVXVlxtIX +qyU3X9ps8ZfjLZ45l6cGhaJQYZHOde3JEMhNRcVFMO8dJDaFeo +f9Oeos0UUothgiDktdQHxdNEwLjQf7lJJBzV+5OtwswCWA== +-----END RSA T \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/1b9a02e9a48fea1d2fc3fb77946ada278e152079-4 b/tests/fuzzers/txfetcher/corpus/1b9a02e9a48fea1d2fc3fb77946ada278e152079-4 new file mode 100644 index 0000000000000000000000000000000000000000..4a56f93d3ba9dc043d85983f6938371424a18831 GIT binary patch literal 17 ZcmdmX;%zzm%{7eWMGPsad6Su6002*L2o3-M literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/1e14c7ea1faef92890988061b5abe96db7190f98-7 b/tests/fuzzers/txfetcher/corpus/1e14c7ea1faef92890988061b5abe96db7190f98-7 new file mode 100644 index 0000000000..d2442fc5a6 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/1e14c7ea1faef92890988061b5abe96db7190f98-7 @@ -0,0 +1 @@ +0000000000000000000000000000000000000000000000000000000000000000000000000 \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/1e7d05f00e99cbf3ff0ef1cd7ea8dd07ad6dff23-6 b/tests/fuzzers/txfetcher/corpus/1e7d05f00e99cbf3ff0ef1cd7ea8dd07ad6dff23-6 new file mode 100644 index 0000000000000000000000000000000000000000..1c342ff53a36366885c5a06494aef83308ed3126 GIT binary patch literal 19 acmZShus5-wASW|9G1$Y=m$7*Nxl#a8k_aaN literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/1ec95e347fd522e6385b5091aa81aa2485be4891-4 b/tests/fuzzers/txfetcher/corpus/1ec95e347fd522e6385b5091aa81aa2485be4891-4 new file mode 100644 index 0000000000000000000000000000000000000000..b0c776bd4d996c4b423b42a871caae498a5bb695 GIT binary patch literal 833 zcmXZb$+DtI0EJvY?uiaW5OC3pLs{H8WBf3>{n2Nad>~2{-XpBKtKy*z7 z+VO+jwx&2CUP!XIdknPs5+C#S?OhhhOc6DhAMLg0*+iY_+G)y&nbovF_e?O(ua@TL z53V#@R+WX6$WV_ED&vV!h4+holrDdwKI_IuW%nq6*C?^Dx@qfOx|0Ugox~tD;mehJ}a=??6)q2HCOwz$325 zp*wt=oTNRZ+sCr=sREM0R}W-Hl{c{bhjYd9bd6;`b{<-^D2sHoI(L^|N=6$OvJI<$ z7^OL`7;Jwyd}`UXMDK@NUHasXIoCPN!xUX=I%YJ9w*(qoqh~Bcc*}BqGwPPF6%qIZ zVNlJ)2&vEF%kO~;ABVP=;sV?8S(JY`Dv!VJqe@8iM`FS#KAp3iGgar|UeiEx&Qhi% zTHa8_;hZ=9)^AApR9+;5r#W-g$;Pm#OjBM1dVeNUKOMpUKXIt;*A@IJ=Id`R@Ha}< B56=Jq literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/1fbfa5d214060d2a0905846a589fd6f78d411451-4 b/tests/fuzzers/txfetcher/corpus/1fbfa5d214060d2a0905846a589fd6f78d411451-4 new file mode 100644 index 0000000000000000000000000000000000000000..75de835c98de4015fd4c9874b4a469e18a3b87d3 GIT binary patch literal 22 ecmdn?ww(Ru8piS>hLqI2$;>Zw^z=$MDF6U-W(gGl literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/1fd84ee194e791783a7f18f0a6deab8efe05fc04-2 b/tests/fuzzers/txfetcher/corpus/1fd84ee194e791783a7f18f0a6deab8efe05fc04-2 new file mode 100644 index 0000000000..3b6d2560ae --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/1fd84ee194e791783a7f18f0a6deab8efe05fc04-2 @@ -0,0 +1 @@ +ďż˝& \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/21e76b9fca21d94d97f860c1c82f40697a83471b-8 b/tests/fuzzers/txfetcher/corpus/21e76b9fca21d94d97f860c1c82f40697a83471b-8 new file mode 100644 index 0000000000..1d4620f49f --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/21e76b9fca21d94d97f860c1c82f40697a83471b-8 @@ -0,0 +1,3 @@ +DtQvfQ+MULKZTXk78c +/fWkpxlQQ/+hgNzVtx9vWgJsafG7b0dA4AFjwVbFLmQcj2PprIMmPNQrooX +L \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/220a87fed0c92474923054094eb7aff14289cf5e-4 b/tests/fuzzers/txfetcher/corpus/220a87fed0c92474923054094eb7aff14289cf5e-4 new file mode 100644 index 0000000000000000000000000000000000000000..175f74fd5aa8883bffb6d3d64727d2265f7bee35 GIT binary patch literal 4 LcmZQz*~^ literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/23ddcd66aa92fe3d78b7f5b6e7cddb1b55c5f5df-3 b/tests/fuzzers/txfetcher/corpus/23ddcd66aa92fe3d78b7f5b6e7cddb1b55c5f5df-3 new file mode 100644 index 0000000000..95892c7b00 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/23ddcd66aa92fe3d78b7f5b6e7cddb1b55c5f5df-3 @@ -0,0 +1,12 @@ +4txjeVE6myuDqkM/uGlfjb9 +SjY1bIw4iA5sBBZzHi3z0h1YV8QPuxEbi4nW91IJm2gsvvZeIrCHS3l6afab4pZB +l2+XsDlrKBxKKtD1rGxlG4jncdabFn9gvLZad2bSysqz/qTAUSTvqJQIDAQAB +AoGAGRzwwXvBOAy5tM/uV6e+Zf6aZzus1s1Y1ClbjbE6HXbnWWF/wbZGOpet +3Z4vD6Mc7pLryzTQIVdfQbRc6+MUVeLKZaTXtdZru+Jk70PJJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+gN4H +qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQ2PprIMPcQroo8vpjSHg1Ev14KxmQeDydfsgeuN8UBESJwm7F +UtuL7Xvjw50pNEbc6Iuyty4QJA21su4sjXNueLQphy2U +fQtuUE9txblTu14qN7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xlp/DoCzjA0CQQDU +y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6ARYiZPYj1oGUFfYAVVxtI +qyBnu3X9pfLZOAkEAlT4R5Yl6cJQYZHOde3JEhNRcVFMO8dJFo +f9Oeos0UUhgiDkQxdEwLjQf7lJJz5OtwC= +-NRSA TESINGKEY-Q_ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/2441d249faf9a859e38c49f6e305b394280c6ea5-1 b/tests/fuzzers/txfetcher/corpus/2441d249faf9a859e38c49f6e305b394280c6ea5-1 new file mode 100644 index 0000000000000000000000000000000000000000..d76207e992a6e50a53966fa06c881f7c9cab5eb4 GIT binary patch literal 55 zcmV~$fenBl3`IeCEThRMsijI7#|fQ)Fo#p<2;O~ZN#mV^=f=b?TvQR_5T|IO)NpG( K!syDY>)L*r_6#ck literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/2da1f0635e11283b1927974f418aadd8837ad31e-7 b/tests/fuzzers/txfetcher/corpus/2da1f0635e11283b1927974f418aadd8837ad31e-7 new file mode 100644 index 0000000000000000000000000000000000000000..73ae7057014ffcb8c913914888047969958df11c GIT binary patch literal 15 WcmZShus5-wAjgAo?|Zhr`%3{g@&`u% literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/2e1853fbf8efe40098b1583224fe3b5f335e7037-6 b/tests/fuzzers/txfetcher/corpus/2e1853fbf8efe40098b1583224fe3b5f335e7037-6 new file mode 100644 index 0000000000000000000000000000000000000000..692981e614155c59f5ba80f40e832734383d8901 GIT binary patch literal 26 icmZShz~Eha=sv^$|L^zjWze;-urM}dU|?{tumAv_9to%b literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/2f25490dc49c103d653843ed47324b310ee7105e-7 b/tests/fuzzers/txfetcher/corpus/2f25490dc49c103d653843ed47324b310ee7105e-7 new file mode 100644 index 0000000000000000000000000000000000000000..5cf7da75df2dc6c74b3353ec1054ae730ab2a4f6 GIT binary patch literal 23 fcmZShz~Eha=sv^$|L^zjWiVziWnf@%u&@9Cesl=a literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/30494b85bb60ad7f099fa49d427007a761620d8f-5 b/tests/fuzzers/txfetcher/corpus/30494b85bb60ad7f099fa49d427007a761620d8f-5 new file mode 100644 index 0000000000..7ff9d39752 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/30494b85bb60ad7f099fa49d427007a761620d8f-5 @@ -0,0 +1,10 @@ +jXbnWWF/wbZGOpet +3Zm4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKwZatTXtdZrhu+Jk7hx0nTPy8Jcb +uJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+hgN4H +qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz8vp +jy4SHEg1AkEA/v13/5M47K9vCxb8QeD/asydfsgS5TeuNi8DoUBEmiSJwma7FXY +fFUtxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su4b3sjXNueLKH85Q+phy2U +fQtuUE9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xl/DoCzjA0CQQDU +y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYiZ6Yj013sovGKUFfYAqVXVlxtIX +qyUBnu3Xh9ps8ZfjLZO7BAkEAlT4R5Yl6cGhaJQYZHOde3JEMhNRcVFMO8dDaFeo +f9Oeos0UotgiDktdQHxdNEwLjQfl \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/316024ca3aaf09c1de5258733ff5fe3d799648d3-4 b/tests/fuzzers/txfetcher/corpus/316024ca3aaf09c1de5258733ff5fe3d799648d3-4 new file mode 100644 index 0000000000..61f7d78f34 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/316024ca3aaf09c1de5258733ff5fe3d799648d3-4 @@ -0,0 +1,15 @@ +ďż˝^ďż˝o�ȗ----BEGIN RA TTING KEY----- +IIXgIBAAKBQDuLnQI3mDgey3VBzWnB2L39JUU4txjeVE6myuDqkM/uGlfjb9 +SjY1bIw4iA5sBBZzHi3z0h1YV8QPuxEbi4nW91IJmgsvvZhrCHSl6afab4pZB +l2+XsDulrKBxKKtD1rGxlG4LjcdabF9gvLZad2bSysqz/qTAUStTvqJQDAQAB +AoGAGRzwwir7XvBOAy5tM/uV6e+Zf6anZzus1s1Y1ClbjbE6HXbnWWF/wbZGOpet +3Z4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKwZatTXtdZrhu+Jk7hx0nTPy8Jcb +uJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+hgN4H +qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz8vp +jy4SHEg1AkEA/v13/5M47K9vCxmb8QeD/asydfsgS5TeuNi8DoUBEmiSJwma7FXY +fFUtxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su4b3sjXNueLKH85Q+phy2U +fQtuUE9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xlp/DoCzjA0CQQDU +y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYiZ6PYj043sovGKUFfYAqVXVlxtIX +qyUBnu3X9ps8ZfjLZO7BAkEAlT4R5Yl6cGhaJQYZHOde3JEMhNRcVFMO8dJDaFeo +f9Oeos0UUothgiDktdQHxdNEwLjQf7lJJBzV+5OtwswCWA== +-----END RSA TESTING KEY-----Q_ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/32a089e2c439a91f4c1b67a13d52429bcded0dd9-7 b/tests/fuzzers/txfetcher/corpus/32a089e2c439a91f4c1b67a13d52429bcded0dd9-7 new file mode 100644 index 0000000000000000000000000000000000000000..a986a9d8e753a8e307ab9d904781ec872fa2eec0 GIT binary patch literal 1665 zcmeIzM@|DV7>3~lGKAg+QhHf%TnTO?MT0U^PHGqB?TNL+#c z#cNIglKp(YXFNsLXl$@(kIip7o$l-T$>8&HrI*x$7kGmKH1Gi<_<|q!LjVLq5ClUA zghCjELj*)Z6qq0yVjvdcARZFH42h5g7D$E^NQE>=hYZLBD`Y`7js-XsIp$_Vy0UDtRnxO^k&F2J5f^o3I7j;DH_3g+17Z12}{u zIEE8Ag)=yZ3%Jz3UQ7J@zXE@B1?20O){(7pq*Rl_RHH`5I+Y9?jY3qOmpbB7@$&~) zrP5(rkfPl!cw&pRD#;afM)}(f-XEf6f}dH?%sehr%@^(CpEq}szIMSBsdu{6#v!AR h`=_R1$ph)-(xt6h1&?%ISK8aH%b}_?r%Ji{`5V}g$;JQx literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/33ec1dc0bfeb93d16edee3c07125fec6ac1aa17d-2 b/tests/fuzzers/txfetcher/corpus/33ec1dc0bfeb93d16edee3c07125fec6ac1aa17d-2 new file mode 100644 index 0000000000..d41771b86c --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/33ec1dc0bfeb93d16edee3c07125fec6ac1aa17d-2 @@ -0,0 +1 @@ +ďż˝ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/37a0d207700b52caa005ec8aeb344dcb13150ed2-5 b/tests/fuzzers/txfetcher/corpus/37a0d207700b52caa005ec8aeb344dcb13150ed2-5 new file mode 100644 index 0000000000000000000000000000000000000000..2f09c6e28f03ccb6b2e40fe56b8d26768544aab6 GIT binary patch literal 55 zcmZShu=hRV-uL_WGO(qVWay`*mLzAS7U`E1CFT_;CYNO9=jkUEu2ZX Lm*?%>|GpFetm7Di literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/382f59c66d0ddb6747d3177263279789ca15c2db-5 b/tests/fuzzers/txfetcher/corpus/382f59c66d0ddb6747d3177263279789ca15c2db-5 new file mode 100644 index 0000000000000000000000000000000000000000..84441ac374622d3d0e6dcb6c31a11adb110de593 GIT binary patch literal 14 VcmZSCtH8S2`8Wdu!~VStr2rs*1dRXy literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/3a010483a4ad8d7215447ce27e0fac3791235c99-4 b/tests/fuzzers/txfetcher/corpus/3a010483a4ad8d7215447ce27e0fac3791235c99-4 new file mode 100644 index 0000000000..28f5d99b98 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/3a010483a4ad8d7215447ce27e0fac3791235c99-4 @@ -0,0 +1,7 @@ + +lGAGRzwwir7XvBOAy5tM/uV6e+Zf6anZzus1s1Y1ClbjbE6HXbnWWF/wbZGOpet +3Zm4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKwZatTXtdZrhu+Jk7hx0nTPy8Jcb +uJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+hgN4H +qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz8vp +jy4SHEg1AkEA/v13/5M47K9vCxmb8QeD/asydfsgS5TeuNi8DoUBEmiSJwma7FXY +fFUtxuvL7XvjwjN5 \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/3a3b717fcfe7ffb000b906e5a76f32248a576bf7-6 b/tests/fuzzers/txfetcher/corpus/3a3b717fcfe7ffb000b906e5a76f32248a576bf7-6 new file mode 100644 index 0000000000000000000000000000000000000000..022de3c61d4bfaa977cfd74616bc29c66c8a8ea3 GIT binary patch literal 1141 zcmc)IS5g&G5CBj|P|OiT6tjXNAZ8I2vx16ZCYuvR5HaiQhj0I;UKR7h7F>iiTd)tV z;gvPFPxnVp_tdTVHS=e(>i1MKHm4|M6%o@vlQ6+tRG<>`FdtP|fQ49u#aM!+Scc_T z5v)|K!fI5b25V4@wWz~7tVcaIU?VnRGqzwWwqZMVpaG58iCt*IZZx9>t=NM$?8QFp z#{nF~AsogLwBsl`a16(B0-ZRCQ#g$?IE!;Qj|;enE_CA(F5?P%a240ki|e?7o4AGB zxP!asLq7&Eh#?GP1ov)j7;^7kVj6m_hh9^u&+GEa^+4;>wl!_zth7@Gy09bpfvi z7cnX2@R_lyU9|*QJ5)>JHWpBo?ymdemqN0iiYN;UD2Hy}!*N}E=E}{_9`$PBK&^G> z+wjUC9HcS<2r^niC>w5yXs7K}vvJZMW2HC?6{oKanP04L9tSt6`O+3S%Gb%@B7`ms z)mkuR)oB|yV7e-Oo&YnTNLPrqq=Vd@f-9{3=?pr?;tIvPfy6B3coR4NuJ;z5It3}u zBNf#@NfF<7BvAhYVsO~#_AyLfO%aSo#8~EaotD^Xh=i1RptFSF#or0*#E3dG1D+px zAm_2rg;7|qqxu*Bi4I-xVWRTd4t$a$BeA)gP)S1V1Md>oZ2Bl)nj0(jo6*eD^3u?> cGZDY>5a6O3Gifo9tL^r<2)|FK(^2sKe=I1K@c;k- literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/3d8b5bf36c80d6f65802280039f85421f32b5055-6 b/tests/fuzzers/txfetcher/corpus/3d8b5bf36c80d6f65802280039f85421f32b5055-6 new file mode 100644 index 0000000000000000000000000000000000000000..eacd269f317b144dc5fdf3eb138007bbd6270192 GIT binary patch literal 23 ecmZShus5-wASW|9G1$Y=l(9H94mgL_|SBMWiSKDl$?WA&`JdB8U{Rp(r+p6h%Oy2ny<$9C-uY&Q4~C zSSU&nq$nJ40r3)e|9NuaHONe!@3-Gha{5E|h6G52J+K#&AQ|?-emDRra8LxTRF*@K2I-IinQ%BfKh>g7Xgk80 zEI10;kOR4JEW&Y?6OacdAs?8hpa2S?2#Vn}l)#yPwo*0+ltDQ-!37mi31{IPoQEod zn&pa-4;vk&^=84Q5=0qFG{Ucex{gdrG)SMVA}U=-fK zTX+Z2JMvyzD8Q*Lx2}Wv)`bFN{!k#KKB;lPFYsA@QD4v literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/608200b402488b3989ec8ec5f4190ccb537b8ea4-4 b/tests/fuzzers/txfetcher/corpus/608200b402488b3989ec8ec5f4190ccb537b8ea4-4 new file mode 100644 index 0000000000000000000000000000000000000000..d37b018515b8c82be6564a0e1f3e772bbefffd89 GIT binary patch literal 24 gcmdmXqMZHa8piS>hLqI2$;>Zw^z^o8mu^x30EF%e5&!@I literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/61e89c3fbdf9eff74bd250ea73cc2e61f8ca0d97-5 b/tests/fuzzers/txfetcher/corpus/61e89c3fbdf9eff74bd250ea73cc2e61f8ca0d97-5 new file mode 100644 index 0000000000..155744bccc --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/61e89c3fbdf9eff74bd250ea73cc2e61f8ca0d97-5 @@ -0,0 +1 @@ +88242871'392752200424491531672177074144720616417147514758635765020556616ďż˝ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/62817a48c78fbf2c12fcdc5ca58e2ca60c43543a-7 b/tests/fuzzers/txfetcher/corpus/62817a48c78fbf2c12fcdc5ca58e2ca60c43543a-7 new file mode 100644 index 0000000000000000000000000000000000000000..795608a789579add52324341abf22fac2b156d73 GIT binary patch literal 27 jcmZShus5;b|Ns9DiH1hHL5#(zDeoEgzGvIJ|6C~m!tV{; literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/6782da8f1a432a77306d60d2ac2470c35b98004f-3 b/tests/fuzzers/txfetcher/corpus/6782da8f1a432a77306d60d2ac2470c35b98004f-3 new file mode 100644 index 0000000000..f44949e6ae --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/6782da8f1a432a77306d60d2ac2470c35b98004f-3 @@ -0,0 +1 @@ +21888242871'392752200424452601091531672177074144720616417147514758635765020556616�� \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/68fb55290cb9d6da5b259017c34bcecf96c944aa-5 b/tests/fuzzers/txfetcher/corpus/68fb55290cb9d6da5b259017c34bcecf96c944aa-5 new file mode 100644 index 0000000000000000000000000000000000000000..23d905b827e2a5c4a04388da93af9bb5dafca84d GIT binary patch literal 49 zcmeZ>axzIaN)AZQ$xO{JOD!sJFAmQ#_G9pk42aTqvnY9J8kyr4WRc?HmY?a85*U~R E0HKW##sB~S literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/6a5059bc86872526241d21ab5dae9f0afd3b9ae1-3 b/tests/fuzzers/txfetcher/corpus/6a5059bc86872526241d21ab5dae9f0afd3b9ae1-3 new file mode 100644 index 0000000000..b71d5dff51 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/6a5059bc86872526241d21ab5dae9f0afd3b9ae1-3 @@ -0,0 +1 @@ +�� \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/717928e0e2d478c680c6409b173552ca98469ba5-6 b/tests/fuzzers/txfetcher/corpus/717928e0e2d478c680c6409b173552ca98469ba5-6 new file mode 100644 index 0000000000..dce5106115 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/717928e0e2d478c680c6409b173552ca98469ba5-6 @@ -0,0 +1 @@ +LvhaJcdaFofenogkjQfJB \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/71d22f25419543e437f249ca437823b87ac926b1-6 b/tests/fuzzers/txfetcher/corpus/71d22f25419543e437f249ca437823b87ac926b1-6 new file mode 100644 index 0000000000000000000000000000000000000000..d07a6c2f3244799a3c836c749bfc3fa614936172 GIT binary patch literal 27 jcmZShus5-wAV)X3?EU_|zKq4GDeoEgzGvIJ|6C~mw;2t$ literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/7312a0f31ae5d773ed4fd74abc7521eb14754683-8 b/tests/fuzzers/txfetcher/corpus/7312a0f31ae5d773ed4fd74abc7521eb14754683-8 new file mode 100644 index 0000000000..3593ce2e19 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/7312a0f31ae5d773ed4fd74abc7521eb14754683-8 @@ -0,0 +1,2 @@ +DtQvfQ+MULKZTXk78c +/fWkpxlyEQQ/+hgNzVtx9vWgJsafG7b0dA4AFjwVbFLmQcj2PprIMmPNQg1AkS5TDEmSJwFVlXsjLZ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/76e413a50dc8861e3756e556f796f1737bec2675-4 b/tests/fuzzers/txfetcher/corpus/76e413a50dc8861e3756e556f796f1737bec2675-4 new file mode 100644 index 0000000000000000000000000000000000000000..623fcf9601e516398a6bce4104435dc4556d99a2 GIT binary patch literal 24 gcmdmX;w}5lHH_s&3@NF3lbK)W=;>|GF5RR60FM$1hyVZp literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/78480977d5c07386b06e9b37f5c82f5ed86c2f09-3 b/tests/fuzzers/txfetcher/corpus/78480977d5c07386b06e9b37f5c82f5ed86c2f09-3 new file mode 100644 index 0000000000..e92863a1c7 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/78480977d5c07386b06e9b37f5c82f5ed86c2f09-3 @@ -0,0 +1,14 @@ + TESTING KEY----- +MIICXgIBAAKBgQDuLnQAI3mDgey3VBzWnB2L39JUU4txjeVE6myuDqkM/uGlfjb9 +SjY1bIw4iAJm2gsvvZhIrCHS3l6afab4pZB +l2+XsDulrKBxKKtD1rGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTvqJQIDAQAB +AoGAGRzwwir7XvBOAy5tM/uV6e+Zf6anZzus1s1Y1ClbjbE6HXbnWWF/wbZGOpet +3Zm4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKwZatTXtdZrhu+Jk7hx0nTPy8Jcb +uJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+hgN4H +qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz8vp +jy4SHEg1AkEA/v13/5M47K9vCxmb8QeD/asydfsgS5TeuNi8DoUBEmiSJwma7FXY +fFUtxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su4b3sjXNueLKH85Q+phy2U +fQtuUE9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xl/DoCzjA0CQQDU +y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYiZ6PYj013sovGKUFfYAqVXVlxtIX +qyUBnu3Xh9ps8ZfjLZO7BAkEAlT4R5Yl6cGhaJQYZHOde3JEMhNRcVFMO8dDaFeo +f9Oeos0UotgiDktdQHxdNEwLjQflJJBzV+5OtwswCA=----EN RATESTI EY-----Q \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/7a113cd3c178934cdb64353af86d51462d7080a4-5 b/tests/fuzzers/txfetcher/corpus/7a113cd3c178934cdb64353af86d51462d7080a4-5 new file mode 100644 index 0000000000..16818128ae --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/7a113cd3c178934cdb64353af86d51462d7080a4-5 @@ -0,0 +1,10 @@ +l6afab4pZB +l2+XsDlrKBxKKtDrGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTtqJQIDAQAB +AoGAGRzwwir7XvBOAy5tuV6ef6anZzus1s1Y1Clb6HbnWWF/wbZGOpet +3m4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKZTXtdZrh+k7hx0nTP8Jcb +uqFk541awmMogY/EfbWd6IOkp+4xqjlFBEDytgbIECQQDvH/6nk+hgN4H +qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz84SHEg1Ak/7KCxmD/sfgS5TeuNi8DoUBEmiSJwm7FX +ftxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su43sjXNueLKH8+ph2UfQuU9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xl/DoCzjA0CQQDU +y2pGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYiZ6PYj13sovGKUFfYAqVXVlxtIďż˝oďż˝X +qUn3X9ps8ZfjLZO7BAkEAlT4R5Yl6cGhaJQYZHOde3JMhNRcVFMO8dDaFo +f9Oeos0UotgiDktdQHxdNEwLjQlJBz+OtwwA=---E ATTIEY- \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/7ea9f71020f3eb783f743f744eba8d8ca4b2582f-3 b/tests/fuzzers/txfetcher/corpus/7ea9f71020f3eb783f743f744eba8d8ca4b2582f-3 new file mode 100644 index 0000000000..08f5bb99f5 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/7ea9f71020f3eb783f743f744eba8d8ca4b2582f-3 @@ -0,0 +1,9 @@ + +l2+DulrKBxKKtD1rGxlG4LjncdabFn9gvLZad2bSysqz/qTAUStTvqJQIDAQAB +AoGAGRzwwir7XvBOAy5tM/uV6e+Zf6anZzus1s1Y1ClbjbE6HXbnWWF/wbZGOpet +3Zm4vD6MXc7jpTLryzTQIvVdfQbRc6+MUVeLKwZatTXtdZrhu+Jk7hx0nTPy8Jcb +uJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+hgN4H +qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprIMmPcQrooz8vp +jy4SHEg1AkEA/v13/5M47K9vCxmb8QeD/asydfsgS5TeuNi8DoUBEmiSJwma7FXY +fFUtxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su4b3sjXNueLKH85Q+phy2U +fQtuUE9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xlp/DoCzjA0CQQDU diff --git a/tests/fuzzers/txfetcher/corpus/84f8c275f3ffbaf8c32c21782af13de10e7de28b-3 b/tests/fuzzers/txfetcher/corpus/84f8c275f3ffbaf8c32c21782af13de10e7de28b-3 new file mode 100644 index 0000000000..2d6060c406 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/84f8c275f3ffbaf8c32c21782af13de10e7de28b-3 @@ -0,0 +1 @@ +KKtDlbjVeLKwZatTXtdZrhu+Jk7hx0xxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLQcmPcQETT YQ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/85dfe7ddee0e52aa19115c0ebb9ed28a14e488c6-5 b/tests/fuzzers/txfetcher/corpus/85dfe7ddee0e52aa19115c0ebb9ed28a14e488c6-5 new file mode 100644 index 0000000000000000000000000000000000000000..9b6fe78029e7d3b85115abd651f2bbbff34bbca8 GIT binary patch literal 11 TcmZSh@Sbt+`~746aLQ)#y_k0wLP=ZpFp&S*+VI@{!HP)aKYf*)2 ztP8An*no|w!6t0R7Hq{fY)372pbk4xj|S|*ZtTHc?8AN>KqC&K35ReP&1k_99K|sl z#|fOoDV#T*eh##Wh^V4cx>n+(s|@a0mUki+i|_ z2Y84_7{DNgFpLq5VhoS*1W)k{&+!5;@d~f;25<2W?=g-Ee85LcVhYpvgwObbJig)^ zX7C-e_<^68i_$ox;+Uj`&^t|*N5o-@Nm7`%6iF6RmPBzDg%xrkRuWDjma4NtoTQ;D VPO=bj7Pc!8L%xIiR8uYqk6qWf{iHWl(kl6J{wndw<-Ac6{ii@NU(Cfa={Fu^7* zx5~U7WoYoN>G=4rzJ__}!e_>lb`TcUmZW}!^JJyuKV*IPCxbV1}L-4ze?foTaeL+vV(%Lwzs-QMz~ulq=q0<1aKfIzn}OCL)*E zwHhDtagof)VX{X>sq2AHQS+OUd+CF~Npf8|AP#20+Prve72t2gI`(y6w)oMK38h{* zlPAG5ekJ@PMkkt``&BaowT!_SMq#sKJtD><5W9tq>iLqC$V%>l2;K^4eC~OU8$Q$O zIJw>Pf;AMo&O+6`U-DEfdZN7Ei+(-@J+5Bz8|UtA&mCp6kXtr}A$~&ge8=0oN#MU9 Qxr+1k1is?=`qz);e`P)HIRF3v literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/8ff3bd49f93079e5e1c7f8f2461ba7ee612900c3-5 b/tests/fuzzers/txfetcher/corpus/8ff3bd49f93079e5e1c7f8f2461ba7ee612900c3-5 new file mode 100644 index 0000000000000000000000000000000000000000..a86a66593b4647dd930c1f2c9f07b668e0f6ee50 GIT binary patch literal 40 wcmZShus5-wASW|9G1$Y=)X+#bDA-Xqx$OP^y}pdasVVOn_r7P_yZ>A%07&f-YXATM literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/9034aaf45143996a2b14465c352ab0c6fa26b221-2 b/tests/fuzzers/txfetcher/corpus/9034aaf45143996a2b14465c352ab0c6fa26b221-2 new file mode 100644 index 0000000000..9c95a6ba6a --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/9034aaf45143996a2b14465c352ab0c6fa26b221-2 @@ -0,0 +1 @@ +ďż˝ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/92cefdc6251d04896349a464b29be03d6bb04c3d-2 b/tests/fuzzers/txfetcher/corpus/92cefdc6251d04896349a464b29be03d6bb04c3d-2 new file mode 100644 index 0000000000..9b78e45707 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/92cefdc6251d04896349a464b29be03d6bb04c3d-2 @@ -0,0 +1 @@ +ďż˝39402006196394479212279040100143613805079739270465446667948293404245721771496870329047266088258938001861606973112319�� \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/9613e580ccb69df7c9074f0e2f6886ac6b34ca55-5 b/tests/fuzzers/txfetcher/corpus/9613e580ccb69df7c9074f0e2f6886ac6b34ca55-5 new file mode 100644 index 0000000000000000000000000000000000000000..681adc6a9cd91ac14946e3aa426206220289f31b GIT binary patch literal 446 zcmXBPOOm26007YGEvvahFIj*R0hd7%0U>~jf;&M$5r$s^!Mn}f#;Lx*t9q`1Oj@Ki z+7tv>LdUtXE_=I3opv1cOkhs80*fn}YOg60iK8q}&o|ZfU#$puS60~l2=fmShUREV z>f?uczfY9&URi77x&LnlA2b||;hJx?RK7s~aXFn*w+^#1XnWhMgDmWu=BEgs9MBr8L1 z2}(1S%~YF#7ebpZ29_#A854GQEj<+5$1)P~kg^D^DNm~-TYyd>T4C)CaPb~Enax?> z?0T}yfqv$XQQMx+#g#e|{lH zMKOTPW!FgViBwLj=q}ecwxu-XnYBtgkQd+x-I*ha*f2hI~qI;_h1*kx0-P>Kt^Mwrk=>C!>J)7i) K4MF?hIPM>cYfrWS literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/a2684adccf16e036b051c12f283734fa803746e8-6 b/tests/fuzzers/txfetcher/corpus/a2684adccf16e036b051c12f283734fa803746e8-6 new file mode 100644 index 0000000000000000000000000000000000000000..4e12af2da8e9fc8fbb4f49a1b09323698fdaed26 GIT binary patch literal 25 hcmZShz~HTAUV7+0!`{6N3=S3+#-^6$7W?0q0swI!2@n7P literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/a37305974cf477ecfe65fa92f37b1f51dea25910-4 b/tests/fuzzers/txfetcher/corpus/a37305974cf477ecfe65fa92f37b1f51dea25910-4 new file mode 100644 index 0000000000000000000000000000000000000000..75cb14e8d98ea165366cc682e0b9b72d0510bd70 GIT binary patch literal 15 XcmZShu=hRV-uL_WGO+F4|GpFeM3o4v literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/a7eb43926bd14b1f62a66a33107776e487434d32-7 b/tests/fuzzers/txfetcher/corpus/a7eb43926bd14b1f62a66a33107776e487434d32-7 new file mode 100644 index 0000000000000000000000000000000000000000..88e6127355dd8492243d27d231165c9a7694c32f GIT binary patch literal 516 zcmWmBHBtlt6a-KPTio5<-QC^YT`oYfDF-4Zo7yTOuD~hw3d{@s>+bk}|Hfl~UqSpM zC7B4*qWk#_l3{9KCBk%>flS1Zg>2*?7kS7>0SZwRQLL1p6lEw!1u9X6YSf?>b*M)J z8qtJiw4fDjXh#P+(S>gGpcj4U#{dQ~gkg+e6k{021STAGX#{mv;gkzlG6lXZc1uk)gYuw-#ceuv`9`S@{yaeHE^y4@EJsAdx Ee|Mk`cK`qY literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/a8f7c254eb64a40fd2a77b79979c7bbdac6a760c-4 b/tests/fuzzers/txfetcher/corpus/a8f7c254eb64a40fd2a77b79979c7bbdac6a760c-4 new file mode 100644 index 0000000000..da61777c22 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/a8f7c254eb64a40fd2a77b79979c7bbdac6a760c-4 @@ -0,0 +1,2 @@ +lxtIX +qyU3X9ps8ZfjLZ45l6cGhaJQYZHOde3JEMhNRcVFMO8dJDaFe \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/a9a8f287d6af24e47d8db468e8f967aa44fb5a1f-7 b/tests/fuzzers/txfetcher/corpus/a9a8f287d6af24e47d8db468e8f967aa44fb5a1f-7 new file mode 100644 index 0000000000000000000000000000000000000000..7811921b79e98dab88b68f52779d560f6ef77ced GIT binary patch literal 106 zcmZQkU|sE8oX8UPv~4=`)3zndPj>eJ=`}#SmHBBq5O*{)Kb|}vNbhET+Svxg+nFCv znacdMyB&x-nVC5l7+At{Qxy`674q`)bc<5ca#E8^6x7P}(lS$XQjRl(rymq!V2A<$ D%DO9) literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/aa7444d8e326158046862590a0db993c07aef372-7 b/tests/fuzzers/txfetcher/corpus/aa7444d8e326158046862590a0db993c07aef372-7 new file mode 100644 index 0000000000..870e12ffbc --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/aa7444d8e326158046862590a0db993c07aef372-7 @@ -0,0 +1 @@ +00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000@0000000000000 \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/ae4593626d8796e079a358c2395a4f6c9ddd6a44-6 b/tests/fuzzers/txfetcher/corpus/ae4593626d8796e079a358c2395a4f6c9ddd6a44-6 new file mode 100644 index 0000000000..845deedd0e --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/ae4593626d8796e079a358c2395a4f6c9ddd6a44-6 @@ -0,0 +1,8 @@ +9pmM gY/xEcfbWd6IOkp+4xqjlFLBEDytgbparsing /E6nk+hgN4H +qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLmQcj2PprLANGcQrooz8vp +jy4SHEg1AkEA/v13/@M47K9vCxb8QeD/asydfsgS5TeuNi8DoUBEmiSJwma7FXY +fFUtxuvL7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su4b3sjXNueLKH85Q+phy2U +fQtuUE9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xl/DoCzďż˝ jA0CQQDU +y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYiZ6Yj013sovGKUFfYAqVXVlxtIX +qyUBnu3Xh9ps8ZfjLZO7BAkEAlT4R5Yl6cGhaJQYFZHOde3JEMhNRcVFMO8dDaFeo +f9Oeos0Uot \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/b2942d4413a66939cda7db93020dee79eb17788c-9 b/tests/fuzzers/txfetcher/corpus/b2942d4413a66939cda7db93020dee79eb17788c-9 new file mode 100644 index 0000000000000000000000000000000000000000..10aca6512180bfd927400162598662c4589b3c52 GIT binary patch literal 18 ZcmZQ**qfYaXrvp&Se%;5$+-7D8vr(g1>OJv literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/b4614117cdfd147d38f4e8a4d85f5a2bb99a6a4f-5 b/tests/fuzzers/txfetcher/corpus/b4614117cdfd147d38f4e8a4d85f5a2bb99a6a4f-5 new file mode 100644 index 0000000000000000000000000000000000000000..af69eef9b08634dc5bf91be2dc262c5e7a52f334 GIT binary patch literal 838 zcmXZbNv^9#0EJ;^7)Duw=jaJzFxZ1gDNJKp8yhe%qv-51Jw-dp#-g{Rw}spr<*eZ7 zm%gndJsj0iwz?Gj_#R=Z>embv=IO$C!uqU)y}_I`*JbgHQcvQ_ z&EK2N$D{_O$Phdc4;0W)o!B?f%6;CCWB5|L^X0)5T$DwSZ9;)4_N3p7L(%ENtm{~$ zJ7q2u)<9L9L{P3y`DlbuvWJhQZ&#{yfcwDC*lM;a0viYfs1=~x*J*lpygv$8upIE$ z1&>QNo6R3l6|O^1_jEv?3y7e^U3^$QpE5PUP`NfKCYZ2fN0BrAm`9Ccbj#}4>gXkz z4ev2Wny!}``V>ak@o^G%uYuG){A=D-&TTnrF(d|~l|KiTj--h!&21h~vlM6WnQ-d$ z(~#`kqCz$VNnCIao2cvGRlcPpuKJL#++r_8yRIId=Z-n?{+A-SSn$7^065Cx}ug9?=(84}0Qci=91gMc_sJ~2>*H7blhDSRUsiabvbVNF{TfQt8Y zKd_MLk#7xOO)kfSO*XG(?-H4f2Uj}b85Yh!vn^xs`Q;u8Y-qmZU=f$&S?{cUe9I|$ zT=32>exto2-7rv7e4tqZY1s5x?NV5Yuyj+u% z(IoR_-D6KSug64;1O8om=DlEq-ufU*^P;CIi`7p=rYEk%udu`sob*{1zfv N+dKG=P5r+a`(NZO69fPN literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/b631ef3291fa405cd6517d11f4d1b9b6d02912d4-2 b/tests/fuzzers/txfetcher/corpus/b631ef3291fa405cd6517d11f4d1b9b6d02912d4-2 new file mode 100644 index 0000000000..a6b8858b40 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/b631ef3291fa405cd6517d11f4d1b9b6d02912d4-2 @@ -0,0 +1 @@ +&ďż˝oďż˝ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/b7a91e338cc11f50ebdb2c414610efc4d5be3137-4 b/tests/fuzzers/txfetcher/corpus/b7a91e338cc11f50ebdb2c414610efc4d5be3137-4 new file mode 100644 index 0000000000000000000000000000000000000000..9709a1fcb82b88b52db14626b9d8855646ebbd96 GIT binary patch literal 34 ncmV~$!2tju2m(O2@f(O}`wv!kyA*ejbvZ;A9Pth;45V^@e8~qu literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/b858cb282617fb0956d960215c8e84d1ccf909c6-2 b/tests/fuzzers/txfetcher/corpus/b858cb282617fb0956d960215c8e84d1ccf909c6-2 new file mode 100644 index 0000000000..0519ecba6e --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/b858cb282617fb0956d960215c8e84d1ccf909c6-2 @@ -0,0 +1 @@ + \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/bc9d570aacf3acd39600feda8e72a293a4667da4-1 b/tests/fuzzers/txfetcher/corpus/bc9d570aacf3acd39600feda8e72a293a4667da4-1 new file mode 100644 index 0000000000..aab27c5909 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/bc9d570aacf3acd39600feda8e72a293a4667da4-1 @@ -0,0 +1 @@ +ďż˝ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/be7eed35b245b5d5d2adcdb4c67f07794eb86b24-3 b/tests/fuzzers/txfetcher/corpus/be7eed35b245b5d5d2adcdb4c67f07794eb86b24-3 new file mode 100644 index 0000000000..47c996d33f --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/be7eed35b245b5d5d2adcdb4c67f07794eb86b24-3 @@ -0,0 +1,2 @@ +4LZmbRc6+MUVeLKXtdZr+Jk7hhgN4H +qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLQcmPcQ SN_ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/c010b0cd70c7edbc5bd332fc9e2e91c6a1cbcdc4-5 b/tests/fuzzers/txfetcher/corpus/c010b0cd70c7edbc5bd332fc9e2e91c6a1cbcdc4-5 new file mode 100644 index 0000000000..474f14d89b --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/c010b0cd70c7edbc5bd332fc9e2e91c6a1cbcdc4-5 @@ -0,0 +1,4 @@ + +Xc7jpTLryzTQIvVdfQbRc6+MUVeLKwZatTXtdZrhu+Jk7hx0nTPy8Jcb +uJqFk541aEw+mMogY/xEcfbWd6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nhgN4H +qzzVtxx7vWrjrIgPbJpvfb \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/c1690698607eb0f4c4244e9f9629968be4beb6bc-8 b/tests/fuzzers/txfetcher/corpus/c1690698607eb0f4c4244e9f9629968be4beb6bc-8 new file mode 100644 index 0000000000..d184a2d8a4 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/c1690698607eb0f4c4244e9f9629968be4beb6bc-8 @@ -0,0 +1,2 @@ +&Ƚ�� ďż˝ +ďż˝ ďż˝ ďż˝ ��������������������� ďż˝!ďż˝"ďż˝#ďż˝$ďż˝%ďż˝&ďż˝'ďż˝(ďż˝)ďż˝*ďż˝+ďż˝,ďż˝-ďż˝.ďż˝/��0 \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/c1f435e4f53a9a17578d9e8c4789860f962a1379-6 b/tests/fuzzers/txfetcher/corpus/c1f435e4f53a9a17578d9e8c4789860f962a1379-6 new file mode 100644 index 0000000000000000000000000000000000000000..f2a68ec3de94cc9015080f2c057f1a707aff0269 GIT binary patch literal 1889 zcmeIz>sCx*9LMn)iAW_1MM#BCqMDk?p&1=?PDfE_bRfbcA!*gT>IU4-tk!BG6iG^o zyyybPCG>mx?z^b9_V@o;`+2q3vuB=d`ycT7LX&|oH8pP6w41TZj4(5Hn-Olt9y9iu z5n;wYGxjT8k#GQ_AR1!eAjHBUI1F(R4@clA9D@W%gyV1ml0u7ft%0ovImyZ?I1R~g z22vmu(%>wdgY%FM8ITECkPR0g2QI=T$b~$}2L}{DAvnPWMQ|Cez*V>g#oz`HT!#`U zg)%6I8&Cn2PzBXc1GP{G^>7m!pb?s&8CswfZb2K|hC9#>9dH-!!F}iiFFb%Q=!PEX zg@^D6`rt7bqJ{Kh&c7sg~5T`lVLXZ?&q{)F1U% zt*d`(d+-0o`-y7n@QP12=SZ>792M3l#FLQB_Ly&*b@rEAXMx*hv-$e_y1Y#l9*2Ff Xr#lqLF35Lj)$MfVIb4O#$__5uJO4h75r literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/c4cdbb891f3ee76476b7375d5ed51691fed95421-10 b/tests/fuzzers/txfetcher/corpus/c4cdbb891f3ee76476b7375d5ed51691fed95421-10 new file mode 100644 index 0000000000000000000000000000000000000000..e365cc52623e848cc2ab0c87573b6653e95477e4 GIT binary patch literal 16 XcmZQ**lTE{8^lQPAnG-^4J!*)#!PJ7f|ad}aSP5n z|K1JAnRA}^tiSo5-oO2tj|bWBm404d)ZztiWuXizP(v7mLj-6b5~3g)v=9Ta5C`#) z0Ev(U$)JN2NQE@eLpo$YCS*Z2VFX5D48~ysCSeMu zVFqSl4(4G27GVjNVFgxU4c1`;oZx~@*n(}?fn9LJ9_)h$4&V@uzzfH40zNneKb*li zTtEOW;R>$dMt-{!2^pHeKb?TsesQfzbW%Y1*i1?lwIWu%-6*1x{yq+0@KbWSJooA^ Mv&Hi4_B|SY0k_#*-v9sr literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/cd1d73b4e101bc7b979e3f6f135cb12d4594d348-5 b/tests/fuzzers/txfetcher/corpus/cd1d73b4e101bc7b979e3f6f135cb12d4594d348-5 new file mode 100644 index 0000000000..3079de5557 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/cd1d73b4e101bc7b979e3f6f135cb12d4594d348-5 @@ -0,0 +1 @@ +822452601031714757585602556 \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/d0acdc8fca32bbd58d368eeac3bd9eaa46f59d27-5 b/tests/fuzzers/txfetcher/corpus/d0acdc8fca32bbd58d368eeac3bd9eaa46f59d27-5 new file mode 100644 index 0000000000000000000000000000000000000000..794d5d86c6a1e18b942a16765778be42f4f583fb GIT binary patch literal 9 QcmZShu$O^t@Ba6t02EdPlmGw# literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/d0e43b715fd00953f7bdd6dfad95811985e81396-4 b/tests/fuzzers/txfetcher/corpus/d0e43b715fd00953f7bdd6dfad95811985e81396-4 new file mode 100644 index 0000000000000000000000000000000000000000..742db5fb3ba97f3f55edce2971d6eabd209d96ef GIT binary patch literal 10 PcmZSC`+h$NF_Z!TB1r}f literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/d925fbd22c8bc0de34d6a9d1258ce3d2928d0927-8 b/tests/fuzzers/txfetcher/corpus/d925fbd22c8bc0de34d6a9d1258ce3d2928d0927-8 new file mode 100644 index 0000000000000000000000000000000000000000..5920dfe6012899e440c8de696128fefd9265fe4e GIT binary patch literal 22 ccmZSS*vmFwS0BO(&rvLx| literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/d9ba78cb7425724185d5fa300cd5c03aec2683bb-7 b/tests/fuzzers/txfetcher/corpus/d9ba78cb7425724185d5fa300cd5c03aec2683bb-7 new file mode 100644 index 0000000000000000000000000000000000000000..c4df1cf210ebbfadf9ae2b676c3950c61f099bd7 GIT binary patch literal 510 zcmWO2OM+uS007WlTg?%AZy`P*c2nXLf<%H~rw9r0uR^(8A8D>I{f_WtJ4thnOvA={bOtbP+p@ezDVmiFAbaF6I24~>UtfRHHrepRPHfB`Kd^$Dc#2^A zc<9Um6XVi95ydBsFsq^yWm47q_DDA0Gv_cbJb24Q+6ls^eO{6j`4P&K@r7xtmR{?w z^{?b$Wst)5px1kWO|^w?obNJbRxK;wBNxp}-P6K4R3F_|G?kzd_USmi+wz;y#I~kO zS2$6_wuNY2M_!iZ$5@SKd4*Q*&B~M(h6Hyx5P)7SA=K^0%PQ{Lk2Y|P%p1FIJIp;* z5#fH)N*2@_U}lzQ{yKtmmgnON58hK{(uw57RNnXg6T9z&+69W%yzkc1q>>)RRG|lG z#k2kUS1pyhSNL7;YA_Q`Oprtb$Fme*B95J~0QsqkT>M*dI_y2Ve-SRA3J>8xlhF!4 zK4yi1cct=lOB6j;$&t$#eU2ElJO2CG9Rsd6a2Y&!8OIHC7U|nN-tkix6d;Vk4>F0; Q@{!z6hbp;93aG*7KfceZ2LJ#7 literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/da39a3ee5e6b4b0d3255bfef95601890afd80709 b/tests/fuzzers/txfetcher/corpus/da39a3ee5e6b4b0d3255bfef95601890afd80709 new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tests/fuzzers/txfetcher/corpus/dcdb7758b87648b5d766b1b341a65834420cf621-7 b/tests/fuzzers/txfetcher/corpus/dcdb7758b87648b5d766b1b341a65834420cf621-7 new file mode 100644 index 0000000000000000000000000000000000000000..78cf11ae2170686d1cd83f0d1b68c1a83552f9cb GIT binary patch literal 22 ecmZShz~Eha=sv^$|L^zjWiU2nU|?{tumAvdh6vLD literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/dd441bd24581332c9ce19e008260a69287aa3cbc-6 b/tests/fuzzers/txfetcher/corpus/dd441bd24581332c9ce19e008260a69287aa3cbc-6 new file mode 100644 index 0000000000..4e0c14006e --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/dd441bd24581332c9ce19e008260a69287aa3cbc-6 @@ -0,0 +1,2 @@ +Dtf1nWk78c +/fWklyEQQ/+hgNzVtxxmDgS5TDETgeXVlXsjLZ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/def879fe0fd637a745c00c8f1da340518db8688c-2 b/tests/fuzzers/txfetcher/corpus/def879fe0fd637a745c00c8f1da340518db8688c-2 new file mode 100644 index 0000000000..555752f0ed --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/def879fe0fd637a745c00c8f1da340518db8688c-2 @@ -0,0 +1 @@ +ďż˝ ďż˝ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/df6c30a9781b93bd6d2f5e97e5592d5945210003-7 b/tests/fuzzers/txfetcher/corpus/df6c30a9781b93bd6d2f5e97e5592d5945210003-7 new file mode 100644 index 0000000000000000000000000000000000000000..2a7adb093bcf95dd675fd92635bfc1c3c9f4f75f GIT binary patch literal 1881 zcmeIz>sCy07{~D$Nks@HDLOc|5~-$UoSG4dP>O^cMx}ENDHKcdsvB@Sd(B!6IdxJx z=)j9EV0Q`q{_~x?&|33-KKt2g=FQ$SdmQVQKVXNf7&SWV)O4D$#f%s;wwe)Z#x^tJ z%!oH5!Hn%nuN{yGJ7E{>h9uYndto0WLkjE{AuE;T0Hnb|NQVqKWY3P*>l4}zb0!nA zARBVv2po-YjO93-fRm65%se;+`EVM}KmnYELMVb_a6t)_f*U+=4$i{`xCob^%wTz0 zd~g}c;R;-Z3b+Q9a2;+y72Jeda2u+j2JS#D)WKc22la42az`3i9>7C*1dWk*JZ5+;%FX0vR!E5M;0eAy%;T;UZ5WI&E z5WOQGwS|M6+H~_em~UM;I652-hSg{FMUAPi>YMtmeyE>nTurD+HKnH2FEyiP)o(SY z=GB5)RDaZxT2?D+RsB`})c<|`*FKF@MIT%#x#lra7UuJ$H>U8T?E>}zSZt^AT=kIs7C?jo0`)E%<>Hvsh1Ntyrv literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/dfc1c3a2e3ccdaf6f88c515fd00e8ad08421e431-6 b/tests/fuzzers/txfetcher/corpus/dfc1c3a2e3ccdaf6f88c515fd00e8ad08421e431-6 new file mode 100644 index 0000000000000000000000000000000000000000..59f3442c053c1cbb924c8c024d5eb054e4a74e3c GIT binary patch literal 10 RcmZQkU|sEeoPmL1KL89{0-gW> literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/e1dcc4e7ead6dfd1139ece7bf57d776cb9dac72d-7 b/tests/fuzzers/txfetcher/corpus/e1dcc4e7ead6dfd1139ece7bf57d776cb9dac72d-7 new file mode 100644 index 0000000000000000000000000000000000000000..5ba489f99ddd2944bc6ad575d0f6072858f7e209 GIT binary patch literal 20 ccmZShus5+lH;Az~HRV0y-uG;K_n#{T09*nIoB#j- literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/e39c2de2c8937d2cbd4339b13d6a0ce94d94f8d2-8 b/tests/fuzzers/txfetcher/corpus/e39c2de2c8937d2cbd4339b13d6a0ce94d94f8d2-8 new file mode 100644 index 0000000000000000000000000000000000000000..0e9508938e4ffbe154a5365977bf7aa023fa99c9 GIT binary patch literal 1275 zcmeIwNlF7z6ouhDNu_kI5eESWf+9Fj5J5yx2nl5y`)KTqCb2D>*kC(x;tHG!E~5zO zz%@8?0YYuT^Ybt>S76}Y@7(vQrKo6YH1>9H!b+fuDq|*aQ!G3-$4oe`n3${t850Im z>`kj@7-nG(<{=Azh5=lX*1jFXowHkl{;vY<&Z0yy6WtX zXCo4^SvA#@*F2IDeoMF0uPjy%YWV90pRdh2+K>=Vg2%P1Z6$NxkIx$GlOo<|(q82E hv0pF2@Pc;B>OQrq)FFH|udDia@=|tcOWV}dl^?;h0WSam literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/e72f76b9579c792e545d02fe405d9186f0d6c39b-6 b/tests/fuzzers/txfetcher/corpus/e72f76b9579c792e545d02fe405d9186f0d6c39b-6 new file mode 100644 index 0000000000000000000000000000000000000000..c4d34b1732a239d78e8e6f9e7700ed4b90db1110 GIT binary patch literal 47 zcmZShu=hRV-uL_WGO(qVWay`*mLzASCYNO9=jkUEu2ZXm*?%>|GpFe D|5z1S literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/eb70814d6355a4498b8f301ba8dbc34f895a9947-5 b/tests/fuzzers/txfetcher/corpus/eb70814d6355a4498b8f301ba8dbc34f895a9947-5 new file mode 100644 index 0000000000000000000000000000000000000000..bd57a22fb1e19cda19fcc5ebd923af9d758e9ed0 GIT binary patch literal 57 zcmd0(C`)zG_j2?!GE7SK@h-?P;z|oFDGhbCEU8G!2`Mc!_A^iS2#RttiSl)etaPc! MGS@a`WME(b03AaSMF0Q* literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/ebdc17efe343e412634dca57cecd5a0e1ce1c1c7-5 b/tests/fuzzers/txfetcher/corpus/ebdc17efe343e412634dca57cecd5a0e1ce1c1c7-5 new file mode 100644 index 0000000000000000000000000000000000000000..aaa3f695ab36ddb550af56dbb03615218a420336 GIT binary patch literal 54 zcmd0(C`)zG_j2?!GE7SK@h-?P;z|oFDGhbCEU8G!2`Mc!_A^iS2#RttiSl)etaPc! JGS_Be008Lz5fK0Y literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/ec0a25eba8966b8f628d821b3cfbdf2dfd4bbb4c-3 b/tests/fuzzers/txfetcher/corpus/ec0a25eba8966b8f628d821b3cfbdf2dfd4bbb4c-3 new file mode 100644 index 0000000000..65cf0df801 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/ec0a25eba8966b8f628d821b3cfbdf2dfd4bbb4c-3 @@ -0,0 +1,13 @@ +ďż˝&^��o�ȗ-----BEGIN RSA TESTING KEY----- +MIICXgIBAAKBgQDuLnQAI3mDgey3VBzWnB2L39JUU4txjeVE6myuDqkM/uGlfjb9 +SjY1bIw4iA5sBBZzHi3z0h1YV8PuxEbi4nW91IJm2gsvvZhIrHS3l6afab4pZB +l2+XsDulrKBxKKtD1rGxlG4Ljncdabn9vLZad2bSysqz/qTAUStvqJQIDAQAB +AoGAGRzwwir7XvBOAy5tM/uV6e+Zf6anZzus1s1K1ClbjbE6HXbnWWF/wbZGOpet +3Zm4vD6MXc7jpTLryzQIvVdfQbRc6+MUVeLKwZatTXtZru+Jk7hx0nTPy8Jcb +uJqFk541aEw+mMogY/xEcfbW6IOkp+4xqjlFLBEDytgbIECQQDvH/E6nk+hg4 +qzzVtxxr397vWrjrIgPbJpQvBsafG7b0dA4AFjwVbFLcj2pIMPQroozvjg1AkEA/v13/5M47K9vCxmb8QeD/aydfsgS5TeuNi8DoUBEmiSJwmaXY +fFUtxv7XvjwjN5B30pNEbc6Iuyt7y4MQJBAIt21su4bjeLKH8Q+ph2 +fQtuUE9txblTu14q3N7gHRZB4ZMhFYyDy8CKrN2cPg/Fvyt0Xlp/DoCzjA0CQQDU +y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6+AYiZ6PYj013sovGKFYqVXVlxtIX +qyUBnu3X9s8ZfjZO7BAkl4R5Yl6cGhaJQYZHOe3JEMhVFaFf9Oes0UUothgiDktdQxdNLj7+5CWA== +-----END RSASQ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/eebe3b76aeba6deed965d17d2b024f7eae1a43f1-5 b/tests/fuzzers/txfetcher/corpus/eebe3b76aeba6deed965d17d2b024f7eae1a43f1-5 new file mode 100644 index 0000000000000000000000000000000000000000..20d62e15b32dce93e2c88cc2c140831d5f22e7b0 GIT binary patch literal 10 Scmeyc_dVm@_xtxUumJ!i;04S8 literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/ef8741a9faf030794d98ff113f556c68a24719a5-6 b/tests/fuzzers/txfetcher/corpus/ef8741a9faf030794d98ff113f556c68a24719a5-6 new file mode 100644 index 0000000000000000000000000000000000000000..09fcd86d77c210e105c543a4eefdbd18e2b25612 GIT binary patch literal 31 lcmZShpkiKn=>Grz@AvOzXxpo6VPRox>Hs1v%`Nu72LSSb4Y>dS literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/efb7410d02418befeba25a43d676cc6124129125-4 b/tests/fuzzers/txfetcher/corpus/efb7410d02418befeba25a43d676cc6124129125-4 new file mode 100644 index 0000000000..2191a7324a --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/efb7410d02418befeba25a43d676cc6124129125-4 @@ -0,0 +1 @@ +88242871'392752200424452601091531672177074144720616417147514758635765020556616ďż˝ \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/f6f97d781a5a749903790e07db8619866cb7c3a1-6 b/tests/fuzzers/txfetcher/corpus/f6f97d781a5a749903790e07db8619866cb7c3a1-6 new file mode 100644 index 0000000000000000000000000000000000000000..219a8d3682f5e40e4c1406ecc1b95909f680975d GIT binary patch literal 22 dcmZShus5-wATc;7*ikpR?EU_|zKq4G?*VUO3J(AP literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/f7a3cd00fa0e57742e7dbbb8283dcaea067eaf7b-5 b/tests/fuzzers/txfetcher/corpus/f7a3cd00fa0e57742e7dbbb8283dcaea067eaf7b-5 new file mode 100644 index 0000000000..f01ccd89ef --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/f7a3cd00fa0e57742e7dbbb8283dcaea067eaf7b-5 @@ -0,0 +1,2 @@ +Xyt0Xl/DoCzjA0CQQDU +y2ptGsuSmgUtWj3NM9xuwYPm+Z/F84K6+ARYi \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/f94d60a6c556ce485ab60088291760b8be25776c-6 b/tests/fuzzers/txfetcher/corpus/f94d60a6c556ce485ab60088291760b8be25776c-6 new file mode 100644 index 0000000000..58d841ff03 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/f94d60a6c556ce485ab60088291760b8be25776c-6 @@ -0,0 +1,2 @@ +HZB4cQZde3JMNRcVFMO8dDFo +f9OeosiDdQQl \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/f9e627b2cb82ffa1ea5e0c6d7f2802f3000b18a8-6 b/tests/fuzzers/txfetcher/corpus/f9e627b2cb82ffa1ea5e0c6d7f2802f3000b18a8-6 new file mode 100644 index 0000000000000000000000000000000000000000..b5dfecc1e9d1cf74363f1eb9213f1fecebdc11eb GIT binary patch literal 11 ScmZSC%evb6I0FO2{=EPd*aP?g literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/corpus/fb3775aa24e5667e658920c05ba4b7b19ff256fb-5 b/tests/fuzzers/txfetcher/corpus/fb3775aa24e5667e658920c05ba4b7b19ff256fb-5 new file mode 100644 index 0000000000..6f4927d822 --- /dev/null +++ b/tests/fuzzers/txfetcher/corpus/fb3775aa24e5667e658920c05ba4b7b19ff256fb-5 @@ -0,0 +1 @@ +HZB4c2cPclieoverpGsumgUtWj3NMYPZ/F8tďż˝5YlNR8dDFoiDdQQl \ No newline at end of file diff --git a/tests/fuzzers/txfetcher/corpus/fd6386548e119a50db96b2fa406e54924c45a2d5-6 b/tests/fuzzers/txfetcher/corpus/fd6386548e119a50db96b2fa406e54924c45a2d5-6 new file mode 100644 index 0000000000000000000000000000000000000000..6fff60edd4f0d637706fae63e784667804e3d071 GIT binary patch literal 28 kcmZShus7Jl(bUi=DA-Xqx$OP^y}pdasVVOn_r7NX0G`(ir2qf` literal 0 HcmV?d00001 diff --git a/tests/fuzzers/txfetcher/txfetcher_fuzzer.go b/tests/fuzzers/txfetcher/txfetcher_fuzzer.go new file mode 100644 index 0000000000..10c7eb9424 --- /dev/null +++ b/tests/fuzzers/txfetcher/txfetcher_fuzzer.go @@ -0,0 +1,199 @@ +// Copyright 2020 The go-ethereum Authors +// This file is part of the go-ethereum library. +// +// The go-ethereum library is free software: you can redistribute it and/or modify +// it under the terms of the GNU Lesser General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// The go-ethereum library is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Lesser General Public License for more details. +// +// You should have received a copy of the GNU Lesser General Public License +// along with the go-ethereum library. If not, see . + +package txfetcher + +import ( + "bytes" + "fmt" + "math/big" + "math/rand" + "time" + + "github.com/ethereum/go-ethereum/common" + "github.com/ethereum/go-ethereum/common/mclock" + "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/fetcher" +) + +var ( + peers []string + txs []*types.Transaction +) + +func init() { + // Random is nice, but we need it deterministic + rand := rand.New(rand.NewSource(0x3a29)) + + peers = make([]string, 10) + for i := 0; i < len(peers); i++ { + peers[i] = fmt.Sprintf("Peer #%d", i) + } + txs = make([]*types.Transaction, 65536) // We need to bump enough to hit all the limits + for i := 0; i < len(txs); i++ { + txs[i] = types.NewTransaction(rand.Uint64(), common.Address{byte(rand.Intn(256))}, new(big.Int), 0, new(big.Int), nil) + } +} + +func Fuzz(input []byte) int { + // Don't generate insanely large test cases, not much value in them + if len(input) > 16*1024 { + return -1 + } + r := bytes.NewReader(input) + + // Reduce the problem space for certain fuzz runs. Small tx space is better + // for testing clashes and in general the fetcher, but we should still run + // some tests with large spaces to hit potential issues on limits. + limit, err := r.ReadByte() + if err != nil { + return 0 + } + switch limit % 4 { + case 0: + txs = txs[:4] + case 1: + txs = txs[:256] + case 2: + txs = txs[:4096] + case 3: + // Full run + } + // Create a fetcher and hook into it's simulated fields + clock := new(mclock.Simulated) + rand := rand.New(rand.NewSource(0x3a29)) // Same used in package tests!!! + + f := fetcher.NewTxFetcherForTests( + func(common.Hash) bool { return false }, + func(txs []*types.Transaction) []error { + return make([]error, len(txs)) + }, + func(string, []common.Hash) error { return nil }, + clock, rand, + ) + f.Start() + defer f.Stop() + + // Try to throw random junk at the fetcher + for { + // Read the next command and abort if we're done + cmd, err := r.ReadByte() + if err != nil { + return 0 + } + switch cmd % 4 { + case 0: + // Notify a new set of transactions: + // Byte 1: Peer index to announce with + // Byte 2: Number of hashes to announce + // Byte 3-4, 5-6, etc: Transaction indices (2 byte) to announce + peerIdx, err := r.ReadByte() + if err != nil { + return 0 + } + peer := peers[int(peerIdx)%len(peers)] + + announceCnt, err := r.ReadByte() + if err != nil { + return 0 + } + announce := int(announceCnt) % (2 * len(txs)) // No point in generating too many duplicates + + var ( + announceIdxs = make([]int, announce) + announces = make([]common.Hash, announce) + ) + for i := 0; i < len(announces); i++ { + annBuf := make([]byte, 2) + if n, err := r.Read(annBuf); err != nil || n != 2 { + return 0 + } + announceIdxs[i] = (int(annBuf[0])*256 + int(annBuf[1])) % len(txs) + announces[i] = txs[announceIdxs[i]].Hash() + } + fmt.Println("Notify", peer, announceIdxs) + if err := f.Notify(peer, announces); err != nil { + panic(err) + } + + case 1: + // Deliver a new set of transactions: + // Byte 1: Peer index to announce with + // Byte 2: Number of hashes to announce + // Byte 3-4, 5-6, etc: Transaction indices (2 byte) to announce + peerIdx, err := r.ReadByte() + if err != nil { + return 0 + } + peer := peers[int(peerIdx)%len(peers)] + + deliverCnt, err := r.ReadByte() + if err != nil { + return 0 + } + deliver := int(deliverCnt) % (2 * len(txs)) // No point in generating too many duplicates + + var ( + deliverIdxs = make([]int, deliver) + deliveries = make([]*types.Transaction, deliver) + ) + for i := 0; i < len(deliveries); i++ { + deliverBuf := make([]byte, 2) + if n, err := r.Read(deliverBuf); err != nil || n != 2 { + return 0 + } + deliverIdxs[i] = (int(deliverBuf[0])*256 + int(deliverBuf[1])) % len(txs) + deliveries[i] = txs[deliverIdxs[i]] + } + directFlag, err := r.ReadByte() + if err != nil { + return 0 + } + direct := (directFlag % 2) == 0 + + fmt.Println("Enqueue", peer, deliverIdxs, direct) + if err := f.Enqueue(peer, deliveries, direct); err != nil { + panic(err) + } + + case 2: + // Drop a peer: + // Byte 1: Peer index to drop + peerIdx, err := r.ReadByte() + if err != nil { + return 0 + } + peer := peers[int(peerIdx)%len(peers)] + + fmt.Println("Drop", peer) + if err := f.Drop(peer); err != nil { + panic(err) + } + + case 3: + // Move the simulated clock forward + // Byte 1: 100ms increment to move forward + tickCnt, err := r.ReadByte() + if err != nil { + return 0 + } + tick := time.Duration(tickCnt) * 100 * time.Millisecond + + fmt.Println("Sleep", tick) + clock.Run(tick) + } + } +} From 38d1b0cba277db17129a4870158115c5a428cffc Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Thu, 13 Feb 2020 14:38:30 +0100 Subject: [PATCH 108/128] cmd/geth: enable DNS discovery by default (#20660) * node: expose config in service context * eth: integrate p2p/dnsdisc * cmd/geth: add some DNS flags * eth: remove DNS URLs * cmd/utils: configure DNS names for testnets * params: update DNS URLs * cmd/geth: configure mainnet DNS * cmd/utils: rename DNS flag and fix flag processing * cmd/utils: remove debug print * node: fix test --- cmd/geth/main.go | 1 + cmd/geth/usage.go | 1 + cmd/utils/flags.go | 36 ++++++++++++++++++++++++++---- eth/backend.go | 8 +++++++ eth/config.go | 8 +++++-- eth/{enr_entry.go => discovery.go} | 12 ++++++++++ eth/gen_config.go | 18 +++++++++++++++ node/node.go | 2 +- node/service.go | 20 ++++++++--------- node/service_test.go | 4 ++-- params/bootnodes.go | 13 +++++++++++ 11 files changed, 104 insertions(+), 19 deletions(-) rename eth/{enr_entry.go => discovery.go} (79%) diff --git a/cmd/geth/main.go b/cmd/geth/main.go index 0369e527b5..99ef78238f 100644 --- a/cmd/geth/main.go +++ b/cmd/geth/main.go @@ -131,6 +131,7 @@ var ( utils.NetrestrictFlag, utils.NodeKeyFileFlag, utils.NodeKeyHexFlag, + utils.DNSDiscoveryFlag, utils.DeveloperFlag, utils.DeveloperPeriodFlag, utils.TestnetFlag, diff --git a/cmd/geth/usage.go b/cmd/geth/usage.go index b3b6b5f93d..6f3197b9c6 100644 --- a/cmd/geth/usage.go +++ b/cmd/geth/usage.go @@ -182,6 +182,7 @@ var AppHelpFlagGroups = []flagGroup{ utils.BootnodesFlag, utils.BootnodesV4Flag, utils.BootnodesV5Flag, + utils.DNSDiscoveryFlag, utils.ListenPortFlag, utils.MaxPeersFlag, utils.MaxPendingPeersFlag, diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index d019c1fdcc..bdadebd852 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -658,6 +658,10 @@ var ( Name: "netrestrict", Usage: "Restricts network communication to the given IP networks (CIDR masks)", } + DNSDiscoveryFlag = cli.StringFlag{ + Name: "discovery.dns", + Usage: "Sets DNS discovery entry points (use \"\" to disable DNS)", + } // ATM the url is left to the user and deployment to JSpathFlag = cli.StringFlag{ @@ -811,9 +815,9 @@ func setBootstrapNodes(ctx *cli.Context, cfg *p2p.Config) { switch { case ctx.GlobalIsSet(BootnodesFlag.Name) || ctx.GlobalIsSet(BootnodesV4Flag.Name): if ctx.GlobalIsSet(BootnodesV4Flag.Name) { - urls = strings.Split(ctx.GlobalString(BootnodesV4Flag.Name), ",") + urls = splitAndTrim(ctx.GlobalString(BootnodesV4Flag.Name)) } else { - urls = strings.Split(ctx.GlobalString(BootnodesFlag.Name), ",") + urls = splitAndTrim(ctx.GlobalString(BootnodesFlag.Name)) } case ctx.GlobalBool(TestnetFlag.Name): urls = params.TestnetBootnodes @@ -845,9 +849,9 @@ func setBootstrapNodesV5(ctx *cli.Context, cfg *p2p.Config) { switch { case ctx.GlobalIsSet(BootnodesFlag.Name) || ctx.GlobalIsSet(BootnodesV5Flag.Name): if ctx.GlobalIsSet(BootnodesV5Flag.Name) { - urls = strings.Split(ctx.GlobalString(BootnodesV5Flag.Name), ",") + urls = splitAndTrim(ctx.GlobalString(BootnodesV5Flag.Name)) } else { - urls = strings.Split(ctx.GlobalString(BootnodesFlag.Name), ",") + urls = splitAndTrim(ctx.GlobalString(BootnodesFlag.Name)) } case ctx.GlobalBool(RinkebyFlag.Name): urls = params.RinkebyBootnodes @@ -1477,6 +1481,14 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { if ctx.GlobalIsSet(RPCGlobalGasCap.Name) { cfg.RPCGasCap = new(big.Int).SetUint64(ctx.GlobalUint64(RPCGlobalGasCap.Name)) } + if ctx.GlobalIsSet(DNSDiscoveryFlag.Name) { + urls := ctx.GlobalString(DNSDiscoveryFlag.Name) + if urls == "" { + cfg.DiscoveryURLs = []string{} + } else { + cfg.DiscoveryURLs = splitAndTrim(urls) + } + } // Override any default configs for hard coded networks. switch { @@ -1485,16 +1497,19 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { cfg.NetworkId = 3 } cfg.Genesis = core.DefaultTestnetGenesisBlock() + setDNSDiscoveryDefaults(cfg, params.KnownDNSNetworks[params.TestnetGenesisHash]) case ctx.GlobalBool(RinkebyFlag.Name): if !ctx.GlobalIsSet(NetworkIdFlag.Name) { cfg.NetworkId = 4 } cfg.Genesis = core.DefaultRinkebyGenesisBlock() + setDNSDiscoveryDefaults(cfg, params.KnownDNSNetworks[params.RinkebyGenesisHash]) case ctx.GlobalBool(GoerliFlag.Name): if !ctx.GlobalIsSet(NetworkIdFlag.Name) { cfg.NetworkId = 5 } cfg.Genesis = core.DefaultGoerliGenesisBlock() + setDNSDiscoveryDefaults(cfg, params.KnownDNSNetworks[params.GoerliGenesisHash]) case ctx.GlobalBool(DeveloperFlag.Name): if !ctx.GlobalIsSet(NetworkIdFlag.Name) { cfg.NetworkId = 1337 @@ -1521,7 +1536,20 @@ func SetEthConfig(ctx *cli.Context, stack *node.Node, cfg *eth.Config) { if !ctx.GlobalIsSet(MinerGasPriceFlag.Name) && !ctx.GlobalIsSet(MinerLegacyGasPriceFlag.Name) { cfg.Miner.GasPrice = big.NewInt(1) } + default: + if cfg.NetworkId == 1 { + setDNSDiscoveryDefaults(cfg, params.KnownDNSNetworks[params.MainnetGenesisHash]) + } + } +} + +// setDNSDiscoveryDefaults configures DNS discovery with the given URL if +// no URLs are set. +func setDNSDiscoveryDefaults(cfg *eth.Config, url string) { + if cfg.DiscoveryURLs != nil { + return } + cfg.DiscoveryURLs = []string{url} } // RegisterEthService adds an Ethereum client to the stack. diff --git a/eth/backend.go b/eth/backend.go index adde609de0..bda307d95b 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -47,6 +47,7 @@ import ( "github.com/ethereum/go-ethereum/miner" "github.com/ethereum/go-ethereum/node" "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/p2p/enr" "github.com/ethereum/go-ethereum/params" "github.com/ethereum/go-ethereum/rlp" @@ -74,6 +75,7 @@ type Ethereum struct { blockchain *core.BlockChain protocolManager *ProtocolManager lesServer LesServer + dialCandiates enode.Iterator // DB interfaces chainDb ethdb.Database // Block chain database @@ -220,6 +222,11 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { } eth.APIBackend.gpo = gasprice.NewOracle(eth.APIBackend, gpoParams) + eth.dialCandiates, err = eth.setupDiscovery(&ctx.Config.P2P) + if err != nil { + return nil, err + } + return eth, nil } @@ -510,6 +517,7 @@ func (s *Ethereum) Protocols() []p2p.Protocol { for i, vsn := range ProtocolVersions { protos[i] = s.protocolManager.makeProtocol(vsn) protos[i].Attributes = []enr.Entry{s.currentEthEntry()} + protos[i].DialCandidates = s.dialCandiates } if s.lesServer != nil { protos = append(protos, s.lesServer.Protocols()...) diff --git a/eth/config.go b/eth/config.go index 8240391114..2eaf21fbc3 100644 --- a/eth/config.go +++ b/eth/config.go @@ -95,6 +95,10 @@ type Config struct { NetworkId uint64 // Network ID to use for selecting peers to connect to SyncMode downloader.SyncMode + // This can be set to list of enrtree:// URLs which will be queried for + // for nodes to connect to. + DiscoveryURLs []string + NoPruning bool // Whether to disable pruning and flush everything to disk NoPrefetch bool // Whether to disable prefetching and only load state on demand @@ -156,8 +160,8 @@ type Config struct { CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` // Istanbul block override (TODO: remove after the fork) - OverrideIstanbul *big.Int + OverrideIstanbul *big.Int `toml:",omitempty"` // MuirGlacier block override (TODO: remove after the fork) - OverrideMuirGlacier *big.Int + OverrideMuirGlacier *big.Int `toml:",omitempty"` } diff --git a/eth/enr_entry.go b/eth/discovery.go similarity index 79% rename from eth/enr_entry.go rename to eth/discovery.go index d9e7b95784..97d6322ca1 100644 --- a/eth/enr_entry.go +++ b/eth/discovery.go @@ -19,6 +19,8 @@ package eth import ( "github.com/ethereum/go-ethereum/core" "github.com/ethereum/go-ethereum/core/forkid" + "github.com/ethereum/go-ethereum/p2p" + "github.com/ethereum/go-ethereum/p2p/dnsdisc" "github.com/ethereum/go-ethereum/p2p/enode" "github.com/ethereum/go-ethereum/rlp" ) @@ -37,6 +39,7 @@ func (e ethEntry) ENRKey() string { return "eth" } +// startEthEntryUpdate starts the ENR updater loop. func (eth *Ethereum) startEthEntryUpdate(ln *enode.LocalNode) { var newHead = make(chan core.ChainHeadEvent, 10) sub := eth.blockchain.SubscribeChainHeadEvent(newHead) @@ -59,3 +62,12 @@ func (eth *Ethereum) startEthEntryUpdate(ln *enode.LocalNode) { func (eth *Ethereum) currentEthEntry() *ethEntry { return ðEntry{ForkID: forkid.NewID(eth.blockchain)} } + +// setupDiscovery creates the node discovery source for the eth protocol. +func (eth *Ethereum) setupDiscovery(cfg *p2p.Config) (enode.Iterator, error) { + if cfg.NoDiscovery || len(eth.config.DiscoveryURLs) == 0 { + return nil, nil + } + client := dnsdisc.NewClient(dnsdisc.Config{}) + return client.NewIterator(eth.config.DiscoveryURLs...) +} diff --git a/eth/gen_config.go b/eth/gen_config.go index bc4b55b120..1c659c393c 100644 --- a/eth/gen_config.go +++ b/eth/gen_config.go @@ -21,6 +21,7 @@ func (c Config) MarshalTOML() (interface{}, error) { Genesis *core.Genesis `toml:",omitempty"` NetworkId uint64 SyncMode downloader.SyncMode + DiscoveryURLs []string NoPruning bool NoPrefetch bool Whitelist map[uint64]common.Hash `toml:"-"` @@ -49,11 +50,14 @@ func (c Config) MarshalTOML() (interface{}, error) { RPCGasCap *big.Int `toml:",omitempty"` Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` + OverrideIstanbul *big.Int `toml:",omitempty"` + OverrideMuirGlacier *big.Int `toml:",omitempty"` } var enc Config enc.Genesis = c.Genesis enc.NetworkId = c.NetworkId enc.SyncMode = c.SyncMode + enc.DiscoveryURLs = c.DiscoveryURLs enc.NoPruning = c.NoPruning enc.NoPrefetch = c.NoPrefetch enc.Whitelist = c.Whitelist @@ -82,6 +86,8 @@ func (c Config) MarshalTOML() (interface{}, error) { enc.RPCGasCap = c.RPCGasCap enc.Checkpoint = c.Checkpoint enc.CheckpointOracle = c.CheckpointOracle + enc.OverrideIstanbul = c.OverrideIstanbul + enc.OverrideMuirGlacier = c.OverrideMuirGlacier return &enc, nil } @@ -91,6 +97,7 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { Genesis *core.Genesis `toml:",omitempty"` NetworkId *uint64 SyncMode *downloader.SyncMode + DiscoveryURLs []string NoPruning *bool NoPrefetch *bool Whitelist map[uint64]common.Hash `toml:"-"` @@ -119,6 +126,8 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { RPCGasCap *big.Int `toml:",omitempty"` Checkpoint *params.TrustedCheckpoint `toml:",omitempty"` CheckpointOracle *params.CheckpointOracleConfig `toml:",omitempty"` + OverrideIstanbul *big.Int `toml:",omitempty"` + OverrideMuirGlacier *big.Int `toml:",omitempty"` } var dec Config if err := unmarshal(&dec); err != nil { @@ -133,6 +142,9 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.SyncMode != nil { c.SyncMode = *dec.SyncMode } + if dec.DiscoveryURLs != nil { + c.DiscoveryURLs = dec.DiscoveryURLs + } if dec.NoPruning != nil { c.NoPruning = *dec.NoPruning } @@ -217,5 +229,11 @@ func (c *Config) UnmarshalTOML(unmarshal func(interface{}) error) error { if dec.CheckpointOracle != nil { c.CheckpointOracle = dec.CheckpointOracle } + if dec.OverrideIstanbul != nil { + c.OverrideIstanbul = dec.OverrideIstanbul + } + if dec.OverrideMuirGlacier != nil { + c.OverrideMuirGlacier = dec.OverrideMuirGlacier + } return nil } diff --git a/node/node.go b/node/node.go index c9c27d8262..1bdd5af7d2 100644 --- a/node/node.go +++ b/node/node.go @@ -194,7 +194,7 @@ func (n *Node) Start() error { for _, constructor := range n.serviceFuncs { // Create a new context for the particular service ctx := &ServiceContext{ - config: n.config, + Config: *n.config, services: make(map[reflect.Type]Service), EventMux: n.eventmux, AccountManager: n.accman, diff --git a/node/service.go b/node/service.go index 4dea00995c..ef5b995e4b 100644 --- a/node/service.go +++ b/node/service.go @@ -32,20 +32,20 @@ import ( // the protocol stack, that is passed to all constructors to be optionally used; // as well as utility methods to operate on the service environment. type ServiceContext struct { - config *Config services map[reflect.Type]Service // Index of the already constructed services - EventMux *event.TypeMux // Event multiplexer used for decoupled notifications - AccountManager *accounts.Manager // Account manager created by the node. + Config Config + EventMux *event.TypeMux // Event multiplexer used for decoupled notifications + AccountManager *accounts.Manager // Account manager created by the node. } // OpenDatabase opens an existing database with the given name (or creates one // if no previous can be found) from within the node's data directory. If the // node is an ephemeral one, a memory database is returned. func (ctx *ServiceContext) OpenDatabase(name string, cache int, handles int, namespace string) (ethdb.Database, error) { - if ctx.config.DataDir == "" { + if ctx.Config.DataDir == "" { return rawdb.NewMemoryDatabase(), nil } - return rawdb.NewLevelDBDatabase(ctx.config.ResolvePath(name), cache, handles, namespace) + return rawdb.NewLevelDBDatabase(ctx.Config.ResolvePath(name), cache, handles, namespace) } // OpenDatabaseWithFreezer opens an existing database with the given name (or @@ -54,16 +54,16 @@ func (ctx *ServiceContext) OpenDatabase(name string, cache int, handles int, nam // database to immutable append-only files. If the node is an ephemeral one, a // memory database is returned. func (ctx *ServiceContext) OpenDatabaseWithFreezer(name string, cache int, handles int, freezer string, namespace string) (ethdb.Database, error) { - if ctx.config.DataDir == "" { + if ctx.Config.DataDir == "" { return rawdb.NewMemoryDatabase(), nil } - root := ctx.config.ResolvePath(name) + root := ctx.Config.ResolvePath(name) switch { case freezer == "": freezer = filepath.Join(root, "ancient") case !filepath.IsAbs(freezer): - freezer = ctx.config.ResolvePath(freezer) + freezer = ctx.Config.ResolvePath(freezer) } return rawdb.NewLevelDBDatabaseWithFreezer(root, cache, handles, freezer, namespace) } @@ -72,7 +72,7 @@ func (ctx *ServiceContext) OpenDatabaseWithFreezer(name string, cache int, handl // and if the user actually uses persistent storage. It will return an empty string // for emphemeral storage and the user's own input for absolute paths. func (ctx *ServiceContext) ResolvePath(path string) string { - return ctx.config.ResolvePath(path) + return ctx.Config.ResolvePath(path) } // Service retrieves a currently running service registered of a specific type. @@ -88,7 +88,7 @@ func (ctx *ServiceContext) Service(service interface{}) error { // ExtRPCEnabled returns the indicator whether node enables the external // RPC(http, ws or graphql). func (ctx *ServiceContext) ExtRPCEnabled() bool { - return ctx.config.ExtRPCEnabled() + return ctx.Config.ExtRPCEnabled() } // ServiceConstructor is the function signature of the constructors needed to be diff --git a/node/service_test.go b/node/service_test.go index 63004a51ab..5da8e9e434 100644 --- a/node/service_test.go +++ b/node/service_test.go @@ -38,7 +38,7 @@ func TestContextDatabases(t *testing.T) { t.Fatalf("non-created database already exists") } // Request the opening/creation of a database and ensure it persists to disk - ctx := &ServiceContext{config: &Config{Name: "unit-test", DataDir: dir}} + ctx := &ServiceContext{Config: Config{Name: "unit-test", DataDir: dir}} db, err := ctx.OpenDatabase("persistent", 0, 0, "") if err != nil { t.Fatalf("failed to open persistent database: %v", err) @@ -49,7 +49,7 @@ func TestContextDatabases(t *testing.T) { t.Fatalf("persistent database doesn't exists: %v", err) } // Request th opening/creation of an ephemeral database and ensure it's not persisted - ctx = &ServiceContext{config: &Config{DataDir: ""}} + ctx = &ServiceContext{Config: Config{DataDir: ""}} db, err = ctx.OpenDatabase("ephemeral", 0, 0, "") if err != nil { t.Fatalf("failed to open ephemeral database: %v", err) diff --git a/params/bootnodes.go b/params/bootnodes.go index f92b487ff5..f27e5b7d18 100644 --- a/params/bootnodes.go +++ b/params/bootnodes.go @@ -16,6 +16,8 @@ package params +import "github.com/ethereum/go-ethereum/common" + // MainnetBootnodes are the enode URLs of the P2P bootstrap nodes running on // the main Ethereum network. var MainnetBootnodes = []string{ @@ -69,3 +71,14 @@ var DiscoveryV5Bootnodes = []string{ "enode://1c7a64d76c0334b0418c004af2f67c50e36a3be60b5e4790bdac0439d21603469a85fad36f2473c9a80eb043ae60936df905fa28f1ff614c3e5dc34f15dcd2dc@40.118.3.223:30306", "enode://85c85d7143ae8bb96924f2b54f1b3e70d8c4d367af305325d30a61385a432f247d2c75c45c6b4a60335060d072d7f5b35dd1d4c45f76941f62a4f83b6e75daaf@40.118.3.223:30307", } + +const dnsPrefix = "enrtree://AKA3AM6LPBYEUDMVNU3BSVQJ5AD45Y7YPOHJLEF6W26QOE4VTUDPE@" + +// These DNS names provide bootstrap connectivity for public testnets and the mainnet. +// See https://github.com/ethereum/discv4-dns-lists for more information. +var KnownDNSNetworks = map[common.Hash]string{ + MainnetGenesisHash: dnsPrefix + "all.mainnet.ethdisco.net", + TestnetGenesisHash: dnsPrefix + "all.ropsten.ethdisco.net", + RinkebyGenesisHash: dnsPrefix + "all.rinkeby.ethdisco.net", + GoerliGenesisHash: dnsPrefix + "all.goerli.ethdisco.net", +} From 855690523a6bd98ce2eef9489d9f76cf2bc3868c Mon Sep 17 00:00:00 2001 From: Martin Holst Swende Date: Fri, 14 Feb 2020 09:54:02 +0100 Subject: [PATCH 109/128] core: ensure state exists for prefetcher (#20627) --- core/blockchain.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/core/blockchain.go b/core/blockchain.go index f083f53e04..d7fcbd5e31 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1654,18 +1654,17 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er // If we have a followup block, run that against the current state to pre-cache // transactions and probabilistically some of the account/storage trie nodes. var followupInterrupt uint32 - if !bc.cacheConfig.TrieCleanNoPrefetch { if followup, err := it.peek(); followup != nil && err == nil { - go func(start time.Time) { - throwaway, _ := state.New(parent.Root, bc.stateCache) - bc.prefetcher.Prefetch(followup, throwaway, bc.vmConfig, &followupInterrupt) + throwaway, _ := state.New(parent.Root, bc.stateCache) + go func(start time.Time, followup *types.Block, throwaway *state.StateDB, interrupt *uint32) { + bc.prefetcher.Prefetch(followup, throwaway, bc.vmConfig, interrupt) blockPrefetchExecuteTimer.Update(time.Since(start)) - if atomic.LoadUint32(&followupInterrupt) == 1 { + if atomic.LoadUint32(interrupt) == 1 { blockPrefetchInterruptMeter.Mark(1) } - }(time.Now()) + }(time.Now(), followup, throwaway, &followupInterrupt) } } // Process block using the parent state as reference point From 37531b18847774a7f057496bc34f2a48053fa417 Mon Sep 17 00:00:00 2001 From: Boqin Qin Date: Sun, 16 Feb 2020 02:14:29 +0800 Subject: [PATCH 110/128] cmd/faucet: protect f.reqs with Rlock to prevent data race (#20669) * cmd/faucet: add Rlock to protect f.reqs in apiHandler * cmd/faucet: make a locked copy of f.reqs --- cmd/faucet/faucet.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/cmd/faucet/faucet.go b/cmd/faucet/faucet.go index 77938efabd..e86f8533d0 100644 --- a/cmd/faucet/faucet.go +++ b/cmd/faucet/faucet.go @@ -360,11 +360,14 @@ func (f *faucet) apiHandler(w http.ResponseWriter, r *http.Request) { } } // Send over the initial stats and the latest header + f.lock.RLock() + reqs := f.reqs + f.lock.RUnlock() if err = send(conn, map[string]interface{}{ "funds": new(big.Int).Div(balance, ether), "funded": nonce, "peers": f.stack.Server().PeerCount(), - "requests": f.reqs, + "requests": reqs, }, 3*time.Second); err != nil { log.Warn("Failed to send initial stats to client", "err", err) return From 36a1e0b67d4f8d8b8861f8ee1de78eb881241127 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 17 Feb 2020 12:01:03 +0200 Subject: [PATCH 111/128] eth: don't enforce minimum broadcast, fix broadcast test --- eth/handler.go | 21 ++------------------- eth/handler_test.go | 43 ++++++++++++++++++------------------------- 2 files changed, 20 insertions(+), 44 deletions(-) diff --git a/eth/handler.go b/eth/handler.go index fc6c74cfe6..f103f1c37f 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -50,9 +50,6 @@ const ( // txChanSize is the size of channel listening to NewTxsEvent. // The number is referenced from the size of tx pool. txChanSize = 4096 - - // minimim number of peers to broadcast entire blocks and transactions too. - minBroadcastPeers = 4 ) var ( @@ -830,14 +827,7 @@ func (pm *ProtocolManager) BroadcastBlock(block *types.Block, propagate bool) { return } // Send the block to a subset of our peers - transferLen := int(math.Sqrt(float64(len(peers)))) - if transferLen < minBroadcastPeers { - transferLen = minBroadcastPeers - } - if transferLen > len(peers) { - transferLen = len(peers) - } - transfer := peers[:transferLen] + transfer := peers[:int(math.Sqrt(float64(len(peers))))] for _, peer := range transfer { peer.AsyncSendNewBlock(block, td) } @@ -866,14 +856,7 @@ func (pm *ProtocolManager) BroadcastTransactions(txs types.Transactions, propaga peers := pm.peers.PeersWithoutTx(tx.Hash()) // Send the block to a subset of our peers - transferLen := int(math.Sqrt(float64(len(peers)))) - if transferLen < minBroadcastPeers { - transferLen = minBroadcastPeers - } - if transferLen > len(peers) { - transferLen = len(peers) - } - transfer := peers[:transferLen] + transfer := peers[:int(math.Sqrt(float64(len(peers))))] for _, peer := range transfer { txset[peer] = append(txset[peer], tx.Hash()) } diff --git a/eth/handler_test.go b/eth/handler_test.go index 97613a9834..60bb1f0831 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -554,12 +554,12 @@ func TestBroadcastBlock(t *testing.T) { broadcastExpected int }{ {1, 1}, - {2, 2}, - {3, 3}, - {4, 4}, - {5, 4}, - {9, 4}, - {12, 4}, + {2, 1}, + {3, 1}, + {4, 2}, + {5, 2}, + {9, 3}, + {12, 3}, {16, 4}, {26, 5}, {100, 10}, @@ -592,6 +592,7 @@ func testBroadcastBlock(t *testing.T, totalPeers, broadcastExpected int) { for i := 0; i < totalPeers; i++ { peer, _ := newTestPeer(fmt.Sprintf("peer %d", i), eth63, pm, true) defer peer.close() + peers = append(peers, peer) } chain, _ := core.GenerateChain(gspec.Config, genesis, ethash.NewFaker(), db, 1, func(i int, gen *core.BlockGen) {}) @@ -608,31 +609,23 @@ func testBroadcastBlock(t *testing.T, totalPeers, broadcastExpected int) { } }(peer) } - timeout := time.After(2 * time.Second) - var receivedCount int -outer: + var received int for { select { - case err = <-errCh: - break outer case <-doneCh: - receivedCount++ - if receivedCount == totalPeers { - break outer + received++ + + case <-time.After(100 * time.Millisecond): + if received != broadcastExpected { + t.Errorf("broadcast count mismatch: have %d, want %d", received, broadcastExpected) } - case <-timeout: - break outer + return + + case err = <-errCh: + t.Fatalf("broadcast failed: %v", err) } } - for _, peer := range peers { - peer.app.Close() - } - if err != nil { - t.Errorf("error matching block by peer: %v", err) - } - if receivedCount != broadcastExpected { - t.Errorf("block broadcast to %d peers, expected %d", receivedCount, broadcastExpected) - } + } // Tests that a propagated malformed block (uncles or transactions don't match From fef8c985bc03df86ed9fc35e660c81fb3eed0330 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Mon, 17 Feb 2020 13:13:24 +0200 Subject: [PATCH 112/128] travis, appveyor, build: bump builder Go to 1.13.8 --- .travis.yml | 4 ++-- appveyor.yml | 4 ++-- build/checksums.txt | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 416a83018d..288c57959e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -93,7 +93,7 @@ jobs: - python-paramiko script: - echo '|1|7SiYPr9xl3uctzovOTj4gMwAC1M=|t6ReES75Bo/PxlOPJ6/GsGbTrM0= ssh-rsa AAAAB3NzaC1yc2EAAAABIwAAAQEA0aKz5UTUndYgIGG7dQBV+HaeuEZJ2xPHo2DS2iSKvUL4xNMSAY4UguNW+pX56nAQmZKIZZ8MaEvSj6zMEDiq6HFfn5JcTlM80UwlnyKe8B8p7Nk06PPQLrnmQt5fh0HmEcZx+JU9TZsfCHPnX7MNz4ELfZE6cFsclClrKim3BHUIGq//t93DllB+h4O9LHjEUsQ1Sr63irDLSutkLJD6RXchjROXkNirlcNVHH/jwLWR5RcYilNX7S5bIkK8NlWPjsn/8Ua5O7I9/YoE97PpO6i73DTGLh5H9JN/SITwCKBkgSDWUt61uPK3Y11Gty7o2lWsBjhBUm2Y38CBsoGmBw==' >> ~/.ssh/known_hosts - - go run build/ci.go debsrc -goversion 1.13.6 -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder " + - go run build/ci.go debsrc -goversion 1.13.8 -upload ethereum/ethereum -sftp-user geth-ci -signer "Go Ethereum Linux Builder " # This builder does the Linux Azure uploads - stage: build @@ -183,7 +183,7 @@ jobs: git: submodules: false # avoid cloning ethereum/tests before_install: - - curl https://dl.google.com/go/go1.13.6.linux-amd64.tar.gz | tar -xz + - curl https://dl.google.com/go/go1.13.8.linux-amd64.tar.gz | tar -xz - export PATH=`pwd`/go/bin:$PATH - export GOROOT=`pwd`/go - export GOPATH=$HOME/go diff --git a/appveyor.yml b/appveyor.yml index 8fcacfd5fe..90a862abe7 100644 --- a/appveyor.yml +++ b/appveyor.yml @@ -23,8 +23,8 @@ environment: install: - git submodule update --init - rmdir C:\go /s /q - - appveyor DownloadFile https://dl.google.com/go/go1.13.6.windows-%GETH_ARCH%.zip - - 7z x go1.13.6.windows-%GETH_ARCH%.zip -y -oC:\ > NUL + - appveyor DownloadFile https://dl.google.com/go/go1.13.8.windows-%GETH_ARCH%.zip + - 7z x go1.13.8.windows-%GETH_ARCH%.zip -y -oC:\ > NUL - go version - gcc --version diff --git a/build/checksums.txt b/build/checksums.txt index 44530ce4be..553d1bbefe 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -1,6 +1,6 @@ # This file contains sha256 checksums of optional build dependencies. -aae5be954bdc40bcf8006eb77e8d8a5dde412722bc8effcdaf9772620d06420c go1.13.6.src.tar.gz +b13bf04633d4d8cf53226ebeaace8d4d2fd07ae6fa676d0844a688339debec34 go1.13.8.src.tar.gz 478994633b0f5121a7a8d4f368078093e21014fdc7fb2c0ceeae63668c13c5b6 golangci-lint-1.22.2-freebsd-amd64.tar.gz fcf80824c21567eb0871055711bf9bdca91cf9a081122e2a45f1d11fed754600 golangci-lint-1.22.2-darwin-amd64.tar.gz From ac7278776838ba4dc69f57f8dbbef219d8bc3b6f Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 17 Feb 2020 12:22:15 +0100 Subject: [PATCH 113/128] p2p: remove MeteredPeerEvent (#20679) This event was added for the dashboard, but we don't need it anymore since the dashboard is gone. --- p2p/metrics.go | 166 ++++--------------------------------------------- p2p/peer.go | 3 +- p2p/rlpx.go | 3 +- p2p/server.go | 3 - 4 files changed, 15 insertions(+), 160 deletions(-) diff --git a/p2p/metrics.go b/p2p/metrics.go index 30bd56bd4d..44946473fa 100644 --- a/p2p/metrics.go +++ b/p2p/metrics.go @@ -20,102 +20,37 @@ package p2p import ( "net" - "sync" - "sync/atomic" - "time" - "github.com/ethereum/go-ethereum/event" - "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" ) const ( - MetricsInboundTraffic = "p2p/ingress" // Name for the registered inbound traffic meter - MetricsOutboundTraffic = "p2p/egress" // Name for the registered outbound traffic meter - MetricsOutboundConnects = "p2p/dials" // Name for the registered outbound connects meter - MetricsInboundConnects = "p2p/serves" // Name for the registered inbound connects meter - - MeteredPeerLimit = 1024 // This amount of peers are individually metered + ingressMeterName = "p2p/ingress" + egressMeterName = "p2p/egress" ) var ( - ingressConnectMeter = metrics.NewRegisteredMeter(MetricsInboundConnects, nil) // Meter counting the ingress connections - ingressTrafficMeter = metrics.NewRegisteredMeter(MetricsInboundTraffic, nil) // Meter metering the cumulative ingress traffic - egressConnectMeter = metrics.NewRegisteredMeter(MetricsOutboundConnects, nil) // Meter counting the egress connections - egressTrafficMeter = metrics.NewRegisteredMeter(MetricsOutboundTraffic, nil) // Meter metering the cumulative egress traffic - activePeerGauge = metrics.NewRegisteredGauge("p2p/peers", nil) // Gauge tracking the current peer count - - PeerIngressRegistry = metrics.NewPrefixedChildRegistry(metrics.EphemeralRegistry, MetricsInboundTraffic+"/") // Registry containing the peer ingress - PeerEgressRegistry = metrics.NewPrefixedChildRegistry(metrics.EphemeralRegistry, MetricsOutboundTraffic+"/") // Registry containing the peer egress - - meteredPeerFeed event.Feed // Event feed for peer metrics - meteredPeerCount int32 // Actually stored peer connection count -) - -// MeteredPeerEventType is the type of peer events emitted by a metered connection. -type MeteredPeerEventType int - -const ( - // PeerHandshakeSucceeded is the type of event - // emitted when a peer successfully makes the handshake. - PeerHandshakeSucceeded MeteredPeerEventType = iota - - // PeerHandshakeFailed is the type of event emitted when a peer fails to - // make the handshake or disconnects before it. - PeerHandshakeFailed - - // PeerDisconnected is the type of event emitted when a peer disconnects. - PeerDisconnected + ingressConnectMeter = metrics.NewRegisteredMeter("p2p/serves", nil) + ingressTrafficMeter = metrics.NewRegisteredMeter(ingressMeterName, nil) + egressConnectMeter = metrics.NewRegisteredMeter("p2p/dials", nil) + egressTrafficMeter = metrics.NewRegisteredMeter(egressMeterName, nil) + activePeerGauge = metrics.NewRegisteredGauge("p2p/peers", nil) ) -// MeteredPeerEvent is an event emitted when peers connect or disconnect. -type MeteredPeerEvent struct { - Type MeteredPeerEventType // Type of peer event - Addr string // TCP address of the peer - Elapsed time.Duration // Time elapsed between the connection and the handshake/disconnection - Peer *Peer // Connected remote node instance - Ingress uint64 // Ingress count at the moment of the event - Egress uint64 // Egress count at the moment of the event -} - -// SubscribeMeteredPeerEvent registers a subscription for peer life-cycle events -// if metrics collection is enabled. -func SubscribeMeteredPeerEvent(ch chan<- MeteredPeerEvent) event.Subscription { - return meteredPeerFeed.Subscribe(ch) -} - // meteredConn is a wrapper around a net.Conn that meters both the // inbound and outbound network traffic. type meteredConn struct { - net.Conn // Network connection to wrap with metering - - connected time.Time // Connection time of the peer - addr *net.TCPAddr // TCP address of the peer - peer *Peer // Peer instance - - // trafficMetered denotes if the peer is registered in the traffic registries. - // Its value is true if the metered peer count doesn't reach the limit in the - // moment of the peer's connection. - trafficMetered bool - ingressMeter metrics.Meter // Meter for the read bytes of the peer - egressMeter metrics.Meter // Meter for the written bytes of the peer - - lock sync.RWMutex // Lock protecting the metered connection's internals + net.Conn } // newMeteredConn creates a new metered connection, bumps the ingress or egress // connection meter and also increases the metered peer count. If the metrics -// system is disabled or the IP address is unspecified, this function returns -// the original object. +// system is disabled, function returns the original connection. func newMeteredConn(conn net.Conn, ingress bool, addr *net.TCPAddr) net.Conn { // Short circuit if metrics are disabled if !metrics.Enabled { return conn } - if addr == nil || addr.IP.IsUnspecified() { - log.Warn("Peer address is unspecified") - return conn - } // Bump the connection counters and wrap the connection if ingress { ingressConnectMeter.Mark(1) @@ -123,12 +58,7 @@ func newMeteredConn(conn net.Conn, ingress bool, addr *net.TCPAddr) net.Conn { egressConnectMeter.Mark(1) } activePeerGauge.Inc(1) - - return &meteredConn{ - Conn: conn, - addr: addr, - connected: time.Now(), - } + return &meteredConn{Conn: conn} } // Read delegates a network read to the underlying connection, bumping the common @@ -136,11 +66,6 @@ func newMeteredConn(conn net.Conn, ingress bool, addr *net.TCPAddr) net.Conn { func (c *meteredConn) Read(b []byte) (n int, err error) { n, err = c.Conn.Read(b) ingressTrafficMeter.Mark(int64(n)) - c.lock.RLock() - if c.trafficMetered { - c.ingressMeter.Mark(int64(n)) - } - c.lock.RUnlock() return n, err } @@ -149,84 +74,15 @@ func (c *meteredConn) Read(b []byte) (n int, err error) { func (c *meteredConn) Write(b []byte) (n int, err error) { n, err = c.Conn.Write(b) egressTrafficMeter.Mark(int64(n)) - c.lock.RLock() - if c.trafficMetered { - c.egressMeter.Mark(int64(n)) - } - c.lock.RUnlock() return n, err } -// handshakeDone is called after the connection passes the handshake. -func (c *meteredConn) handshakeDone(peer *Peer) { - if atomic.AddInt32(&meteredPeerCount, 1) >= MeteredPeerLimit { - // Don't register the peer in the traffic registries. - atomic.AddInt32(&meteredPeerCount, -1) - c.lock.Lock() - c.peer, c.trafficMetered = peer, false - c.lock.Unlock() - log.Warn("Metered peer count reached the limit") - } else { - enode := peer.Node().String() - c.lock.Lock() - c.peer, c.trafficMetered = peer, true - c.ingressMeter = metrics.NewRegisteredMeter(enode, PeerIngressRegistry) - c.egressMeter = metrics.NewRegisteredMeter(enode, PeerEgressRegistry) - c.lock.Unlock() - } - meteredPeerFeed.Send(MeteredPeerEvent{ - Type: PeerHandshakeSucceeded, - Addr: c.addr.String(), - Peer: peer, - Elapsed: time.Since(c.connected), - }) -} - // Close delegates a close operation to the underlying connection, unregisters // the peer from the traffic registries and emits close event. func (c *meteredConn) Close() error { err := c.Conn.Close() - c.lock.RLock() - if c.peer == nil { - // If the peer disconnects before/during the handshake. - c.lock.RUnlock() - meteredPeerFeed.Send(MeteredPeerEvent{ - Type: PeerHandshakeFailed, - Addr: c.addr.String(), - Elapsed: time.Since(c.connected), - }) - activePeerGauge.Dec(1) - return err - } - peer := c.peer - if !c.trafficMetered { - // If the peer isn't registered in the traffic registries. - c.lock.RUnlock() - meteredPeerFeed.Send(MeteredPeerEvent{ - Type: PeerDisconnected, - Addr: c.addr.String(), - Peer: peer, - }) + if err == nil { activePeerGauge.Dec(1) - return err } - ingress, egress, enode := uint64(c.ingressMeter.Count()), uint64(c.egressMeter.Count()), c.peer.Node().String() - c.lock.RUnlock() - - // Decrement the metered peer count - atomic.AddInt32(&meteredPeerCount, -1) - - // Unregister the peer from the traffic registries - PeerIngressRegistry.Unregister(enode) - PeerEgressRegistry.Unregister(enode) - - meteredPeerFeed.Send(MeteredPeerEvent{ - Type: PeerDisconnected, - Addr: c.addr.String(), - Peer: peer, - Ingress: ingress, - Egress: egress, - }) - activePeerGauge.Dec(1) return err } diff --git a/p2p/peer.go b/p2p/peer.go index 9a9788bc17..4398ad0f23 100644 --- a/p2p/peer.go +++ b/p2p/peer.go @@ -302,7 +302,8 @@ func (p *Peer) handle(msg Msg) error { return fmt.Errorf("msg code out of range: %v", msg.Code) } if metrics.Enabled { - metrics.GetOrRegisterMeter(fmt.Sprintf("%s/%s/%d/%#02x", MetricsInboundTraffic, proto.Name, proto.Version, msg.Code-proto.offset), nil).Mark(int64(msg.meterSize)) + m := fmt.Sprintf("%s/%s/%d/%#02x", ingressMeterName, proto.Name, proto.Version, msg.Code-proto.offset) + metrics.GetOrRegisterMeter(m, nil).Mark(int64(msg.meterSize)) } select { case proto.in <- msg: diff --git a/p2p/rlpx.go b/p2p/rlpx.go index c9ca6ea42f..c134aec1de 100644 --- a/p2p/rlpx.go +++ b/p2p/rlpx.go @@ -595,7 +595,8 @@ func (rw *rlpxFrameRW) WriteMsg(msg Msg) error { } msg.meterSize = msg.Size if metrics.Enabled && msg.meterCap.Name != "" { // don't meter non-subprotocol messages - metrics.GetOrRegisterMeter(fmt.Sprintf("%s/%s/%d/%#02x", MetricsOutboundTraffic, msg.meterCap.Name, msg.meterCap.Version, msg.meterCode), nil).Mark(int64(msg.meterSize)) + m := fmt.Sprintf("%s/%s/%d/%#02x", egressMeterName, msg.meterCap.Name, msg.meterCap.Version, msg.meterCode) + metrics.GetOrRegisterMeter(m, nil).Mark(int64(msg.meterSize)) } // write header headbuf := make([]byte, 32) diff --git a/p2p/server.go b/p2p/server.go index 1ed1be2ac6..c87b7758df 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -755,9 +755,6 @@ running: peers[c.node.ID()] = p srv.log.Debug("Adding p2p peer", "peercount", len(peers), "id", p.ID(), "conn", c.flags, "addr", p.RemoteAddr(), "name", truncateName(c.name)) srv.dialsched.peerAdded(c) - if conn, ok := c.fd.(*meteredConn); ok { - conn.handshakeDone(p) - } if p.Inbound() { inboundCount++ } From 1c4c486a85ef304e060da592bf46f81d015aa662 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 17 Feb 2020 12:22:52 +0100 Subject: [PATCH 114/128] cmd/ethkey: speed up test by using weaker scrypt parameters (#20680) --- cmd/ethkey/generate.go | 10 +++++++++- cmd/ethkey/message_test.go | 2 +- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/cmd/ethkey/generate.go b/cmd/ethkey/generate.go index fe9a0c1519..09b00cc9dc 100644 --- a/cmd/ethkey/generate.go +++ b/cmd/ethkey/generate.go @@ -52,6 +52,10 @@ If you want to encrypt an existing private key, it can be specified by setting Name: "privatekey", Usage: "file containing a raw private key to encrypt", }, + cli.BoolFlag{ + Name: "lightkdf", + Usage: "use less secure scrypt parameters", + }, }, Action: func(ctx *cli.Context) error { // Check if keyfile path given and make sure it doesn't already exist. @@ -91,7 +95,11 @@ If you want to encrypt an existing private key, it can be specified by setting // Encrypt key with passphrase. passphrase := promptPassphrase(true) - keyjson, err := keystore.EncryptKey(key, passphrase, keystore.StandardScryptN, keystore.StandardScryptP) + scryptN, scryptP := keystore.StandardScryptN, keystore.StandardScryptP + if ctx.Bool("lightkdf") { + scryptN, scryptP = keystore.LightScryptN, keystore.LightScryptP + } + keyjson, err := keystore.EncryptKey(key, passphrase, scryptN, scryptP) if err != nil { utils.Fatalf("Error encrypting key: %v", err) } diff --git a/cmd/ethkey/message_test.go b/cmd/ethkey/message_test.go index e9e8eeeafb..9d242ac002 100644 --- a/cmd/ethkey/message_test.go +++ b/cmd/ethkey/message_test.go @@ -34,7 +34,7 @@ func TestMessageSignVerify(t *testing.T) { message := "test message" // Create the key. - generate := runEthkey(t, "generate", keyfile) + generate := runEthkey(t, "generate", "--lightkdf", keyfile) generate.Expect(` !! Unsupported terminal, password will be echoed. Password: {{.InputLine "foobar"}} From 57d4898e2992a46fc2deab93a2666a2979b6704c Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 17 Feb 2020 15:23:25 +0100 Subject: [PATCH 115/128] p2p/dnsdisc: re-check tree root when leaf resolution fails (#20682) This adds additional logic to re-resolve the root name of a tree when a couple of leaf requests have failed. We need this change to avoid getting into a failure state where leaf requests keep failing for half an hour when the tree has been updated. --- p2p/dnsdisc/client_test.go | 66 ++++++++++++++++++++++++++++++++------ p2p/dnsdisc/sync.go | 65 +++++++++++++++++++++++++++++++------ 2 files changed, 113 insertions(+), 18 deletions(-) diff --git a/p2p/dnsdisc/client_test.go b/p2p/dnsdisc/client_test.go index 1a1d5ade95..6a6705abf2 100644 --- a/p2p/dnsdisc/client_test.go +++ b/p2p/dnsdisc/client_test.go @@ -19,6 +19,7 @@ package dnsdisc import ( "context" "crypto/ecdsa" + "errors" "math/rand" "reflect" "testing" @@ -176,11 +177,62 @@ func TestIteratorNodeUpdates(t *testing.T) { t.Fatal(err) } - // sync the original tree. + // Sync the original tree. resolver.add(tree1.ToTXT("n")) checkIterator(t, it, nodes[:25]) - // Update some nodes and ensure RandomNode returns the new nodes as well. + // Ensure RandomNode returns the new nodes after the tree is updated. + updateSomeNodes(nodesSeed1, nodes) + tree2, _ := makeTestTree("n", nodes, nil) + resolver.clear() + resolver.add(tree2.ToTXT("n")) + t.Log("tree updated") + + clock.Run(c.cfg.RecheckInterval + 1*time.Second) + checkIterator(t, it, nodes) +} + +// This test checks that the tree root is rechecked when a couple of leaf +// requests have failed. The test is just like TestIteratorNodeUpdates, but +// without advancing the clock by recheckInterval after the tree update. +func TestIteratorRootRecheckOnFail(t *testing.T) { + var ( + clock = new(mclock.Simulated) + nodes = testNodes(nodesSeed1, 30) + resolver = newMapResolver() + c = NewClient(Config{ + Resolver: resolver, + Logger: testlog.Logger(t, log.LvlTrace), + RecheckInterval: 20 * time.Minute, + RateLimit: 500, + // Disabling the cache is required for this test because the client doesn't + // notice leaf failures if all records are cached. + CacheLimit: 1, + }) + ) + c.clock = clock + tree1, url := makeTestTree("n", nodes[:25], nil) + it, err := c.NewIterator(url) + if err != nil { + t.Fatal(err) + } + + // Sync the original tree. + resolver.add(tree1.ToTXT("n")) + checkIterator(t, it, nodes[:25]) + + // Ensure RandomNode returns the new nodes after the tree is updated. + updateSomeNodes(nodesSeed1, nodes) + tree2, _ := makeTestTree("n", nodes, nil) + resolver.clear() + resolver.add(tree2.ToTXT("n")) + t.Log("tree updated") + + checkIterator(t, it, nodes) +} + +// updateSomeNodes applies ENR updates to some of the given nodes. +func updateSomeNodes(keySeed int64, nodes []*enode.Node) { keys := testKeys(nodesSeed1, len(nodes)) for i, n := range nodes[:len(nodes)/2] { r := n.Record() @@ -190,11 +242,6 @@ func TestIteratorNodeUpdates(t *testing.T) { n2, _ := enode.New(enode.ValidSchemes, r) nodes[i] = n2 } - tree2, _ := makeTestTree("n", nodes, nil) - clock.Run(c.cfg.RecheckInterval + 1*time.Second) - resolver.clear() - resolver.add(tree2.ToTXT("n")) - checkIterator(t, it, nodes) } // This test verifies that randomIterator re-checks the root of the tree to catch @@ -230,9 +277,10 @@ func TestIteratorLinkUpdates(t *testing.T) { // Add link to tree3, remove link to tree2. tree1, _ = makeTestTree("t1", nodes[:10], []string{url3}) resolver.add(tree1.ToTXT("t1")) - clock.Run(c.cfg.RecheckInterval + 1*time.Second) t.Log("tree1 updated") + clock.Run(c.cfg.RecheckInterval + 1*time.Second) + var wantNodes []*enode.Node wantNodes = append(wantNodes, tree1.Nodes()...) wantNodes = append(wantNodes, tree3.Nodes()...) @@ -345,5 +393,5 @@ func (mr mapResolver) LookupTXT(ctx context.Context, name string) ([]string, err if record, ok := mr[name]; ok { return []string{record}, nil } - return nil, nil + return nil, errors.New("not found") } diff --git a/p2p/dnsdisc/sync.go b/p2p/dnsdisc/sync.go index 53423527a5..36f02acba6 100644 --- a/p2p/dnsdisc/sync.go +++ b/p2p/dnsdisc/sync.go @@ -25,15 +25,22 @@ import ( "github.com/ethereum/go-ethereum/p2p/enode" ) +const ( + rootRecheckFailCount = 5 // update root if this many leaf requests fail +) + // clientTree is a full tree being synced. type clientTree struct { c *Client loc *linkEntry // link to this tree lastRootCheck mclock.AbsTime // last revalidation of root - root *rootEntry - enrs *subtreeSync - links *subtreeSync + leafFailCount int + rootFailCount int + + root *rootEntry + enrs *subtreeSync + links *subtreeSync lc *linkCache // tracks all links between all trees curLinks map[string]struct{} // links contained in this tree @@ -46,7 +53,7 @@ func newClientTree(c *Client, lc *linkCache, loc *linkEntry) *clientTree { // syncAll retrieves all entries of the tree. func (ct *clientTree) syncAll(dest map[string]entry) error { - if err := ct.updateRoot(); err != nil { + if err := ct.updateRoot(context.Background()); err != nil { return err } if err := ct.links.resolveAll(dest); err != nil { @@ -60,12 +67,20 @@ func (ct *clientTree) syncAll(dest map[string]entry) error { // syncRandom retrieves a single entry of the tree. The Node return value // is non-nil if the entry was a node. -func (ct *clientTree) syncRandom(ctx context.Context) (*enode.Node, error) { +func (ct *clientTree) syncRandom(ctx context.Context) (n *enode.Node, err error) { if ct.rootUpdateDue() { - if err := ct.updateRoot(); err != nil { + if err := ct.updateRoot(ctx); err != nil { return nil, err } } + + // Update fail counter for leaf request errors. + defer func() { + if err != nil { + ct.leafFailCount++ + } + }() + // Link tree sync has priority, run it to completion before syncing ENRs. if !ct.links.done() { err := ct.syncNextLink(ctx) @@ -138,15 +153,22 @@ func removeHash(h []string, index int) []string { } // updateRoot ensures that the given tree has an up-to-date root. -func (ct *clientTree) updateRoot() error { +func (ct *clientTree) updateRoot(ctx context.Context) error { + if !ct.slowdownRootUpdate(ctx) { + return ctx.Err() + } + ct.lastRootCheck = ct.c.clock.Now() - ctx, cancel := context.WithTimeout(context.Background(), ct.c.cfg.Timeout) + ctx, cancel := context.WithTimeout(ctx, ct.c.cfg.Timeout) defer cancel() root, err := ct.c.resolveRoot(ctx, ct.loc) if err != nil { + ct.rootFailCount++ return err } ct.root = &root + ct.rootFailCount = 0 + ct.leafFailCount = 0 // Invalidate subtrees if changed. if ct.links == nil || root.lroot != ct.links.root { @@ -161,7 +183,32 @@ func (ct *clientTree) updateRoot() error { // rootUpdateDue returns true when a root update is needed. func (ct *clientTree) rootUpdateDue() bool { - return ct.root == nil || time.Duration(ct.c.clock.Now()-ct.lastRootCheck) > ct.c.cfg.RecheckInterval + tooManyFailures := ct.leafFailCount > rootRecheckFailCount + scheduledCheck := ct.c.clock.Now().Sub(ct.lastRootCheck) > ct.c.cfg.RecheckInterval + return ct.root == nil || tooManyFailures || scheduledCheck +} + +// slowdownRootUpdate applies a delay to root resolution if is tried +// too frequently. This avoids busy polling when the client is offline. +// Returns true if the timeout passed, false if sync was canceled. +func (ct *clientTree) slowdownRootUpdate(ctx context.Context) bool { + var delay time.Duration + switch { + case ct.rootFailCount > 20: + delay = 10 * time.Second + case ct.rootFailCount > 5: + delay = 5 * time.Second + default: + return true + } + timeout := ct.c.clock.NewTimer(delay) + defer timeout.Stop() + select { + case <-timeout.C(): + return true + case <-ctx.Done(): + return false + } } // subtreeSync is the sync of an ENR or link subtree. From 1b9c5b393b9a52078a25f426af6e1fcc5996ca67 Mon Sep 17 00:00:00 2001 From: Boqin Qin Date: Tue, 18 Feb 2020 00:33:12 +0800 Subject: [PATCH 116/128] all: fix goroutine leaks in unit tests by adding 1-elem channel buffer (#20666) This fixes a bunch of cases where a timeout in the test would leak a goroutine. --- common/mclock/simclock_test.go | 2 +- eth/handler_test.go | 2 +- p2p/server_test.go | 2 +- rpc/client_test.go | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/common/mclock/simclock_test.go b/common/mclock/simclock_test.go index 94aa4f2b39..48f3fd56a0 100644 --- a/common/mclock/simclock_test.go +++ b/common/mclock/simclock_test.go @@ -96,7 +96,7 @@ func TestSimulatedSleep(t *testing.T) { var ( c Simulated timeout = 1 * time.Hour - done = make(chan AbsTime) + done = make(chan AbsTime, 1) ) go func() { c.Sleep(timeout) diff --git a/eth/handler_test.go b/eth/handler_test.go index 60bb1f0831..4a4e1f9559 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -670,7 +670,7 @@ func TestBroadcastMalformedBlock(t *testing.T) { malformedEverything.TxHash[0]++ // Keep listening to broadcasts and notify if any arrives - notify := make(chan struct{}) + notify := make(chan struct{}, 1) go func() { if _, err := sink.app.ReadMsg(); err == nil { notify <- struct{}{} diff --git a/p2p/server_test.go b/p2p/server_test.go index 958eb29129..7dc344a67d 100644 --- a/p2p/server_test.go +++ b/p2p/server_test.go @@ -550,7 +550,7 @@ func TestServerInboundThrottle(t *testing.T) { conn.Close() // Dial again. This time the server should close the connection immediately. - connClosed := make(chan struct{}) + connClosed := make(chan struct{}, 1) conn, err = net.DialTimeout("tcp", srv.ListenAddr, timeout) if err != nil { t.Fatalf("could not dial: %v", err) diff --git a/rpc/client_test.go b/rpc/client_test.go index 9c7f72053b..0287dad088 100644 --- a/rpc/client_test.go +++ b/rpc/client_test.go @@ -297,7 +297,7 @@ func TestClientSubscribeClose(t *testing.T) { var ( nc = make(chan int) - errc = make(chan error) + errc = make(chan error, 1) sub *ClientSubscription err error ) From 91b228966e042d8fad36451d524bbf8b2abde690 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Mon, 17 Feb 2020 17:33:32 +0100 Subject: [PATCH 117/128] rpc: remove startup error for invalid modules, log it instead (#20684) This removes the error added in #20597 in favor of a log message at error level. Failing to start broke a bunch of people's setups and is probably not the right thing to do for this check. --- rpc/endpoints.go | 31 +++++++++++++------------------ 1 file changed, 13 insertions(+), 18 deletions(-) diff --git a/rpc/endpoints.go b/rpc/endpoints.go index 2cf51aedfc..ed295adfc0 100644 --- a/rpc/endpoints.go +++ b/rpc/endpoints.go @@ -17,7 +17,6 @@ package rpc import ( - "fmt" "net" "github.com/ethereum/go-ethereum/log" @@ -25,30 +24,26 @@ import ( // checkModuleAvailability check that all names given in modules are actually // available API services. -func checkModuleAvailability(modules []string, apis []API) error { - available := make(map[string]struct{}) - var availableNames string - for i, api := range apis { - if _, ok := available[api.Namespace]; !ok { - available[api.Namespace] = struct{}{} - if i > 0 { - availableNames += ", " - } - availableNames += api.Namespace +func checkModuleAvailability(modules []string, apis []API) (bad, available []string) { + availableSet := make(map[string]struct{}) + for _, api := range apis { + if _, ok := availableSet[api.Namespace]; !ok { + availableSet[api.Namespace] = struct{}{} + available = append(available, api.Namespace) } } for _, name := range modules { - if _, ok := available[name]; !ok { - return fmt.Errorf("invalid API %q in whitelist (available: %s)", name, availableNames) + if _, ok := availableSet[name]; !ok { + bad = append(bad, name) } } - return nil + return bad, available } // StartHTTPEndpoint starts the HTTP RPC endpoint, configured with cors/vhosts/modules. func StartHTTPEndpoint(endpoint string, apis []API, modules []string, cors []string, vhosts []string, timeouts HTTPTimeouts) (net.Listener, *Server, error) { - if err := checkModuleAvailability(modules, apis); err != nil { - return nil, nil, err + if bad, available := checkModuleAvailability(modules, apis); len(bad) > 0 { + log.Error("Unavailable modules in HTTP API list", "unavailable", bad, "available", available) } // Generate the whitelist based on the allowed modules whitelist := make(map[string]bool) @@ -79,8 +74,8 @@ func StartHTTPEndpoint(endpoint string, apis []API, modules []string, cors []str // StartWSEndpoint starts a websocket endpoint. func StartWSEndpoint(endpoint string, apis []API, modules []string, wsOrigins []string, exposeAll bool) (net.Listener, *Server, error) { - if err := checkModuleAvailability(modules, apis); err != nil { - return nil, nil, err + if bad, available := checkModuleAvailability(modules, apis); len(bad) > 0 { + log.Error("Unavailable modules in WS API list", "unavailable", bad, "available", available) } // Generate the whitelist based on the allowed modules whitelist := make(map[string]bool) From 4f55e24c02011c47feb33129f68284f092bcc06e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 18 Feb 2020 10:55:44 +0200 Subject: [PATCH 118/128] params: update CHTs for the v1.9.11 release --- params/config.go | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/params/config.go b/params/config.go index 5a2078c42c..bcbde40b86 100644 --- a/params/config.go +++ b/params/config.go @@ -72,10 +72,10 @@ var ( // MainnetTrustedCheckpoint contains the light client trusted checkpoint for the main network. MainnetTrustedCheckpoint = &TrustedCheckpoint{ - SectionIndex: 283, - SectionHead: common.HexToHash("0x7da9639e5b378421f2cabd1d3bdae02dbf6d4ba79fc9b3c3916c66412ef7d0b6"), - CHTRoot: common.HexToHash("0xb8c6f06e1d5a4fddf593d1ff787e6b89eff4183d0b40d276f19e7d41178a53cf"), - BloomRoot: common.HexToHash("0xc47d7d6924ba46090d568296bb667a3ea4b6a00dd69bd5fb4f4f78be2fe05ec2"), + SectionIndex: 289, + SectionHead: common.HexToHash("0x5a95eed1a6e01d58b59f86c754cda88e8d6bede65428530eb0bec03267cda6a9"), + CHTRoot: common.HexToHash("0x6d4abf2b0f3c015952e6a3cbd5cc9885aacc29b8e55d4de662d29783c74a62bf"), + BloomRoot: common.HexToHash("0x1af2a8abbaca8048136b02f782cb6476ab546313186a1d1bd2b02df88ea48e7e"), } // MainnetCheckpointOracle contains a set of configs for the main network oracle. @@ -111,10 +111,10 @@ var ( // TestnetTrustedCheckpoint contains the light client trusted checkpoint for the Ropsten test network. TestnetTrustedCheckpoint = &TrustedCheckpoint{ - SectionIndex: 217, - SectionHead: common.HexToHash("0x1895e3cceb6fb201044679db2b9f4f9df4233b52e8d3c5ec4b75ae0ae75c90fa"), - CHTRoot: common.HexToHash("0x8f2016fb336b64bd8ef4e9a73659a0a99476ea8789aacad695d65295a50fdb8d"), - BloomRoot: common.HexToHash("0x57f5b8ecfa10ada7509a45f7e0f2283c6b2dc08d8771163ffbb4ff0e3e6bca1c"), + SectionIndex: 223, + SectionHead: common.HexToHash("0x9aa51ca383f5075f816e0b8ce7125075cd562b918839ee286c03770722147661"), + CHTRoot: common.HexToHash("0x755c6a5931b7bd36e55e47f3f1e81fa79c930ae15c55682d3a85931eedaf8cf2"), + BloomRoot: common.HexToHash("0xabc37762d11b29dc7dde11b89846e2308ba681eeb015b6a202ef5e242bc107e8"), } // TestnetCheckpointOracle contains a set of configs for the Ropsten test network oracle. @@ -152,10 +152,10 @@ var ( // RinkebyTrustedCheckpoint contains the light client trusted checkpoint for the Rinkeby test network. RinkebyTrustedCheckpoint = &TrustedCheckpoint{ - SectionIndex: 176, - SectionHead: common.HexToHash("0xb2cbd3396f25647fd80598fc4f7b015fb4131c0831630d878742b33dd286b641"), - CHTRoot: common.HexToHash("0xd81776495227babd75d8a519cb55e506e7b24265de5aa1847d38cb8f216ac09e"), - BloomRoot: common.HexToHash("0x0ea52b610139fcd6b1bc75351787e21c72922b18ea8a27091a51eb29b6723cd6"), + SectionIndex: 181, + SectionHead: common.HexToHash("0xdda275f3e9ecadf4834a6a682db1ca3db6945fa4014c82dadcad032fc5c1aefa"), + CHTRoot: common.HexToHash("0x0fdfdbdb12e947e838fe26dd3ada4cc3092d6fa22aefec719b83f16004b5e596"), + BloomRoot: common.HexToHash("0xfd8dc404a438eaa5cf93dd58dbaeed648aa49d563b511892262acff77c5db7db"), } // RinkebyCheckpointOracle contains a set of configs for the Rinkeby test network oracle. @@ -191,10 +191,10 @@ var ( // GoerliTrustedCheckpoint contains the light client trusted checkpoint for the Görli test network. GoerliTrustedCheckpoint = &TrustedCheckpoint{ - SectionIndex: 60, - SectionHead: common.HexToHash("0x9bc784b6ad0c944f6aebf930fb5e811497059788b616e211db655a885e65f1cb"), - CHTRoot: common.HexToHash("0x961a811e2843f6196903edc1d04adbf13dee92627a89c21b3e0cdf69e0100638"), - BloomRoot: common.HexToHash("0x1564889bbe8c68011d29d3966439956303283c0fb8b08daa8376028e82dcd763"), + SectionIndex: 66, + SectionHead: common.HexToHash("0xeea3a7b2cb275956f3049dd27e6cdacd8a6ef86738d593d556efee5361019475"), + CHTRoot: common.HexToHash("0x11712af50b4083dc5910e452ca69fbfc0f2940770b9846200a573f87a0af94e6"), + BloomRoot: common.HexToHash("0x331b7a7b273e81daeac8cafb9952a16669d7facc7be3b0ebd3a792b4d8b95cc5"), } // GoerliCheckpointOracle contains a set of configs for the Goerli test network oracle. From dae3aee5ff7ebcd24a50937a1d0ae080c2f1d897 Mon Sep 17 00:00:00 2001 From: Felix Lange Date: Tue, 18 Feb 2020 12:24:05 +0100 Subject: [PATCH 119/128] les: add bootstrap nodes as initial discoveries (#20688) --- les/serverpool.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/les/serverpool.go b/les/serverpool.go index 37621dc634..f8fd721696 100644 --- a/les/serverpool.go +++ b/les/serverpool.go @@ -179,6 +179,19 @@ func (pool *serverPool) start(server *p2p.Server, topic discv5.Topic) { pool.checkDial() pool.wg.Add(1) go pool.eventLoop() + + // Inject the bootstrap nodes as initial dial candiates. + pool.wg.Add(1) + go func() { + defer pool.wg.Done() + for _, n := range server.BootstrapNodes { + select { + case pool.discNodes <- n: + case <-pool.closeCh: + return + } + } + }() } func (pool *serverPool) stop() { From 6a62fe399b68ab9e3625ef5e7900394f389adc3a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?P=C3=A9ter=20Szil=C3=A1gyi?= Date: Tue, 18 Feb 2020 13:26:00 +0200 Subject: [PATCH 120/128] params: release Geth v1.9.11 stable --- params/version.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/params/version.go b/params/version.go index 54e006f480..42bf349df1 100644 --- a/params/version.go +++ b/params/version.go @@ -21,10 +21,10 @@ import ( ) const ( - VersionMajor = 1 // Major version component of the current release - VersionMinor = 9 // Minor version component of the current release - VersionPatch = 11 // Patch version component of the current release - VersionMeta = "unstable" // Version metadata to append to the version string + VersionMajor = 1 // Major version component of the current release + VersionMinor = 9 // Minor version component of the current release + VersionPatch = 11 // Patch version component of the current release + VersionMeta = "stable" // Version metadata to append to the version string ) // Version holds the textual version string. From 9f2254b558348b79e0311b3968a20456b00f24c2 Mon Sep 17 00:00:00 2001 From: amalraj mani Date: Thu, 27 Aug 2020 16:14:46 +0800 Subject: [PATCH 121/128] upgrade 1.9.8 to 1.9.11 --- accounts/abi/bind/backends/simulated.go | 16 ++-- accounts/abi/bind/backends/simulated_test.go | 18 ++-- consensus/protocol.go | 9 +- console/console.go | 91 ++------------------ core/blockchain.go | 8 +- eth/handler.go | 8 +- eth/handler_test.go | 2 +- eth/helper_test.go | 2 +- eth/peer.go | 2 +- go.mod | 4 +- miner/worker.go | 2 +- node/service.go | 2 +- p2p/server.go | 5 -- params/config.go | 2 +- rpc/websocket.go | 11 ++- 15 files changed, 55 insertions(+), 127 deletions(-) diff --git a/accounts/abi/bind/backends/simulated.go b/accounts/abi/bind/backends/simulated.go index c1689b9557..371dc38276 100644 --- a/accounts/abi/bind/backends/simulated.go +++ b/accounts/abi/bind/backends/simulated.go @@ -96,7 +96,7 @@ func NewSimulatedBackendFrom(ethereum *eth.Ethereum) *SimulatedBackend { database: ethereum.ChainDb(), blockchain: ethereum.BlockChain(), config: ethereum.BlockChain().Config(), - events: filters.NewEventSystem(new(event.TypeMux), &filterBackend{ethereum.ChainDb(), ethereum.BlockChain()}, false), + events: filters.NewEventSystem(&filterBackend{ethereum.ChainDb(), ethereum.BlockChain()}, false), } backend.rollback() return backend @@ -145,13 +145,15 @@ func (b *SimulatedBackend) rollback() { // stateByBlockNumber retrieves a state by a given blocknumber. func (b *SimulatedBackend) stateByBlockNumber(ctx context.Context, blockNumber *big.Int) (*state.StateDB, error) { if blockNumber == nil || blockNumber.Cmp(b.blockchain.CurrentBlock().Number()) == 0 { - return b.blockchain.State() + statedb, _, err := b.blockchain.State() + return statedb, err } block, err := b.BlockByNumber(ctx, blockNumber) if err != nil { return nil, err } - return b.blockchain.StateAt(block.Hash()) + statedb, _, err := b.blockchain.StateAt(block.Hash()) + return statedb, err } // CodeAt returns the code associated with a certain account in the blockchain. @@ -163,7 +165,7 @@ func (b *SimulatedBackend) CodeAt(ctx context.Context, contract common.Address, if err != nil { return nil, err } - statedb, _, _ := b.blockchain.State() + statedb, _, _ = b.blockchain.State() return statedb.GetCode(contract), nil } @@ -176,7 +178,7 @@ func (b *SimulatedBackend) BalanceAt(ctx context.Context, contract common.Addres if err != nil { return nil, err } - statedb, _, _ := b.blockchain.State() + statedb, _, _ = b.blockchain.State() return statedb.GetBalance(contract), nil } @@ -189,7 +191,7 @@ func (b *SimulatedBackend) NonceAt(ctx context.Context, contract common.Address, if err != nil { return 0, err } - statedb, _, _ := b.blockchain.State() + statedb, _, _ = b.blockchain.State() return statedb.GetNonce(contract), nil } @@ -202,7 +204,7 @@ func (b *SimulatedBackend) StorageAt(ctx context.Context, contract common.Addres if err != nil { return nil, err } - statedb, _, _ := b.blockchain.State() + statedb, _, _ = b.blockchain.State() val := statedb.GetState(contract, key) return val[:], nil } diff --git a/accounts/abi/bind/backends/simulated_test.go b/accounts/abi/bind/backends/simulated_test.go index f8101e0260..8f6c1b686e 100644 --- a/accounts/abi/bind/backends/simulated_test.go +++ b/accounts/abi/bind/backends/simulated_test.go @@ -123,7 +123,7 @@ func TestNewSimulatedBackend(t *testing.T) { t.Errorf("expected sim blockchain config to equal params.AllEthashProtocolChanges, got %v", sim.config) } - statedb, _ := sim.blockchain.State() + statedb, _, _ := sim.blockchain.State() bal := statedb.GetBalance(testAddr) if bal.Cmp(expectedBal) != 0 { t.Errorf("expected balance for test address not received. expected: %v actual: %v", expectedBal, bal) @@ -253,7 +253,7 @@ func TestSimulatedBackend_NonceAt(t *testing.T) { } // send tx to simulated backend - err = sim.SendTransaction(bgCtx, signedTx) + err = sim.SendTransaction(bgCtx, signedTx, bind.PrivateTxArgs{}) if err != nil { t.Errorf("could not add tx to pending block: %v", err) } @@ -288,7 +288,7 @@ func TestSimulatedBackend_SendTransaction(t *testing.T) { } // send tx to simulated backend - err = sim.SendTransaction(bgCtx, signedTx) + err = sim.SendTransaction(bgCtx, signedTx, bind.PrivateTxArgs{}) if err != nil { t.Errorf("could not add tx to pending block: %v", err) } @@ -323,7 +323,7 @@ func TestSimulatedBackend_TransactionByHash(t *testing.T) { } // send tx to simulated backend - err = sim.SendTransaction(bgCtx, signedTx) + err = sim.SendTransaction(bgCtx, signedTx, bind.PrivateTxArgs{}) if err != nil { t.Errorf("could not add tx to pending block: %v", err) } @@ -486,7 +486,7 @@ func TestSimulatedBackend_TransactionCount(t *testing.T) { } // send tx to simulated backend - err = sim.SendTransaction(bgCtx, signedTx) + err = sim.SendTransaction(bgCtx, signedTx, bind.PrivateTxArgs{}) if err != nil { t.Errorf("could not add tx to pending block: %v", err) } @@ -545,7 +545,7 @@ func TestSimulatedBackend_TransactionInBlock(t *testing.T) { } // send tx to simulated backend - err = sim.SendTransaction(bgCtx, signedTx) + err = sim.SendTransaction(bgCtx, signedTx, bind.PrivateTxArgs{}) if err != nil { t.Errorf("could not add tx to pending block: %v", err) } @@ -604,7 +604,7 @@ func TestSimulatedBackend_PendingNonceAt(t *testing.T) { } // send tx to simulated backend - err = sim.SendTransaction(bgCtx, signedTx) + err = sim.SendTransaction(bgCtx, signedTx, bind.PrivateTxArgs{}) if err != nil { t.Errorf("could not add tx to pending block: %v", err) } @@ -625,7 +625,7 @@ func TestSimulatedBackend_PendingNonceAt(t *testing.T) { if err != nil { t.Errorf("could not sign tx: %v", err) } - err = sim.SendTransaction(bgCtx, signedTx) + err = sim.SendTransaction(bgCtx, signedTx, bind.PrivateTxArgs{}) if err != nil { t.Errorf("could not send tx: %v", err) } @@ -660,7 +660,7 @@ func TestSimulatedBackend_TransactionReceipt(t *testing.T) { } // send tx to simulated backend - err = sim.SendTransaction(bgCtx, signedTx) + err = sim.SendTransaction(bgCtx, signedTx, bind.PrivateTxArgs{}) if err != nil { t.Errorf("could not add tx to pending block: %v", err) } diff --git a/consensus/protocol.go b/consensus/protocol.go index 368e358aee..8ccacdd2c7 100644 --- a/consensus/protocol.go +++ b/consensus/protocol.go @@ -13,6 +13,7 @@ import ( const ( eth63 = 63 eth64 = 64 + eth65 = 65 Istanbul64 = 64 Istanbul99 = 99 ) @@ -26,15 +27,15 @@ var ( CliqueProtocol = Protocol{ Name: "eth", - Versions: []uint{eth64, eth63}, - Lengths: map[uint]uint64{eth64: 17, eth63: 17}, + Versions: []uint{eth65, eth64, eth63}, + Lengths: map[uint]uint64{eth65: 17, eth64: 17, eth63: 17}, } // Default: Keep up-to-date with eth/protocol.go EthProtocol = Protocol{ Name: "eth", - Versions: []uint{eth64, eth63}, - Lengths: map[uint]uint64{eth64: 17, eth63: 17}, + Versions: []uint{eth65, eth64, eth63}, + Lengths: map[uint]uint64{eth65: 17, eth64: 17, eth63: 17}, } NorewardsProtocol = Protocol{ diff --git a/console/console.go b/console/console.go index f2e6107a8c..fb03cf9b09 100644 --- a/console/console.go +++ b/console/console.go @@ -115,91 +115,8 @@ func (c *Console) init(preload []string) error { // Initialize the JavaScript <-> Go RPC bridge. bridge := newBridge(c.client, c.prompter, c.printer) - c.jsre.Set("jeth", struct{}{}) - - jethObj, _ := c.jsre.Get("jeth") - jethObj.Object().Set("send", bridge.Send) - jethObj.Object().Set("sendAsync", bridge.Send) - - consoleObj, _ := c.jsre.Get("console") - consoleObj.Object().Set("log", c.consoleOutput) - consoleObj.Object().Set("error", c.consoleOutput) - - // Load all the internal utility JavaScript libraries - if err := c.jsre.Compile("bignumber.js", jsre.BignumberJs); err != nil { - return fmt.Errorf("bignumber.js: %v", err) - } - if err := c.jsre.Compile("web3.js", jsre.Web3Js); err != nil { - return fmt.Errorf("web3.js: %v", err) - } - if _, err := c.jsre.Run("var Web3 = require('web3');"); err != nil { - return fmt.Errorf("web3 require: %v", err) - } - if _, err := c.jsre.Run("var web3 = new Web3(jeth);"); err != nil { - return fmt.Errorf("web3 provider: %v", err) - } - // Load the supported APIs into the JavaScript runtime environment - apis, err := c.client.SupportedModules() - if err != nil { - return fmt.Errorf("api modules: %v", err) - } - flatten := "var eth = web3.eth; var personal = web3.personal; " - for api := range apis { - if api == "web3" { - continue // manually mapped or ignore - } - //quorum - // the @ symbol results in errors that prevent the extension from being added to the web3 object - api = strings.Replace(api, "plugin@", "plugin_", 1) - //!quorum - - if file, ok := web3ext.Modules[api]; ok { - // Load our extension for the module. - if err = c.jsre.Compile(fmt.Sprintf("%s.js", api), file); err != nil { - return fmt.Errorf("%s.js: %v", api, err) - } - flatten += fmt.Sprintf("var %s = web3.%s; ", api, api) - } else if obj, err := c.jsre.Run("web3." + api); err == nil && obj.IsObject() { - // Enable web3.js built-in extension if available. - flatten += fmt.Sprintf("var %s = web3.%s; ", api, api) - } - } - if _, err = c.jsre.Run(flatten); err != nil { - return fmt.Errorf("namespace flattening: %v", err) - } - // Initialize the global name register (disabled for now) - //c.jsre.Run(`var GlobalRegistrar = eth.contract(` + registrar.GlobalRegistrarAbi + `); registrar = GlobalRegistrar.at("` + registrar.GlobalRegistrarAddr + `");`) - - // If the console is in interactive mode, instrument password related methods to query the user - if c.prompter != nil { - // Retrieve the account management object to instrument - personal, err := c.jsre.Get("personal") - if err != nil { - return err - } - // Override the openWallet, unlockAccount, newAccount and sign methods since - // these require user interaction. Assign these method in the Console the - // original web3 callbacks. These will be called by the jeth.* methods after - // they got the password from the user and send the original web3 request to - // the backend. - if obj := personal.Object(); obj != nil { // make sure the personal api is enabled over the interface - if _, err = c.jsre.Run(`jeth.openWallet = personal.openWallet;`); err != nil { - return fmt.Errorf("personal.openWallet: %v", err) - } - if _, err = c.jsre.Run(`jeth.unlockAccount = personal.unlockAccount;`); err != nil { - return fmt.Errorf("personal.unlockAccount: %v", err) - } - if _, err = c.jsre.Run(`jeth.newAccount = personal.newAccount;`); err != nil { - return fmt.Errorf("personal.newAccount: %v", err) - } - if _, err = c.jsre.Run(`jeth.sign = personal.sign;`); err != nil { - return fmt.Errorf("personal.sign: %v", err) - } - obj.Set("openWallet", bridge.OpenWallet) - obj.Set("unlockAccount", bridge.UnlockAccount) - obj.Set("newAccount", bridge.NewAccount) - obj.Set("sign", bridge.Sign) - } + if err := c.initWeb3(bridge); err != nil { + return err } if err := c.initExtensions(); err != nil { return err @@ -279,6 +196,10 @@ func (c *Console) initExtensions() error { if api == "web3" { continue } + //quorum + // the @ symbol results in errors that prevent the extension from being added to the web3 object + api = strings.Replace(api, "plugin@", "plugin_", 1) + //!quorum aliases[api] = struct{}{} if file, ok := web3ext.Modules[api]; ok { if err = c.jsre.Compile(api+".js", file); err != nil { diff --git a/core/blockchain.go b/core/blockchain.go index 495a25544d..9b19617fbc 100644 --- a/core/blockchain.go +++ b/core/blockchain.go @@ -1369,7 +1369,7 @@ func (bc *BlockChain) CommitBlockWithState(deleteEmptyObjects bool, state, priva // writeBlockWithState writes the block and all associated state to the database, // but is expects the chain mutex to be held. -func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.Receipt, logs []*types.Log, state, privateState *state.StateDB) (status WriteStatus, err error) { +func (bc *BlockChain) writeBlockWithState(block *types.Block, receipts []*types.Receipt, logs []*types.Log, state, privateState *state.StateDB, emitHeadEvent bool) (status WriteStatus, err error) { bc.wg.Add(1) defer bc.wg.Done() @@ -1727,7 +1727,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er // QUORUM if bc.isRaft() { // Only returns an error for raft mode - return it.index, events, coalescedLogs, ErrAbortBlocksProcessing + return it.index, ErrAbortBlocksProcessing } // END QUORUM break @@ -1779,7 +1779,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er privateStateRoot := rawdb.GetPrivateStateRoot(bc.db, parent.Root) privateState, err := stateNew(privateStateRoot, bc.privateStateCache) if err != nil { - return it.index, events, coalescedLogs, err + return it.index, err } // /Quorum @@ -1845,7 +1845,7 @@ func (bc *BlockChain) insertChain(chain types.Blocks, verifySeals bool) (int, er } atomic.StoreUint32(&followupInterrupt, 1) if err := rawdb.WritePrivateBlockBloom(bc.db, block.NumberU64(), privateReceipts); err != nil { - return it.index, events, coalescedLogs, err + return it.index, err } // Update the metrics touched during block commit accountCommitTimer.Update(statedb.AccountCommits) // Account commits are complete, we can mark them diff --git a/eth/handler.go b/eth/handler.go index ae74b80fac..586f683ab0 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -415,7 +415,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // Quorum if pm.raftMode { - if msg.Code != TxMsg && + if msg.Code != TransactionMsg && msg.Code != GetBlockHeadersMsg && msg.Code != BlockHeadersMsg && msg.Code != GetBlockBodiesMsg && msg.Code != BlockBodiesMsg { @@ -860,7 +860,7 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // Quorum func (pm *ProtocolManager) Enqueue(id string, block *types.Block) { - pm.fetcher.Enqueue(id, block) + pm.blockFetcher.Enqueue(id, block) } // BroadcastBlock will either propagate a block to a subset of its peers, or @@ -913,16 +913,18 @@ func (pm *ProtocolManager) BroadcastTransactions(txs types.Transactions, propaga if propagate { for _, tx := range txs { peers := pm.peers.PeersWithoutTx(tx.Hash()) + // Send the block to a subset of our peers transfer := peers[:int(math.Sqrt(float64(len(peers))))] for _, peer := range transfer { - txset[peer] = append(txset[peer], tx) + txset[peer] = append(txset[peer], tx.Hash()) } log.Trace("Broadcast transaction", "hash", tx.Hash(), "recipients", len(peers)) } for peer, hashes := range txset { peer.AsyncSendTransactions(hashes) } + return } // Otherwise only broadcast the announcement to peers for _, tx := range txs { diff --git a/eth/handler_test.go b/eth/handler_test.go index 9557deeed7..3f1468671d 100644 --- a/eth/handler_test.go +++ b/eth/handler_test.go @@ -683,7 +683,7 @@ func TestBroadcastMalformedBlock(t *testing.T) { if err != nil { t.Fatalf("failed to create new blockchain: %v", err) } - pm, err := NewProtocolManager(config, nil, downloader.FullSync, DefaultConfig.NetworkId, new(event.TypeMux), new(testTxPool), engine, blockchain, db, 1, nil) + pm, err := NewProtocolManager(config, nil, downloader.FullSync, DefaultConfig.NetworkId, new(event.TypeMux), new(testTxPool), engine, blockchain, db, 1, nil, false) if err != nil { t.Fatalf("failed to start test protocol manager: %v", err) } diff --git a/eth/helper_test.go b/eth/helper_test.go index 7a233a16ae..67d525aa59 100644 --- a/eth/helper_test.go +++ b/eth/helper_test.go @@ -73,7 +73,7 @@ func newTestProtocolManager(mode downloader.SyncMode, blocks int, generator func if _, err := blockchain.InsertChain(chain); err != nil { panic(err) } - pm, err := NewProtocolManager(gspec.Config, nil, mode, DefaultConfig.NetworkId, evmux, &testTxPool{added: newtx}, engine, blockchain, db, 1, nil, false) + pm, err := NewProtocolManager(gspec.Config, nil, mode, DefaultConfig.NetworkId, evmux, &testTxPool{added: newtx, pool: make(map[common.Hash]*types.Transaction)}, engine, blockchain, db, 1, nil, false) if err != nil { return nil, nil, err } diff --git a/eth/peer.go b/eth/peer.go index 7c786d986c..865b932584 100644 --- a/eth/peer.go +++ b/eth/peer.go @@ -587,7 +587,7 @@ func (p *peer) Handshake(network uint64, td *big.Int, head common.Hash, genesis CurrentBlock: head, GenesisBlock: genesis, }) - case p.version == eth64 || istanbulNew: + case p.version >= eth64 || istanbulNew: errc <- p2p.Send(p.rw, StatusMsg, &statusData{ ProtocolVersion: uint32(p.version), NetworkID: network, diff --git a/go.mod b/go.mod index 7d085b16b1..e9b89133be 100644 --- a/go.mod +++ b/go.mod @@ -25,9 +25,9 @@ require ( github.com/deckarep/golang-set v0.0.0-20180603214616-504e848d77ea github.com/dlclark/regexp2 v1.2.0 // indirect github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf + github.com/dop251/goja v0.0.0-20200106141417-aaec0e7bde29 github.com/eapache/channels v1.1.0 github.com/eapache/queue v1.1.0 // indirect - github.com/dop251/goja v0.0.0-20200106141417-aaec0e7bde29 github.com/edsrzf/mmap-go v0.0.0-20160512033002-935e0e8a636c github.com/elastic/gosigar v0.8.1-0.20180330100440-37f05ff46ffa github.com/ethereum/go-ethereum/crypto/secp256k1 v0.0.0-00010101000000-000000000000 @@ -82,10 +82,10 @@ require ( golang.org/x/sync v0.0.0-20190423024810-112230192c58 golang.org/x/sys v0.0.0-20191008105621-543471e840be golang.org/x/text v0.3.2 + golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 google.golang.org/grpc v1.29.1 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 gopkg.in/karalabe/cookiejar.v2 v2.0.0-20150724131613-8dcd6a7f4951 - golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce gopkg.in/olebedev/go-duktape.v3 v3.0.0-20190213234257-ec84240a7772 gopkg.in/oleiade/lane.v1 v1.0.0 diff --git a/miner/worker.go b/miner/worker.go index edacf230db..45eb0eea05 100644 --- a/miner/worker.go +++ b/miner/worker.go @@ -633,7 +633,7 @@ func (w *worker) resultLoop() { allReceipts := mergeReceipts(pubReceipts, prvReceipts) // Commit block and state to database. - stat, err := w.chain.WriteBlockWithState(block, allReceipts, logs, task.state, task.privateState, true) + _, err := w.chain.WriteBlockWithState(block, allReceipts, logs, task.state, task.privateState, true) if err != nil { log.Error("Failed writing block to chain", "err", err) continue diff --git a/node/service.go b/node/service.go index 65cf549aa4..d3d5ddb42a 100644 --- a/node/service.go +++ b/node/service.go @@ -88,7 +88,7 @@ func (ctx *ServiceContext) Service(service interface{}) error { // NodeKey returns node key from config func (ctx *ServiceContext) NodeKey() *ecdsa.PrivateKey { - return ctx.config.NodeKey() + return ctx.Config.NodeKey() } // ExtRPCEnabled returns the indicator whether node enables the external diff --git a/p2p/server.go b/p2p/server.go index baacca7232..a569c01b5b 100644 --- a/p2p/server.go +++ b/p2p/server.go @@ -1001,11 +1001,6 @@ func (srv *Server) setupConn(c *conn, flags connFlag, dialDest *enode.Node) erro //END - QUORUM Permissioning - if conn, ok := c.fd.(*meteredConn); ok { - p := newPeer(srv.log, c, srv.Protocols) - conn.handshakeDone(p) - } - err = srv.checkpoint(c, srv.checkpointPostHandshake) if err != nil { clog.Trace("Rejected peer", "err", err) diff --git a/params/config.go b/params/config.go index f23507d951..e094d60c1c 100644 --- a/params/config.go +++ b/params/config.go @@ -228,7 +228,7 @@ var ( TestChainConfig = &ChainConfig{big.NewInt(10), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil, nil, false, 32, 32, big.NewInt(0), big.NewInt(0), nil} TestRules = TestChainConfig.Rules(new(big.Int)) - QuorumTestChainConfig = &ChainConfig{big.NewInt(10), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, new(EthashConfig), nil, nil, true, 64, 32, big.NewInt(0), big.NewInt(0), nil} + QuorumTestChainConfig = &ChainConfig{big.NewInt(10), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil, nil, true, 64, 32, big.NewInt(0), big.NewInt(0), nil} ) // TrustedCheckpoint represents a set of post-processed trie roots (CHT and diff --git a/rpc/websocket.go b/rpc/websocket.go index 84b5abf248..535c07bbcb 100644 --- a/rpc/websocket.go +++ b/rpc/websocket.go @@ -134,7 +134,7 @@ func (e wsHandshakeError) Error() string { // The context is used for the initial connection establishment. It does not // affect subsequent interactions with the client. func DialWebsocketWithDialer(ctx context.Context, endpoint, origin string, dialer websocket.Dialer) (*Client, error) { - return DialWebsocketWithCustomTLS(ctx, endpoint, origin, dialer, nil) + return DialWebsocketWithCustomTLS(ctx, endpoint, origin, nil) } // Quorum @@ -145,7 +145,13 @@ func DialWebsocketWithDialer(ctx context.Context, endpoint, origin string, diale // // The context is used for the initial connection establishment. It does not // affect subsequent interactions with the client. -func DialWebsocketWithCustomTLS(ctx context.Context, endpoint, origin string, dialer websocket.Dialer, tlsConfig *tls.Config) (*Client, error) { +func DialWebsocketWithCustomTLS(ctx context.Context, endpoint, origin string, tlsConfig *tls.Config) (*Client, error) { + dialer := websocket.Dialer{ + ReadBufferSize: wsReadBuffer, + WriteBufferSize: wsWriteBuffer, + WriteBufferPool: wsBufferPool, + } + endpoint, header, err := wsClientHeaders(endpoint, origin) if err != nil { return nil, err @@ -153,6 +159,7 @@ func DialWebsocketWithCustomTLS(ctx context.Context, endpoint, origin string, di if tlsConfig != nil { dialer.TLSClientConfig = tlsConfig } + credProviderFunc, hasCredProviderFunc := ctx.Value(CtxCredentialsProvider).(HttpCredentialsProviderFunc) return newClient(ctx, func(ctx context.Context) (ServerCodec, error) { if hasCredProviderFunc { From eb2b72bb5522dc1a193b1310c20f3688aa660062 Mon Sep 17 00:00:00 2001 From: amalraj mani Date: Fri, 28 Aug 2020 16:51:02 +0800 Subject: [PATCH 122/128] add make bootnode --- Makefile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 2b424e929f..0cac496449 100644 --- a/Makefile +++ b/Makefile @@ -18,11 +18,10 @@ geth: @echo "Run \"$(GOBIN)/geth\" to launch geth." bootnode: - build/env.sh go run build/ci.go install ./cmd/bootnode + $(GORUN) build/ci.go install ./cmd/bootnode @echo "Done building." @echo "Run \"$(GOBIN)/bootnode\" to launch bootnode." - all: $(GORUN) build/ci.go install From 039269555dac6515d4f53ffe5eb5e4e159c5f3e4 Mon Sep 17 00:00:00 2001 From: Ricardo Silva Date: Thu, 14 Jan 2021 17:19:02 +0000 Subject: [PATCH 123/128] fix(unit test): tx pool tests to use the quourum chain config. refactor code to remove duplication fix(unit test): handler test that access internal config value --- core/tx_pool.go | 27 ++++----------------------- core/tx_pool_test.go | 29 ++++++++++++++--------------- eth/handler.go | 5 ++++- params/config.go | 40 ++++++++++++++++++++++++++-------------- raft/handler_test.go | 7 ++++--- 5 files changed, 52 insertions(+), 56 deletions(-) diff --git a/core/tx_pool.go b/core/tx_pool.go index 2c5363981f..f7ddcdbbac 100644 --- a/core/tx_pool.go +++ b/core/tx_pool.go @@ -32,6 +32,7 @@ import ( "github.com/ethereum/go-ethereum/log" "github.com/ethereum/go-ethereum/metrics" "github.com/ethereum/go-ethereum/params" + pcore "github.com/ethereum/go-ethereum/permission/core" ) const ( @@ -49,6 +50,7 @@ const ( // more expensive to propagate; larger transactions also take more resources // to validate whether they fit into the pool or not. txMaxSize = 2 * txSlotSize // 64KB, don't bump without EIP-2464 support + // Quorum - value above is not used. instead, ChainConfig.TransactionSizeLimit is used ) var ( @@ -169,7 +171,6 @@ type TxPoolConfig struct { // Quorum TransactionSizeLimit uint64 // Maximum size allowed for valid transaction (in KB) MaxCodeSize uint64 // Maximum size allowed of contract code that can be deployed (in KB) - } // DefaultTxPoolConfig contains the default configurations for the transaction @@ -576,8 +577,8 @@ func (pool *TxPool) validateTx(tx *types.Transaction, local bool) error { if tx.IsPrivate() && (len(tx.Data()) == 0 || tx.Value().Sign() != 0) { return ErrEtherValueUnsupported } - // Check if the sender account is authorized to perform the transaction - if err := checkAccount(from, tx.To()); err != nil { + // Quorum - check if the sender account is authorized to perform the transaction + if err := pcore.CheckAccountPermission(tx.From(), tx.To(), tx.Value(), tx.Data(), tx.Gas(), tx.GasPrice()); err != nil { return err } } else { @@ -1632,26 +1633,6 @@ func (t *txLookup) Remove(hash common.Hash) { delete(t.all, hash) } -// checks if the account is has the necessary access for the transaction -func checkAccount(fromAcct common.Address, toAcct *common.Address) error { - access := types.GetAcctAccess(fromAcct) - - switch access { - case types.ReadOnly: - return errors.New("read only account. cannot transact") - - case types.Transact: - if toAcct == nil { - return errors.New("account does not have contract create permissions") - } - - case types.FullAccess, types.ContractDeploy: - return nil - - } - return nil -} - // helper function to return chainHeadChannel size func GetChainHeadChannleSize() int { return chainHeadChanSize diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index d0b312c15d..6230ee2bfc 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -88,24 +88,22 @@ func pricedDataTransaction(nonce uint64, gaslimit uint64, gasprice *big.Int, key } func setupTxPool() (*TxPool, *ecdsa.PrivateKey) { + return setupTxPoolWithConfig(params.TestChainConfig) +} + +func setupTxPoolWithConfig(config *params.ChainConfig) (*TxPool, *ecdsa.PrivateKey) { statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) - blockchain := &testBlockChain{statedb, statedb, 1000000, new(event.Feed)} + // TODO ricardolyn: investigate why 1m is not enough for gasLimit. changed to 10m for now + blockchain := &testBlockChain{statedb, statedb, 10000000, new(event.Feed)} key, _ := crypto.GenerateKey() - pool := NewTxPool(testTxPoolConfig, params.TestChainConfig, blockchain) + pool := NewTxPool(testTxPoolConfig, config, blockchain) return pool, key } func setupQuorumTxPool() (*TxPool, *ecdsa.PrivateKey) { - db := rawdb.NewMemoryDatabase() - statedb, _ := state.New(common.Hash{}, state.NewDatabase(db)) - blockchain := &testBlockChain{statedb, statedb, 1000000, new(event.Feed)} - - key, _ := crypto.GenerateKey() - pool := NewTxPool(testTxPoolConfig, params.QuorumTestChainConfig, blockchain) - - return pool, key + return setupTxPoolWithConfig(params.QuorumTestChainConfig) } // validateTxPoolInternals checks various consistency invariants within the pool. @@ -1133,7 +1131,7 @@ func TestTransactionAllowedTxSize(t *testing.T) { t.Parallel() // Create a test account and fund it - pool, key := setupTxPool() + pool, key := setupQuorumTxPool() defer pool.Stop() account := crypto.PubkeyToAddress(key.PublicKey) @@ -1153,20 +1151,21 @@ func TestTransactionAllowedTxSize(t *testing.T) { dataSize := txMaxSize - baseSize // Try adding a transaction with maximal allowed size - tx := pricedDataTransaction(0, pool.currentMaxGas, big.NewInt(1), key, dataSize) + gasPrice := big.NewInt(0) + tx := pricedDataTransaction(0, pool.currentMaxGas, gasPrice, key, dataSize) if err := pool.addRemoteSync(tx); err != nil { t.Fatalf("failed to add transaction of size %d, close to maximal: %v", int(tx.Size()), err) } // Try adding a transaction with random allowed size - if err := pool.addRemoteSync(pricedDataTransaction(1, pool.currentMaxGas, big.NewInt(1), key, uint64(rand.Intn(int(dataSize))))); err != nil { + if err := pool.addRemoteSync(pricedDataTransaction(1, pool.currentMaxGas, gasPrice, key, uint64(rand.Intn(int(dataSize))))); err != nil { t.Fatalf("failed to add transaction of random allowed size: %v", err) } // Try adding a transaction of minimal not allowed size - if err := pool.addRemoteSync(pricedDataTransaction(2, pool.currentMaxGas, big.NewInt(1), key, txMaxSize)); err == nil { + if err := pool.addRemoteSync(pricedDataTransaction(2, pool.currentMaxGas, gasPrice, key, txMaxSize)); err == nil { t.Fatalf("expected rejection on slightly oversize transaction") } // Try adding a transaction of random not allowed size - if err := pool.addRemoteSync(pricedDataTransaction(2, pool.currentMaxGas, big.NewInt(1), key, dataSize+1+uint64(rand.Intn(int(10*txMaxSize))))); err == nil { + if err := pool.addRemoteSync(pricedDataTransaction(2, pool.currentMaxGas, gasPrice, key, dataSize+1+uint64(rand.Intn(int(10*txMaxSize))))); err == nil { t.Fatalf("expected rejection on oversize transaction") } // Run some sanity checks on the pool internals diff --git a/eth/handler.go b/eth/handler.go index 586f683ab0..52700e9959 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -415,10 +415,13 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // Quorum if pm.raftMode { - if msg.Code != TransactionMsg && + // TODO ricardolyn: refactor to reduce duplication of msg.Code + if msg.Code != TransactionMsg && msg.Code != PooledTransactionsMsg && + msg.Code != GetPooledTransactionsMsg && msg.Code != NewPooledTransactionHashesMsg && msg.Code != GetBlockHeadersMsg && msg.Code != BlockHeadersMsg && msg.Code != GetBlockBodiesMsg && msg.Code != BlockBodiesMsg { + // Error coming from here log.Info("raft: ignoring message", "code", msg.Code) return nil diff --git a/params/config.go b/params/config.go index e094d60c1c..08d9f315da 100644 --- a/params/config.go +++ b/params/config.go @@ -216,19 +216,19 @@ var ( // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil, nil, false, 32, 32, big.NewInt(0), big.NewInt(0), nil} + AllEthashProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil, nil, false, 32, 35, big.NewInt(0), big.NewInt(0), nil, nil} // AllCliqueProtocolChanges contains every protocol change (EIPs) introduced // and accepted by the Ethereum core developers into the Clique consensus. // // This configuration is intentionally not using keyed fields to force anyone // adding flags to the config to also have to set these fields. - AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}, nil, false, 32, 32, big.NewInt(0), big.NewInt(0), nil} + AllCliqueProtocolChanges = &ChainConfig{big.NewInt(1337), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, nil, &CliqueConfig{Period: 0, Epoch: 30000}, nil, false, 32, 32, big.NewInt(0), big.NewInt(0), nil, nil} - TestChainConfig = &ChainConfig{big.NewInt(10), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil, nil, false, 32, 32, big.NewInt(0), big.NewInt(0), nil} + TestChainConfig = &ChainConfig{big.NewInt(10), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil, nil, false, 32, 32, big.NewInt(0), big.NewInt(0), nil, nil} TestRules = TestChainConfig.Rules(new(big.Int)) - QuorumTestChainConfig = &ChainConfig{big.NewInt(10), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil, nil, true, 64, 32, big.NewInt(0), big.NewInt(0), nil} + QuorumTestChainConfig = &ChainConfig{big.NewInt(10), big.NewInt(0), nil, false, big.NewInt(0), common.Hash{}, big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), big.NewInt(0), nil, nil, new(EthashConfig), nil, nil, true, 64, 32, big.NewInt(0), big.NewInt(0), nil, big.NewInt(0)} ) // TrustedCheckpoint represents a set of post-processed trie roots (CHT and @@ -321,6 +321,7 @@ type ChainConfig struct { // to track multiple changes to maxCodeSize MaxCodeSizeConfig []MaxCodeConfigStruct `json:"maxCodeSizeConfig,omitempty"` // Quorum + PrivacyEnhancementsBlock *big.Int `json:"privacyEnhancementsBlock,omitempty"` } // EthashConfig is the consensus engine configs for proof-of-work based sealing. @@ -368,7 +369,7 @@ func (c *ChainConfig) String() string { default: engine = "unknown" } - return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v IsQuorum: %v Constantinople: %v TransactionSizeLimit: %v MaxCodeSize: %v Petersburg: %v Istanbul: %v Muir Glacier: %v Engine: %v}", + return fmt.Sprintf("{ChainID: %v Homestead: %v DAO: %v DAOSupport: %v EIP150: %v EIP155: %v EIP158: %v Byzantium: %v IsQuorum: %v Constantinople: %v TransactionSizeLimit: %v MaxCodeSize: %v Petersburg: %v Istanbul: %v Muir Glacier: %v PrivacyEnhancements: %v Engine: %v}", c.ChainID, c.HomesteadBlock, c.DAOForkBlock, @@ -384,6 +385,7 @@ func (c *ChainConfig) String() string { c.PetersburgBlock, c.IstanbulBlock, c.MuirGlacierBlock, + c.PrivacyEnhancementsBlock, engine, ) } @@ -575,6 +577,11 @@ func isMaxCodeSizeConfigCompatible(c1, c2 *ChainConfig, head *big.Int) (error, * return nil, big.NewInt(0), big.NewInt(0) } +// IsPrivacyEnhancementsEnabled returns whether num represents a block number after the PrivacyEnhancementsEnabled fork +func (c *ChainConfig) IsPrivacyEnhancementsEnabled(num *big.Int) bool { + return isForked(c.PrivacyEnhancementsBlock, num) +} + // /Quorum // CheckCompatible checks whether scheduled fork transitions have been imported @@ -695,6 +702,9 @@ func (c *ChainConfig) checkCompatible(newcfg *ChainConfig, head *big.Int, isQuor if newcfg.MaxCodeSizeChangeBlock != nil && isForkIncompatible(c.MaxCodeSizeChangeBlock, newcfg.MaxCodeSizeChangeBlock, head) { return newCompatError("max code size change fork block", c.MaxCodeSizeChangeBlock, newcfg.MaxCodeSizeChangeBlock) } + if isForkIncompatible(c.PrivacyEnhancementsBlock, newcfg.PrivacyEnhancementsBlock, head) { + return newCompatError("Privacy Enhancements fork block", c.PrivacyEnhancementsBlock, newcfg.PrivacyEnhancementsBlock) + } return nil } @@ -762,6 +772,7 @@ type Rules struct { ChainID *big.Int IsHomestead, IsEIP150, IsEIP155, IsEIP158 bool IsByzantium, IsConstantinople, IsPetersburg, IsIstanbul bool + IsPrivacyEnhancementsEnabled bool } // Rules ensures c's ChainID is not nil. @@ -771,14 +782,15 @@ func (c *ChainConfig) Rules(num *big.Int) Rules { chainID = new(big.Int) } return Rules{ - ChainID: new(big.Int).Set(chainID), - IsHomestead: c.IsHomestead(num), - IsEIP150: c.IsEIP150(num), - IsEIP155: c.IsEIP155(num), - IsEIP158: c.IsEIP158(num), - IsByzantium: c.IsByzantium(num), - IsConstantinople: c.IsConstantinople(num), - IsPetersburg: c.IsPetersburg(num), - IsIstanbul: c.IsIstanbul(num), + ChainID: new(big.Int).Set(chainID), + IsHomestead: c.IsHomestead(num), + IsEIP150: c.IsEIP150(num), + IsEIP155: c.IsEIP155(num), + IsEIP158: c.IsEIP158(num), + IsByzantium: c.IsByzantium(num), + IsConstantinople: c.IsConstantinople(num), + IsPetersburg: c.IsPetersburg(num), + IsIstanbul: c.IsIstanbul(num), + IsPrivacyEnhancementsEnabled: c.IsPrivacyEnhancementsEnabled(num), } } diff --git a/raft/handler_test.go b/raft/handler_test.go index 57b0381347..424bd400c4 100644 --- a/raft/handler_test.go +++ b/raft/handler_test.go @@ -4,7 +4,6 @@ import ( "crypto/ecdsa" "encoding/binary" "fmt" - etcdRaft "github.com/coreos/etcd/raft" "io/ioutil" "net" "os" @@ -13,6 +12,8 @@ import ( "time" "unsafe" + etcdRaft "github.com/coreos/etcd/raft" + "github.com/coreos/etcd/wal" "github.com/coreos/etcd/wal/walpb" "github.com/ethereum/go-ethereum/core" @@ -155,9 +156,9 @@ func prepareServiceContext(key *ecdsa.PrivateKey) (ctx *node.ServiceContext, cfg EventMux: new(event.TypeMux), } // config is private field so we need some workaround to set the value - configField := reflect.ValueOf(ctx).Elem().FieldByName("config") + configField := reflect.ValueOf(ctx).Elem().FieldByName("Config") configField = reflect.NewAt(configField.Type(), unsafe.Pointer(configField.UnsafeAddr())).Elem() - configField.Set(reflect.ValueOf(cfg)) + configField.Set(reflect.ValueOf(*cfg)) return } From c8d63d17b07b88972fd6ce5824a96b815ae6614d Mon Sep 17 00:00:00 2001 From: Ricardo Silva Date: Fri, 15 Jan 2021 11:15:28 +0000 Subject: [PATCH 124/128] fix: set the max gas to 10m on tx_pool test to match what is on geth refactor: tidy up the code that checks the message type when using raft --- core/tx_pool_test.go | 1 - eth/handler.go | 15 +++++++-------- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/core/tx_pool_test.go b/core/tx_pool_test.go index 6230ee2bfc..22bf9314ef 100644 --- a/core/tx_pool_test.go +++ b/core/tx_pool_test.go @@ -93,7 +93,6 @@ func setupTxPool() (*TxPool, *ecdsa.PrivateKey) { func setupTxPoolWithConfig(config *params.ChainConfig) (*TxPool, *ecdsa.PrivateKey) { statedb, _ := state.New(common.Hash{}, state.NewDatabase(rawdb.NewMemoryDatabase())) - // TODO ricardolyn: investigate why 1m is not enough for gasLimit. changed to 10m for now blockchain := &testBlockChain{statedb, statedb, 10000000, new(event.Feed)} key, _ := crypto.GenerateKey() diff --git a/eth/handler.go b/eth/handler.go index 52700e9959..e763eb3bbe 100644 --- a/eth/handler.go +++ b/eth/handler.go @@ -415,15 +415,14 @@ func (pm *ProtocolManager) handleMsg(p *peer) error { // Quorum if pm.raftMode { - // TODO ricardolyn: refactor to reduce duplication of msg.Code - if msg.Code != TransactionMsg && msg.Code != PooledTransactionsMsg && - msg.Code != GetPooledTransactionsMsg && msg.Code != NewPooledTransactionHashesMsg && - msg.Code != GetBlockHeadersMsg && msg.Code != BlockHeadersMsg && - msg.Code != GetBlockBodiesMsg && msg.Code != BlockBodiesMsg { - - // Error coming from here + switch msg.Code { + case TransactionMsg, PooledTransactionsMsg, + GetPooledTransactionsMsg, NewPooledTransactionHashesMsg, + GetBlockHeadersMsg, BlockHeadersMsg, + GetBlockBodiesMsg, BlockBodiesMsg: + // supported by Raft + default: log.Info("raft: ignoring message", "code", msg.Code) - return nil } } else if handler, ok := pm.engine.(consensus.Handler); ok { From 3a5ef1d3739425f958568c6da37829eef1fd0fe3 Mon Sep 17 00:00:00 2001 From: Ricardo Silva Date: Wed, 3 Feb 2021 10:44:48 +0000 Subject: [PATCH 125/128] fix: add stub method to the dummy chain for test --- core/vm/runtime/runtime_test.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index f2d05118ce..19d6a7b969 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -17,6 +17,8 @@ package runtime import ( + "context" + "github.com/jpmorganchase/quorum-security-plugin-sdk-go/proto" "math/big" "strings" "testing" @@ -242,6 +244,10 @@ func (d *dummyChain) GetHeader(h common.Hash, n uint64) *types.Header { return fakeHeader(n, parentHash) } +func (d *dummyChain) SupportsMultitenancy(context.Context) (*proto.PreAuthenticatedAuthenticationToken, bool) { + return nil, false +} + // TestBlockhash tests the blockhash operation. It's a bit special, since it internally // requires access to a chain reader. func TestBlockhash(t *testing.T) { From 20b9e79514b0fb2d87d58869139ebd0ccc7b6d15 Mon Sep 17 00:00:00 2001 From: Ricardo Silva Date: Wed, 3 Feb 2021 10:53:21 +0000 Subject: [PATCH 126/128] lint: upgraded golangci to version 1.27.0, same version as used by go-ethereum with go1.15 --- build/checksums.txt | 31 ++++++++++++++++--------------- build/ci.go | 2 +- 2 files changed, 17 insertions(+), 16 deletions(-) diff --git a/build/checksums.txt b/build/checksums.txt index 553d1bbefe..e126f71b3e 100644 --- a/build/checksums.txt +++ b/build/checksums.txt @@ -2,18 +2,19 @@ b13bf04633d4d8cf53226ebeaace8d4d2fd07ae6fa676d0844a688339debec34 go1.13.8.src.tar.gz -478994633b0f5121a7a8d4f368078093e21014fdc7fb2c0ceeae63668c13c5b6 golangci-lint-1.22.2-freebsd-amd64.tar.gz -fcf80824c21567eb0871055711bf9bdca91cf9a081122e2a45f1d11fed754600 golangci-lint-1.22.2-darwin-amd64.tar.gz -cda85c72fc128b2ea0ae05baea7b91172c63aea34064829f65285f1dd536f1e0 golangci-lint-1.22.2-windows-386.zip -94f04899f620aadc9c1524e5482e415efdbd993fa2b2918c4fec2798f030ac1c golangci-lint-1.22.2-linux-armv7.tar.gz -0e72a87d71edde00b6e37e84a99841833ad55fee83e20d21130a7a622b2860bb golangci-lint-1.22.2-freebsd-386.tar.gz -86def2f31fe8fd7c05674104ed2a4bef3e44b7132b93c6ad2f52f198b3d01801 golangci-lint-1.22.2-linux-s390x.tar.gz -b0df4546d36be94e8107733ba290b98dd9b7e41a42d3fb202e87fc7e4ee800c3 golangci-lint-1.22.2-freebsd-armv6.tar.gz -3d45958dcf6a8d195086d2fced1a21db42a90815dfd156d180efa62dbdda6724 golangci-lint-1.22.2-darwin-386.tar.gz -7ee29f35c74fab017a454237990c74d984ce3855960f2c10509238992bb781f9 golangci-lint-1.22.2-linux-arm64.tar.gz -52086ac52a502b68578e58e35d3964f127c16d7a90b9ffcb399a004d055ded51 golangci-lint-1.22.2-linux-386.tar.gz -c2e4df1fab2ae53762f9baac6041503eeeaa968ce38ea41779f7cb526751c667 golangci-lint-1.22.2-windows-amd64.zip -109d38cdc89f271392f5a138d6782657157f9f496fd4801956efa2d0428e0cbe golangci-lint-1.22.2-linux-amd64.tar.gz -f08aae4868d4828c8f07deb0dcd941a1da695b97e58d15e9f3d1d07dcc7a0c84 golangci-lint-1.22.2-linux-armv6.tar.gz -37af03d9c144d527cb15c46a07e6a22d3f62b5491e34ad6f3bfe6bb0b0b597d4 golangci-lint-1.22.2-linux-ppc64le.tar.gz -251a1081d53944f1d5f86216d752837b23079f90605c9d1cc628da1ffcd2e749 golangci-lint-1.22.2-freebsd-armv7.tar.gz +d998a84eea42f2271aca792a7b027ca5c1edfcba229e8e5a844c9ac3f336df35 golangci-lint-1.27.0-linux-armv7.tar.gz +bf781f05b0d393b4bf0a327d9e62926949a4f14d7774d950c4e009fc766ed1d4 golangci-lint.exe-1.27.0-windows-amd64.zip +bf781f05b0d393b4bf0a327d9e62926949a4f14d7774d950c4e009fc766ed1d4 golangci-lint-1.27.0-windows-amd64.zip +0e2a57d6ba709440d3ed018ef1037465fa010ed02595829092860e5cf863042e golangci-lint-1.27.0-freebsd-386.tar.gz +90205fc42ab5ed0096413e790d88ac9b4ed60f4c47e576d13dc0660f7ed4b013 golangci-lint-1.27.0-linux-arm64.tar.gz +8d345e4e88520e21c113d81978e89ad77fc5b13bfdf20e5bca86b83fc4261272 golangci-lint-1.27.0-linux-amd64.tar.gz +cc619634a77f18dc73df2a0725be13116d64328dc35131ca1737a850d6f76a59 golangci-lint-1.27.0-freebsd-armv7.tar.gz +fe683583cfc9eeec83e498c0d6159d87b5e1919dbe4b6c3b3913089642906069 golangci-lint-1.27.0-linux-s390x.tar.gz +058f5579bee75bdaacbaf75b75e1369f7ad877fd8b3b145aed17a17545de913e golangci-lint-1.27.0-freebsd-armv6.tar.gz +38e1e3dadbe3f56ab62b4de82ee0b88e8fad966d8dfd740a26ef94c2edef9818 golangci-lint-1.27.0-linux-armv6.tar.gz +071b34af5516f4e1ddcaea6011e18208f4f043e1af8ba21eeccad4585cb3d095 golangci-lint.exe-1.27.0-windows-386.zip +071b34af5516f4e1ddcaea6011e18208f4f043e1af8ba21eeccad4585cb3d095 golangci-lint-1.27.0-windows-386.zip +5f37e2b33914ecddb7cad38186ef4ec61d88172fc04f930fa0267c91151ff306 golangci-lint-1.27.0-linux-386.tar.gz +4d94cfb51fdebeb205f1d5a349ac2b683c30591c5150708073c1c329e15965f0 golangci-lint-1.27.0-freebsd-amd64.tar.gz +52572ba8ff07d5169c2365d3de3fec26dc55a97522094d13d1596199580fa281 golangci-lint-1.27.0-linux-ppc64le.tar.gz +3fb1a1683a29c6c0a8cd76135f62b606fbdd538d5a7aeab94af1af70ffdc2fd4 golangci-lint-1.27.0-darwin-amd64.tar.gz diff --git a/build/ci.go b/build/ci.go index a900776571..0fa3781b5a 100644 --- a/build/ci.go +++ b/build/ci.go @@ -360,7 +360,7 @@ func doLint(cmdline []string) { // downloadLinter downloads and unpacks golangci-lint. func downloadLinter(cachedir string) string { - const version = "1.22.2" + const version = "1.27.0" csdb := build.MustLoadChecksums("build/checksums.txt") base := fmt.Sprintf("golangci-lint-%s-%s-%s", version, runtime.GOOS, runtime.GOARCH) From d929a3ff9772a44fa325f39510f4e71daa950e36 Mon Sep 17 00:00:00 2001 From: Ricardo Silva Date: Wed, 3 Feb 2021 11:41:02 +0000 Subject: [PATCH 127/128] lint:fix lint issues lint: ignored for now the issue with "S1039: unnecessary use of fmt.Sprintf (gosimple)" as it required many changes in geth code base. we will remove this exclusion when we get to geth 1.9.20 --- .golangci.yml | 4 ++++ core/vm/runtime/runtime_test.go | 3 ++- .../privacyExtension/state_set_utilities.go | 2 +- internal/ethapi/api_test.go | 21 ------------------- 4 files changed, 7 insertions(+), 23 deletions(-) diff --git a/.golangci.yml b/.golangci.yml index 24d00da6ec..42f4f59aac 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -32,6 +32,10 @@ linters-settings: min-occurrences: 6 # minimum number of occurrences issues: + # Quorum - Disabling check for "S1039: unnecessary use of fmt.Sprintf (gosimple)" until we upgrade to geth 1.9.20 and have the lint fixes that also were applied in go-ethereum, as it was creating many necessary changes in geth codebase + exclude: + - S1039 + # End Quorum exclude-rules: - path: crypto/blake2b/ linters: diff --git a/core/vm/runtime/runtime_test.go b/core/vm/runtime/runtime_test.go index 19d6a7b969..25d6464da0 100644 --- a/core/vm/runtime/runtime_test.go +++ b/core/vm/runtime/runtime_test.go @@ -18,11 +18,12 @@ package runtime import ( "context" - "github.com/jpmorganchase/quorum-security-plugin-sdk-go/proto" "math/big" "strings" "testing" + "github.com/jpmorganchase/quorum-security-plugin-sdk-go/proto" + "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus" diff --git a/extension/privacyExtension/state_set_utilities.go b/extension/privacyExtension/state_set_utilities.go index 1d6143735b..5b1814d191 100644 --- a/extension/privacyExtension/state_set_utilities.go +++ b/extension/privacyExtension/state_set_utilities.go @@ -68,7 +68,7 @@ func setManagedParties(ptm private.PrivateTransactionManager, privateState *stat return } - _, managedParties, _, _, err := ptm.Receive(ptmHash) + _, managedParties, _, _, _ := ptm.Receive(ptmHash) newManagedParties := common.AppendSkipDuplicates(existingManagedParties, managedParties...) privateState.SetManagedParties(address, newManagedParties) } diff --git a/internal/ethapi/api_test.go b/internal/ethapi/api_test.go index 77e0b5eec0..4eb2006766 100644 --- a/internal/ethapi/api_test.go +++ b/internal/ethapi/api_test.go @@ -445,27 +445,6 @@ func TestHandlePrivateTransaction_whenRawStandardPrivateMessageCall(t *testing.T } -// Copy and set private -func copyTransaction(tx *types.Transaction) *types.Transaction { - var privateTx *types.Transaction - if tx.To() == nil { - privateTx = types.NewContractCreation(tx.Nonce(), - tx.Value(), - tx.Gas(), - tx.GasPrice(), - tx.Data()) - } else { - privateTx = types.NewTransaction(tx.Nonce(), - *tx.To(), - tx.Value(), - tx.Gas(), - tx.GasPrice(), - tx.Data()) - } - privateTx.SetPrivate() - return privateTx -} - type StubBackend struct { getEVMCalled bool mockAccountExtraDataStateGetter *vm.MockAccountExtraDataStateGetter From 54612f5cb994107205eea9478a702634b63de02a Mon Sep 17 00:00:00 2001 From: Ricardo Silva Date: Wed, 3 Feb 2021 14:38:17 +0000 Subject: [PATCH 128/128] fix: adding back missing code in eth/backend.go that was added in PR https://github.com/ethereum/go-ethereum/pull/20660/files --- eth/backend.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/eth/backend.go b/eth/backend.go index 061262430d..7a8ac949bf 100644 --- a/eth/backend.go +++ b/eth/backend.go @@ -266,6 +266,11 @@ func New(ctx *node.ServiceContext, config *Config) (*Ethereum, error) { } eth.APIBackend.gpo = gasprice.NewOracle(eth.APIBackend, gpoParams) + eth.dialCandiates, err = eth.setupDiscovery(&ctx.Config.P2P) + if err != nil { + return nil, err + } + return eth, nil }