From 449b5d23e6ea42aa8871b0dbd9d6cea142995fa3 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 23 Feb 2023 09:02:29 +0700 Subject: [PATCH 001/100] save --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 939851cb199..a3611b09569 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230222083020-fe85438cb4e2 + github.com/ledgerwatch/erigon-lib v0.0.0-20230223015813-429ae81cbd44 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230215122237-5e33484e9be4 github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index 59e720b38b3..62329f5eb4a 100644 --- a/go.sum +++ b/go.sum @@ -521,8 +521,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230222083020-fe85438cb4e2 h1:Ws0LhbV7tpK03UAz2e2OCoK88RcUsOT5zp9H50BJ9PM= -github.com/ledgerwatch/erigon-lib v0.0.0-20230222083020-fe85438cb4e2/go.mod h1:y9a/PjMYVygqnGHD4mmaBePuI5P7kgtVnfiOuc2c1pU= +github.com/ledgerwatch/erigon-lib v0.0.0-20230223015813-429ae81cbd44 h1:OWtZNNSi6t9KbMgxJeGQpKJB1164p6nPV0cRbkQyiww= +github.com/ledgerwatch/erigon-lib v0.0.0-20230223015813-429ae81cbd44/go.mod h1:y9a/PjMYVygqnGHD4mmaBePuI5P7kgtVnfiOuc2c1pU= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230215122237-5e33484e9be4 h1:KH4/JnCdg52tcTqBfv4dVwa3jryncGXjSByrzkNUXxc= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230215122237-5e33484e9be4/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From 986ccdb846130a7c3b0e05fe3095db1dc5b13be5 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 23 Feb 2023 09:42:18 +0700 Subject: [PATCH 002/100] save --- go.mod | 4 ++-- go.sum | 10 ++++------ 2 files changed, 6 insertions(+), 8 deletions(-) diff --git a/go.mod b/go.mod index a3611b09569..e571ab3eb35 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/99designs/gqlgen v0.17.24 github.com/Giulio2002/bls v0.0.0-20230217173148-c87a29266b6c github.com/RoaringBitmap/roaring v1.2.3 - github.com/VictoriaMetrics/fastcache v1.12.0 + github.com/VictoriaMetrics/fastcache v1.12.1 github.com/VictoriaMetrics/metrics v1.23.1 github.com/anacrolix/sync v0.4.0 github.com/anacrolix/torrent v1.48.1-0.20230219022425-e8971ea0f1bf @@ -79,7 +79,7 @@ require ( github.com/tidwall/btree v1.6.0 github.com/ugorji/go/codec v1.1.13 github.com/ugorji/go/codec/codecgen v1.1.13 - github.com/urfave/cli/v2 v2.24.3 + github.com/urfave/cli/v2 v2.24.4 github.com/valyala/fastjson v1.6.4 github.com/vektah/gqlparser/v2 v2.5.1 github.com/xsleonard/go-merkle v1.1.0 diff --git a/go.sum b/go.sum index 62329f5eb4a..ecc814620b6 100644 --- a/go.sum +++ b/go.sum @@ -27,8 +27,8 @@ github.com/RoaringBitmap/roaring v1.2.3 h1:yqreLINqIrX22ErkKI0vY47/ivtJr6n+kMhVO github.com/RoaringBitmap/roaring v1.2.3/go.mod h1:plvDsJQpxOC5bw8LRteu/MLWHsHez/3y6cubLI4/1yE= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= -github.com/VictoriaMetrics/fastcache v1.12.0 h1:vnVi/y9yKDcD9akmc4NqAoqgQhJrOwUF+j9LTgn4QDE= -github.com/VictoriaMetrics/fastcache v1.12.0/go.mod h1:tjiYeEfYXCqacuvYw/7UoDIeJaNxq6132xHICNP77w8= +github.com/VictoriaMetrics/fastcache v1.12.1 h1:i0mICQuojGDL3KblA7wUNlY5lOK6a4bwt3uRKnkZU40= +github.com/VictoriaMetrics/fastcache v1.12.1/go.mod h1:tX04vaqcNoQeGLD+ra5pU5sWkuxnzWhEzLwhP9w653o= github.com/VictoriaMetrics/metrics v1.23.1 h1:/j8DzeJBxSpL2qSIdqnRFLvQQhbJyJbbEi22yMm7oL0= github.com/VictoriaMetrics/metrics v1.23.1/go.mod h1:rAr/llLpEnAdTehiNlUxKgnjcOuROSzpw0GvjpEbvFc= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= @@ -166,7 +166,6 @@ github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/cespare/xxhash/v2 v2.1.2/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= @@ -932,8 +931,8 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.8.1/go.mod h1:Z41J9TPoffeoqP0Iza0YbAhGvymRdZAd2uPmZ5JxRdY= -github.com/urfave/cli/v2 v2.24.3 h1:7Q1w8VN8yE0MJEHP06bv89PjYsN4IHWED2s1v/Zlfm0= -github.com/urfave/cli/v2 v2.24.3/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= +github.com/urfave/cli/v2 v2.24.4 h1:0gyJJEBYtCV87zI/x2nZCPyDxD51K6xM8SkwjHFCNEU= +github.com/urfave/cli/v2 v2.24.4/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/valyala/fastjson v1.6.4 h1:uAUNq9Z6ymTgGhcm0UynUAB6tlbakBrz6CQFax3BXVQ= github.com/valyala/fastjson v1.6.4/go.mod h1:CLCAqky6SMuOcxStkYQvblddUtoRxhYMGLrsQns1aXY= github.com/valyala/fastrand v1.1.0 h1:f+5HkLW4rsgzdNoleUOB69hyT9IlD2ZQh9GyDMfb5G8= @@ -1140,7 +1139,6 @@ golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211020174200-9d6173849985/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220405052023-b1e9470b6e64/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220608164250-635b8c9b7f68/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= From a1a6512daa50a687d0973e513a1389ed9c619fec Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 23 Feb 2023 11:28:43 +0700 Subject: [PATCH 003/100] save --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e571ab3eb35..b02b8f7e5ba 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230223015813-429ae81cbd44 + github.com/ledgerwatch/erigon-lib v0.0.0-20230223042814-51800cbdd34e github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230215122237-5e33484e9be4 github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index ecc814620b6..e81749a6bb3 100644 --- a/go.sum +++ b/go.sum @@ -520,8 +520,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230223015813-429ae81cbd44 h1:OWtZNNSi6t9KbMgxJeGQpKJB1164p6nPV0cRbkQyiww= -github.com/ledgerwatch/erigon-lib v0.0.0-20230223015813-429ae81cbd44/go.mod h1:y9a/PjMYVygqnGHD4mmaBePuI5P7kgtVnfiOuc2c1pU= +github.com/ledgerwatch/erigon-lib v0.0.0-20230223042814-51800cbdd34e h1:KBzChPSEt0t54hJQqVZdqj+6PGuMJgTzkYi3ms9rU/0= +github.com/ledgerwatch/erigon-lib v0.0.0-20230223042814-51800cbdd34e/go.mod h1:y9a/PjMYVygqnGHD4mmaBePuI5P7kgtVnfiOuc2c1pU= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230215122237-5e33484e9be4 h1:KH4/JnCdg52tcTqBfv4dVwa3jryncGXjSByrzkNUXxc= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230215122237-5e33484e9be4/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From 00da25580913324f84427316ca15171949f51dc3 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 23 Feb 2023 12:00:50 +0700 Subject: [PATCH 004/100] save --- turbo/stages/blockchain_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/turbo/stages/blockchain_test.go b/turbo/stages/blockchain_test.go index 4e51261b504..e1068d3cf8c 100644 --- a/turbo/stages/blockchain_test.go +++ b/turbo/stages/blockchain_test.go @@ -385,6 +385,7 @@ func testBadHashes(t *testing.T) { // Tests that chain reorganisations handle transaction removals and reinsertions. func TestChainTxReorgs(t *testing.T) { + defer t.Fail() var ( key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") key2, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") From 9d2306baa0a2e4f2bc3b35a98adc6071ca9b32c8 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 23 Feb 2023 13:37:24 +0700 Subject: [PATCH 005/100] save --- turbo/stages/blockchain_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/turbo/stages/blockchain_test.go b/turbo/stages/blockchain_test.go index e1068d3cf8c..4e51261b504 100644 --- a/turbo/stages/blockchain_test.go +++ b/turbo/stages/blockchain_test.go @@ -385,7 +385,6 @@ func testBadHashes(t *testing.T) { // Tests that chain reorganisations handle transaction removals and reinsertions. func TestChainTxReorgs(t *testing.T) { - defer t.Fail() var ( key1, _ = crypto.HexToECDSA("b71c71a67e1177ad4e901695e1b4b9ee17ae16c6668d313eac2f96dbcda3f291") key2, _ = crypto.HexToECDSA("8a1f9a8f95be41cd7ccb6168179afb4504aefe388d1e14474d32c45c72ce7b7a") From b93c6ba92ad846e56f2497aa2e1a0ec38752f3b2 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 23 Feb 2023 16:07:55 +0700 Subject: [PATCH 006/100] save --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b02b8f7e5ba..b8bb5540b0d 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230223042814-51800cbdd34e + github.com/ledgerwatch/erigon-lib v0.0.0-20230223050205-6e845070fb10 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230215122237-5e33484e9be4 github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index e81749a6bb3..a22bda9ceb1 100644 --- a/go.sum +++ b/go.sum @@ -520,8 +520,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230223042814-51800cbdd34e h1:KBzChPSEt0t54hJQqVZdqj+6PGuMJgTzkYi3ms9rU/0= -github.com/ledgerwatch/erigon-lib v0.0.0-20230223042814-51800cbdd34e/go.mod h1:y9a/PjMYVygqnGHD4mmaBePuI5P7kgtVnfiOuc2c1pU= +github.com/ledgerwatch/erigon-lib v0.0.0-20230223050205-6e845070fb10 h1:xpXAXPoeqlYEqQxxlS1hcQQhxgS0d6zPIXQmjffawz0= +github.com/ledgerwatch/erigon-lib v0.0.0-20230223050205-6e845070fb10/go.mod h1:y9a/PjMYVygqnGHD4mmaBePuI5P7kgtVnfiOuc2c1pU= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230215122237-5e33484e9be4 h1:KH4/JnCdg52tcTqBfv4dVwa3jryncGXjSByrzkNUXxc= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230215122237-5e33484e9be4/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From dafe41e298ab948f6141bbaab396c8faf4b542f1 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 23 Feb 2023 16:20:40 +0700 Subject: [PATCH 007/100] save --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b8bb5540b0d..02a201cbc62 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230223050205-6e845070fb10 + github.com/ledgerwatch/erigon-lib v0.0.0-20230223091953-ec4c391abd98 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230215122237-5e33484e9be4 github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index a22bda9ceb1..5fcc94773f9 100644 --- a/go.sum +++ b/go.sum @@ -520,8 +520,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230223050205-6e845070fb10 h1:xpXAXPoeqlYEqQxxlS1hcQQhxgS0d6zPIXQmjffawz0= -github.com/ledgerwatch/erigon-lib v0.0.0-20230223050205-6e845070fb10/go.mod h1:y9a/PjMYVygqnGHD4mmaBePuI5P7kgtVnfiOuc2c1pU= +github.com/ledgerwatch/erigon-lib v0.0.0-20230223091953-ec4c391abd98 h1:NdD7rK0Q7apWbdMU0nZcinexkgF91uWeH3lV8iaTgTk= +github.com/ledgerwatch/erigon-lib v0.0.0-20230223091953-ec4c391abd98/go.mod h1:y9a/PjMYVygqnGHD4mmaBePuI5P7kgtVnfiOuc2c1pU= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230215122237-5e33484e9be4 h1:KH4/JnCdg52tcTqBfv4dVwa3jryncGXjSByrzkNUXxc= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230215122237-5e33484e9be4/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From 15eca45868e6489430ccb3db0f50d8948aa5f6d7 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Fri, 24 Feb 2023 09:10:00 +0700 Subject: [PATCH 008/100] save --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 02a201cbc62..208d9f0475b 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230223091953-ec4c391abd98 + github.com/ledgerwatch/erigon-lib v0.0.0-20230224020920-e3d49fa5631d github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230215122237-5e33484e9be4 github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index 5fcc94773f9..58f3442ccca 100644 --- a/go.sum +++ b/go.sum @@ -520,8 +520,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230223091953-ec4c391abd98 h1:NdD7rK0Q7apWbdMU0nZcinexkgF91uWeH3lV8iaTgTk= -github.com/ledgerwatch/erigon-lib v0.0.0-20230223091953-ec4c391abd98/go.mod h1:y9a/PjMYVygqnGHD4mmaBePuI5P7kgtVnfiOuc2c1pU= +github.com/ledgerwatch/erigon-lib v0.0.0-20230224020920-e3d49fa5631d h1:WkjtJuEetB0T2wZM5dElXC8kN757XM9hNXVSrnmQIzk= +github.com/ledgerwatch/erigon-lib v0.0.0-20230224020920-e3d49fa5631d/go.mod h1:y9a/PjMYVygqnGHD4mmaBePuI5P7kgtVnfiOuc2c1pU= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230215122237-5e33484e9be4 h1:KH4/JnCdg52tcTqBfv4dVwa3jryncGXjSByrzkNUXxc= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230215122237-5e33484e9be4/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From d640ad94ca5ffd88b8b134a0dba3d71650b9160a Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Fri, 24 Feb 2023 09:57:06 +0700 Subject: [PATCH 009/100] save --- eth/integrity/trie.go | 12 ++++++++---- eth/stagedsync/exec3.go | 21 ++++++++++++++------- eth/stagedsync/stage_interhashes.go | 6 ++++-- turbo/snapshotsync/block_snapshots.go | 6 ++++-- 4 files changed, 30 insertions(+), 15 deletions(-) diff --git a/eth/integrity/trie.go b/eth/integrity/trie.go index 2e8ed71df5f..805b9c7178e 100644 --- a/eth/integrity/trie.go +++ b/eth/integrity/trie.go @@ -43,7 +43,8 @@ func Trie(db kv.RoDB, tx kv.Tx, slowChecks bool, ctx context.Context) { panic(err) } defer c.Close() - kv.ReadAhead(readAheadCtx, db, atomic.NewBool(false), kv.TrieOfAccounts, nil, math.MaxInt32) + clear := kv.ReadAhead(readAheadCtx, db, atomic.NewBool(false), kv.TrieOfAccounts, nil, math.MaxInt32) + defer clear() trieAcc2, err := tx.Cursor(kv.TrieOfAccounts) if err != nil { @@ -56,7 +57,8 @@ func Trie(db kv.RoDB, tx kv.Tx, slowChecks bool, ctx context.Context) { panic(err) } defer accC.Close() - kv.ReadAhead(readAheadCtx, db, atomic.NewBool(false), kv.HashedAccounts, nil, math.MaxInt32) + clear2 := kv.ReadAhead(readAheadCtx, db, atomic.NewBool(false), kv.HashedAccounts, nil, math.MaxInt32) + defer clear2() for k, v, errc := c.First(); k != nil; k, v, errc = c.Next() { if errc != nil { @@ -157,7 +159,8 @@ func Trie(db kv.RoDB, tx kv.Tx, slowChecks bool, ctx context.Context) { panic(err) } defer c.Close() - kv.ReadAhead(readAheadCtx, db, atomic.NewBool(false), kv.TrieOfStorage, nil, math.MaxInt32) + clear := kv.ReadAhead(readAheadCtx, db, atomic.NewBool(false), kv.TrieOfStorage, nil, math.MaxInt32) + defer clear() trieStorage, err := tx.Cursor(kv.TrieOfStorage) if err != nil { @@ -170,7 +173,8 @@ func Trie(db kv.RoDB, tx kv.Tx, slowChecks bool, ctx context.Context) { panic(err) } defer storageC.Close() - kv.ReadAhead(readAheadCtx, db, atomic.NewBool(false), kv.HashedStorage, nil, math.MaxInt32) + clear2 := kv.ReadAhead(readAheadCtx, db, atomic.NewBool(false), kv.HashedStorage, nil, math.MaxInt32) + defer clear2() for k, v, errc := c.First(); k != nil; k, v, errc = c.Next() { if errc != nil { diff --git a/eth/stagedsync/exec3.go b/eth/stagedsync/exec3.go index 37813b9dfd3..9604e93fd37 100644 --- a/eth/stagedsync/exec3.go +++ b/eth/stagedsync/exec3.go @@ -27,13 +27,13 @@ import ( "github.com/ledgerwatch/erigon-lib/kv/rawdbv3" libstate "github.com/ledgerwatch/erigon-lib/state" state2 "github.com/ledgerwatch/erigon-lib/state" + "github.com/ledgerwatch/erigon/common/math" "github.com/ledgerwatch/log/v3" "github.com/torquem-ch/mdbx-go/mdbx" atomic2 "go.uber.org/atomic" "github.com/ledgerwatch/erigon/cmd/state/exec22" "github.com/ledgerwatch/erigon/cmd/state/exec3" - "github.com/ledgerwatch/erigon/common/math" "github.com/ledgerwatch/erigon/consensus" "github.com/ledgerwatch/erigon/core" "github.com/ledgerwatch/erigon/core/rawdb" @@ -1072,8 +1072,10 @@ func reconstituteStep(last bool, plainContractCollector := etl.NewCollector(fmt.Sprintf("%s recon plainContract", s.LogPrefix()), dirs.Tmp, etl.NewSortableBuffer(etl.BufferOptimalSize)) defer plainContractCollector.Close() var transposedKey []byte + if err = db.View(ctx, func(roTx kv.Tx) error { - kv.ReadAhead(ctx, db, atomic2.NewBool(false), kv.PlainStateR, nil, math.MaxUint32) + clear := kv.ReadAhead(ctx, db, atomic2.NewBool(false), kv.PlainStateR, nil, math.MaxUint32) + defer clear() if err = roTx.ForEach(kv.PlainStateR, nil, func(k, v []byte) error { transposedKey = append(transposedKey[:0], k[8:]...) transposedKey = append(transposedKey, k[:8]...) @@ -1081,7 +1083,8 @@ func reconstituteStep(last bool, }); err != nil { return err } - kv.ReadAhead(ctx, db, atomic2.NewBool(false), kv.PlainStateD, nil, math.MaxUint32) + clear2 := kv.ReadAhead(ctx, db, atomic2.NewBool(false), kv.PlainStateD, nil, math.MaxUint32) + defer clear2() if err = roTx.ForEach(kv.PlainStateD, nil, func(k, v []byte) error { transposedKey = append(transposedKey[:0], v...) transposedKey = append(transposedKey, k...) @@ -1089,7 +1092,8 @@ func reconstituteStep(last bool, }); err != nil { return err } - kv.ReadAhead(ctx, db, atomic2.NewBool(false), kv.CodeR, nil, math.MaxUint32) + clear3 := kv.ReadAhead(ctx, db, atomic2.NewBool(false), kv.CodeR, nil, math.MaxUint32) + defer clear3() if err = roTx.ForEach(kv.CodeR, nil, func(k, v []byte) error { transposedKey = append(transposedKey[:0], k[8:]...) transposedKey = append(transposedKey, k[:8]...) @@ -1097,7 +1101,8 @@ func reconstituteStep(last bool, }); err != nil { return err } - kv.ReadAhead(ctx, db, atomic2.NewBool(false), kv.CodeD, nil, math.MaxUint32) + clear4 := kv.ReadAhead(ctx, db, atomic2.NewBool(false), kv.CodeD, nil, math.MaxUint32) + defer clear4() if err = roTx.ForEach(kv.CodeD, nil, func(k, v []byte) error { transposedKey = append(transposedKey[:0], v...) transposedKey = append(transposedKey, k...) @@ -1105,7 +1110,8 @@ func reconstituteStep(last bool, }); err != nil { return err } - kv.ReadAhead(ctx, db, atomic2.NewBool(false), kv.PlainContractR, nil, math.MaxUint32) + clear5 := kv.ReadAhead(ctx, db, atomic2.NewBool(false), kv.PlainContractR, nil, math.MaxUint32) + defer clear5() if err = roTx.ForEach(kv.PlainContractR, nil, func(k, v []byte) error { transposedKey = append(transposedKey[:0], k[8:]...) transposedKey = append(transposedKey, k[:8]...) @@ -1113,7 +1119,8 @@ func reconstituteStep(last bool, }); err != nil { return err } - kv.ReadAhead(ctx, db, atomic2.NewBool(false), kv.PlainContractD, nil, math.MaxUint32) + clear6 := kv.ReadAhead(ctx, db, atomic2.NewBool(false), kv.PlainContractD, nil, math.MaxUint32) + defer clear6() if err = roTx.ForEach(kv.PlainContractD, nil, func(k, v []byte) error { transposedKey = append(transposedKey[:0], v...) transposedKey = append(transposedKey, k...) diff --git a/eth/stagedsync/stage_interhashes.go b/eth/stagedsync/stage_interhashes.go index 174ab41ec2a..794c4730e09 100644 --- a/eth/stagedsync/stage_interhashes.go +++ b/eth/stagedsync/stage_interhashes.go @@ -150,8 +150,10 @@ func RegenerateIntermediateHashes(logPrefix string, db kv.RwTx, cfg TrieCfg, exp defer log.Info(fmt.Sprintf("[%s] Regeneration ended", logPrefix)) _ = db.ClearBucket(kv.TrieOfAccounts) _ = db.ClearBucket(kv.TrieOfStorage) - kv.ReadAhead(ctx, cfg.db, atomic.NewBool(false), kv.HashedAccounts, nil, math.MaxUint32) - kv.ReadAhead(ctx, cfg.db, atomic.NewBool(false), kv.HashedStorage, nil, math.MaxUint32) + clean := kv.ReadAhead(ctx, cfg.db, atomic.NewBool(false), kv.HashedAccounts, nil, math.MaxUint32) + defer clean() + clean2 := kv.ReadAhead(ctx, cfg.db, atomic.NewBool(false), kv.HashedStorage, nil, math.MaxUint32) + defer clean2() accTrieCollector := etl.NewCollector(logPrefix, cfg.tmpDir, etl.NewSortableBuffer(etl.BufferOptimalSize)) defer accTrieCollector.Close() diff --git a/turbo/snapshotsync/block_snapshots.go b/turbo/snapshotsync/block_snapshots.go index c7a4c119c1b..1ff08a0660c 100644 --- a/turbo/snapshotsync/block_snapshots.go +++ b/turbo/snapshotsync/block_snapshots.go @@ -1395,10 +1395,12 @@ func DumpTxs(ctx context.Context, db kv.RoDB, segmentFile, tmpDir string, blockF return true, nil } if doWarmup && !warmupSenders.Load() && blockNum%1_000 == 0 { - kv.ReadAhead(warmupCtx, db, warmupSenders, kv.Senders, hexutility.EncodeTs(blockNum), 10_000) + clean := kv.ReadAhead(warmupCtx, db, warmupSenders, kv.Senders, hexutility.EncodeTs(blockNum), 10_000) + defer clean() } if doWarmup && !warmupTxs.Load() && blockNum%1_000 == 0 { - kv.ReadAhead(warmupCtx, db, warmupTxs, kv.EthTx, hexutility.EncodeTs(body.BaseTxId), 100*10_000) + clean := kv.ReadAhead(warmupCtx, db, warmupTxs, kv.EthTx, hexutility.EncodeTs(body.BaseTxId), 100*10_000) + defer clean() } senders, err := rawdb.ReadSenders(tx, h, blockNum) if err != nil { From a1cb6e9ffcb519bb91f1af01cc612a114c91fb8d Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Fri, 24 Feb 2023 09:58:54 +0700 Subject: [PATCH 010/100] save --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 208d9f0475b..fc133fe1a86 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230224020920-e3d49fa5631d + github.com/ledgerwatch/erigon-lib v0.0.0-20230224025706-e195f9a0b546 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230215122237-5e33484e9be4 github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index 58f3442ccca..42da8ede2f7 100644 --- a/go.sum +++ b/go.sum @@ -520,8 +520,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230224020920-e3d49fa5631d h1:WkjtJuEetB0T2wZM5dElXC8kN757XM9hNXVSrnmQIzk= -github.com/ledgerwatch/erigon-lib v0.0.0-20230224020920-e3d49fa5631d/go.mod h1:y9a/PjMYVygqnGHD4mmaBePuI5P7kgtVnfiOuc2c1pU= +github.com/ledgerwatch/erigon-lib v0.0.0-20230224025706-e195f9a0b546 h1:ooworLkikTUhy6ZVrYdd7aKz5om1oS9sYgiWKAQsXhg= +github.com/ledgerwatch/erigon-lib v0.0.0-20230224025706-e195f9a0b546/go.mod h1:y9a/PjMYVygqnGHD4mmaBePuI5P7kgtVnfiOuc2c1pU= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230215122237-5e33484e9be4 h1:KH4/JnCdg52tcTqBfv4dVwa3jryncGXjSByrzkNUXxc= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230215122237-5e33484e9be4/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From 3b7a46fb4395336b25be1dfd9275e32a6f716e56 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 27 Feb 2023 14:09:42 +0700 Subject: [PATCH 011/100] save --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 9a830b86dc8..0a6561b2683 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230226023032-d4a346dce6cd + github.com/ledgerwatch/erigon-lib v0.0.0-20230227070911-8e13fc364662 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230224032817-e29f5ad218a2 github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index 71e43f1986e..b362b1f56db 100644 --- a/go.sum +++ b/go.sum @@ -520,8 +520,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230226023032-d4a346dce6cd h1:z2/2dMtDTTbxXXf8Jcn90g7T7gjaSbJoXr2T/f8+0sU= -github.com/ledgerwatch/erigon-lib v0.0.0-20230226023032-d4a346dce6cd/go.mod h1:xcrJLJiNfthKFectsWBUtRJK9d6XtjiyQRGVeImRYRs= +github.com/ledgerwatch/erigon-lib v0.0.0-20230227070911-8e13fc364662 h1:p1bwH1AFNZrFngsvcjIDrhdVCbZT9b9mOOjDWRAguPE= +github.com/ledgerwatch/erigon-lib v0.0.0-20230227070911-8e13fc364662/go.mod h1:xcrJLJiNfthKFectsWBUtRJK9d6XtjiyQRGVeImRYRs= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230224032817-e29f5ad218a2 h1:Vf5oUCq9s4JSLd5/WJxq8zTuMY5Brn8So0yeJDvUauc= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230224032817-e29f5ad218a2/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From b13604d9d3b8797a0d835e61a93f747cc50b3751 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 11:23:01 +0700 Subject: [PATCH 012/100] save --- cmd/state/exec3/calltracer_v3.go | 4 ++++ cmd/state/exec3/state.go | 19 ++++++++++++------- core/gaspool.go | 4 ++++ 3 files changed, 20 insertions(+), 7 deletions(-) diff --git a/cmd/state/exec3/calltracer_v3.go b/cmd/state/exec3/calltracer_v3.go index 2785db7de7e..2393500d125 100644 --- a/cmd/state/exec3/calltracer_v3.go +++ b/cmd/state/exec3/calltracer_v3.go @@ -18,6 +18,10 @@ func NewCallTracer() *CallTracer { tos: map[libcommon.Address]struct{}{}, } } +func (ct *CallTracer) Reset() { + ct.froms = map[libcommon.Address]struct{}{} + ct.tos = map[libcommon.Address]struct{}{} +} func (ct *CallTracer) Froms() map[libcommon.Address]struct{} { return ct.froms } func (ct *CallTracer) Tos() map[libcommon.Address]struct{} { return ct.tos } diff --git a/cmd/state/exec3/state.go b/cmd/state/exec3/state.go index 50e243a7694..bb86cbcffb7 100644 --- a/cmd/state/exec3/state.go +++ b/cmd/state/exec3/state.go @@ -45,6 +45,9 @@ type Worker struct { isPoSA bool posa consensus.PoSA + callTracer *CallTracer + taskGasPool *core.GasPool + evm *vm.EVM ibs *state.IntraBlockState } @@ -66,7 +69,9 @@ func NewWorker(lock sync.Locker, ctx context.Context, background bool, chainDb k resultCh: resultCh, engine: engine, - evm: vm.NewEVM(evmtypes.BlockContext{}, evmtypes.TxContext{}, nil, chainConfig, vm.Config{}), + evm: vm.NewEVM(evmtypes.BlockContext{}, evmtypes.TxContext{}, nil, chainConfig, vm.Config{}), + callTracer: NewCallTracer(), + taskGasPool: new(core.GasPool), } w.getHeader = func(hash libcommon.Hash, number uint64) *types.Header { h, err := blockReader.Header(ctx, w.chainTx, hash, number) @@ -189,9 +194,9 @@ func (rw *Worker) RunTxTaskNoLock(txTask *exec22.TxTask) { } } txHash := txTask.Tx.Hash() - gp := new(core.GasPool).AddGas(txTask.Tx.GetGas()) - ct := NewCallTracer() - vmConfig := vm.Config{Debug: true, Tracer: ct, SkipAnalysis: txTask.SkipAnalysis} + rw.taskGasPool.Reset(txTask.Tx.GetGas()) + rw.callTracer.Reset() + vmConfig := vm.Config{Debug: true, Tracer: rw.callTracer, SkipAnalysis: txTask.SkipAnalysis} ibs.Prepare(txHash, txTask.BlockHash, txTask.TxIndex) msg := txTask.TxAsMessage @@ -203,7 +208,7 @@ func (rw *Worker) RunTxTaskNoLock(txTask *exec22.TxTask) { rw.evm.ResetBetweenBlocks(blockContext, core.NewEVMTxContext(msg), ibs, vmConfig, rules) vmenv := rw.evm - applyRes, err := core.ApplyMessage(vmenv, msg, gp, true /* refunds */, false /* gasBailout */) + applyRes, err := core.ApplyMessage(vmenv, msg, rw.taskGasPool, true /* refunds */, false /* gasBailout */) if err != nil { txTask.Error = err //fmt.Printf("error=%v\n", err) @@ -212,8 +217,8 @@ func (rw *Worker) RunTxTaskNoLock(txTask *exec22.TxTask) { // Update the state with pending changes ibs.SoftFinalise() txTask.Logs = ibs.GetLogs(txHash) - txTask.TraceFroms = ct.froms - txTask.TraceTos = ct.tos + txTask.TraceFroms = rw.callTracer.froms + txTask.TraceTos = rw.callTracer.tos } } // Prepare read set, write set and balanceIncrease set and send for serialisation diff --git a/core/gaspool.go b/core/gaspool.go index e3795c1ee9e..05b403cb118 100644 --- a/core/gaspool.go +++ b/core/gaspool.go @@ -25,6 +25,10 @@ import ( // in a block. The zero value is a pool with zero gas available. type GasPool uint64 +func (gp *GasPool) Reset(amount uint64) { + *gp = GasPool(amount) +} + // AddGas makes gas available for execution. func (gp *GasPool) AddGas(amount uint64) *GasPool { if uint64(*gp) > math.MaxUint64-amount { From 63b20c47751fca448356e469af17dce407cd68ce Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 11:33:56 +0700 Subject: [PATCH 013/100] save --- cmd/state/exec3/state.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/state/exec3/state.go b/cmd/state/exec3/state.go index 50e243a7694..88b49deb9d7 100644 --- a/cmd/state/exec3/state.go +++ b/cmd/state/exec3/state.go @@ -191,7 +191,7 @@ func (rw *Worker) RunTxTaskNoLock(txTask *exec22.TxTask) { txHash := txTask.Tx.Hash() gp := new(core.GasPool).AddGas(txTask.Tx.GetGas()) ct := NewCallTracer() - vmConfig := vm.Config{Debug: true, Tracer: ct, SkipAnalysis: txTask.SkipAnalysis} + vmConfig := vm.Config{Debug: false, Tracer: ct, SkipAnalysis: txTask.SkipAnalysis} ibs.Prepare(txHash, txTask.BlockHash, txTask.TxIndex) msg := txTask.TxAsMessage From fa79bcd513a4564f0bdf40502a2373fec0b7f1e4 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 11:54:46 +0700 Subject: [PATCH 014/100] save --- core/vm/contracts_test.go | 24 +++++++++++++++++++++++ core/vm/evm.go | 40 ++++++++++++++++++--------------------- core/vm/interpreter.go | 31 +++++++++++++++--------------- 3 files changed, 57 insertions(+), 38 deletions(-) diff --git a/core/vm/contracts_test.go b/core/vm/contracts_test.go index 0c5b065e926..629c5987e70 100644 --- a/core/vm/contracts_test.go +++ b/core/vm/contracts_test.go @@ -29,6 +29,30 @@ import ( "github.com/ledgerwatch/erigon/common" ) +var aaa, bbb int + +func aa() { + defer func() { aaa++ }() +} +func incB() { + bbb++ +} +func bb() { + defer incB() +} +func BenchmarkName(b *testing.B) { + b.Run("1", func(b *testing.B) { + for i := 0; i < b.N; i++ { + aa() + } + }) + b.Run("b", func(b *testing.B) { + for i := 0; i < b.N; i++ { + bb() + } + }) +} + // precompiledTest defines the input/output pairs for precompiled contract tests. type precompiledTest struct { Input, Expected string diff --git a/core/vm/evm.go b/core/vm/evm.go index 8d6cdd736e3..201092b1368 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -199,14 +199,10 @@ func (evm *EVM) call(typ OpCode, caller ContractRef, addr libcommon.Address, inp // Calling a non existing account, don't do anything, but ping the tracer if evm.interpreter.Depth() == 0 { evm.config.Tracer.CaptureStart(evm, caller.Address(), addr, isPrecompile, false /* create */, input, gas, v, code) - defer func(startGas uint64) { // Lazy evaluation of the parameters - evm.config.Tracer.CaptureEnd(ret, 0, err) - }(gas) + evm.config.Tracer.CaptureEnd(ret, 0, nil) } else { evm.config.Tracer.CaptureEnter(typ, caller.Address(), addr, isPrecompile, false /* create */, input, gas, v, code) - defer func(startGas uint64) { // Lazy evaluation of the parameters - evm.config.Tracer.CaptureExit(ret, 0, err) - }(gas) + evm.config.Tracer.CaptureExit(ret, 0, nil) } } return nil, gas, nil @@ -221,6 +217,7 @@ func (evm *EVM) call(typ OpCode, caller ContractRef, addr libcommon.Address, inp // future scenarios evm.intraBlockState.AddBalance(addr, u256.Num0) } + var startGas = gas if evm.config.Debug { v := value if typ == STATICCALL { @@ -228,14 +225,8 @@ func (evm *EVM) call(typ OpCode, caller ContractRef, addr libcommon.Address, inp } if evm.interpreter.Depth() == 0 { evm.config.Tracer.CaptureStart(evm, caller.Address(), addr, isPrecompile, false /* create */, input, gas, v, code) - defer func(startGas uint64) { // Lazy evaluation of the parameters - evm.config.Tracer.CaptureEnd(ret, startGas-gas, err) - }(gas) } else { evm.config.Tracer.CaptureEnter(typ, caller.Address(), addr, isPrecompile, false /* create */, input, gas, v, code) - defer func(startGas uint64) { // Lazy evaluation of the parameters - evm.config.Tracer.CaptureExit(ret, startGas-gas, err) - }(gas) } } @@ -282,6 +273,14 @@ func (evm *EVM) call(typ OpCode, caller ContractRef, addr libcommon.Address, inp //} else { // evm.StateDB.DiscardSnapshot(snapshot) } + + if evm.config.Debug { + if evm.interpreter.Depth() == 0 { + evm.config.Tracer.CaptureEnd(ret, startGas-gas, err) + } else { + evm.config.Tracer.CaptureExit(ret, startGas-gas, err) + } + } return ret, gas, err } @@ -337,19 +336,12 @@ func (c *codeAndHash) Hash() libcommon.Hash { func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, value *uint256.Int, address libcommon.Address, typ OpCode, incrementNonce bool) ([]byte, libcommon.Address, uint64, error) { var ret []byte var err error - var gasConsumption uint64 if evm.config.Debug { if evm.interpreter.Depth() == 0 { evm.config.Tracer.CaptureStart(evm, caller.Address(), address, false /* precompile */, true /* create */, codeAndHash.code, gas, value, nil) - defer func() { - evm.config.Tracer.CaptureEnd(ret, gasConsumption, err) - }() } else { evm.config.Tracer.CaptureEnter(typ, caller.Address(), address, false /* precompile */, true /* create */, codeAndHash.code, gas, value, nil) - defer func() { - evm.config.Tracer.CaptureExit(ret, gasConsumption, err) - }() } } @@ -437,9 +429,13 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, } } - // calculate gasConsumption for deferred captures - gasConsumption = gas - contract.Gas - + if evm.config.Debug { + if evm.interpreter.Depth() == 0 { + evm.config.Tracer.CaptureEnd(ret, gas-contract.Gas, err) + } else { + evm.config.Tracer.CaptureExit(ret, gas-contract.Gas, err) + } + } return ret, address, contract.Gas, err } diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 60e5b787933..38ee2b5404f 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -160,6 +160,8 @@ func NewEVMInterpreter(evm VMInterpreter, cfg Config) *EVMInterpreter { } } +func (in *EVMInterpreter) decrementDepth() { in.depth-- } + // Run loops and evaluates the contract's code with the given input data and returns // the return byte-slice and an error if one occurred. // @@ -169,7 +171,7 @@ func NewEVMInterpreter(evm VMInterpreter, cfg Config) *EVMInterpreter { func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) (ret []byte, err error) { // Increment the call depth which is restricted to 1024 in.depth++ - defer func() { in.depth-- }() + defer in.decrementDepth() // Make sure the readOnly is only set if we aren't in readOnly yet. // This makes also sure that the readOnly flag isn't removed for child calls. @@ -212,17 +214,6 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( defer stack.ReturnNormalStack(locStack) contract.Input = input - if in.cfg.Debug { - defer func() { - if err != nil { - if !logged { - in.cfg.Tracer.CaptureState(pcCopy, op, gasCopy, cost, callContext, in.returnData, in.depth, err) //nolint:errcheck - } else { - in.cfg.Tracer.CaptureFault(pcCopy, op, gasCopy, cost, callContext, in.depth, err) - } - } - }() - } // The Interpreter main run loop (contextual). This loop runs until either an // explicit STOP, RETURN or SELFDESTRUCT is executed, an error occurred during // the execution of one of the operations or until the done flag is set by the @@ -298,6 +289,13 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( err = nil // clear stop token error } + if in.cfg.Debug && err != nil { + if !logged { + in.cfg.Tracer.CaptureState(pcCopy, op, gasCopy, cost, callContext, in.returnData, in.depth, err) //nolint:errcheck + } else { + in.cfg.Tracer.CaptureFault(pcCopy, op, gasCopy, cost, callContext, in.depth, err) + } + } return res, err } @@ -306,14 +304,15 @@ func (in *EVMInterpreter) Depth() int { return in.depth } +func (vm *VM) disableReadonly() { vm.readOnly = false } +func (vm *VM) noop() {} + func (vm *VM) setReadonly(outerReadonly bool) func() { if outerReadonly && !vm.readOnly { vm.readOnly = true - return func() { - vm.readOnly = false - } + return vm.disableReadonly } - return func() {} + return vm.noop } func (vm *VM) getReadonly() bool { From 8604d4a82488a84598c2215cc4be0b1fe817553d Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 11:55:58 +0700 Subject: [PATCH 015/100] save --- cmd/state/exec3/state.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/state/exec3/state.go b/cmd/state/exec3/state.go index 88b49deb9d7..50e243a7694 100644 --- a/cmd/state/exec3/state.go +++ b/cmd/state/exec3/state.go @@ -191,7 +191,7 @@ func (rw *Worker) RunTxTaskNoLock(txTask *exec22.TxTask) { txHash := txTask.Tx.Hash() gp := new(core.GasPool).AddGas(txTask.Tx.GetGas()) ct := NewCallTracer() - vmConfig := vm.Config{Debug: false, Tracer: ct, SkipAnalysis: txTask.SkipAnalysis} + vmConfig := vm.Config{Debug: true, Tracer: ct, SkipAnalysis: txTask.SkipAnalysis} ibs.Prepare(txHash, txTask.BlockHash, txTask.TxIndex) msg := txTask.TxAsMessage From fb168cb65ceff3ca934c151c49ed2cbbbf185f51 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 12:00:00 +0700 Subject: [PATCH 016/100] save --- core/vm/interpreter.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 38ee2b5404f..b90e78337e1 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -169,6 +169,11 @@ func (in *EVMInterpreter) decrementDepth() { in.depth-- } // considered a revert-and-consume-all-gas operation except for // ErrExecutionReverted which means revert-and-keep-gas-left. func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) (ret []byte, err error) { + // Don't bother with the execution if there's no code. + if len(contract.Code) == 0 { + return nil, nil + } + // Increment the call depth which is restricted to 1024 in.depth++ defer in.decrementDepth() @@ -182,11 +187,6 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( // as every returning call will return new data anyway. in.returnData = nil - // Don't bother with the execution if there's no code. - if len(contract.Code) == 0 { - return nil, nil - } - var ( op OpCode // current opcode mem = NewMemory() // bound memory From a6cc8c93d09caa7eeb4575e550d7ee9dd8e1ec06 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 12:03:20 +0700 Subject: [PATCH 017/100] save --- core/vm/evm.go | 22 ++++++++++++---------- 1 file changed, 12 insertions(+), 10 deletions(-) diff --git a/core/vm/evm.go b/core/vm/evm.go index 201092b1368..41a05c9c369 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -165,11 +165,13 @@ func (evm *EVM) Interpreter() Interpreter { } func (evm *EVM) call(typ OpCode, caller ContractRef, addr libcommon.Address, input []byte, gas uint64, value *uint256.Int, bailout bool) (ret []byte, leftOverGas uint64, err error) { - if evm.config.NoRecursion && evm.interpreter.Depth() > 0 { + depth := evm.interpreter.Depth() + + if evm.config.NoRecursion && depth > 0 { return nil, gas, nil } // Fail if we're trying to execute above the call depth limit - if evm.interpreter.Depth() > int(params.CallCreateDepth) { + if depth > int(params.CallCreateDepth) { return nil, gas, ErrDepth } if typ == CALL || typ == CALLCODE { @@ -197,7 +199,7 @@ func (evm *EVM) call(typ OpCode, caller ContractRef, addr libcommon.Address, inp v = nil } // Calling a non existing account, don't do anything, but ping the tracer - if evm.interpreter.Depth() == 0 { + if depth == 0 { evm.config.Tracer.CaptureStart(evm, caller.Address(), addr, isPrecompile, false /* create */, input, gas, v, code) evm.config.Tracer.CaptureEnd(ret, 0, nil) } else { @@ -223,7 +225,7 @@ func (evm *EVM) call(typ OpCode, caller ContractRef, addr libcommon.Address, inp if typ == STATICCALL { v = nil } - if evm.interpreter.Depth() == 0 { + if depth == 0 { evm.config.Tracer.CaptureStart(evm, caller.Address(), addr, isPrecompile, false /* create */, input, gas, v, code) } else { evm.config.Tracer.CaptureEnter(typ, caller.Address(), addr, isPrecompile, false /* create */, input, gas, v, code) @@ -275,7 +277,7 @@ func (evm *EVM) call(typ OpCode, caller ContractRef, addr libcommon.Address, inp } if evm.config.Debug { - if evm.interpreter.Depth() == 0 { + if depth == 0 { evm.config.Tracer.CaptureEnd(ret, startGas-gas, err) } else { evm.config.Tracer.CaptureExit(ret, startGas-gas, err) @@ -336,9 +338,9 @@ func (c *codeAndHash) Hash() libcommon.Hash { func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, value *uint256.Int, address libcommon.Address, typ OpCode, incrementNonce bool) ([]byte, libcommon.Address, uint64, error) { var ret []byte var err error - + depth := evm.interpreter.Depth() if evm.config.Debug { - if evm.interpreter.Depth() == 0 { + if depth == 0 { evm.config.Tracer.CaptureStart(evm, caller.Address(), address, false /* precompile */, true /* create */, codeAndHash.code, gas, value, nil) } else { evm.config.Tracer.CaptureEnter(typ, caller.Address(), address, false /* precompile */, true /* create */, codeAndHash.code, gas, value, nil) @@ -347,7 +349,7 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, // Depth check execution. Fail if we're trying to execute above the // limit. - if evm.interpreter.Depth() > int(params.CallCreateDepth) { + if depth > int(params.CallCreateDepth) { err = ErrDepth return nil, libcommon.Address{}, gas, err } @@ -387,7 +389,7 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, contract := NewContract(caller, AccountRef(address), value, gas, evm.config.SkipAnalysis) contract.SetCodeOptionalHash(&address, codeAndHash) - if evm.config.NoRecursion && evm.interpreter.Depth() > 0 { + if evm.config.NoRecursion && depth > 0 { return nil, address, gas, nil } @@ -430,7 +432,7 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, } if evm.config.Debug { - if evm.interpreter.Depth() == 0 { + if depth == 0 { evm.config.Tracer.CaptureEnd(ret, gas-contract.Gas, err) } else { evm.config.Tracer.CaptureExit(ret, gas-contract.Gas, err) From 4706bb87aab835780af694b5fb5589ae7f09ca46 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 12:22:43 +0700 Subject: [PATCH 018/100] save --- core/vm/interpreter.go | 24 +++++++++++++++--------- 1 file changed, 15 insertions(+), 9 deletions(-) diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index b90e78337e1..fa455079a65 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -180,8 +180,10 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( // Make sure the readOnly is only set if we aren't in readOnly yet. // This makes also sure that the readOnly flag isn't removed for child calls. - callback := in.setReadonly(readOnly) - defer callback() + if readOnly && !in.readOnly { + in.readOnly = true + defer func() { in.readOnly = false }() + } // Reset the previous call's return data. It's unimportant to preserve the old buffer // as every returning call will return new data anyway. @@ -214,6 +216,17 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( defer stack.ReturnNormalStack(locStack) contract.Input = input + if in.cfg.Debug { + defer func() { + if err != nil { + if !logged { + in.cfg.Tracer.CaptureState(pcCopy, op, gasCopy, cost, callContext, in.returnData, in.depth, err) //nolint:errcheck + } else { + in.cfg.Tracer.CaptureFault(pcCopy, op, gasCopy, cost, callContext, in.depth, err) + } + } + }() + } // The Interpreter main run loop (contextual). This loop runs until either an // explicit STOP, RETURN or SELFDESTRUCT is executed, an error occurred during // the execution of one of the operations or until the done flag is set by the @@ -289,13 +302,6 @@ func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) ( err = nil // clear stop token error } - if in.cfg.Debug && err != nil { - if !logged { - in.cfg.Tracer.CaptureState(pcCopy, op, gasCopy, cost, callContext, in.returnData, in.depth, err) //nolint:errcheck - } else { - in.cfg.Tracer.CaptureFault(pcCopy, op, gasCopy, cost, callContext, in.depth, err) - } - } return res, err } From 6e4f35c24561ad0b5b4962cf2761de3b3be09f81 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 12:26:20 +0700 Subject: [PATCH 019/100] save --- core/vm/contracts_test.go | 24 ------------------------ 1 file changed, 24 deletions(-) diff --git a/core/vm/contracts_test.go b/core/vm/contracts_test.go index 629c5987e70..0c5b065e926 100644 --- a/core/vm/contracts_test.go +++ b/core/vm/contracts_test.go @@ -29,30 +29,6 @@ import ( "github.com/ledgerwatch/erigon/common" ) -var aaa, bbb int - -func aa() { - defer func() { aaa++ }() -} -func incB() { - bbb++ -} -func bb() { - defer incB() -} -func BenchmarkName(b *testing.B) { - b.Run("1", func(b *testing.B) { - for i := 0; i < b.N; i++ { - aa() - } - }) - b.Run("b", func(b *testing.B) { - for i := 0; i < b.N; i++ { - bb() - } - }) -} - // precompiledTest defines the input/output pairs for precompiled contract tests. type precompiledTest struct { Input, Expected string From 062f93ebe2e92487b7f49d7a3f255494225c2112 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 12:52:21 +0700 Subject: [PATCH 020/100] save --- cmd/state/exec3/state.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/state/exec3/state.go b/cmd/state/exec3/state.go index bb86cbcffb7..1e8921b3a02 100644 --- a/cmd/state/exec3/state.go +++ b/cmd/state/exec3/state.go @@ -217,8 +217,8 @@ func (rw *Worker) RunTxTaskNoLock(txTask *exec22.TxTask) { // Update the state with pending changes ibs.SoftFinalise() txTask.Logs = ibs.GetLogs(txHash) - txTask.TraceFroms = rw.callTracer.froms - txTask.TraceTos = rw.callTracer.tos + txTask.TraceFroms = rw.callTracer.Froms() + txTask.TraceTos = rw.callTracer.Tos() } } // Prepare read set, write set and balanceIncrease set and send for serialisation From 6ba244862efc29c648374966b1494a6dd2a6b8fb Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 12:53:26 +0700 Subject: [PATCH 021/100] save --- cmd/state/exec3/calltracer_v3.go | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/cmd/state/exec3/calltracer_v3.go b/cmd/state/exec3/calltracer_v3.go index 2393500d125..aa6f7192156 100644 --- a/cmd/state/exec3/calltracer_v3.go +++ b/cmd/state/exec3/calltracer_v3.go @@ -3,6 +3,7 @@ package exec3 import ( "github.com/holiman/uint256" libcommon "github.com/ledgerwatch/erigon-lib/common" + "golang.org/x/exp/maps" "github.com/ledgerwatch/erigon/core/vm" ) @@ -19,11 +20,11 @@ func NewCallTracer() *CallTracer { } } func (ct *CallTracer) Reset() { - ct.froms = map[libcommon.Address]struct{}{} - ct.tos = map[libcommon.Address]struct{}{} + maps.Clear(ct.froms) + maps.Clear(ct.tos) } -func (ct *CallTracer) Froms() map[libcommon.Address]struct{} { return ct.froms } -func (ct *CallTracer) Tos() map[libcommon.Address]struct{} { return ct.tos } +func (ct *CallTracer) Froms() map[libcommon.Address]struct{} { return maps.Clone(ct.froms) } +func (ct *CallTracer) Tos() map[libcommon.Address]struct{} { return maps.Clone(ct.tos) } func (ct *CallTracer) CaptureTxStart(gasLimit uint64) {} func (ct *CallTracer) CaptureTxEnd(restGas uint64) {} From 316110a9d3896aa1a742b60105fdc8c910a51381 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 12:54:03 +0700 Subject: [PATCH 022/100] save --- cmd/state/exec3/calltracer_v3.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cmd/state/exec3/calltracer_v3.go b/cmd/state/exec3/calltracer_v3.go index aa6f7192156..99f7e82d304 100644 --- a/cmd/state/exec3/calltracer_v3.go +++ b/cmd/state/exec3/calltracer_v3.go @@ -29,12 +29,10 @@ func (ct *CallTracer) Tos() map[libcommon.Address]struct{} { return maps.Clone func (ct *CallTracer) CaptureTxStart(gasLimit uint64) {} func (ct *CallTracer) CaptureTxEnd(restGas uint64) {} func (ct *CallTracer) CaptureStart(env vm.VMInterface, from libcommon.Address, to libcommon.Address, precompile bool, create bool, input []byte, gas uint64, value *uint256.Int, code []byte) { - ct.froms[from] = struct{}{} - ct.tos[to] = struct{}{} + ct.froms[from], ct.tos[to] = struct{}{}, struct{}{} } func (ct *CallTracer) CaptureEnter(typ vm.OpCode, from libcommon.Address, to libcommon.Address, precompile bool, create bool, input []byte, gas uint64, value *uint256.Int, code []byte) { - ct.froms[from] = struct{}{} - ct.tos[to] = struct{}{} + ct.froms[from], ct.tos[to] = struct{}{}, struct{}{} } func (ct *CallTracer) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { } From b26f113b909ba5cdfc06d93c5e859787c6e28b78 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 12:59:28 +0700 Subject: [PATCH 023/100] save --- cmd/state/exec3/calltracer_v3.go | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/cmd/state/exec3/calltracer_v3.go b/cmd/state/exec3/calltracer_v3.go index 99f7e82d304..0ac31cc2488 100644 --- a/cmd/state/exec3/calltracer_v3.go +++ b/cmd/state/exec3/calltracer_v3.go @@ -9,22 +9,36 @@ import ( ) type CallTracer struct { - froms map[libcommon.Address]struct{} - tos map[libcommon.Address]struct{} + froms map[libcommon.Address]struct{} + tos map[libcommon.Address]struct{} + zerocopy bool } -func NewCallTracer() *CallTracer { +func NewCallTracer(zerocopy bool) *CallTracer { return &CallTracer{ - froms: map[libcommon.Address]struct{}{}, - tos: map[libcommon.Address]struct{}{}, + froms: map[libcommon.Address]struct{}{}, + tos: map[libcommon.Address]struct{}{}, + zerocopy: zerocopy, } } func (ct *CallTracer) Reset() { maps.Clear(ct.froms) maps.Clear(ct.tos) } -func (ct *CallTracer) Froms() map[libcommon.Address]struct{} { return maps.Clone(ct.froms) } -func (ct *CallTracer) Tos() map[libcommon.Address]struct{} { return maps.Clone(ct.tos) } +func (ct *CallTracer) Froms() map[libcommon.Address]struct{} { + if ct.zerocopy { + return ct.froms + } else { + return maps.Clone(ct.froms) + } +} +func (ct *CallTracer) Tos() map[libcommon.Address]struct{} { + if ct.zerocopy { + return ct.froms + } else { + return maps.Clone(ct.froms) + } +} func (ct *CallTracer) CaptureTxStart(gasLimit uint64) {} func (ct *CallTracer) CaptureTxEnd(restGas uint64) {} From 11570cda2a1fcac17dcd43ca9b9aad703c24246b Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 13:00:02 +0700 Subject: [PATCH 024/100] save --- cmd/state/commands/erigon4.go | 2 +- cmd/state/exec3/calltracer_v3.go | 4 ++-- cmd/state/exec3/state.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/state/commands/erigon4.go b/cmd/state/commands/erigon4.go index 6cfdf26df3e..26516554fd7 100644 --- a/cmd/state/commands/erigon4.go +++ b/cmd/state/commands/erigon4.go @@ -373,7 +373,7 @@ func processBlock23(startTxNum uint64, trace bool, txNumStart uint64, rw *Reader if txNum >= startTxNum { ibs := state.New(rw) ibs.Prepare(tx.Hash(), block.Hash(), i) - ct := exec3.NewCallTracer() + ct := exec3.NewCallTracer(false) vmConfig.Tracer = ct receipt, _, err := core.ApplyTransaction(chainConfig, getHashFn, engine, nil, gp, ibs, ww, header, tx, usedGas, vmConfig) if err != nil { diff --git a/cmd/state/exec3/calltracer_v3.go b/cmd/state/exec3/calltracer_v3.go index 0ac31cc2488..5d4f0acade7 100644 --- a/cmd/state/exec3/calltracer_v3.go +++ b/cmd/state/exec3/calltracer_v3.go @@ -34,9 +34,9 @@ func (ct *CallTracer) Froms() map[libcommon.Address]struct{} { } func (ct *CallTracer) Tos() map[libcommon.Address]struct{} { if ct.zerocopy { - return ct.froms + return ct.tos } else { - return maps.Clone(ct.froms) + return maps.Clone(ct.tos) } } diff --git a/cmd/state/exec3/state.go b/cmd/state/exec3/state.go index 1e8921b3a02..814639ed88b 100644 --- a/cmd/state/exec3/state.go +++ b/cmd/state/exec3/state.go @@ -70,7 +70,7 @@ func NewWorker(lock sync.Locker, ctx context.Context, background bool, chainDb k engine: engine, evm: vm.NewEVM(evmtypes.BlockContext{}, evmtypes.TxContext{}, nil, chainConfig, vm.Config{}), - callTracer: NewCallTracer(), + callTracer: NewCallTracer(!background), taskGasPool: new(core.GasPool), } w.getHeader = func(hash libcommon.Hash, number uint64) *types.Header { From 40a43361ba6f72f5088651152df65035e6896496 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 13:01:52 +0700 Subject: [PATCH 025/100] save --- cmd/state/exec3/state.go | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cmd/state/exec3/state.go b/cmd/state/exec3/state.go index 814639ed88b..b7db6c844e4 100644 --- a/cmd/state/exec3/state.go +++ b/cmd/state/exec3/state.go @@ -53,6 +53,8 @@ type Worker struct { } func NewWorker(lock sync.Locker, ctx context.Context, background bool, chainDb kv.RoDB, rs *state.StateV3, blockReader services.FullBlockReader, chainConfig *chain.Config, logger log.Logger, genesis *core.Genesis, resultCh chan *exec22.TxTask, engine consensus.Engine) *Worker { + zeroCopyCallTracer := !background + w := &Worker{ lock: lock, chainDb: chainDb, @@ -70,7 +72,7 @@ func NewWorker(lock sync.Locker, ctx context.Context, background bool, chainDb k engine: engine, evm: vm.NewEVM(evmtypes.BlockContext{}, evmtypes.TxContext{}, nil, chainConfig, vm.Config{}), - callTracer: NewCallTracer(!background), + callTracer: NewCallTracer(zeroCopyCallTracer), taskGasPool: new(core.GasPool), } w.getHeader = func(hash libcommon.Hash, number uint64) *types.Header { From f9f00cab3cccb87dac82d7166ced0e973b207181 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 13:02:25 +0700 Subject: [PATCH 026/100] save --- cmd/state/exec3/state.go | 4 ++-- core/state/rw_v3.go | 4 ++-- eth/stagedsync/stage_execute_test.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/state/exec3/state.go b/cmd/state/exec3/state.go index b7db6c844e4..a80a86bfc7d 100644 --- a/cmd/state/exec3/state.go +++ b/cmd/state/exec3/state.go @@ -61,8 +61,8 @@ func NewWorker(lock sync.Locker, ctx context.Context, background bool, chainDb k rs: rs, background: background, blockReader: blockReader, - stateWriter: state.NewStateWriter22(rs), - stateReader: state.NewStateReader22(rs), + stateWriter: state.NewStateWriterV3(rs), + stateReader: state.NewStateReaderV3(rs), chainConfig: chainConfig, ctx: ctx, diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index 8f51de96d68..2aa21c74ca7 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -596,7 +596,7 @@ type StateWriterV3 struct { codePrevs map[string]uint64 } -func NewStateWriter22(rs *StateV3) *StateWriterV3 { +func NewStateWriterV3(rs *StateV3) *StateWriterV3 { return &StateWriterV3{ rs: rs, writeLists: newWriteList(), @@ -698,7 +698,7 @@ type StateReaderV3 struct { readLists map[string]*exec22.KvList } -func NewStateReader22(rs *StateV3) *StateReaderV3 { +func NewStateReaderV3(rs *StateV3) *StateReaderV3 { return &StateReaderV3{ rs: rs, readLists: newReadList(), diff --git a/eth/stagedsync/stage_execute_test.go b/eth/stagedsync/stage_execute_test.go index d818555564b..8b4a028758b 100644 --- a/eth/stagedsync/stage_execute_test.go +++ b/eth/stagedsync/stage_execute_test.go @@ -130,7 +130,7 @@ func apply(tx kv.RwTx, agg *libstate.AggregatorV3) (beforeBlock, afterBlock test agg.StartWrites() rs := state.NewStateV3() - stateWriter := state.NewStateWriter22(rs) + stateWriter := state.NewStateWriterV3(rs) return func(n, from, numberOfBlocks uint64) { stateWriter.SetTxNum(n) stateWriter.ResetWriteSet() From ecefb12664bc696636ef14b4f496c067c39572fd Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 13:03:36 +0700 Subject: [PATCH 027/100] save --- cmd/state/exec3/calltracer_v3.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cmd/state/exec3/calltracer_v3.go b/cmd/state/exec3/calltracer_v3.go index 5d4f0acade7..8e8b8f08243 100644 --- a/cmd/state/exec3/calltracer_v3.go +++ b/cmd/state/exec3/calltracer_v3.go @@ -28,16 +28,14 @@ func (ct *CallTracer) Reset() { func (ct *CallTracer) Froms() map[libcommon.Address]struct{} { if ct.zerocopy { return ct.froms - } else { - return maps.Clone(ct.froms) } + return maps.Clone(ct.froms) } func (ct *CallTracer) Tos() map[libcommon.Address]struct{} { if ct.zerocopy { return ct.tos - } else { - return maps.Clone(ct.tos) } + return maps.Clone(ct.tos) } func (ct *CallTracer) CaptureTxStart(gasLimit uint64) {} From 70965f8dc97b0a84a8c9229e208134b7719df637 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 13:09:09 +0700 Subject: [PATCH 028/100] save --- cmd/state/exec3/calltracer_v3.go | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/cmd/state/exec3/calltracer_v3.go b/cmd/state/exec3/calltracer_v3.go index 8e8b8f08243..87375d59f35 100644 --- a/cmd/state/exec3/calltracer_v3.go +++ b/cmd/state/exec3/calltracer_v3.go @@ -3,8 +3,6 @@ package exec3 import ( "github.com/holiman/uint256" libcommon "github.com/ledgerwatch/erigon-lib/common" - "golang.org/x/exp/maps" - "github.com/ledgerwatch/erigon/core/vm" ) @@ -22,20 +20,14 @@ func NewCallTracer(zerocopy bool) *CallTracer { } } func (ct *CallTracer) Reset() { - maps.Clear(ct.froms) - maps.Clear(ct.tos) + ct.froms = map[libcommon.Address]struct{}{} + ct.tos = map[libcommon.Address]struct{}{} } func (ct *CallTracer) Froms() map[libcommon.Address]struct{} { - if ct.zerocopy { - return ct.froms - } - return maps.Clone(ct.froms) + return ct.froms } func (ct *CallTracer) Tos() map[libcommon.Address]struct{} { - if ct.zerocopy { - return ct.tos - } - return maps.Clone(ct.tos) + return ct.tos } func (ct *CallTracer) CaptureTxStart(gasLimit uint64) {} From 89753f77bad07f5b79f4a85bfeb89fa3b2d0f691 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 13:15:38 +0700 Subject: [PATCH 029/100] save --- cmd/state/exec3/calltracer_v3.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/cmd/state/exec3/calltracer_v3.go b/cmd/state/exec3/calltracer_v3.go index 87375d59f35..8033d1cc589 100644 --- a/cmd/state/exec3/calltracer_v3.go +++ b/cmd/state/exec3/calltracer_v3.go @@ -20,13 +20,19 @@ func NewCallTracer(zerocopy bool) *CallTracer { } } func (ct *CallTracer) Reset() { - ct.froms = map[libcommon.Address]struct{}{} - ct.tos = map[libcommon.Address]struct{}{} + ct.froms = nil + ct.tos = nil } func (ct *CallTracer) Froms() map[libcommon.Address]struct{} { + if ct.froms == nil { + ct.froms = map[libcommon.Address]struct{}{} + } return ct.froms } func (ct *CallTracer) Tos() map[libcommon.Address]struct{} { + if ct.tos == nil { + ct.tos = map[libcommon.Address]struct{}{} + } return ct.tos } From 538370639b0c47c6ddeea53b62011f47988b252e Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 13:16:40 +0700 Subject: [PATCH 030/100] save --- cmd/state/exec3/calltracer_v3.go | 18 ++++++++++++------ 1 file changed, 12 insertions(+), 6 deletions(-) diff --git a/cmd/state/exec3/calltracer_v3.go b/cmd/state/exec3/calltracer_v3.go index 8033d1cc589..0c7fc6d5df9 100644 --- a/cmd/state/exec3/calltracer_v3.go +++ b/cmd/state/exec3/calltracer_v3.go @@ -24,24 +24,30 @@ func (ct *CallTracer) Reset() { ct.tos = nil } func (ct *CallTracer) Froms() map[libcommon.Address]struct{} { - if ct.froms == nil { - ct.froms = map[libcommon.Address]struct{}{} - } return ct.froms } func (ct *CallTracer) Tos() map[libcommon.Address]struct{} { - if ct.tos == nil { - ct.tos = map[libcommon.Address]struct{}{} - } return ct.tos } func (ct *CallTracer) CaptureTxStart(gasLimit uint64) {} func (ct *CallTracer) CaptureTxEnd(restGas uint64) {} func (ct *CallTracer) CaptureStart(env vm.VMInterface, from libcommon.Address, to libcommon.Address, precompile bool, create bool, input []byte, gas uint64, value *uint256.Int, code []byte) { + if ct.froms == nil { + ct.froms = map[libcommon.Address]struct{}{} + } + if ct.tos == nil { + ct.tos = map[libcommon.Address]struct{}{} + } ct.froms[from], ct.tos[to] = struct{}{}, struct{}{} } func (ct *CallTracer) CaptureEnter(typ vm.OpCode, from libcommon.Address, to libcommon.Address, precompile bool, create bool, input []byte, gas uint64, value *uint256.Int, code []byte) { + if ct.froms == nil { + ct.froms = map[libcommon.Address]struct{}{} + } + if ct.tos == nil { + ct.tos = map[libcommon.Address]struct{}{} + } ct.froms[from], ct.tos[to] = struct{}{}, struct{}{} } func (ct *CallTracer) CaptureState(pc uint64, op vm.OpCode, gas, cost uint64, scope *vm.ScopeContext, rData []byte, depth int, err error) { From f62e95633bc5f2aac22d0c96b49622803a12bdab Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 13:20:25 +0700 Subject: [PATCH 031/100] save --- cmd/state/exec3/calltracer_v3.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cmd/state/exec3/calltracer_v3.go b/cmd/state/exec3/calltracer_v3.go index 0c7fc6d5df9..74ddc3fe0b0 100644 --- a/cmd/state/exec3/calltracer_v3.go +++ b/cmd/state/exec3/calltracer_v3.go @@ -14,14 +14,11 @@ type CallTracer struct { func NewCallTracer(zerocopy bool) *CallTracer { return &CallTracer{ - froms: map[libcommon.Address]struct{}{}, - tos: map[libcommon.Address]struct{}{}, zerocopy: zerocopy, } } func (ct *CallTracer) Reset() { - ct.froms = nil - ct.tos = nil + ct.froms, ct.tos = nil, nil } func (ct *CallTracer) Froms() map[libcommon.Address]struct{} { return ct.froms From 49a0b668ed62e2b976bd653428c9f7997b6209e8 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 13:22:55 +0700 Subject: [PATCH 032/100] save --- core/state/rw_v3.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index 2aa21c74ca7..d2e58399c4e 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -601,9 +601,7 @@ func NewStateWriterV3(rs *StateV3) *StateWriterV3 { rs: rs, writeLists: newWriteList(), accountPrevs: map[string][]byte{}, - accountDels: map[string]*accounts.Account{}, storagePrevs: map[string][]byte{}, - codePrevs: map[string]uint64{}, } } @@ -614,9 +612,9 @@ func (w *StateWriterV3) SetTxNum(txNum uint64) { func (w *StateWriterV3) ResetWriteSet() { w.writeLists = newWriteList() w.accountPrevs = map[string][]byte{} - w.accountDels = map[string]*accounts.Account{} + w.accountDels = nil w.storagePrevs = map[string][]byte{} - w.codePrevs = map[string]uint64{} + w.codePrevs = nil } func (w *StateWriterV3) WriteSet() map[string]*exec22.KvList { @@ -651,6 +649,10 @@ func (w *StateWriterV3) UpdateAccountCode(address common.Address, incarnation ui w.writeLists[kv.PlainContractCode].Keys = append(w.writeLists[kv.PlainContractCode].Keys, dbutils.PlainGenerateStoragePrefix(addressBytes, incarnation)) w.writeLists[kv.PlainContractCode].Vals = append(w.writeLists[kv.PlainContractCode].Vals, codeHashBytes) } + + if w.codePrevs == nil { + w.codePrevs = map[string]uint64{} + } w.codePrevs[string(addressBytes)] = incarnation return nil } @@ -666,6 +668,9 @@ func (w *StateWriterV3) DeleteAccount(address common.Address, original *accounts w.writeLists[kv.IncarnationMap].Vals = append(w.writeLists[kv.IncarnationMap].Vals, b[:]) } if original.Initialised { + if w.accountDels == nil { + w.accountDels = map[string]*accounts.Account{} + } w.accountDels[string(addressBytes)] = original } return nil From 877375a5d63fa662fb26327cb9c6f953e6bb841d Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 13:24:41 +0700 Subject: [PATCH 033/100] save --- core/state/rw_v3.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index d2e58399c4e..8c9ad89e686 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -601,7 +601,6 @@ func NewStateWriterV3(rs *StateV3) *StateWriterV3 { rs: rs, writeLists: newWriteList(), accountPrevs: map[string][]byte{}, - storagePrevs: map[string][]byte{}, } } @@ -613,7 +612,7 @@ func (w *StateWriterV3) ResetWriteSet() { w.writeLists = newWriteList() w.accountPrevs = map[string][]byte{} w.accountDels = nil - w.storagePrevs = map[string][]byte{} + w.storagePrevs = nil w.codePrevs = nil } @@ -684,6 +683,9 @@ func (w *StateWriterV3) WriteAccountStorage(address common.Address, incarnation w.writeLists[kv.PlainState].Keys = append(w.writeLists[kv.PlainState].Keys, composite) w.writeLists[kv.PlainState].Vals = append(w.writeLists[kv.PlainState].Vals, value.Bytes()) //fmt.Printf("storage [%x] [%x] => [%x], txNum: %d\n", address, *key, v, w.txNum) + if w.storagePrevs == nil { + w.storagePrevs = map[string][]byte{} + } w.storagePrevs[string(composite)] = original.Bytes() return nil } From fa6a6f2f45dbf6e305d143928477eba71332ccff Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 13:27:42 +0700 Subject: [PATCH 034/100] save --- core/state/rw_v3.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index 8c9ad89e686..63107ce3f9c 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -598,9 +598,8 @@ type StateWriterV3 struct { func NewStateWriterV3(rs *StateV3) *StateWriterV3 { return &StateWriterV3{ - rs: rs, - writeLists: newWriteList(), - accountPrevs: map[string][]byte{}, + rs: rs, + writeLists: newWriteList(), } } @@ -610,7 +609,7 @@ func (w *StateWriterV3) SetTxNum(txNum uint64) { func (w *StateWriterV3) ResetWriteSet() { w.writeLists = newWriteList() - w.accountPrevs = map[string][]byte{} + w.accountPrevs = nil w.accountDels = nil w.storagePrevs = nil w.codePrevs = nil @@ -635,6 +634,9 @@ func (w *StateWriterV3) UpdateAccountData(address common.Address, original, acco if original.Initialised { prev = accounts.SerialiseV3(original) } + if w.accountPrevs == nil { + w.accountPrevs = map[string][]byte{} + } w.accountPrevs[string(addressBytes)] = prev return nil } From b20ccdd1183f20e82e69e12e6d4a2a1f8be0ed65 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 13:29:36 +0700 Subject: [PATCH 035/100] save --- cmd/state/exec3/calltracer_v3.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/cmd/state/exec3/calltracer_v3.go b/cmd/state/exec3/calltracer_v3.go index 74ddc3fe0b0..3fd6963b2c3 100644 --- a/cmd/state/exec3/calltracer_v3.go +++ b/cmd/state/exec3/calltracer_v3.go @@ -32,8 +32,6 @@ func (ct *CallTracer) CaptureTxEnd(restGas uint64) {} func (ct *CallTracer) CaptureStart(env vm.VMInterface, from libcommon.Address, to libcommon.Address, precompile bool, create bool, input []byte, gas uint64, value *uint256.Int, code []byte) { if ct.froms == nil { ct.froms = map[libcommon.Address]struct{}{} - } - if ct.tos == nil { ct.tos = map[libcommon.Address]struct{}{} } ct.froms[from], ct.tos[to] = struct{}{}, struct{}{} @@ -41,8 +39,6 @@ func (ct *CallTracer) CaptureStart(env vm.VMInterface, from libcommon.Address, t func (ct *CallTracer) CaptureEnter(typ vm.OpCode, from libcommon.Address, to libcommon.Address, precompile bool, create bool, input []byte, gas uint64, value *uint256.Int, code []byte) { if ct.froms == nil { ct.froms = map[libcommon.Address]struct{}{} - } - if ct.tos == nil { ct.tos = map[libcommon.Address]struct{}{} } ct.froms[from], ct.tos[to] = struct{}{}, struct{}{} From 96388945bd2493c941f05afe4c93d615ea561c3d Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 14:23:42 +0700 Subject: [PATCH 036/100] save --- core/state/intra_block_state.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/core/state/intra_block_state.go b/core/state/intra_block_state.go index 2ad2e3c5c49..15f2a503978 100644 --- a/core/state/intra_block_state.go +++ b/core/state/intra_block_state.go @@ -30,7 +30,6 @@ import ( "github.com/ledgerwatch/erigon/core/types/accounts" "github.com/ledgerwatch/erigon/crypto" "github.com/ledgerwatch/erigon/turbo/trie" - "golang.org/x/exp/maps" ) type revision struct { @@ -118,15 +117,15 @@ func (sdb *IntraBlockState) Error() error { // Reset clears out all ephemeral state objects from the state db, but keeps // the underlying state trie to avoid reloading data for the next operations. func (sdb *IntraBlockState) Reset() { - maps.Clear(sdb.nilAccounts) - maps.Clear(sdb.stateObjects) - maps.Clear(sdb.stateObjectsDirty) + sdb.nilAccounts = make(map[libcommon.Address]struct{}) + sdb.stateObjects = make(map[libcommon.Address]*stateObject) + sdb.stateObjectsDirty = make(map[libcommon.Address]struct{}) + sdb.logs = make(map[libcommon.Hash][]*types.Log) + sdb.balanceInc = make(map[libcommon.Address]*BalanceIncrease) sdb.thash = libcommon.Hash{} sdb.bhash = libcommon.Hash{} sdb.txIndex = 0 - maps.Clear(sdb.logs) sdb.logSize = 0 - maps.Clear(sdb.balanceInc) } func (sdb *IntraBlockState) AddLog(log2 *types.Log) { From db8ac571cbf17e45ad103b79d581641fb6f2dfda Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 14:31:58 +0700 Subject: [PATCH 037/100] save --- core/state/intra_block_state.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/core/state/intra_block_state.go b/core/state/intra_block_state.go index 15f2a503978..c8aa3378293 100644 --- a/core/state/intra_block_state.go +++ b/core/state/intra_block_state.go @@ -139,6 +139,9 @@ func (sdb *IntraBlockState) AddLog(log2 *types.Log) { } func (sdb *IntraBlockState) GetLogs(hash libcommon.Hash) []*types.Log { + if sdb.logs == nil { + return nil + } return sdb.logs[hash] } From c9c9a3dec5712c60aeb6a13bf4ec41e5c887f622 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 14:34:15 +0700 Subject: [PATCH 038/100] save --- core/state/intra_block_state.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/core/state/intra_block_state.go b/core/state/intra_block_state.go index c8aa3378293..31bb8165837 100644 --- a/core/state/intra_block_state.go +++ b/core/state/intra_block_state.go @@ -30,6 +30,7 @@ import ( "github.com/ledgerwatch/erigon/core/types/accounts" "github.com/ledgerwatch/erigon/crypto" "github.com/ledgerwatch/erigon/turbo/trie" + "github.com/ledgerwatch/log/v3" ) type revision struct { @@ -117,6 +118,12 @@ func (sdb *IntraBlockState) Error() error { // Reset clears out all ephemeral state objects from the state db, but keeps // the underlying state trie to avoid reloading data for the next operations. func (sdb *IntraBlockState) Reset() { + if len(sdb.nilAccounts) == 0 || len(sdb.stateObjects) == 0 || len(sdb.stateObjectsDirty) == 0 || len(sdb.balanceInc) == 0 { + log.Warn("zero", "len(sdb.nilAccounts)", len(sdb.nilAccounts), + "len(sdb.stateObjects)", len(sdb.stateObjects), + "len(sdb.stateObjectsDirty)", len(sdb.stateObjectsDirty), + "len(sdb.balanceInc)", len(sdb.balanceInc)) + } sdb.nilAccounts = make(map[libcommon.Address]struct{}) sdb.stateObjects = make(map[libcommon.Address]*stateObject) sdb.stateObjectsDirty = make(map[libcommon.Address]struct{}) From adb7a82d9a884052c9b56a841ad187cd28e41ddb Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 15:23:18 +0700 Subject: [PATCH 039/100] save --- cmd/state/commands/erigon4.go | 2 +- cmd/state/exec3/calltracer_v3.go | 19 ++++++------------- cmd/state/exec3/state.go | 2 +- 3 files changed, 8 insertions(+), 15 deletions(-) diff --git a/cmd/state/commands/erigon4.go b/cmd/state/commands/erigon4.go index 26516554fd7..6cfdf26df3e 100644 --- a/cmd/state/commands/erigon4.go +++ b/cmd/state/commands/erigon4.go @@ -373,7 +373,7 @@ func processBlock23(startTxNum uint64, trace bool, txNumStart uint64, rw *Reader if txNum >= startTxNum { ibs := state.New(rw) ibs.Prepare(tx.Hash(), block.Hash(), i) - ct := exec3.NewCallTracer(false) + ct := exec3.NewCallTracer() vmConfig.Tracer = ct receipt, _, err := core.ApplyTransaction(chainConfig, getHashFn, engine, nil, gp, ibs, ww, header, tx, usedGas, vmConfig) if err != nil { diff --git a/cmd/state/exec3/calltracer_v3.go b/cmd/state/exec3/calltracer_v3.go index 3fd6963b2c3..31e25fa0007 100644 --- a/cmd/state/exec3/calltracer_v3.go +++ b/cmd/state/exec3/calltracer_v3.go @@ -7,25 +7,18 @@ import ( ) type CallTracer struct { - froms map[libcommon.Address]struct{} - tos map[libcommon.Address]struct{} - zerocopy bool + froms map[libcommon.Address]struct{} + tos map[libcommon.Address]struct{} } -func NewCallTracer(zerocopy bool) *CallTracer { - return &CallTracer{ - zerocopy: zerocopy, - } +func NewCallTracer() *CallTracer { + return &CallTracer{} } func (ct *CallTracer) Reset() { ct.froms, ct.tos = nil, nil } -func (ct *CallTracer) Froms() map[libcommon.Address]struct{} { - return ct.froms -} -func (ct *CallTracer) Tos() map[libcommon.Address]struct{} { - return ct.tos -} +func (ct *CallTracer) Froms() map[libcommon.Address]struct{} { return ct.froms } +func (ct *CallTracer) Tos() map[libcommon.Address]struct{} { return ct.tos } func (ct *CallTracer) CaptureTxStart(gasLimit uint64) {} func (ct *CallTracer) CaptureTxEnd(restGas uint64) {} diff --git a/cmd/state/exec3/state.go b/cmd/state/exec3/state.go index a80a86bfc7d..5cedc598973 100644 --- a/cmd/state/exec3/state.go +++ b/cmd/state/exec3/state.go @@ -72,7 +72,7 @@ func NewWorker(lock sync.Locker, ctx context.Context, background bool, chainDb k engine: engine, evm: vm.NewEVM(evmtypes.BlockContext{}, evmtypes.TxContext{}, nil, chainConfig, vm.Config{}), - callTracer: NewCallTracer(zeroCopyCallTracer), + callTracer: NewCallTracer(), taskGasPool: new(core.GasPool), } w.getHeader = func(hash libcommon.Hash, number uint64) *types.Header { From 4dbc24063f4ee1a5d5bb1015aa73f244c1b53f13 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 15:27:17 +0700 Subject: [PATCH 040/100] save --- cmd/state/exec22/txtask.go | 7 +++---- cmd/state/exec3/state.go | 2 -- core/state/rw_v3.go | 36 +++++++++++++++++++++++------------- 3 files changed, 26 insertions(+), 19 deletions(-) diff --git a/cmd/state/exec22/txtask.go b/cmd/state/exec22/txtask.go index 7fb4b9dc4a0..3353080e371 100644 --- a/cmd/state/exec22/txtask.go +++ b/cmd/state/exec22/txtask.go @@ -1,8 +1,6 @@ package exec22 import ( - "bytes" - "github.com/holiman/uint256" "github.com/ledgerwatch/erigon-lib/chain" libcommon "github.com/ledgerwatch/erigon-lib/common" @@ -78,7 +76,8 @@ func (h *TxTaskQueue) Pop() interface{} { // KvList sort.Interface to sort write list by keys type KvList struct { - Keys, Vals [][]byte + Keys []string + Vals [][]byte } func (l KvList) Len() int { @@ -86,7 +85,7 @@ func (l KvList) Len() int { } func (l KvList) Less(i, j int) bool { - return bytes.Compare(l.Keys[i], l.Keys[j]) < 0 + return l.Keys[i] < l.Keys[j] } func (l *KvList) Swap(i, j int) { diff --git a/cmd/state/exec3/state.go b/cmd/state/exec3/state.go index 5cedc598973..df643448993 100644 --- a/cmd/state/exec3/state.go +++ b/cmd/state/exec3/state.go @@ -53,8 +53,6 @@ type Worker struct { } func NewWorker(lock sync.Locker, ctx context.Context, background bool, chainDb kv.RoDB, rs *state.StateV3, blockReader services.FullBlockReader, chainConfig *chain.Config, logger log.Logger, genesis *core.Genesis, resultCh chan *exec22.TxTask, engine consensus.Engine) *Worker { - zeroCopyCallTracer := !background - w := &Worker{ lock: lock, chainDb: chainDb, diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index 63107ce3f9c..8ca0a7e23ce 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -72,6 +72,15 @@ func (rs *StateV3) put(table string, key, val []byte) { } } +func (rs *StateV3) puts(table string, key string, val []byte) { + old, ok := rs.changes[table].Set(key, val) + if ok { + rs.sizeEstimate += len(val) - len(old) + } else { + rs.sizeEstimate += len(key) + len(val) + } +} + func (rs *StateV3) Get(table string, key []byte) []byte { rs.lock.RLock() v := rs.get(table, key) @@ -379,7 +388,7 @@ func (rs *StateV3) appplyState(roTx kv.Tx, txTask *exec22.TxTask, agg *libstate. if txTask.WriteLists != nil { for table, list := range txTask.WriteLists { for i, key := range list.Keys { - rs.put(table, key, list.Vals[i]) + rs.puts(table, key, list.Vals[i]) } } } @@ -628,7 +637,7 @@ func (w *StateWriterV3) UpdateAccountData(address common.Address, original, acco value := make([]byte, account.EncodingLengthForStorage()) account.EncodeForStorage(value) //fmt.Printf("account [%x]=>{Balance: %d, Nonce: %d, Root: %x, CodeHash: %x} txNum: %d\n", address, &account.Balance, account.Nonce, account.Root, account.CodeHash, w.txNum) - w.writeLists[kv.PlainState].Keys = append(w.writeLists[kv.PlainState].Keys, addressBytes) + w.writeLists[kv.PlainState].Keys = append(w.writeLists[kv.PlainState].Keys, string(addressBytes)) w.writeLists[kv.PlainState].Vals = append(w.writeLists[kv.PlainState].Vals, value) var prev []byte if original.Initialised { @@ -643,11 +652,11 @@ func (w *StateWriterV3) UpdateAccountData(address common.Address, original, acco func (w *StateWriterV3) UpdateAccountCode(address common.Address, incarnation uint64, codeHash common.Hash, code []byte) error { addressBytes, codeHashBytes := address.Bytes(), codeHash.Bytes() - w.writeLists[kv.Code].Keys = append(w.writeLists[kv.Code].Keys, codeHashBytes) + w.writeLists[kv.Code].Keys = append(w.writeLists[kv.Code].Keys, string(codeHashBytes)) w.writeLists[kv.Code].Vals = append(w.writeLists[kv.Code].Vals, code) if len(code) > 0 { //fmt.Printf("code [%x] => [%x] CodeHash: %x, txNum: %d\n", address, code, codeHash, w.txNum) - w.writeLists[kv.PlainContractCode].Keys = append(w.writeLists[kv.PlainContractCode].Keys, dbutils.PlainGenerateStoragePrefix(addressBytes, incarnation)) + w.writeLists[kv.PlainContractCode].Keys = append(w.writeLists[kv.PlainContractCode].Keys, string(dbutils.PlainGenerateStoragePrefix(addressBytes, incarnation))) w.writeLists[kv.PlainContractCode].Vals = append(w.writeLists[kv.PlainContractCode].Vals, codeHashBytes) } @@ -660,12 +669,12 @@ func (w *StateWriterV3) UpdateAccountCode(address common.Address, incarnation ui func (w *StateWriterV3) DeleteAccount(address common.Address, original *accounts.Account) error { addressBytes := address.Bytes() - w.writeLists[kv.PlainState].Keys = append(w.writeLists[kv.PlainState].Keys, addressBytes) + w.writeLists[kv.PlainState].Keys = append(w.writeLists[kv.PlainState].Keys, string(addressBytes)) w.writeLists[kv.PlainState].Vals = append(w.writeLists[kv.PlainState].Vals, []byte{}) if original.Incarnation > 0 { var b [8]byte binary.BigEndian.PutUint64(b[:], original.Incarnation) - w.writeLists[kv.IncarnationMap].Keys = append(w.writeLists[kv.IncarnationMap].Keys, addressBytes) + w.writeLists[kv.IncarnationMap].Keys = append(w.writeLists[kv.IncarnationMap].Keys, string(addressBytes)) w.writeLists[kv.IncarnationMap].Vals = append(w.writeLists[kv.IncarnationMap].Vals, b[:]) } if original.Initialised { @@ -682,13 +691,14 @@ func (w *StateWriterV3) WriteAccountStorage(address common.Address, incarnation return nil } composite := dbutils.PlainGenerateCompositeStorageKey(address[:], incarnation, key.Bytes()) - w.writeLists[kv.PlainState].Keys = append(w.writeLists[kv.PlainState].Keys, composite) + cmpositeS := string(composite) + w.writeLists[kv.PlainState].Keys = append(w.writeLists[kv.PlainState].Keys, cmpositeS) w.writeLists[kv.PlainState].Vals = append(w.writeLists[kv.PlainState].Vals, value.Bytes()) //fmt.Printf("storage [%x] [%x] => [%x], txNum: %d\n", address, *key, v, w.txNum) if w.storagePrevs == nil { w.storagePrevs = map[string][]byte{} } - w.storagePrevs[string(composite)] = original.Bytes() + w.storagePrevs[cmpositeS] = original.Bytes() return nil } @@ -733,7 +743,7 @@ func (r *StateReaderV3) ReadAccountData(address common.Address) (*accounts.Accou } if !r.discardReadList { // lifecycle of `r.readList` is less than lifecycle of `r.rs` and `r.tx`, also `r.rs` and `r.tx` do store data immutable way - r.readLists[kv.PlainState].Keys = append(r.readLists[kv.PlainState].Keys, addr) + r.readLists[kv.PlainState].Keys = append(r.readLists[kv.PlainState].Keys, string(addr)) r.readLists[kv.PlainState].Vals = append(r.readLists[kv.PlainState].Vals, enc) } if len(enc) == 0 { @@ -760,7 +770,7 @@ func (r *StateReaderV3) ReadAccountStorage(address common.Address, incarnation u } } if !r.discardReadList { - r.readLists[kv.PlainState].Keys = append(r.readLists[kv.PlainState].Keys, composite) + r.readLists[kv.PlainState].Keys = append(r.readLists[kv.PlainState].Keys, string(composite)) r.readLists[kv.PlainState].Vals = append(r.readLists[kv.PlainState].Vals, enc) } if r.trace { @@ -787,7 +797,7 @@ func (r *StateReaderV3) ReadAccountCode(address common.Address, incarnation uint } } if !r.discardReadList { - r.readLists[kv.Code].Keys = append(r.readLists[kv.Code].Keys, addr) + r.readLists[kv.Code].Keys = append(r.readLists[kv.Code].Keys, string(addr)) r.readLists[kv.Code].Vals = append(r.readLists[kv.Code].Vals, enc) } if r.trace { @@ -809,7 +819,7 @@ func (r *StateReaderV3) ReadAccountCodeSize(address common.Address, incarnation var sizebuf [8]byte binary.BigEndian.PutUint64(sizebuf[:], uint64(len(enc))) if !r.discardReadList { - r.readLists[CodeSizeTable].Keys = append(r.readLists[CodeSizeTable].Keys, address[:]) + r.readLists[CodeSizeTable].Keys = append(r.readLists[CodeSizeTable].Keys, string(address[:])) r.readLists[CodeSizeTable].Vals = append(r.readLists[CodeSizeTable].Vals, sizebuf[:]) } size := len(enc) @@ -830,7 +840,7 @@ func (r *StateReaderV3) ReadAccountIncarnation(address common.Address) (uint64, } } if !r.discardReadList { - r.readLists[kv.IncarnationMap].Keys = append(r.readLists[kv.IncarnationMap].Keys, addrBytes) + r.readLists[kv.IncarnationMap].Keys = append(r.readLists[kv.IncarnationMap].Keys, string(addrBytes)) r.readLists[kv.IncarnationMap].Vals = append(r.readLists[kv.IncarnationMap].Vals, enc) } if len(enc) == 0 { From 2f54fbb8884ee741b0bc2b2794340acc6810cf8d Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 15:32:01 +0700 Subject: [PATCH 041/100] save --- core/state/intra_block_state.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/core/state/intra_block_state.go b/core/state/intra_block_state.go index 31bb8165837..9ead59fe2f3 100644 --- a/core/state/intra_block_state.go +++ b/core/state/intra_block_state.go @@ -30,7 +30,6 @@ import ( "github.com/ledgerwatch/erigon/core/types/accounts" "github.com/ledgerwatch/erigon/crypto" "github.com/ledgerwatch/erigon/turbo/trie" - "github.com/ledgerwatch/log/v3" ) type revision struct { @@ -118,12 +117,12 @@ func (sdb *IntraBlockState) Error() error { // Reset clears out all ephemeral state objects from the state db, but keeps // the underlying state trie to avoid reloading data for the next operations. func (sdb *IntraBlockState) Reset() { - if len(sdb.nilAccounts) == 0 || len(sdb.stateObjects) == 0 || len(sdb.stateObjectsDirty) == 0 || len(sdb.balanceInc) == 0 { - log.Warn("zero", "len(sdb.nilAccounts)", len(sdb.nilAccounts), - "len(sdb.stateObjects)", len(sdb.stateObjects), - "len(sdb.stateObjectsDirty)", len(sdb.stateObjectsDirty), - "len(sdb.balanceInc)", len(sdb.balanceInc)) - } + //if len(sdb.nilAccounts) == 0 || len(sdb.stateObjects) == 0 || len(sdb.stateObjectsDirty) == 0 || len(sdb.balanceInc) == 0 { + // log.Warn("zero", "len(sdb.nilAccounts)", len(sdb.nilAccounts), + // "len(sdb.stateObjects)", len(sdb.stateObjects), + // "len(sdb.stateObjectsDirty)", len(sdb.stateObjectsDirty), + // "len(sdb.balanceInc)", len(sdb.balanceInc)) + //} sdb.nilAccounts = make(map[libcommon.Address]struct{}) sdb.stateObjects = make(map[libcommon.Address]*stateObject) sdb.stateObjectsDirty = make(map[libcommon.Address]struct{}) From 19cc32d3fe7e3cd7cc243b0b607d3ab279ce94ef Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 16:10:01 +0700 Subject: [PATCH 042/100] save --- cmd/state/exec3/state.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmd/state/exec3/state.go b/cmd/state/exec3/state.go index df643448993..649583b54ba 100644 --- a/cmd/state/exec3/state.go +++ b/cmd/state/exec3/state.go @@ -3,6 +3,7 @@ package exec3 import ( "context" "math/big" + "sort" "sync" "github.com/ledgerwatch/erigon-lib/chain" @@ -251,6 +252,11 @@ func (rw *Worker) RunTxTaskNoLock(txTask *exec22.TxTask) { } } txTask.ResultsSize = int64(size) + if rw.background { + for _, set := range txTask.WriteLists { + sort.Sort(set) + } + } } } From c664c26e8beb1bcf53154873f9df1b289c7b9af5 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 16:10:10 +0700 Subject: [PATCH 043/100] save --- cmd/state/exec3/state.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/cmd/state/exec3/state.go b/cmd/state/exec3/state.go index 649583b54ba..d60b3cb1b67 100644 --- a/cmd/state/exec3/state.go +++ b/cmd/state/exec3/state.go @@ -3,7 +3,6 @@ package exec3 import ( "context" "math/big" - "sort" "sync" "github.com/ledgerwatch/erigon-lib/chain" @@ -252,11 +251,11 @@ func (rw *Worker) RunTxTaskNoLock(txTask *exec22.TxTask) { } } txTask.ResultsSize = int64(size) - if rw.background { - for _, set := range txTask.WriteLists { - sort.Sort(set) - } - } + //if rw.background { + // for _, set := range txTask.WriteLists { + // sort.Sort(set) + // } + //} } } From c52fd98ed2600292ff69c1b3d3753841c5b1fef5 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 16:21:07 +0700 Subject: [PATCH 044/100] save --- core/state/rw_v3.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index 8ca0a7e23ce..50e4345788e 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -579,9 +579,7 @@ func (rs *StateV3) ReadsValid(readLists map[string]*exec22.KvList) bool { continue } for i, key := range list.Keys { - key := key - keyS := *(*string)(unsafe.Pointer(&key)) - if val, ok := t.Get(keyS); ok { + if val, ok := t.Get(key); ok { if table == CodeSizeTable { if binary.BigEndian.Uint64(list.Vals[i]) != uint64(len(val)) { return false From 7349408188ff3a4495a849f73daf023eeba5e006 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 16:29:02 +0700 Subject: [PATCH 045/100] save --- cmd/state/exec3/state.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/cmd/state/exec3/state.go b/cmd/state/exec3/state.go index d60b3cb1b67..df643448993 100644 --- a/cmd/state/exec3/state.go +++ b/cmd/state/exec3/state.go @@ -251,11 +251,6 @@ func (rw *Worker) RunTxTaskNoLock(txTask *exec22.TxTask) { } } txTask.ResultsSize = int64(size) - //if rw.background { - // for _, set := range txTask.WriteLists { - // sort.Sort(set) - // } - //} } } From dcf5d4c50e87212aa907c3a2de6c676d58cd6c05 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 28 Feb 2023 16:29:23 +0700 Subject: [PATCH 046/100] save --- core/state/intra_block_state.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/core/state/intra_block_state.go b/core/state/intra_block_state.go index 9ead59fe2f3..ff8028ca579 100644 --- a/core/state/intra_block_state.go +++ b/core/state/intra_block_state.go @@ -145,9 +145,6 @@ func (sdb *IntraBlockState) AddLog(log2 *types.Log) { } func (sdb *IntraBlockState) GetLogs(hash libcommon.Hash) []*types.Log { - if sdb.logs == nil { - return nil - } return sdb.logs[hash] } From be73c6421ffbc01339db1a774d1bfc9c39265f5a Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 1 Mar 2023 09:22:16 +0700 Subject: [PATCH 047/100] save --- cmd/rpcdaemon/commands/gen_traces_test.go | 4 ++-- core/vm/evm.go | 8 ++++++++ core/vm/interpreter.go | 6 ++++-- 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/cmd/rpcdaemon/commands/gen_traces_test.go b/cmd/rpcdaemon/commands/gen_traces_test.go index 40a12777569..241acf35d56 100644 --- a/cmd/rpcdaemon/commands/gen_traces_test.go +++ b/cmd/rpcdaemon/commands/gen_traces_test.go @@ -9,7 +9,7 @@ import ( jsoniter "github.com/json-iterator/go" "github.com/stretchr/testify/assert" - libcommon "github.com/ledgerwatch/erigon-lib/common" + "github.com/ledgerwatch/erigon-lib/common" "github.com/ledgerwatch/erigon-lib/kv/kvcache" "github.com/ledgerwatch/erigon/cmd/rpcdaemon/cli/httpcfg" "github.com/ledgerwatch/erigon/cmd/rpcdaemon/rpcdaemontest" @@ -281,7 +281,7 @@ func TestGeneratedTraceApiCollision(t *testing.T) { stateCache := kvcache.New(kvcache.DefaultCoherentConfig) baseApi := NewBaseApi(nil, stateCache, br, agg, false, rpccfg.DefaultEvmCallTimeout, m.Engine) api := NewTraceAPI(baseApi, m.DB, &httpcfg.HttpCfg{}) - traces, err := api.Transaction(context.Background(), libcommon.HexToHash("0xb2b9fa4c999c1c8370ce1fbd1c4315a9ce7f8421fe2ebed8a9051ff2e4e7e3da")) + traces, err := api.Transaction(context.Background(), common.HexToHash("0xb2b9fa4c999c1c8370ce1fbd1c4315a9ce7f8421fe2ebed8a9051ff2e4e7e3da")) if err != nil { t.Errorf("trace_block %d: %v", 0, err) } diff --git a/core/vm/evm.go b/core/vm/evm.go index 41a05c9c369..06283e1ccd6 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -338,12 +338,20 @@ func (c *codeAndHash) Hash() libcommon.Hash { func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, value *uint256.Int, address libcommon.Address, typ OpCode, incrementNonce bool) ([]byte, libcommon.Address, uint64, error) { var ret []byte var err error + var gasConsumption uint64 depth := evm.interpreter.Depth() + if evm.config.Debug { if depth == 0 { evm.config.Tracer.CaptureStart(evm, caller.Address(), address, false /* precompile */, true /* create */, codeAndHash.code, gas, value, nil) + defer func() { + evm.config.Tracer.CaptureEnd(ret, gasConsumption, err) + }() } else { evm.config.Tracer.CaptureEnter(typ, caller.Address(), address, false /* precompile */, true /* create */, codeAndHash.code, gas, value, nil) + defer func() { + evm.config.Tracer.CaptureExit(ret, gasConsumption, err) + }() } } diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index fa455079a65..f1867ddab39 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -316,9 +316,11 @@ func (vm *VM) noop() {} func (vm *VM) setReadonly(outerReadonly bool) func() { if outerReadonly && !vm.readOnly { vm.readOnly = true - return vm.disableReadonly + return func() { + vm.readOnly = false + } } - return vm.noop + return func() {} } func (vm *VM) getReadonly() bool { From db5950c06e0cd2bf80b6094f66602f1af9e48e33 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 2 Mar 2023 10:34:59 +0700 Subject: [PATCH 048/100] save --- core/state/rw_v3.go | 288 ++++++++++++++++++++++++++++++-------------- 1 file changed, 199 insertions(+), 89 deletions(-) diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index e81aac9718f..59e58c541d4 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -28,6 +28,7 @@ import ( ) const CodeSizeTable = "CodeSize" +const StorageTable = "Storage" type StateV3 struct { lock sync.RWMutex @@ -37,22 +38,34 @@ type StateV3 struct { triggerLock sync.Mutex queue exec22.TxTaskQueue queueLock sync.Mutex - changes map[string]*btree2.Map[string, []byte] + //changes map[string]*btree2.Map[string, []byte] sizeEstimate int txsDone *atomic2.Uint64 finished atomic2.Bool + + chCode map[string][]byte + chAccs map[string][]byte + chStorage *btree2.Map[string, []byte] + chIncs map[string][]byte + chContractCode map[string][]byte } func NewStateV3() *StateV3 { rs := &StateV3{ triggers: map[uint64]*exec22.TxTask{}, senderTxNums: map[common.Address]uint64{}, - changes: map[string]*btree2.Map[string, []byte]{ - kv.PlainState: btree2.NewMap[string, []byte](128), - kv.Code: btree2.NewMap[string, []byte](128), - kv.IncarnationMap: btree2.NewMap[string, []byte](128), - kv.PlainContractCode: btree2.NewMap[string, []byte](128), - }, + //changes: map[string]*btree2.Map[string, []byte]{ + // kv.PlainState: btree2.NewMap[string, []byte](128), + // kv.Code: btree2.NewMap[string, []byte](128), + // kv.IncarnationMap: btree2.NewMap[string, []byte](128), + // kv.PlainContractCode: btree2.NewMap[string, []byte](128), + //}, + chCode: map[string][]byte{}, + chAccs: map[string][]byte{}, + chStorage: btree2.NewMap[string, []byte](128), + chIncs: map[string][]byte{}, + chContractCode: map[string][]byte{}, + txsDone: atomic2.NewUint64(0), } rs.receiveWork = sync.NewCond(&rs.queueLock) @@ -60,20 +73,30 @@ func NewStateV3() *StateV3 { } func (rs *StateV3) put(table string, key, val []byte) { - old, ok := rs.changes[table].Set(string(key), val) - if ok { - rs.sizeEstimate += len(val) - len(old) - } else { - rs.sizeEstimate += len(key) + len(val) - } + rs.puts(table, string(key), val) } func (rs *StateV3) puts(table string, key string, val []byte) { - old, ok := rs.changes[table].Set(key, val) - if ok { - rs.sizeEstimate += len(val) - len(old) - } else { - rs.sizeEstimate += len(key) + len(val) + switch table { + case kv.PlainState: + if len(key) == 20 { + rs.chAccs[key] = val + } else { + old, ok := rs.chStorage.Set(key, val) + if ok { + rs.sizeEstimate += len(val) - len(old) + } else { + rs.sizeEstimate += len(key) + len(val) + } + } + case kv.Code: + rs.chCode[key] = val + case kv.IncarnationMap: + rs.chIncs[key] = val + case kv.PlainContractCode: + rs.chContractCode[key] = val + default: + panic(table) } } @@ -85,46 +108,107 @@ func (rs *StateV3) Get(table string, key []byte) []byte { } func (rs *StateV3) get(table string, key []byte) (v []byte) { - v, _ = rs.changes[table].Get(*(*string)(unsafe.Pointer(&key))) + keyS := *(*string)(unsafe.Pointer(&key)) + switch table { + case kv.PlainState: + if len(key) == 20 { + v, _ = rs.chAccs[keyS] + } else { + v, _ = rs.chStorage.Get(keyS) + } + case kv.Code: + v, _ = rs.chCode[keyS] + case kv.IncarnationMap: + v, _ = rs.chIncs[keyS] + case kv.PlainContractCode: + v, _ = rs.chContractCode[keyS] + default: + panic(table) + } return v } -func (rs *StateV3) Flush(ctx context.Context, rwTx kv.RwTx, logPrefix string, logEvery *time.Ticker) error { - rs.lock.Lock() - defer rs.lock.Unlock() - for table, t := range rs.changes { - c, err := rwTx.RwCursor(table) - if err != nil { - return err +func (rs *StateV3) flushMap(ctx context.Context, rwTx kv.RwTx, table string, m map[string][]byte, logPrefix string, logEvery *time.Ticker) error { + c, err := rwTx.RwCursor(table) + if err != nil { + return err + } + defer c.Close() + for k, v := range m { + if len(v) == 0 { + if err = c.Delete([]byte(k)); err != nil { + return err + } + } else { + if err = c.Put([]byte(k), v); err != nil { + return err + } } - iter := t.Iter() - for ok := iter.First(); ok; ok = iter.Next() { - if len(iter.Value()) == 0 { - if err = c.Delete([]byte(iter.Key())); err != nil { - return err - } - //fmt.Printf("Flush [%x]=>\n", item.key) - } else { - if err = c.Put([]byte(iter.Key()), iter.Value()); err != nil { - return err - } - //fmt.Printf("Flush [%x]=>[%x]\n", item.key, item.val) + select { + case <-logEvery.C: + log.Info(fmt.Sprintf("[%s] Flush", logPrefix), "table", table, "current_prefix", hex.EncodeToString([]byte(k)[:4])) + case <-ctx.Done(): + return ctx.Err() + default: + } + } + return nil +} +func (rs *StateV3) flushBtree(ctx context.Context, rwTx kv.RwTx, table string, m *btree2.Map[string, []byte], logPrefix string, logEvery *time.Ticker) error { + c, err := rwTx.RwCursor(table) + if err != nil { + return err + } + defer c.Close() + iter := m.Iter() + for ok := iter.First(); ok; ok = iter.Next() { + if len(iter.Value()) == 0 { + if err = c.Delete([]byte(iter.Key())); err != nil { + return err } - - select { - case <-logEvery.C: - log.Info(fmt.Sprintf("[%s] Flush", logPrefix), "table", table, "current_prefix", hex.EncodeToString([]byte(iter.Key())[:4])) - case <-ctx.Done(): - return ctx.Err() - default: + } else { + if err = c.Put([]byte(iter.Key()), iter.Value()); err != nil { + return err } } - if err != nil { - return err + + select { + case <-logEvery.C: + log.Info(fmt.Sprintf("[%s] Flush", logPrefix), "table", table, "current_prefix", hex.EncodeToString([]byte(iter.Key())[:4])) + case <-ctx.Done(): + return ctx.Err() + default: } - t.Clear() } + return nil +} + +func (rs *StateV3) Flush(ctx context.Context, rwTx kv.RwTx, logPrefix string, logEvery *time.Ticker) error { + rs.lock.Lock() + defer rs.lock.Unlock() + + if err := rs.flushMap(ctx, rwTx, kv.PlainState, rs.chAccs, logPrefix, logEvery); err != nil { + return err + } + rs.chAccs = map[string][]byte{} + if err := rs.flushBtree(ctx, rwTx, kv.PlainState, rs.chStorage, logPrefix, logEvery); err != nil { + return err + } + rs.chStorage.Clear() + if err := rs.flushMap(ctx, rwTx, kv.Code, rs.chCode, logPrefix, logEvery); err != nil { + return err + } + rs.chCode = map[string][]byte{} + if err := rs.flushMap(ctx, rwTx, kv.PlainContractCode, rs.chContractCode, logPrefix, logEvery); err != nil { + return err + } + rs.chContractCode = map[string][]byte{} + if err := rs.flushMap(ctx, rwTx, kv.IncarnationMap, rs.chIncs, logPrefix, logEvery); err != nil { + return err + } + rs.chIncs = map[string][]byte{} + rs.sizeEstimate = 0 return nil } @@ -241,7 +325,6 @@ func (rs *StateV3) appplyState1(roTx kv.Tx, txTask *exec22.TxTask, agg *libstate } defer cursor.Close() addr1 := make([]byte, 20+8) - psChanges := rs.changes[kv.PlainState] for addrS, original := range txTask.AccountDels { addr := []byte(addrS) copy(addr1, addr) @@ -273,28 +356,26 @@ func (rs *StateV3) appplyState1(roTx kv.Tx, txTask *exec22.TxTask, agg *libstate if !bytes.HasPrefix(k, addr1) { k = nil } - if psChanges != nil { - //TODO: try full-scan, then can replace btree by map - iter := psChanges.Iter() - for ok := iter.Seek(string(addr1)); ok; ok = iter.Next() { - key := []byte(iter.Key()) - if !bytes.HasPrefix(key, addr1) { - break - } - for ; e == nil && k != nil && bytes.HasPrefix(k, addr1) && bytes.Compare(k, key) <= 0; k, v, e = cursor.Next() { - if !bytes.Equal(k, key) { - // Skip the cursor item when the key is equal, i.e. prefer the item from the changes tree - if e = agg.AddStoragePrev(addr, k[28:], v); e != nil { - return e - } + //TODO: try full-scan, then can replace btree by map + iter := rs.chStorage.Iter() + for ok := iter.Seek(string(addr1)); ok; ok = iter.Next() { + key := []byte(iter.Key()) + if !bytes.HasPrefix(key, addr1) { + break + } + for ; e == nil && k != nil && bytes.HasPrefix(k, addr1) && bytes.Compare(k, key) <= 0; k, v, e = cursor.Next() { + if !bytes.Equal(k, key) { + // Skip the cursor item when the key is equal, i.e. prefer the item from the changes tree + if e = agg.AddStoragePrev(addr, k[28:], v); e != nil { + return e } } - if e != nil { - return e - } - if e = agg.AddStoragePrev(addr, key[28:], iter.Value()); e != nil { - break - } + } + if e != nil { + return e + } + if e = agg.AddStoragePrev(addr, key[28:], iter.Value()); e != nil { + break } } for ; e == nil && k != nil && bytes.HasPrefix(k, addr1); k, v, e = cursor.Next() { @@ -555,31 +636,59 @@ func (rs *StateV3) SizeEstimate() uint64 { } func (rs *StateV3) ReadsValid(readLists map[string]*exec22.KvList) bool { - var t *btree2.Map[string, []byte] - rs.lock.RLock() defer rs.lock.RUnlock() - //fmt.Printf("ValidReads\n") for table, list := range readLists { - //fmt.Printf("Table %s\n", table) - var ok bool - if table == CodeSizeTable { - t, ok = rs.changes[kv.Code] - } else { - t, ok = rs.changes[table] + switch table { + case kv.PlainState: + if !rs.readsValidMap(table, list, rs.chAccs) { + return false + } + case CodeSizeTable: + if !rs.readsValidMap(table, list, rs.chCode) { + return false + } + case StorageTable: + if !rs.readsValidBtree(table, list, rs.chStorage) { + return false + } + case kv.Code: + if !rs.readsValidMap(table, list, rs.chCode) { + return false + } + case kv.IncarnationMap: + if !rs.readsValidMap(table, list, rs.chIncs) { + return false + } } - if !ok { - continue + } + return true +} + +func (rs *StateV3) readsValidMap(table string, list *exec22.KvList, m map[string][]byte) bool { + for i, key := range list.Keys { + if val, ok := m[key]; ok { + if table == CodeSizeTable { + if binary.BigEndian.Uint64(list.Vals[i]) != uint64(len(val)) { + return false + } + } else if !bytes.Equal(list.Vals[i], val) { + return false + } } - for i, key := range list.Keys { - if val, ok := t.Get(key); ok { - if table == CodeSizeTable { - if binary.BigEndian.Uint64(list.Vals[i]) != uint64(len(val)) { - return false - } - } else if !bytes.Equal(list.Vals[i], val) { + } + return true +} + +func (rs *StateV3) readsValidBtree(table string, list *exec22.KvList, m *btree2.Map[string, []byte]) bool { + for i, key := range list.Keys { + if val, ok := m.Get(key); ok { + if table == CodeSizeTable { + if binary.BigEndian.Uint64(list.Vals[i]) != uint64(len(val)) { return false } + } else if !bytes.Equal(list.Vals[i], val) { + return false } } } @@ -761,8 +870,8 @@ func (r *StateReaderV3) ReadAccountStorage(address common.Address, incarnation u } } if !r.discardReadList { - r.readLists[kv.PlainState].Keys = append(r.readLists[kv.PlainState].Keys, string(composite)) - r.readLists[kv.PlainState].Vals = append(r.readLists[kv.PlainState].Vals, enc) + r.readLists[StorageTable].Keys = append(r.readLists[StorageTable].Keys, string(composite)) + r.readLists[StorageTable].Vals = append(r.readLists[StorageTable].Vals, enc) } if r.trace { if enc == nil { @@ -871,6 +980,7 @@ var readListPool = sync.Pool{ kv.PlainState: {}, kv.Code: {}, CodeSizeTable: {}, + StorageTable: {}, kv.IncarnationMap: {}, } }, From 0b5b66a6bdbd22f95fbfa038ef9e2fcde7358931 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 2 Mar 2023 10:56:17 +0700 Subject: [PATCH 049/100] save --- core/state/history_test.go | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/core/state/history_test.go b/core/state/history_test.go index a1a545fb1af..693f5b74bdd 100644 --- a/core/state/history_test.go +++ b/core/state/history_test.go @@ -28,6 +28,31 @@ import ( "github.com/ledgerwatch/erigon/turbo/trie" ) +func BenchmarkName1(b *testing.B) { + b.Run("1", func(b *testing.B) { + rl := newReadList() + for i := 0; i < b.N; i++ { + rl[kv.PlainState].Keys = append(rl[kv.PlainState].Keys, "") + rl[kv.PlainState].Vals = append(rl[kv.PlainState].Vals, []byte{}) + } + }) + b.Run("2", func(b *testing.B) { + rl := newReadList() + for i := 0; i < b.N; i++ { + l := rl[kv.PlainState] + l.Keys = append(l.Keys, "") + l.Vals = append(l.Vals, []byte{}) + rl[kv.PlainState] = l + } + }) +} +func TestA(t *testing.T) { + var a []int + _ = append(a, 1) + _ = append(a, 2) + fmt.Printf("a: %d\n", a) +} + func TestMutationDeleteTimestamp(t *testing.T) { _, tx := memdb.NewTestTx(t) From 76a9b32ca74933f5f4ee1202e1d64f4361288def Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 2 Mar 2023 11:49:53 +0700 Subject: [PATCH 050/100] save --- core/state/rw_v3.go | 35 +++++++++++++++-------------------- 1 file changed, 15 insertions(+), 20 deletions(-) diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index 59e58c541d4..76bc6994910 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -31,35 +31,30 @@ const CodeSizeTable = "CodeSize" const StorageTable = "Storage" type StateV3 struct { - lock sync.RWMutex - receiveWork *sync.Cond - triggers map[uint64]*exec22.TxTask - senderTxNums map[common.Address]uint64 - triggerLock sync.Mutex - queue exec22.TxTaskQueue - queueLock sync.Mutex - //changes map[string]*btree2.Map[string, []byte] - sizeEstimate int - txsDone *atomic2.Uint64 - finished atomic2.Bool - + lock sync.RWMutex + sizeEstimate int chCode map[string][]byte chAccs map[string][]byte chStorage *btree2.Map[string, []byte] chIncs map[string][]byte chContractCode map[string][]byte + + receiveWork *sync.Cond + triggers map[uint64]*exec22.TxTask + senderTxNums map[common.Address]uint64 + triggerLock sync.Mutex + + queue exec22.TxTaskQueue + queueLock sync.Mutex + + txsDone *atomic2.Uint64 + finished atomic2.Bool } func NewStateV3() *StateV3 { rs := &StateV3{ - triggers: map[uint64]*exec22.TxTask{}, - senderTxNums: map[common.Address]uint64{}, - //changes: map[string]*btree2.Map[string, []byte]{ - // kv.PlainState: btree2.NewMap[string, []byte](128), - // kv.Code: btree2.NewMap[string, []byte](128), - // kv.IncarnationMap: btree2.NewMap[string, []byte](128), - // kv.PlainContractCode: btree2.NewMap[string, []byte](128), - //}, + triggers: map[uint64]*exec22.TxTask{}, + senderTxNums: map[common.Address]uint64{}, chCode: map[string][]byte{}, chAccs: map[string][]byte{}, chStorage: btree2.NewMap[string, []byte](128), From 25bd4b824e33b356abffceb99d97d46265445677 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 2 Mar 2023 11:54:51 +0700 Subject: [PATCH 051/100] save --- core/state/rw_v3.go | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index 76bc6994910..36ca3fa3923 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -75,6 +75,11 @@ func (rs *StateV3) puts(table string, key string, val []byte) { switch table { case kv.PlainState: if len(key) == 20 { + if old, ok := rs.chAccs[key]; !ok { + rs.sizeEstimate += len(val) - len(old) + } else { + rs.sizeEstimate += len(key) + len(val) + } rs.chAccs[key] = val } else { old, ok := rs.chStorage.Set(key, val) @@ -85,10 +90,25 @@ func (rs *StateV3) puts(table string, key string, val []byte) { } } case kv.Code: + if old, ok := rs.chCode[key]; !ok { + rs.sizeEstimate += len(val) - len(old) + } else { + rs.sizeEstimate += len(key) + len(val) + } rs.chCode[key] = val case kv.IncarnationMap: + if old, ok := rs.chIncs[key]; !ok { + rs.sizeEstimate += len(val) - len(old) + } else { + rs.sizeEstimate += len(key) + len(val) + } rs.chIncs[key] = val case kv.PlainContractCode: + if old, ok := rs.chContractCode[key]; !ok { + rs.sizeEstimate += len(val) - len(old) + } else { + rs.sizeEstimate += len(key) + len(val) + } rs.chContractCode[key] = val default: panic(table) From f662b36352bb81b798aaa620d8945dd9534f57f2 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 2 Mar 2023 11:56:03 +0700 Subject: [PATCH 052/100] save --- core/state/rw_v3.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index 36ca3fa3923..91d552aa22f 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -75,36 +75,35 @@ func (rs *StateV3) puts(table string, key string, val []byte) { switch table { case kv.PlainState: if len(key) == 20 { - if old, ok := rs.chAccs[key]; !ok { + if old, ok := rs.chAccs[key]; ok { rs.sizeEstimate += len(val) - len(old) } else { rs.sizeEstimate += len(key) + len(val) } rs.chAccs[key] = val } else { - old, ok := rs.chStorage.Set(key, val) - if ok { + if old, ok := rs.chStorage.Set(key, val); ok { rs.sizeEstimate += len(val) - len(old) } else { rs.sizeEstimate += len(key) + len(val) } } case kv.Code: - if old, ok := rs.chCode[key]; !ok { + if old, ok := rs.chCode[key]; ok { rs.sizeEstimate += len(val) - len(old) } else { rs.sizeEstimate += len(key) + len(val) } rs.chCode[key] = val case kv.IncarnationMap: - if old, ok := rs.chIncs[key]; !ok { + if old, ok := rs.chIncs[key]; ok { rs.sizeEstimate += len(val) - len(old) } else { rs.sizeEstimate += len(key) + len(val) } rs.chIncs[key] = val case kv.PlainContractCode: - if old, ok := rs.chContractCode[key]; !ok { + if old, ok := rs.chContractCode[key]; ok { rs.sizeEstimate += len(val) - len(old) } else { rs.sizeEstimate += len(key) + len(val) From 6fde10c08c4824f65c409b0121df5ff2cf0846b6 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 2 Mar 2023 11:57:29 +0700 Subject: [PATCH 053/100] save --- core/state/rw_v3.go | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index 91d552aa22f..7977430d7f0 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -697,11 +697,7 @@ func (rs *StateV3) readsValidMap(table string, list *exec22.KvList, m map[string func (rs *StateV3) readsValidBtree(table string, list *exec22.KvList, m *btree2.Map[string, []byte]) bool { for i, key := range list.Keys { if val, ok := m.Get(key); ok { - if table == CodeSizeTable { - if binary.BigEndian.Uint64(list.Vals[i]) != uint64(len(val)) { - return false - } - } else if !bytes.Equal(list.Vals[i], val) { + if !bytes.Equal(list.Vals[i], val) { return false } } From 3d814cebcb7ab8b687f8895e4245aaf7e9efacd3 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 2 Mar 2023 11:58:44 +0700 Subject: [PATCH 054/100] save --- core/state/rw_v3.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index 7977430d7f0..51d0cb219db 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -680,14 +680,21 @@ func (rs *StateV3) ReadsValid(readLists map[string]*exec22.KvList) bool { } func (rs *StateV3) readsValidMap(table string, list *exec22.KvList, m map[string][]byte) bool { - for i, key := range list.Keys { - if val, ok := m[key]; ok { - if table == CodeSizeTable { + switch table { + case CodeSizeTable: + for i, key := range list.Keys { + if val, ok := m[key]; ok { if binary.BigEndian.Uint64(list.Vals[i]) != uint64(len(val)) { return false } - } else if !bytes.Equal(list.Vals[i], val) { - return false + } + } + default: + for i, key := range list.Keys { + if val, ok := m[key]; ok { + if !bytes.Equal(list.Vals[i], val) { + return false + } } } } From 4eff532dfa1cd27cc1d25c800373db91a5993e43 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 2 Mar 2023 12:04:19 +0700 Subject: [PATCH 055/100] save --- core/state/rw_v3.go | 38 ++++++++++++++-------------- eth/stagedsync/stage_execute.go | 4 +-- eth/stagedsync/stage_execute_test.go | 2 +- 3 files changed, 22 insertions(+), 22 deletions(-) diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index 51d0cb219db..aeda646eee1 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -49,10 +49,13 @@ type StateV3 struct { txsDone *atomic2.Uint64 finished atomic2.Bool + + tmpdir string } -func NewStateV3() *StateV3 { +func NewStateV3(tmpdir string) *StateV3 { rs := &StateV3{ + tmpdir: tmpdir, triggers: map[uint64]*exec22.TxTask{}, senderTxNums: map[common.Address]uint64{}, chCode: map[string][]byte{}, @@ -143,30 +146,27 @@ func (rs *StateV3) get(table string, key []byte) (v []byte) { } func (rs *StateV3) flushMap(ctx context.Context, rwTx kv.RwTx, table string, m map[string][]byte, logPrefix string, logEvery *time.Ticker) error { - c, err := rwTx.RwCursor(table) - if err != nil { - return err - } - defer c.Close() + collector := etl.NewCollector(logPrefix, "", etl.NewSortableBuffer(etl.BufferOptimalSize)) + defer collector.Close() + + var count int + total := len(m) for k, v := range m { - if len(v) == 0 { - if err = c.Delete([]byte(k)); err != nil { - return err - } - } else { - if err = c.Put([]byte(k), v); err != nil { - return err - } + if err := collector.Collect([]byte(k), v); err != nil { + return err } - + count++ select { - case <-logEvery.C: - log.Info(fmt.Sprintf("[%s] Flush", logPrefix), "table", table, "current_prefix", hex.EncodeToString([]byte(k)[:4])) - case <-ctx.Done(): - return ctx.Err() default: + case <-logEvery.C: + progress := fmt.Sprintf("%.1fM/%.1fM", float64(count)/1_000_000, total/1_000_000) + log.Info("Write to db", "progress", progress, "current table", table) + rwTx.CollectMetrics() } } + if err := collector.Load(rwTx, table, etl.IdentityLoadFunc, etl.TransformArgs{Quit: ctx.Done()}); err != nil { + return err + } return nil } func (rs *StateV3) flushBtree(ctx context.Context, rwTx kv.RwTx, table string, m *btree2.Map[string, []byte], logPrefix string, logEvery *time.Ticker) error { diff --git a/eth/stagedsync/stage_execute.go b/eth/stagedsync/stage_execute.go index f5998b7aa76..d66d9ec0026 100644 --- a/eth/stagedsync/stage_execute.go +++ b/eth/stagedsync/stage_execute.go @@ -281,7 +281,7 @@ func ExecBlockV3(s *StageState, u Unwinder, tx kv.RwTx, toBlock uint64, ctx cont if to > s.BlockNumber+16 { log.Info(fmt.Sprintf("[%s] Blocks execution", logPrefix), "from", s.BlockNumber, "to", to) } - rs := state.NewStateV3() + rs := state.NewStateV3(cfg.dirs.Tmp) parallel := initialCycle && tx == nil if err := ExecV3(ctx, s, u, workersCount, cfg, tx, parallel, rs, logPrefix, log.New(), to); err != nil { @@ -312,7 +312,7 @@ func reconstituteBlock(agg *libstate.AggregatorV3, db kv.RoDB, tx kv.Tx) (n uint func unwindExec3(u *UnwindState, s *StageState, tx kv.RwTx, ctx context.Context, cfg ExecuteBlockCfg, accumulator *shards.Accumulator) (err error) { cfg.agg.SetLogPrefix(s.LogPrefix()) - rs := state.NewStateV3() + rs := state.NewStateV3(cfg.dirs.Tmp) // unwind all txs of u.UnwindPoint block. 1 txn in begin/end of block - system txs txNum, err := rawdbv3.TxNums.Min(tx, u.UnwindPoint+1) if err != nil { diff --git a/eth/stagedsync/stage_execute_test.go b/eth/stagedsync/stage_execute_test.go index 8b4a028758b..9d4ca52e21c 100644 --- a/eth/stagedsync/stage_execute_test.go +++ b/eth/stagedsync/stage_execute_test.go @@ -129,7 +129,7 @@ func apply(tx kv.RwTx, agg *libstate.AggregatorV3) (beforeBlock, afterBlock test agg.SetTx(tx) agg.StartWrites() - rs := state.NewStateV3() + rs := state.NewStateV3("") stateWriter := state.NewStateWriterV3(rs) return func(n, from, numberOfBlocks uint64) { stateWriter.SetTxNum(n) From 0f9415b9d6ae6c32bf9e8c4eb523459cc6165d1f Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 2 Mar 2023 13:16:57 +0700 Subject: [PATCH 056/100] save --- core/state/history_test.go | 25 ------------------------- 1 file changed, 25 deletions(-) diff --git a/core/state/history_test.go b/core/state/history_test.go index 693f5b74bdd..a1a545fb1af 100644 --- a/core/state/history_test.go +++ b/core/state/history_test.go @@ -28,31 +28,6 @@ import ( "github.com/ledgerwatch/erigon/turbo/trie" ) -func BenchmarkName1(b *testing.B) { - b.Run("1", func(b *testing.B) { - rl := newReadList() - for i := 0; i < b.N; i++ { - rl[kv.PlainState].Keys = append(rl[kv.PlainState].Keys, "") - rl[kv.PlainState].Vals = append(rl[kv.PlainState].Vals, []byte{}) - } - }) - b.Run("2", func(b *testing.B) { - rl := newReadList() - for i := 0; i < b.N; i++ { - l := rl[kv.PlainState] - l.Keys = append(l.Keys, "") - l.Vals = append(l.Vals, []byte{}) - rl[kv.PlainState] = l - } - }) -} -func TestA(t *testing.T) { - var a []int - _ = append(a, 1) - _ = append(a, 2) - fmt.Printf("a: %d\n", a) -} - func TestMutationDeleteTimestamp(t *testing.T) { _, tx := memdb.NewTestTx(t) From b89e3f52de247d3c6ccfe4388f93117eef8a2be3 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 2 Mar 2023 13:49:31 +0700 Subject: [PATCH 057/100] save --- core/state/rw_v3.go | 24 ++++++++----- core/types/accounts/account.go | 66 ++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+), 9 deletions(-) diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index aeda646eee1..2ed3c3b6015 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -50,7 +50,9 @@ type StateV3 struct { txsDone *atomic2.Uint64 finished atomic2.Bool - tmpdir string + tmpdir string + applyPrevAccountBuf []byte // buffer for ApplyState. Doesn't need mutex because Apply is single-threaded + addrIncBuf []byte // buffer for ApplyState. Doesn't need mutex because Apply is single-threaded } func NewStateV3(tmpdir string) *StateV3 { @@ -65,6 +67,9 @@ func NewStateV3(tmpdir string) *StateV3 { chContractCode: map[string][]byte{}, txsDone: atomic2.NewUint64(0), + + applyPrevAccountBuf: make([]byte, 256), + addrIncBuf: make([]byte, 20+8), } rs.receiveWork = sync.NewCond(&rs.queueLock) return rs @@ -129,16 +134,16 @@ func (rs *StateV3) get(table string, key []byte) (v []byte) { switch table { case kv.PlainState: if len(key) == 20 { - v, _ = rs.chAccs[keyS] + v = rs.chAccs[keyS] } else { v, _ = rs.chStorage.Get(keyS) } case kv.Code: - v, _ = rs.chCode[keyS] + v = rs.chCode[keyS] case kv.IncarnationMap: - v, _ = rs.chIncs[keyS] + v = rs.chIncs[keyS] case kv.PlainContractCode: - v, _ = rs.chContractCode[keyS] + v = rs.chContractCode[keyS] default: panic(table) } @@ -338,13 +343,14 @@ func (rs *StateV3) appplyState1(roTx kv.Tx, txTask *exec22.TxTask, agg *libstate return err } defer cursor.Close() - addr1 := make([]byte, 20+8) + addr1 := rs.addrIncBuf for addrS, original := range txTask.AccountDels { addr := []byte(addrS) copy(addr1, addr) binary.BigEndian.PutUint64(addr1[len(addr):], original.Incarnation) - prev := accounts.SerialiseV3(original) + prev := rs.applyPrevAccountBuf[:accounts.SerialiseV3Len(original)] + accounts.SerialiseV3To(original, prev) if err := agg.AddAccountPrev(addr, prev); err != nil { return err } @@ -403,7 +409,7 @@ func (rs *StateV3) appplyState1(roTx kv.Tx, txTask *exec22.TxTask, agg *libstate } } - k := make([]byte, 20+8) + k := rs.addrIncBuf for addrS, incarnation := range txTask.CodePrevs { addr := []byte(addrS) copy(k, addr) @@ -646,7 +652,7 @@ func (rs *StateV3) SizeEstimate() uint64 { rs.lock.RLock() r := rs.sizeEstimate rs.lock.RUnlock() - return uint64(r) + return uint64(r) * 2 // multiply 2 here, to cover data-structures overhead. more precise accounting - expensive. } func (rs *StateV3) ReadsValid(readLists map[string]*exec22.KvList) bool { diff --git a/core/types/accounts/account.go b/core/types/accounts/account.go index 5e1835b1524..e5f65e20f45 100644 --- a/core/types/accounts/account.go +++ b/core/types/accounts/account.go @@ -718,3 +718,69 @@ func SerialiseV3(a *Account) []byte { } return value } + +func SerialiseV3Len(a *Account) (l int) { + l++ + if a.Nonce > 0 { + l += (bits.Len64(a.Nonce) + 7) / 8 + } + l++ + if !a.Balance.IsZero() { + l += a.Balance.ByteLen() + } + l++ + if !a.IsEmptyCodeHash() { + l += 32 + } + l++ + if a.Incarnation > 0 { + l += (bits.Len64(a.Incarnation) + 7) / 8 + } + return l +} +func SerialiseV3To(a *Account, value []byte) { + pos := 0 + if a.Nonce == 0 { + value[pos] = 0 + pos++ + } else { + nonceBytes := (bits.Len64(a.Nonce) + 7) / 8 + value[pos] = byte(nonceBytes) + var nonce = a.Nonce + for i := nonceBytes; i > 0; i-- { + value[pos+i] = byte(nonce) + nonce >>= 8 + } + pos += nonceBytes + 1 + } + if a.Balance.IsZero() { + value[pos] = 0 + pos++ + } else { + balanceBytes := a.Balance.ByteLen() + value[pos] = byte(balanceBytes) + pos++ + a.Balance.WriteToSlice(value[pos : pos+balanceBytes]) + pos += balanceBytes + } + if a.IsEmptyCodeHash() { + value[pos] = 0 + pos++ + } else { + value[pos] = 32 + pos++ + copy(value[pos:pos+32], a.CodeHash[:]) + pos += 32 + } + if a.Incarnation == 0 { + value[pos] = 0 + } else { + incBytes := (bits.Len64(a.Incarnation) + 7) / 8 + value[pos] = byte(incBytes) + var inc = a.Incarnation + for i := incBytes; i > 0; i-- { + value[pos+i] = byte(inc) + inc >>= 8 + } + } +} From a575681179108145c4e50336c52da410532a213b Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 2 Mar 2023 13:52:00 +0700 Subject: [PATCH 058/100] save --- core/state/rw_v3.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index 2ed3c3b6015..0b00dc03230 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -333,7 +333,7 @@ func (rs *StateV3) Finish() { rs.receiveWork.Broadcast() } -func (rs *StateV3) appplyState1(roTx kv.Tx, txTask *exec22.TxTask, agg *libstate.AggregatorV3) error { +func (rs *StateV3) writeStateHistory(roTx kv.Tx, txTask *exec22.TxTask, agg *libstate.AggregatorV3) error { rs.lock.RLock() defer rs.lock.RUnlock() @@ -441,7 +441,7 @@ func (rs *StateV3) appplyState1(roTx kv.Tx, txTask *exec22.TxTask, agg *libstate return nil } -func (rs *StateV3) appplyState(roTx kv.Tx, txTask *exec22.TxTask, agg *libstate.AggregatorV3) error { +func (rs *StateV3) applyState(roTx kv.Tx, txTask *exec22.TxTask, agg *libstate.AggregatorV3) error { emptyRemoval := txTask.Rules.IsSpuriousDragon rs.lock.Lock() defer rs.lock.Unlock() @@ -493,10 +493,10 @@ func (rs *StateV3) ApplyState(roTx kv.Tx, txTask *exec22.TxTask, agg *libstate.A defer agg.BatchHistoryWriteStart().BatchHistoryWriteEnd() agg.SetTxNum(txTask.TxNum) - if err := rs.appplyState1(roTx, txTask, agg); err != nil { + if err := rs.writeStateHistory(roTx, txTask, agg); err != nil { return err } - if err := rs.appplyState(roTx, txTask, agg); err != nil { + if err := rs.applyState(roTx, txTask, agg); err != nil { return err } From d5879a9ba8fd9f6cf67f99f81c254fb256e1127f Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 2 Mar 2023 15:30:30 +0700 Subject: [PATCH 059/100] save --- core/vm/evm.go | 37 ++++++++++++++----------------------- core/vm/interpreter.go | 23 ++++++++--------------- 2 files changed, 22 insertions(+), 38 deletions(-) diff --git a/core/vm/evm.go b/core/vm/evm.go index 06283e1ccd6..11bf28cb94e 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -165,13 +165,11 @@ func (evm *EVM) Interpreter() Interpreter { } func (evm *EVM) call(typ OpCode, caller ContractRef, addr libcommon.Address, input []byte, gas uint64, value *uint256.Int, bailout bool) (ret []byte, leftOverGas uint64, err error) { - depth := evm.interpreter.Depth() - - if evm.config.NoRecursion && depth > 0 { + if evm.config.NoRecursion && evm.interpreter.Depth() > 0 { return nil, gas, nil } // Fail if we're trying to execute above the call depth limit - if depth > int(params.CallCreateDepth) { + if evm.interpreter.Depth() > int(params.CallCreateDepth) { return nil, gas, ErrDepth } if typ == CALL || typ == CALLCODE { @@ -199,7 +197,7 @@ func (evm *EVM) call(typ OpCode, caller ContractRef, addr libcommon.Address, inp v = nil } // Calling a non existing account, don't do anything, but ping the tracer - if depth == 0 { + if evm.interpreter.Depth() == 0 { evm.config.Tracer.CaptureStart(evm, caller.Address(), addr, isPrecompile, false /* create */, input, gas, v, code) evm.config.Tracer.CaptureEnd(ret, 0, nil) } else { @@ -219,16 +217,21 @@ func (evm *EVM) call(typ OpCode, caller ContractRef, addr libcommon.Address, inp // future scenarios evm.intraBlockState.AddBalance(addr, u256.Num0) } - var startGas = gas if evm.config.Debug { v := value if typ == STATICCALL { v = nil } - if depth == 0 { + if evm.interpreter.Depth() == 0 { evm.config.Tracer.CaptureStart(evm, caller.Address(), addr, isPrecompile, false /* create */, input, gas, v, code) + defer func(startGas uint64) { // Lazy evaluation of the parameters + evm.config.Tracer.CaptureEnd(ret, startGas-gas, err) + }(gas) } else { evm.config.Tracer.CaptureEnter(typ, caller.Address(), addr, isPrecompile, false /* create */, input, gas, v, code) + defer func(startGas uint64) { // Lazy evaluation of the parameters + evm.config.Tracer.CaptureExit(ret, startGas-gas, err) + }(gas) } } @@ -275,14 +278,6 @@ func (evm *EVM) call(typ OpCode, caller ContractRef, addr libcommon.Address, inp //} else { // evm.StateDB.DiscardSnapshot(snapshot) } - - if evm.config.Debug { - if depth == 0 { - evm.config.Tracer.CaptureEnd(ret, startGas-gas, err) - } else { - evm.config.Tracer.CaptureExit(ret, startGas-gas, err) - } - } return ret, gas, err } @@ -397,7 +392,7 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, contract := NewContract(caller, AccountRef(address), value, gas, evm.config.SkipAnalysis) contract.SetCodeOptionalHash(&address, codeAndHash) - if evm.config.NoRecursion && depth > 0 { + if evm.config.NoRecursion && evm.interpreter.Depth() > 0 { return nil, address, gas, nil } @@ -439,13 +434,9 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, } } - if evm.config.Debug { - if depth == 0 { - evm.config.Tracer.CaptureEnd(ret, gas-contract.Gas, err) - } else { - evm.config.Tracer.CaptureExit(ret, gas-contract.Gas, err) - } - } + // calculate gasConsumption for deferred captures + gasConsumption = gas - contract.Gas + return ret, address, contract.Gas, err } diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index f1867ddab39..60e5b787933 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -160,8 +160,6 @@ func NewEVMInterpreter(evm VMInterpreter, cfg Config) *EVMInterpreter { } } -func (in *EVMInterpreter) decrementDepth() { in.depth-- } - // Run loops and evaluates the contract's code with the given input data and returns // the return byte-slice and an error if one occurred. // @@ -169,26 +167,24 @@ func (in *EVMInterpreter) decrementDepth() { in.depth-- } // considered a revert-and-consume-all-gas operation except for // ErrExecutionReverted which means revert-and-keep-gas-left. func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) (ret []byte, err error) { - // Don't bother with the execution if there's no code. - if len(contract.Code) == 0 { - return nil, nil - } - // Increment the call depth which is restricted to 1024 in.depth++ - defer in.decrementDepth() + defer func() { in.depth-- }() // Make sure the readOnly is only set if we aren't in readOnly yet. // This makes also sure that the readOnly flag isn't removed for child calls. - if readOnly && !in.readOnly { - in.readOnly = true - defer func() { in.readOnly = false }() - } + callback := in.setReadonly(readOnly) + defer callback() // Reset the previous call's return data. It's unimportant to preserve the old buffer // as every returning call will return new data anyway. in.returnData = nil + // Don't bother with the execution if there's no code. + if len(contract.Code) == 0 { + return nil, nil + } + var ( op OpCode // current opcode mem = NewMemory() // bound memory @@ -310,9 +306,6 @@ func (in *EVMInterpreter) Depth() int { return in.depth } -func (vm *VM) disableReadonly() { vm.readOnly = false } -func (vm *VM) noop() {} - func (vm *VM) setReadonly(outerReadonly bool) func() { if outerReadonly && !vm.readOnly { vm.readOnly = true From eb9f41afd6e9908cbb4b74d0b1166de7192ea864 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 2 Mar 2023 15:34:12 +0700 Subject: [PATCH 060/100] save --- core/state/rw_v3.go | 26 ++++++++++++-------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index 0b00dc03230..8f0653c3653 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -81,21 +81,19 @@ func (rs *StateV3) put(table string, key, val []byte) { func (rs *StateV3) puts(table string, key string, val []byte) { switch table { + case StorageTable: + if old, ok := rs.chStorage.Set(key, val); ok { + rs.sizeEstimate += len(val) - len(old) + } else { + rs.sizeEstimate += len(key) + len(val) + } case kv.PlainState: - if len(key) == 20 { - if old, ok := rs.chAccs[key]; ok { - rs.sizeEstimate += len(val) - len(old) - } else { - rs.sizeEstimate += len(key) + len(val) - } - rs.chAccs[key] = val + if old, ok := rs.chAccs[key]; ok { + rs.sizeEstimate += len(val) - len(old) } else { - if old, ok := rs.chStorage.Set(key, val); ok { - rs.sizeEstimate += len(val) - len(old) - } else { - rs.sizeEstimate += len(key) + len(val) - } + rs.sizeEstimate += len(key) + len(val) } + rs.chAccs[key] = val case kv.Code: if old, ok := rs.chCode[key]; ok { rs.sizeEstimate += len(val) - len(old) @@ -815,8 +813,8 @@ func (w *StateWriterV3) WriteAccountStorage(address common.Address, incarnation } composite := dbutils.PlainGenerateCompositeStorageKey(address[:], incarnation, key.Bytes()) cmpositeS := string(composite) - w.writeLists[kv.PlainState].Keys = append(w.writeLists[kv.PlainState].Keys, cmpositeS) - w.writeLists[kv.PlainState].Vals = append(w.writeLists[kv.PlainState].Vals, value.Bytes()) + w.writeLists[StorageTable].Keys = append(w.writeLists[StorageTable].Keys, cmpositeS) + w.writeLists[StorageTable].Vals = append(w.writeLists[StorageTable].Vals, value.Bytes()) //fmt.Printf("storage [%x] [%x] => [%x], txNum: %d\n", address, *key, v, w.txNum) if w.storagePrevs == nil { w.storagePrevs = map[string][]byte{} From 9769086636e620f9886ed9d5e0d8836793e17e4c Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 2 Mar 2023 15:35:29 +0700 Subject: [PATCH 061/100] save --- core/state/rw_v3.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index 8f0653c3653..e35dcc8936d 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -974,6 +974,7 @@ var writeListPool = sync.Pool{ New: func() any { return map[string]*exec22.KvList{ kv.PlainState: {}, + StorageTable: {}, kv.Code: {}, kv.PlainContractCode: {}, kv.IncarnationMap: {}, From 791f819e8ffce475a305a10575ab376862a0d35f Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 2 Mar 2023 15:36:13 +0700 Subject: [PATCH 062/100] save --- core/state/rw_v3.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index e35dcc8936d..2bf582ecbac 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -882,7 +882,7 @@ func (r *StateReaderV3) ReadAccountData(address common.Address) (*accounts.Accou func (r *StateReaderV3) ReadAccountStorage(address common.Address, incarnation uint64, key *common.Hash) ([]byte, error) { composite := dbutils.PlainGenerateCompositeStorageKey(address.Bytes(), incarnation, key.Bytes()) - enc := r.rs.Get(kv.PlainState, composite) + enc := r.rs.Get(StorageTable, composite) if enc == nil { var err error enc, err = r.tx.GetOne(kv.PlainState, composite) From cae410e75eafafe31b4e9bf26752bd5e3a58deb7 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 2 Mar 2023 15:39:28 +0700 Subject: [PATCH 063/100] save --- core/state/rw_v3.go | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index 2bf582ecbac..804be54fc4e 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -130,12 +130,10 @@ func (rs *StateV3) Get(table string, key []byte) []byte { func (rs *StateV3) get(table string, key []byte) (v []byte) { keyS := *(*string)(unsafe.Pointer(&key)) switch table { + case StorageTable: + v, _ = rs.chStorage.Get(keyS) case kv.PlainState: - if len(key) == 20 { - v = rs.chAccs[keyS] - } else { - v, _ = rs.chStorage.Get(keyS) - } + v = rs.chAccs[keyS] case kv.Code: v = rs.chCode[keyS] case kv.IncarnationMap: From ff081685e1ed520239c5c4e0e1149d90d1409649 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 2 Mar 2023 16:33:28 +0700 Subject: [PATCH 064/100] save --- core/vm/evm.go | 37 +++++++++++++++++++++++-------------- core/vm/interpreter.go | 23 +++++++++++++++-------- 2 files changed, 38 insertions(+), 22 deletions(-) diff --git a/core/vm/evm.go b/core/vm/evm.go index 11bf28cb94e..06283e1ccd6 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -165,11 +165,13 @@ func (evm *EVM) Interpreter() Interpreter { } func (evm *EVM) call(typ OpCode, caller ContractRef, addr libcommon.Address, input []byte, gas uint64, value *uint256.Int, bailout bool) (ret []byte, leftOverGas uint64, err error) { - if evm.config.NoRecursion && evm.interpreter.Depth() > 0 { + depth := evm.interpreter.Depth() + + if evm.config.NoRecursion && depth > 0 { return nil, gas, nil } // Fail if we're trying to execute above the call depth limit - if evm.interpreter.Depth() > int(params.CallCreateDepth) { + if depth > int(params.CallCreateDepth) { return nil, gas, ErrDepth } if typ == CALL || typ == CALLCODE { @@ -197,7 +199,7 @@ func (evm *EVM) call(typ OpCode, caller ContractRef, addr libcommon.Address, inp v = nil } // Calling a non existing account, don't do anything, but ping the tracer - if evm.interpreter.Depth() == 0 { + if depth == 0 { evm.config.Tracer.CaptureStart(evm, caller.Address(), addr, isPrecompile, false /* create */, input, gas, v, code) evm.config.Tracer.CaptureEnd(ret, 0, nil) } else { @@ -217,21 +219,16 @@ func (evm *EVM) call(typ OpCode, caller ContractRef, addr libcommon.Address, inp // future scenarios evm.intraBlockState.AddBalance(addr, u256.Num0) } + var startGas = gas if evm.config.Debug { v := value if typ == STATICCALL { v = nil } - if evm.interpreter.Depth() == 0 { + if depth == 0 { evm.config.Tracer.CaptureStart(evm, caller.Address(), addr, isPrecompile, false /* create */, input, gas, v, code) - defer func(startGas uint64) { // Lazy evaluation of the parameters - evm.config.Tracer.CaptureEnd(ret, startGas-gas, err) - }(gas) } else { evm.config.Tracer.CaptureEnter(typ, caller.Address(), addr, isPrecompile, false /* create */, input, gas, v, code) - defer func(startGas uint64) { // Lazy evaluation of the parameters - evm.config.Tracer.CaptureExit(ret, startGas-gas, err) - }(gas) } } @@ -278,6 +275,14 @@ func (evm *EVM) call(typ OpCode, caller ContractRef, addr libcommon.Address, inp //} else { // evm.StateDB.DiscardSnapshot(snapshot) } + + if evm.config.Debug { + if depth == 0 { + evm.config.Tracer.CaptureEnd(ret, startGas-gas, err) + } else { + evm.config.Tracer.CaptureExit(ret, startGas-gas, err) + } + } return ret, gas, err } @@ -392,7 +397,7 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, contract := NewContract(caller, AccountRef(address), value, gas, evm.config.SkipAnalysis) contract.SetCodeOptionalHash(&address, codeAndHash) - if evm.config.NoRecursion && evm.interpreter.Depth() > 0 { + if evm.config.NoRecursion && depth > 0 { return nil, address, gas, nil } @@ -434,9 +439,13 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, } } - // calculate gasConsumption for deferred captures - gasConsumption = gas - contract.Gas - + if evm.config.Debug { + if depth == 0 { + evm.config.Tracer.CaptureEnd(ret, gas-contract.Gas, err) + } else { + evm.config.Tracer.CaptureExit(ret, gas-contract.Gas, err) + } + } return ret, address, contract.Gas, err } diff --git a/core/vm/interpreter.go b/core/vm/interpreter.go index 60e5b787933..f1867ddab39 100644 --- a/core/vm/interpreter.go +++ b/core/vm/interpreter.go @@ -160,6 +160,8 @@ func NewEVMInterpreter(evm VMInterpreter, cfg Config) *EVMInterpreter { } } +func (in *EVMInterpreter) decrementDepth() { in.depth-- } + // Run loops and evaluates the contract's code with the given input data and returns // the return byte-slice and an error if one occurred. // @@ -167,24 +169,26 @@ func NewEVMInterpreter(evm VMInterpreter, cfg Config) *EVMInterpreter { // considered a revert-and-consume-all-gas operation except for // ErrExecutionReverted which means revert-and-keep-gas-left. func (in *EVMInterpreter) Run(contract *Contract, input []byte, readOnly bool) (ret []byte, err error) { + // Don't bother with the execution if there's no code. + if len(contract.Code) == 0 { + return nil, nil + } + // Increment the call depth which is restricted to 1024 in.depth++ - defer func() { in.depth-- }() + defer in.decrementDepth() // Make sure the readOnly is only set if we aren't in readOnly yet. // This makes also sure that the readOnly flag isn't removed for child calls. - callback := in.setReadonly(readOnly) - defer callback() + if readOnly && !in.readOnly { + in.readOnly = true + defer func() { in.readOnly = false }() + } // Reset the previous call's return data. It's unimportant to preserve the old buffer // as every returning call will return new data anyway. in.returnData = nil - // Don't bother with the execution if there's no code. - if len(contract.Code) == 0 { - return nil, nil - } - var ( op OpCode // current opcode mem = NewMemory() // bound memory @@ -306,6 +310,9 @@ func (in *EVMInterpreter) Depth() int { return in.depth } +func (vm *VM) disableReadonly() { vm.readOnly = false } +func (vm *VM) noop() {} + func (vm *VM) setReadonly(outerReadonly bool) func() { if outerReadonly && !vm.readOnly { vm.readOnly = true From d95dddbab50e0762ed1636b1fe90d4d8de96abe2 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Fri, 3 Mar 2023 09:49:51 +0700 Subject: [PATCH 065/100] save --- core/state/rw_v3.go | 37 +++++++++++++++++-------------------- 1 file changed, 17 insertions(+), 20 deletions(-) diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index 0b00dc03230..804be54fc4e 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -81,21 +81,19 @@ func (rs *StateV3) put(table string, key, val []byte) { func (rs *StateV3) puts(table string, key string, val []byte) { switch table { + case StorageTable: + if old, ok := rs.chStorage.Set(key, val); ok { + rs.sizeEstimate += len(val) - len(old) + } else { + rs.sizeEstimate += len(key) + len(val) + } case kv.PlainState: - if len(key) == 20 { - if old, ok := rs.chAccs[key]; ok { - rs.sizeEstimate += len(val) - len(old) - } else { - rs.sizeEstimate += len(key) + len(val) - } - rs.chAccs[key] = val + if old, ok := rs.chAccs[key]; ok { + rs.sizeEstimate += len(val) - len(old) } else { - if old, ok := rs.chStorage.Set(key, val); ok { - rs.sizeEstimate += len(val) - len(old) - } else { - rs.sizeEstimate += len(key) + len(val) - } + rs.sizeEstimate += len(key) + len(val) } + rs.chAccs[key] = val case kv.Code: if old, ok := rs.chCode[key]; ok { rs.sizeEstimate += len(val) - len(old) @@ -132,12 +130,10 @@ func (rs *StateV3) Get(table string, key []byte) []byte { func (rs *StateV3) get(table string, key []byte) (v []byte) { keyS := *(*string)(unsafe.Pointer(&key)) switch table { + case StorageTable: + v, _ = rs.chStorage.Get(keyS) case kv.PlainState: - if len(key) == 20 { - v = rs.chAccs[keyS] - } else { - v, _ = rs.chStorage.Get(keyS) - } + v = rs.chAccs[keyS] case kv.Code: v = rs.chCode[keyS] case kv.IncarnationMap: @@ -815,8 +811,8 @@ func (w *StateWriterV3) WriteAccountStorage(address common.Address, incarnation } composite := dbutils.PlainGenerateCompositeStorageKey(address[:], incarnation, key.Bytes()) cmpositeS := string(composite) - w.writeLists[kv.PlainState].Keys = append(w.writeLists[kv.PlainState].Keys, cmpositeS) - w.writeLists[kv.PlainState].Vals = append(w.writeLists[kv.PlainState].Vals, value.Bytes()) + w.writeLists[StorageTable].Keys = append(w.writeLists[StorageTable].Keys, cmpositeS) + w.writeLists[StorageTable].Vals = append(w.writeLists[StorageTable].Vals, value.Bytes()) //fmt.Printf("storage [%x] [%x] => [%x], txNum: %d\n", address, *key, v, w.txNum) if w.storagePrevs == nil { w.storagePrevs = map[string][]byte{} @@ -884,7 +880,7 @@ func (r *StateReaderV3) ReadAccountData(address common.Address) (*accounts.Accou func (r *StateReaderV3) ReadAccountStorage(address common.Address, incarnation uint64, key *common.Hash) ([]byte, error) { composite := dbutils.PlainGenerateCompositeStorageKey(address.Bytes(), incarnation, key.Bytes()) - enc := r.rs.Get(kv.PlainState, composite) + enc := r.rs.Get(StorageTable, composite) if enc == nil { var err error enc, err = r.tx.GetOne(kv.PlainState, composite) @@ -976,6 +972,7 @@ var writeListPool = sync.Pool{ New: func() any { return map[string]*exec22.KvList{ kv.PlainState: {}, + StorageTable: {}, kv.Code: {}, kv.PlainContractCode: {}, kv.IncarnationMap: {}, From 9374b1d0753734308a3d67869490ece8ba57514f Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Fri, 3 Mar 2023 10:06:28 +0700 Subject: [PATCH 066/100] save --- cmd/state/exec3/state.go | 5 ++++- core/state/rw_v3.go | 2 +- ethdb/olddb/mapmutation.go | 2 +- ethdb/olddb/mutation.go | 2 +- 4 files changed, 7 insertions(+), 4 deletions(-) diff --git a/cmd/state/exec3/state.go b/cmd/state/exec3/state.go index 8c204dec37e..514f30b444d 100644 --- a/cmd/state/exec3/state.go +++ b/cmd/state/exec3/state.go @@ -70,7 +70,8 @@ func NewWorker(lock sync.Locker, ctx context.Context, background bool, chainDb k resultCh: resultCh, engine: engine, - evm: vm.NewEVM(evmtypes.BlockContext{}, evmtypes.TxContext{}, nil, chainConfig, vm.Config{}), + evm: vm.NewEVM(evmtypes.BlockContext{}, evmtypes.TxContext{}, nil, chainConfig, vm.Config{}), + //callTracer: NewCallTracer(background, agg), callTracer: NewCallTracer(), taskGasPool: new(core.GasPool), } @@ -178,6 +179,8 @@ func (rw *Worker) RunTxTaskNoLock(txTask *exec22.TxTask) { //fmt.Printf("error=%v\n", err) txTask.Error = err } else { + //rw.callTracer.AddCoinbase(txTask.Coinbase, txTask.Uncles) + //txTask.TraceTos = rw.callTracer.Tos() txTask.TraceTos = map[libcommon.Address]struct{}{} txTask.TraceTos[txTask.Coinbase] = struct{}{} for _, uncle := range txTask.Uncles { diff --git a/core/state/rw_v3.go b/core/state/rw_v3.go index 804be54fc4e..8e0e84b1b0b 100644 --- a/core/state/rw_v3.go +++ b/core/state/rw_v3.go @@ -160,7 +160,7 @@ func (rs *StateV3) flushMap(ctx context.Context, rwTx kv.RwTx, table string, m m select { default: case <-logEvery.C: - progress := fmt.Sprintf("%.1fM/%.1fM", float64(count)/1_000_000, total/1_000_000) + progress := fmt.Sprintf("%.1fM/%.1fM", float64(count)/1_000_000, float64(total)/1_000_000) log.Info("Write to db", "progress", progress, "current table", table) rwTx.CollectMetrics() } diff --git a/ethdb/olddb/mapmutation.go b/ethdb/olddb/mapmutation.go index 55efd505eaf..deba908c370 100644 --- a/ethdb/olddb/mapmutation.go +++ b/ethdb/olddb/mapmutation.go @@ -228,7 +228,7 @@ func (m *mapmutation) doCommit(tx kv.RwTx) error { select { default: case <-logEvery.C: - progress := fmt.Sprintf("%.1fM/%.1fM", float64(count)/1_000_000, total/1_000_000) + progress := fmt.Sprintf("%.1fM/%.1fM", float64(count)/1_000_000, float64(total)/1_000_000) log.Info("Write to db", "progress", progress, "current table", table) tx.CollectMetrics() } diff --git a/ethdb/olddb/mutation.go b/ethdb/olddb/mutation.go index ee39868dabb..fcb1b5a043f 100644 --- a/ethdb/olddb/mutation.go +++ b/ethdb/olddb/mutation.go @@ -289,7 +289,7 @@ func (m *mutation) doCommit(tx kv.RwTx) error { select { default: case <-logEvery.C: - progress := fmt.Sprintf("%.1fM/%.1fM", float64(count)/1_000_000, total/1_000_000) + progress := fmt.Sprintf("%.1fM/%.1fM", float64(count)/1_000_000, float64(total)/1_000_000) log.Info("Write to db", "progress", progress, "current table", mi.table) tx.CollectMetrics() case <-m.quit: From bd00e7ac7e4cc3560951904589f6a8584402cca1 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Fri, 3 Mar 2023 10:10:02 +0700 Subject: [PATCH 067/100] save --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f6ca0a4f5f1..64c5a82ba54 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230227070911-8e13fc364662 + github.com/ledgerwatch/erigon-lib v0.0.0-20230303030854-02db97c755a2 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230224032817-e29f5ad218a2 github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index 9382f1d740a..ed8c5b6321e 100644 --- a/go.sum +++ b/go.sum @@ -517,8 +517,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230227070911-8e13fc364662 h1:p1bwH1AFNZrFngsvcjIDrhdVCbZT9b9mOOjDWRAguPE= -github.com/ledgerwatch/erigon-lib v0.0.0-20230227070911-8e13fc364662/go.mod h1:xcrJLJiNfthKFectsWBUtRJK9d6XtjiyQRGVeImRYRs= +github.com/ledgerwatch/erigon-lib v0.0.0-20230303030854-02db97c755a2 h1:ZQtClnrdWJkYQj+5qvh0Xf6gi6TzY5pNHQaKzzjOnT0= +github.com/ledgerwatch/erigon-lib v0.0.0-20230303030854-02db97c755a2/go.mod h1:xcrJLJiNfthKFectsWBUtRJK9d6XtjiyQRGVeImRYRs= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230224032817-e29f5ad218a2 h1:Vf5oUCq9s4JSLd5/WJxq8zTuMY5Brn8So0yeJDvUauc= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230224032817-e29f5ad218a2/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From 1268a8aad0500ff3dcf4f7b3c5ce3b76a3d37d9d Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Fri, 3 Mar 2023 10:17:42 +0700 Subject: [PATCH 068/100] save --- core/vm/evm.go | 25 +++++++++---------------- 1 file changed, 9 insertions(+), 16 deletions(-) diff --git a/core/vm/evm.go b/core/vm/evm.go index 06283e1ccd6..c7ae784ff8a 100644 --- a/core/vm/evm.go +++ b/core/vm/evm.go @@ -219,7 +219,6 @@ func (evm *EVM) call(typ OpCode, caller ContractRef, addr libcommon.Address, inp // future scenarios evm.intraBlockState.AddBalance(addr, u256.Num0) } - var startGas = gas if evm.config.Debug { v := value if typ == STATICCALL { @@ -227,8 +226,14 @@ func (evm *EVM) call(typ OpCode, caller ContractRef, addr libcommon.Address, inp } if depth == 0 { evm.config.Tracer.CaptureStart(evm, caller.Address(), addr, isPrecompile, false /* create */, input, gas, v, code) + defer func(startGas uint64) { // Lazy evaluation of the parameters + evm.config.Tracer.CaptureEnd(ret, startGas-gas, err) + }(gas) } else { evm.config.Tracer.CaptureEnter(typ, caller.Address(), addr, isPrecompile, false /* create */, input, gas, v, code) + defer func(startGas uint64) { // Lazy evaluation of the parameters + evm.config.Tracer.CaptureExit(ret, startGas-gas, err) + }(gas) } } @@ -275,14 +280,6 @@ func (evm *EVM) call(typ OpCode, caller ContractRef, addr libcommon.Address, inp //} else { // evm.StateDB.DiscardSnapshot(snapshot) } - - if evm.config.Debug { - if depth == 0 { - evm.config.Tracer.CaptureEnd(ret, startGas-gas, err) - } else { - evm.config.Tracer.CaptureExit(ret, startGas-gas, err) - } - } return ret, gas, err } @@ -439,13 +436,9 @@ func (evm *EVM) create(caller ContractRef, codeAndHash *codeAndHash, gas uint64, } } - if evm.config.Debug { - if depth == 0 { - evm.config.Tracer.CaptureEnd(ret, gas-contract.Gas, err) - } else { - evm.config.Tracer.CaptureExit(ret, gas-contract.Gas, err) - } - } + // calculate gasConsumption for deferred captures + gasConsumption = gas - contract.Gas + return ret, address, contract.Gas, err } From 80295a311d9e5073d2aa0d581eb3a03933095813 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Fri, 3 Mar 2023 10:18:39 +0700 Subject: [PATCH 069/100] save --- cmd/state/exec3/state.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/state/exec3/state.go b/cmd/state/exec3/state.go index 514f30b444d..740b58ff3c2 100644 --- a/cmd/state/exec3/state.go +++ b/cmd/state/exec3/state.go @@ -70,8 +70,7 @@ func NewWorker(lock sync.Locker, ctx context.Context, background bool, chainDb k resultCh: resultCh, engine: engine, - evm: vm.NewEVM(evmtypes.BlockContext{}, evmtypes.TxContext{}, nil, chainConfig, vm.Config{}), - //callTracer: NewCallTracer(background, agg), + evm: vm.NewEVM(evmtypes.BlockContext{}, evmtypes.TxContext{}, nil, chainConfig, vm.Config{}), callTracer: NewCallTracer(), taskGasPool: new(core.GasPool), } From 6443f4db8d3be64c5b1ef8fd227b6640afe8b053 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Fri, 3 Mar 2023 12:36:03 +0700 Subject: [PATCH 070/100] save --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8a587de4e75..7522c53a774 100644 --- a/Makefile +++ b/Makefile @@ -166,7 +166,7 @@ lintci: ## lintci-deps: (re)installs golangci-lint to build/bin/golangci-lint lintci-deps: rm -f ./build/bin/golangci-lint - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./build/bin v1.51.1 + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./build/bin v1.51.2 ## clean: cleans the go cache, build dir, libmdbx db dir clean: From be3c12a0e373f1dea39c09b7d6df697d2ab4ff7f Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Fri, 3 Mar 2023 12:47:11 +0700 Subject: [PATCH 071/100] save --- ethdb/olddb/mapmutation.go | 2 +- ethdb/olddb/mutation.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ethdb/olddb/mapmutation.go b/ethdb/olddb/mapmutation.go index deba908c370..55efd505eaf 100644 --- a/ethdb/olddb/mapmutation.go +++ b/ethdb/olddb/mapmutation.go @@ -228,7 +228,7 @@ func (m *mapmutation) doCommit(tx kv.RwTx) error { select { default: case <-logEvery.C: - progress := fmt.Sprintf("%.1fM/%.1fM", float64(count)/1_000_000, float64(total)/1_000_000) + progress := fmt.Sprintf("%.1fM/%.1fM", float64(count)/1_000_000, total/1_000_000) log.Info("Write to db", "progress", progress, "current table", table) tx.CollectMetrics() } diff --git a/ethdb/olddb/mutation.go b/ethdb/olddb/mutation.go index fcb1b5a043f..ee39868dabb 100644 --- a/ethdb/olddb/mutation.go +++ b/ethdb/olddb/mutation.go @@ -289,7 +289,7 @@ func (m *mutation) doCommit(tx kv.RwTx) error { select { default: case <-logEvery.C: - progress := fmt.Sprintf("%.1fM/%.1fM", float64(count)/1_000_000, float64(total)/1_000_000) + progress := fmt.Sprintf("%.1fM/%.1fM", float64(count)/1_000_000, total/1_000_000) log.Info("Write to db", "progress", progress, "current table", mi.table) tx.CollectMetrics() case <-m.quit: From 6dda70fc72a086d246eeb3ac09cf8343dc16a377 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Sat, 4 Mar 2023 10:13:22 +0700 Subject: [PATCH 072/100] save --- cmd/integration/commands/stages.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/integration/commands/stages.go b/cmd/integration/commands/stages.go index bf10ac722fe..6fbc8b1eb8e 100644 --- a/cmd/integration/commands/stages.go +++ b/cmd/integration/commands/stages.go @@ -629,6 +629,10 @@ func stageSenders(db kv.RwDB, ctx context.Context) error { must(sync.SetCurrentStage(stages.Senders)) + if reset { + return db.Update(ctx, func(tx kv.RwTx) error { return reset2.ResetSenders(ctx, db, tx) }) + } + tx, err := db.BeginRw(ctx) if err != nil { return err @@ -671,10 +675,6 @@ func stageSenders(db kv.RwDB, ctx context.Context) error { return nil } - if reset { - return db.Update(ctx, func(tx kv.RwTx) error { return reset2.ResetSenders(ctx, db, tx) }) - } - s := stage(sync, tx, nil, stages.Senders) log.Info("Stage", "name", s.ID, "progress", s.BlockNumber) From ff657e2c3b072721d1c531de0f634bae601c7fc7 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 6 Mar 2023 10:14:05 +0700 Subject: [PATCH 073/100] save --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 91cdd15097e..eb6db969119 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/ledgerwatch/erigon -go 1.18 +go 1.20 require ( github.com/ledgerwatch/erigon-lib v0.0.0-20230306025804-0b9ca27fb310 From fb77dea190e9fb1958c275384ab45cdd7d283f54 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 6 Mar 2023 10:14:56 +0700 Subject: [PATCH 074/100] save --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index eb6db969119..de92151177f 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 github.com/ledgerwatch/trackerslist v1.1.0 // indirect - github.com/torquem-ch/mdbx-go v0.27.8 + github.com/torquem-ch/mdbx-go v0.29.1-0.20230306031132-11bfa1d62d85 ) require ( diff --git a/go.sum b/go.sum index 71ac2ca55f5..2256e473bc4 100644 --- a/go.sum +++ b/go.sum @@ -914,8 +914,8 @@ github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDW github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= -github.com/torquem-ch/mdbx-go v0.27.8 h1:rqGmgT2g4S/nKlfmFhLcQUKGQYV6OSog1tT+HHi7LME= -github.com/torquem-ch/mdbx-go v0.27.8/go.mod h1:T2fsoJDVppxfAPTLd1svUgH1kpPmeXdPESmroSHcL1E= +github.com/torquem-ch/mdbx-go v0.29.1-0.20230306031132-11bfa1d62d85 h1:FrgKpeFzv5Yz2hAXw0nulZBOeVvNHKMmLok+QYW/9sY= +github.com/torquem-ch/mdbx-go v0.29.1-0.20230306031132-11bfa1d62d85/go.mod h1:T2fsoJDVppxfAPTLd1svUgH1kpPmeXdPESmroSHcL1E= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go v1.1.13/go.mod h1:jxau1n+/wyTGLQoCkjok9r5zFa/FxT6eI5HiHKQszjc= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= From 2e7da84ab16711abb2cc7aada7cf1668dee62f3c Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 6 Mar 2023 13:11:06 +0700 Subject: [PATCH 075/100] save --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index de92151177f..d4e542c696d 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.20 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230306025804-0b9ca27fb310 + github.com/ledgerwatch/erigon-lib v0.0.0-20230306060912-ec215a766e83 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230224032817-e29f5ad218a2 github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index 2256e473bc4..f4ea80ca854 100644 --- a/go.sum +++ b/go.sum @@ -517,8 +517,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230306025804-0b9ca27fb310 h1:16bg3Tdok6oQg6Bb3KAcTWrqIZJQZ+AAghdyTW8LQyE= -github.com/ledgerwatch/erigon-lib v0.0.0-20230306025804-0b9ca27fb310/go.mod h1:xcrJLJiNfthKFectsWBUtRJK9d6XtjiyQRGVeImRYRs= +github.com/ledgerwatch/erigon-lib v0.0.0-20230306060912-ec215a766e83 h1:6salRGdatHfYhCpYmKN5tyXrvBmQvnTAq3mVfvZKBrQ= +github.com/ledgerwatch/erigon-lib v0.0.0-20230306060912-ec215a766e83/go.mod h1:xcrJLJiNfthKFectsWBUtRJK9d6XtjiyQRGVeImRYRs= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230224032817-e29f5ad218a2 h1:Vf5oUCq9s4JSLd5/WJxq8zTuMY5Brn8So0yeJDvUauc= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230224032817-e29f5ad218a2/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From 4eb266bb3c7370c961888b6c9a3dd2ad5c60ac5f Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 6 Mar 2023 14:57:41 +0700 Subject: [PATCH 076/100] save --- eth/stagedsync/stage_execute.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/eth/stagedsync/stage_execute.go b/eth/stagedsync/stage_execute.go index d66d9ec0026..8b3ee6d3b72 100644 --- a/eth/stagedsync/stage_execute.go +++ b/eth/stagedsync/stage_execute.go @@ -361,6 +361,7 @@ func senderStageProgress(tx kv.Tx, db kv.RoDB) (prevStageProgress uint64, err er // ================ Erigon3 End ================ func SpawnExecuteBlocksStage(s *StageState, u Unwinder, tx kv.RwTx, toBlock uint64, ctx context.Context, cfg ExecuteBlockCfg, initialCycle bool, quiet bool) (err error) { + fmt.Printf("forw\n") if cfg.historyV3 { if err = ExecBlockV3(s, u, tx, toBlock, ctx, cfg, initialCycle); err != nil { return err @@ -549,6 +550,7 @@ func logProgress(logPrefix string, prevBlock uint64, prevTime time.Time, current } func UnwindExecutionStage(u *UnwindState, s *StageState, tx kv.RwTx, ctx context.Context, cfg ExecuteBlockCfg, initialCycle bool) (err error) { + fmt.Printf("un\n") if u.UnwindPoint >= s.BlockNumber { return nil } From ecaa06a3523d14db3c9c616781ce705eeac07b66 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 7 Mar 2023 13:17:47 +0700 Subject: [PATCH 077/100] save --- accounts/abi/bind/bind_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/accounts/abi/bind/bind_test.go b/accounts/abi/bind/bind_test.go index 39032d6e945..ce428433cd3 100644 --- a/accounts/abi/bind/bind_test.go +++ b/accounts/abi/bind/bind_test.go @@ -1791,6 +1791,7 @@ var bindTests = []struct { // Tests that packages generated by the binder can be successfully compiled and // the requested tester run against it. func TestGolangBindings(t *testing.T) { + t.Skip("") // Skip the test if no Go command can be found gocmd := runtime.GOROOT() + "/bin/go" if !dir.FileExist(gocmd) { From e8d37f768d9729e46c79c2568b9c854351e847f0 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 7 Mar 2023 13:19:15 +0700 Subject: [PATCH 078/100] save --- eth/stagedsync/stage_execute.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/eth/stagedsync/stage_execute.go b/eth/stagedsync/stage_execute.go index 8b3ee6d3b72..d66d9ec0026 100644 --- a/eth/stagedsync/stage_execute.go +++ b/eth/stagedsync/stage_execute.go @@ -361,7 +361,6 @@ func senderStageProgress(tx kv.Tx, db kv.RoDB) (prevStageProgress uint64, err er // ================ Erigon3 End ================ func SpawnExecuteBlocksStage(s *StageState, u Unwinder, tx kv.RwTx, toBlock uint64, ctx context.Context, cfg ExecuteBlockCfg, initialCycle bool, quiet bool) (err error) { - fmt.Printf("forw\n") if cfg.historyV3 { if err = ExecBlockV3(s, u, tx, toBlock, ctx, cfg, initialCycle); err != nil { return err @@ -550,7 +549,6 @@ func logProgress(logPrefix string, prevBlock uint64, prevTime time.Time, current } func UnwindExecutionStage(u *UnwindState, s *StageState, tx kv.RwTx, ctx context.Context, cfg ExecuteBlockCfg, initialCycle bool) (err error) { - fmt.Printf("un\n") if u.UnwindPoint >= s.BlockNumber { return nil } From b4214a8de11c82c6ade983cdbd258a47f46ed530 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 7 Mar 2023 13:21:44 +0700 Subject: [PATCH 079/100] save --- accounts/abi/bind/bind_test.go | 1 - eth/stagedsync/exec3.go | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/accounts/abi/bind/bind_test.go b/accounts/abi/bind/bind_test.go index ce428433cd3..39032d6e945 100644 --- a/accounts/abi/bind/bind_test.go +++ b/accounts/abi/bind/bind_test.go @@ -1791,7 +1791,6 @@ var bindTests = []struct { // Tests that packages generated by the binder can be successfully compiled and // the requested tester run against it. func TestGolangBindings(t *testing.T) { - t.Skip("") // Skip the test if no Go command can be found gocmd := runtime.GOROOT() + "/bin/go" if !dir.FileExist(gocmd) { diff --git a/eth/stagedsync/exec3.go b/eth/stagedsync/exec3.go index 5f63c2b7395..9de397bdd6d 100644 --- a/eth/stagedsync/exec3.go +++ b/eth/stagedsync/exec3.go @@ -331,8 +331,8 @@ func ExecV3(ctx context.Context, // it means better spend time for pruning, before flushing more data to db // also better do it now - instead of before Commit() - because Commit does block execution stepsInDB := rawdbhelpers.IdxStepsCountV3(tx) - if stepsInDB > 5 && rs.SizeEstimate() < uint64(float64(commitThreshold)*0.2) { - if err = agg.Prune(ctx, ethconfig.HistoryV3AggregationStep*2); err != nil { // prune part of retired data, before commit + if stepsInDB > 10 && rs.SizeEstimate() < uint64(float64(commitThreshold)*0.2) { + if err = agg.Prune(ctx, ethconfig.HistoryV3AggregationStep); err != nil { // prune part of retired data, before commit panic(err) } } From aeb6d6bb862eafcabd79b8304bc762dd7c532026 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 7 Mar 2023 13:22:23 +0700 Subject: [PATCH 080/100] save --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 04d9d01a0c1..c502a874d73 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230307061542-e5a10c55d9a1 + github.com/ledgerwatch/erigon-lib v0.0.0-20230307062144-6c0eee4ef32f github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index 76de8ee8d19..ef6ecab6ca5 100644 --- a/go.sum +++ b/go.sum @@ -517,8 +517,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230307061542-e5a10c55d9a1 h1:lzSxgafubmxfEvnzIsyeSLYgZW3EVFLIgnf3HHi6QkM= -github.com/ledgerwatch/erigon-lib v0.0.0-20230307061542-e5a10c55d9a1/go.mod h1:nyJqfX9uPm1P/poZB1211DFe5DnAKOhYqvkEPyW7dXM= +github.com/ledgerwatch/erigon-lib v0.0.0-20230307062144-6c0eee4ef32f h1:WUEZ3Rp0PQifuvk0oN5BfDkwb7L2qV0WBEy31zye+hY= +github.com/ledgerwatch/erigon-lib v0.0.0-20230307062144-6c0eee4ef32f/go.mod h1:nyJqfX9uPm1P/poZB1211DFe5DnAKOhYqvkEPyW7dXM= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 h1:tfzawK1gIIgRjVZeANXOr0Ziu+kqCIBuKMe0TXfl5Aw= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From 3dac0cf42be93db80cdae3727f123d0f0c2fbf45 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 8 Mar 2023 09:28:33 +0700 Subject: [PATCH 081/100] save --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 02f1394dc0c..d84ed9354aa 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230307090534-c336676a833f + github.com/ledgerwatch/erigon-lib v0.0.0-20230308022805-01040b0aa87d github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index faef2b8fec3..b76a7685ada 100644 --- a/go.sum +++ b/go.sum @@ -517,8 +517,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230307090534-c336676a833f h1:hieY7Azq1MJ4cRjaYlBQlDVmyRUSSEuTyxsns89HEKQ= -github.com/ledgerwatch/erigon-lib v0.0.0-20230307090534-c336676a833f/go.mod h1:IWw+R6KLaK9a60fdK7/QiWNUAPowtyZQ3Eb0qMbjeHo= +github.com/ledgerwatch/erigon-lib v0.0.0-20230308022805-01040b0aa87d h1:cBWP1VMl/UCGrzoiQwyPxYWjVVwGk9Slf+GUfkzEosI= +github.com/ledgerwatch/erigon-lib v0.0.0-20230308022805-01040b0aa87d/go.mod h1:IWw+R6KLaK9a60fdK7/QiWNUAPowtyZQ3Eb0qMbjeHo= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 h1:tfzawK1gIIgRjVZeANXOr0Ziu+kqCIBuKMe0TXfl5Aw= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From 1bb28fcf808a8e4376c595246fca85328064e787 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 8 Mar 2023 11:30:14 +0700 Subject: [PATCH 082/100] save --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d84ed9354aa..9d5b5314517 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230308022805-01040b0aa87d + github.com/ledgerwatch/erigon-lib v0.0.0-20230308042517-48dde41592c7 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index b76a7685ada..e8c265563b3 100644 --- a/go.sum +++ b/go.sum @@ -517,8 +517,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230308022805-01040b0aa87d h1:cBWP1VMl/UCGrzoiQwyPxYWjVVwGk9Slf+GUfkzEosI= -github.com/ledgerwatch/erigon-lib v0.0.0-20230308022805-01040b0aa87d/go.mod h1:IWw+R6KLaK9a60fdK7/QiWNUAPowtyZQ3Eb0qMbjeHo= +github.com/ledgerwatch/erigon-lib v0.0.0-20230308042517-48dde41592c7 h1:Rlkm0cHO5Bo6AAHkC7Zts2PmmsJ83ErHuiSq5Q0Bk80= +github.com/ledgerwatch/erigon-lib v0.0.0-20230308042517-48dde41592c7/go.mod h1:IWw+R6KLaK9a60fdK7/QiWNUAPowtyZQ3Eb0qMbjeHo= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 h1:tfzawK1gIIgRjVZeANXOr0Ziu+kqCIBuKMe0TXfl5Aw= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From 2bd82bb3af95535704d930a51383ef42ade09852 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 9 Mar 2023 09:59:29 +0700 Subject: [PATCH 083/100] better iter dup --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c09055e7bcf..49cbf364699 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230309023846-59e0f308589d + github.com/ledgerwatch/erigon-lib v0.0.0-20230309025726-c2cc141ac048 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index 8212c169ffb..3daa8f68d25 100644 --- a/go.sum +++ b/go.sum @@ -517,8 +517,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230309023846-59e0f308589d h1:IjdOu4AxbjsO9yxwNIZKxnnMnhGlfPcU063C31xFD5w= -github.com/ledgerwatch/erigon-lib v0.0.0-20230309023846-59e0f308589d/go.mod h1:/Sq4pXPwvnuUpEhryCdm8tt1umhfsOLG36isbz+BweM= +github.com/ledgerwatch/erigon-lib v0.0.0-20230309025726-c2cc141ac048 h1:Z8PiwhcG5tbbqZ60xyN7ufOM7QbutLTld6qtYXxWk9M= +github.com/ledgerwatch/erigon-lib v0.0.0-20230309025726-c2cc141ac048/go.mod h1:/Sq4pXPwvnuUpEhryCdm8tt1umhfsOLG36isbz+BweM= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 h1:tfzawK1gIIgRjVZeANXOr0Ziu+kqCIBuKMe0TXfl5Aw= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From a48bb2533e9bbf52aad8f7b66d61c31cb65a4ab2 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 9 Mar 2023 10:59:27 +0700 Subject: [PATCH 084/100] save --- eth/stagedsync/exec3.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/eth/stagedsync/exec3.go b/eth/stagedsync/exec3.go index 9de397bdd6d..5f63c2b7395 100644 --- a/eth/stagedsync/exec3.go +++ b/eth/stagedsync/exec3.go @@ -331,8 +331,8 @@ func ExecV3(ctx context.Context, // it means better spend time for pruning, before flushing more data to db // also better do it now - instead of before Commit() - because Commit does block execution stepsInDB := rawdbhelpers.IdxStepsCountV3(tx) - if stepsInDB > 10 && rs.SizeEstimate() < uint64(float64(commitThreshold)*0.2) { - if err = agg.Prune(ctx, ethconfig.HistoryV3AggregationStep); err != nil { // prune part of retired data, before commit + if stepsInDB > 5 && rs.SizeEstimate() < uint64(float64(commitThreshold)*0.2) { + if err = agg.Prune(ctx, ethconfig.HistoryV3AggregationStep*2); err != nil { // prune part of retired data, before commit panic(err) } } From 10e4d9bd18cdeccef26aaeb54ce3872aa02a7695 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 9 Mar 2023 11:13:35 +0700 Subject: [PATCH 085/100] save --- eth/stagedsync/exec3.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/eth/stagedsync/exec3.go b/eth/stagedsync/exec3.go index 5f63c2b7395..7a17c354645 100644 --- a/eth/stagedsync/exec3.go +++ b/eth/stagedsync/exec3.go @@ -211,6 +211,8 @@ func ExecV3(ctx context.Context, applyLoopWg := sync.WaitGroup{} // to wait for finishing of applyLoop after applyCtx cancel defer applyLoopWg.Wait() + var stepsInDBAfterCommit float64 + applyLoopInner := func(ctx context.Context) error { tx, err := chainDb.BeginRo(ctx) if err != nil { @@ -307,6 +309,7 @@ func ExecV3(ctx context.Context, } else { defer agg.StartWrites().FinishWrites() } + stepsInDBAfterCommit = rawdbhelpers.IdxStepsCountV3(tx) defer applyLoopWg.Wait() applyCtx, cancelApplyCtx := context.WithCancel(ctx) @@ -330,17 +333,16 @@ func ExecV3(ctx context.Context, // too much steps in db will slow-down everything: flush and prune // it means better spend time for pruning, before flushing more data to db // also better do it now - instead of before Commit() - because Commit does block execution - stepsInDB := rawdbhelpers.IdxStepsCountV3(tx) - if stepsInDB > 5 && rs.SizeEstimate() < uint64(float64(commitThreshold)*0.2) { - if err = agg.Prune(ctx, ethconfig.HistoryV3AggregationStep*2); err != nil { // prune part of retired data, before commit + if stepsInDBAfterCommit > 5 && rs.SizeEstimate() < uint64(float64(commitThreshold)*0.2) { + if err = agg.Prune(ctx, ethconfig.HistoryV3AggregationStep/2); err != nil { // prune part of retired data, before commit panic(err) } } - if err = agg.Flush(ctx, tx); err != nil { + if err = agg.PruneWithTiemout(ctx, 1*time.Second); err != nil { return err } - if err = agg.PruneWithTiemout(ctx, 1*time.Second); err != nil { + if err = agg.Flush(ctx, tx); err != nil { return err } break @@ -447,6 +449,7 @@ func ExecV3(ctx context.Context, } defer tx.Rollback() agg.SetTx(tx) + stepsInDBAfterCommit = rawdbhelpers.IdxStepsCountV3(tx) applyCtx, cancelApplyCtx = context.WithCancel(ctx) defer cancelApplyCtx() From abcc6d0861eb7d18750dd97a076cf7644e0d2a13 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Thu, 9 Mar 2023 14:48:06 +0700 Subject: [PATCH 086/100] save --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 49cbf364699..775c3ac876e 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230309025726-c2cc141ac048 + github.com/ledgerwatch/erigon-lib v0.0.0-20230309074725-f3b315356d06 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index 3daa8f68d25..ece3a563c52 100644 --- a/go.sum +++ b/go.sum @@ -517,8 +517,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230309025726-c2cc141ac048 h1:Z8PiwhcG5tbbqZ60xyN7ufOM7QbutLTld6qtYXxWk9M= -github.com/ledgerwatch/erigon-lib v0.0.0-20230309025726-c2cc141ac048/go.mod h1:/Sq4pXPwvnuUpEhryCdm8tt1umhfsOLG36isbz+BweM= +github.com/ledgerwatch/erigon-lib v0.0.0-20230309074725-f3b315356d06 h1:5SAWNjRYRDb0f4qFOnuQAWLYfZr0Rc/OJbik5oO1a0o= +github.com/ledgerwatch/erigon-lib v0.0.0-20230309074725-f3b315356d06/go.mod h1:/Sq4pXPwvnuUpEhryCdm8tt1umhfsOLG36isbz+BweM= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 h1:tfzawK1gIIgRjVZeANXOr0Ziu+kqCIBuKMe0TXfl5Aw= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From 1004d052125d31665ad7e862a9e706631e4e2cf2 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 13 Mar 2023 11:03:45 +0700 Subject: [PATCH 087/100] save --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2906ae9aa0f..b48e7861bbb 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230313024614-5ea4a4dd136f + github.com/ledgerwatch/erigon-lib v0.0.0-20230313040258-2d3ab6784b3b github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index 125a9544b1f..473e8446663 100644 --- a/go.sum +++ b/go.sum @@ -517,8 +517,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230313024614-5ea4a4dd136f h1:2Hh9YgBrs0tpca/gZLcI0jerMflVAiV4ncTZsI8OlPU= -github.com/ledgerwatch/erigon-lib v0.0.0-20230313024614-5ea4a4dd136f/go.mod h1:sKLWgIyFuajTu7nu+cahKhReP+CZW57R5jFUYtzvn44= +github.com/ledgerwatch/erigon-lib v0.0.0-20230313040258-2d3ab6784b3b h1:/MA6ZZQdrmXUDF5pcv8FCosOrP/l+uyw+JNbk+yBs7U= +github.com/ledgerwatch/erigon-lib v0.0.0-20230313040258-2d3ab6784b3b/go.mod h1:sKLWgIyFuajTu7nu+cahKhReP+CZW57R5jFUYtzvn44= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 h1:tfzawK1gIIgRjVZeANXOr0Ziu+kqCIBuKMe0TXfl5Aw= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From e90a599a6fa43d7cf8b42f77d2158021b626bb86 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 13 Mar 2023 11:08:57 +0700 Subject: [PATCH 088/100] save --- eth/stagedsync/exec3.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/eth/stagedsync/exec3.go b/eth/stagedsync/exec3.go index 3b99669aec4..1e8bc84dd9f 100644 --- a/eth/stagedsync/exec3.go +++ b/eth/stagedsync/exec3.go @@ -212,8 +212,6 @@ func ExecV3(ctx context.Context, applyLoopWg := sync.WaitGroup{} // to wait for finishing of applyLoop after applyCtx cancel defer applyLoopWg.Wait() - var stepsInDBAfterCommit float64 - applyLoopInner := func(ctx context.Context) error { tx, err := chainDb.BeginRo(ctx) if err != nil { @@ -310,7 +308,6 @@ func ExecV3(ctx context.Context, } else { defer agg.StartWrites().FinishWrites() } - stepsInDBAfterCommit = rawdbhelpers.IdxStepsCountV3(tx) defer applyLoopWg.Wait() applyCtx, cancelApplyCtx := context.WithCancel(ctx) @@ -444,7 +441,6 @@ func ExecV3(ctx context.Context, } defer tx.Rollback() agg.SetTx(tx) - stepsInDBAfterCommit = rawdbhelpers.IdxStepsCountV3(tx) applyCtx, cancelApplyCtx = context.WithCancel(ctx) defer cancelApplyCtx() From 9d1da8f81dc63855e3fe1bb5100d64fdad8c4212 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 13 Mar 2023 11:13:26 +0700 Subject: [PATCH 089/100] save --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b48e7861bbb..54b3e12b086 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230313040258-2d3ab6784b3b + github.com/ledgerwatch/erigon-lib v0.0.0-20230313041254-14767b708cc0 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index 473e8446663..23fdbaaa439 100644 --- a/go.sum +++ b/go.sum @@ -517,8 +517,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230313040258-2d3ab6784b3b h1:/MA6ZZQdrmXUDF5pcv8FCosOrP/l+uyw+JNbk+yBs7U= -github.com/ledgerwatch/erigon-lib v0.0.0-20230313040258-2d3ab6784b3b/go.mod h1:sKLWgIyFuajTu7nu+cahKhReP+CZW57R5jFUYtzvn44= +github.com/ledgerwatch/erigon-lib v0.0.0-20230313041254-14767b708cc0 h1:t6rvtzbDDw67rBLQ1kaKx4CFVgke7vM9iUrITh3zNQs= +github.com/ledgerwatch/erigon-lib v0.0.0-20230313041254-14767b708cc0/go.mod h1:sKLWgIyFuajTu7nu+cahKhReP+CZW57R5jFUYtzvn44= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 h1:tfzawK1gIIgRjVZeANXOr0Ziu+kqCIBuKMe0TXfl5Aw= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From db69b96d916ab9c3234869be9532eda766108acd Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 13 Mar 2023 11:16:28 +0700 Subject: [PATCH 090/100] save --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 54b3e12b086..c4219595f09 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230313041254-14767b708cc0 + github.com/ledgerwatch/erigon-lib v0.0.0-20230313041550-16d1f678cbc3 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index 23fdbaaa439..e7fbeb06ed5 100644 --- a/go.sum +++ b/go.sum @@ -517,8 +517,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230313041254-14767b708cc0 h1:t6rvtzbDDw67rBLQ1kaKx4CFVgke7vM9iUrITh3zNQs= -github.com/ledgerwatch/erigon-lib v0.0.0-20230313041254-14767b708cc0/go.mod h1:sKLWgIyFuajTu7nu+cahKhReP+CZW57R5jFUYtzvn44= +github.com/ledgerwatch/erigon-lib v0.0.0-20230313041550-16d1f678cbc3 h1:nK1bk0u4WKlhnf/tcaRcEPRMpt6EzNyGQZfY4ov3jRs= +github.com/ledgerwatch/erigon-lib v0.0.0-20230313041550-16d1f678cbc3/go.mod h1:sKLWgIyFuajTu7nu+cahKhReP+CZW57R5jFUYtzvn44= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 h1:tfzawK1gIIgRjVZeANXOr0Ziu+kqCIBuKMe0TXfl5Aw= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From c71ad85c6e7f170b589cadc1bfc845da6d27dccc Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Mon, 13 Mar 2023 15:44:37 +0700 Subject: [PATCH 091/100] save --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index c4219595f09..3e2a4fe03d1 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230313041550-16d1f678cbc3 + github.com/ledgerwatch/erigon-lib v0.0.0-20230313084152-4227beeefe85 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index e7fbeb06ed5..e92f16f38bd 100644 --- a/go.sum +++ b/go.sum @@ -517,8 +517,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230313041550-16d1f678cbc3 h1:nK1bk0u4WKlhnf/tcaRcEPRMpt6EzNyGQZfY4ov3jRs= -github.com/ledgerwatch/erigon-lib v0.0.0-20230313041550-16d1f678cbc3/go.mod h1:sKLWgIyFuajTu7nu+cahKhReP+CZW57R5jFUYtzvn44= +github.com/ledgerwatch/erigon-lib v0.0.0-20230313084152-4227beeefe85 h1:ZcJXR9+JNZZoM96EZuYVXaq4KoGPNDc/e7jdDzFpaSo= +github.com/ledgerwatch/erigon-lib v0.0.0-20230313084152-4227beeefe85/go.mod h1:sKLWgIyFuajTu7nu+cahKhReP+CZW57R5jFUYtzvn44= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 h1:tfzawK1gIIgRjVZeANXOr0Ziu+kqCIBuKMe0TXfl5Aw= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From 3583552e0a18b88b2457668f0308fbb2386df572 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 14 Mar 2023 10:37:34 +0700 Subject: [PATCH 092/100] save --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2b890a508ff..7b8067a1e5c 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230314021635-7e9b63778bfa + github.com/ledgerwatch/erigon-lib v0.0.0-20230314033653-b5facbb85a59 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index b26a590d3de..5ca7f30e289 100644 --- a/go.sum +++ b/go.sum @@ -517,8 +517,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230314021635-7e9b63778bfa h1:BIV99uawTXusgqTas/DQproZcHsr33Sf5l8dQz/cD4A= -github.com/ledgerwatch/erigon-lib v0.0.0-20230314021635-7e9b63778bfa/go.mod h1:sKLWgIyFuajTu7nu+cahKhReP+CZW57R5jFUYtzvn44= +github.com/ledgerwatch/erigon-lib v0.0.0-20230314033653-b5facbb85a59 h1:ZcMTpT33lNGrAX8qOTBHA3h/kFrBEUYS1v7Y4vngLJM= +github.com/ledgerwatch/erigon-lib v0.0.0-20230314033653-b5facbb85a59/go.mod h1:sKLWgIyFuajTu7nu+cahKhReP+CZW57R5jFUYtzvn44= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 h1:tfzawK1gIIgRjVZeANXOr0Ziu+kqCIBuKMe0TXfl5Aw= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From 5cf3df99404161f8998dbe779f999f4bd416d49a Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 14 Mar 2023 10:53:18 +0700 Subject: [PATCH 093/100] save --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 7b8067a1e5c..6489b0990f3 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230314033653-b5facbb85a59 + github.com/ledgerwatch/erigon-lib v0.0.0-20230314035243-308cebc6d072 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index 5ca7f30e289..19071db09aa 100644 --- a/go.sum +++ b/go.sum @@ -517,8 +517,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230314033653-b5facbb85a59 h1:ZcMTpT33lNGrAX8qOTBHA3h/kFrBEUYS1v7Y4vngLJM= -github.com/ledgerwatch/erigon-lib v0.0.0-20230314033653-b5facbb85a59/go.mod h1:sKLWgIyFuajTu7nu+cahKhReP+CZW57R5jFUYtzvn44= +github.com/ledgerwatch/erigon-lib v0.0.0-20230314035243-308cebc6d072 h1:vmc0/mu/JQz0WUoQ2PC/nAY27wUsaIaHTSknEvbDzNA= +github.com/ledgerwatch/erigon-lib v0.0.0-20230314035243-308cebc6d072/go.mod h1:sKLWgIyFuajTu7nu+cahKhReP+CZW57R5jFUYtzvn44= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 h1:tfzawK1gIIgRjVZeANXOr0Ziu+kqCIBuKMe0TXfl5Aw= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From 779141a0383f0babf3c0769f7c3767d1f29e92a6 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 14 Mar 2023 11:30:42 +0700 Subject: [PATCH 094/100] save --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6489b0990f3..311c58d36e1 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230314035243-308cebc6d072 + github.com/ledgerwatch/erigon-lib v0.0.0-20230314042734-21bd5e3dc95a github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index 19071db09aa..c135b7febbd 100644 --- a/go.sum +++ b/go.sum @@ -517,8 +517,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230314035243-308cebc6d072 h1:vmc0/mu/JQz0WUoQ2PC/nAY27wUsaIaHTSknEvbDzNA= -github.com/ledgerwatch/erigon-lib v0.0.0-20230314035243-308cebc6d072/go.mod h1:sKLWgIyFuajTu7nu+cahKhReP+CZW57R5jFUYtzvn44= +github.com/ledgerwatch/erigon-lib v0.0.0-20230314042734-21bd5e3dc95a h1:mLrw86i/R5sgt/B31BDifMn60B1wbKquibKnMxr7E1o= +github.com/ledgerwatch/erigon-lib v0.0.0-20230314042734-21bd5e3dc95a/go.mod h1:sKLWgIyFuajTu7nu+cahKhReP+CZW57R5jFUYtzvn44= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 h1:tfzawK1gIIgRjVZeANXOr0Ziu+kqCIBuKMe0TXfl5Aw= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From a321181ea9ff1a5dbd1609317354fe4537c041b7 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 14 Mar 2023 14:36:02 +0700 Subject: [PATCH 095/100] save --- eth/stagedsync/exec3.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eth/stagedsync/exec3.go b/eth/stagedsync/exec3.go index 1e8bc84dd9f..e8f381ca0dd 100644 --- a/eth/stagedsync/exec3.go +++ b/eth/stagedsync/exec3.go @@ -329,7 +329,7 @@ func ExecV3(ctx context.Context, case <-pruneEvery.C: if rs.SizeEstimate() < commitThreshold { if agg.CanPrune(tx) { - if err = agg.Prune(ctx, ethconfig.HistoryV3AggregationStep/10); err != nil { // prune part of retired data, before commit + if err = agg.Prune(ctx, ethconfig.HistoryV3AggregationStep*10); err != nil { // prune part of retired data, before commit return err } } else { From d16489202322aace7ae8750b150ecc1cb2e4fd2c Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 14 Mar 2023 14:53:32 +0700 Subject: [PATCH 096/100] save --- go.mod | 4 ++-- go.sum | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index bf96516bbf5..2311e4d1504 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230314065137-d03c28854f53 + github.com/ledgerwatch/erigon-lib v0.0.0-20230314075105-f7a0fc1fc46c github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 @@ -57,6 +57,7 @@ require ( github.com/libp2p/go-libp2p-pubsub v0.9.1-0.20230221111042-3dbc2fd5baca github.com/maticnetwork/crand v1.0.2 github.com/maticnetwork/polyproto v0.0.2 + github.com/minio/sha256-simd v1.0.0 github.com/multiformats/go-multiaddr v0.8.0 github.com/nxadm/tail v1.4.9-0.20211216163028-4472660a31a6 github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 @@ -180,7 +181,6 @@ require ( github.com/miekg/dns v1.1.50 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect - github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.2 // indirect diff --git a/go.sum b/go.sum index 0fca50a0f76..a9b616de5b5 100644 --- a/go.sum +++ b/go.sum @@ -517,8 +517,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230314065137-d03c28854f53 h1:afhHLD++/zGz6IHKgflvvJWcKgP5F7T5wfb8DPN74V4= -github.com/ledgerwatch/erigon-lib v0.0.0-20230314065137-d03c28854f53/go.mod h1:MSCADyq0R0RXYUlIuywRW8XbWsRVTURsNx6bGH2ouGE= +github.com/ledgerwatch/erigon-lib v0.0.0-20230314075105-f7a0fc1fc46c h1:quHV7cJxg14EOC25jBYMhs/R2+qYt2e0UPIWmckKrZ0= +github.com/ledgerwatch/erigon-lib v0.0.0-20230314075105-f7a0fc1fc46c/go.mod h1:MSCADyq0R0RXYUlIuywRW8XbWsRVTURsNx6bGH2ouGE= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 h1:tfzawK1gIIgRjVZeANXOr0Ziu+kqCIBuKMe0TXfl5Aw= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From 22a30ed9d48a5fde83d4e77f5eff4ca8ea0a84ff Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 14 Mar 2023 14:54:02 +0700 Subject: [PATCH 097/100] save --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2311e4d1504..95ef6f28e48 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230314075105-f7a0fc1fc46c + github.com/ledgerwatch/erigon-lib v0.0.0-20230314075332-d2b0c0883227 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index a9b616de5b5..0388f62c6a6 100644 --- a/go.sum +++ b/go.sum @@ -517,8 +517,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230314075105-f7a0fc1fc46c h1:quHV7cJxg14EOC25jBYMhs/R2+qYt2e0UPIWmckKrZ0= -github.com/ledgerwatch/erigon-lib v0.0.0-20230314075105-f7a0fc1fc46c/go.mod h1:MSCADyq0R0RXYUlIuywRW8XbWsRVTURsNx6bGH2ouGE= +github.com/ledgerwatch/erigon-lib v0.0.0-20230314075332-d2b0c0883227 h1:sue8juyJ81YgZEEZE+SDBN+qvbJHYOCp+ntj5U3ZnGk= +github.com/ledgerwatch/erigon-lib v0.0.0-20230314075332-d2b0c0883227/go.mod h1:MSCADyq0R0RXYUlIuywRW8XbWsRVTURsNx6bGH2ouGE= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 h1:tfzawK1gIIgRjVZeANXOr0Ziu+kqCIBuKMe0TXfl5Aw= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From 5a5bf1b929fc5786deeb5a15cf268fd9e533d40a Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Tue, 14 Mar 2023 15:51:45 +0700 Subject: [PATCH 098/100] save --- go.sum | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/go.sum b/go.sum index 0388f62c6a6..b0f913a2197 100644 --- a/go.sum +++ b/go.sum @@ -517,8 +517,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230314075332-d2b0c0883227 h1:sue8juyJ81YgZEEZE+SDBN+qvbJHYOCp+ntj5U3ZnGk= -github.com/ledgerwatch/erigon-lib v0.0.0-20230314075332-d2b0c0883227/go.mod h1:MSCADyq0R0RXYUlIuywRW8XbWsRVTURsNx6bGH2ouGE= +github.com/ledgerwatch/erigon-lib v0.0.0-20230314085041-7e2742f36197 h1:bYASvCVD8zNmH9XANgLYLUak+Oq5VJd/SWhPwlnTRFA= +github.com/ledgerwatch/erigon-lib v0.0.0-20230314085041-7e2742f36197/go.mod h1:MSCADyq0R0RXYUlIuywRW8XbWsRVTURsNx6bGH2ouGE= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 h1:tfzawK1gIIgRjVZeANXOr0Ziu+kqCIBuKMe0TXfl5Aw= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From 2a7b37302fe58e32f8dba851e4ece11f2181ec5e Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 15 Mar 2023 09:13:37 +0700 Subject: [PATCH 099/100] save --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 55f2763a8ae..75ece8559ef 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/ledgerwatch/erigon go 1.18 require ( - github.com/ledgerwatch/erigon-lib v0.0.0-20230314085041-7e2742f36197 + github.com/ledgerwatch/erigon-lib v0.0.0-20230315021245-895401cad630 github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 github.com/ledgerwatch/log/v3 v3.7.0 github.com/ledgerwatch/secp256k1 v1.0.0 diff --git a/go.sum b/go.sum index b0f913a2197..85cb1c95849 100644 --- a/go.sum +++ b/go.sum @@ -517,8 +517,8 @@ github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170224010052-a616ab194758 h1:0D5M2HQSGD3PYPwICLl+/9oulQauOuETfgFvhBDffs0= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/ledgerwatch/erigon-lib v0.0.0-20230314085041-7e2742f36197 h1:bYASvCVD8zNmH9XANgLYLUak+Oq5VJd/SWhPwlnTRFA= -github.com/ledgerwatch/erigon-lib v0.0.0-20230314085041-7e2742f36197/go.mod h1:MSCADyq0R0RXYUlIuywRW8XbWsRVTURsNx6bGH2ouGE= +github.com/ledgerwatch/erigon-lib v0.0.0-20230315021245-895401cad630 h1:l2em+Z6f1Jo183iPBb0a5B8AGfHpHZROoqVUhGYSRWw= +github.com/ledgerwatch/erigon-lib v0.0.0-20230315021245-895401cad630/go.mod h1:MSCADyq0R0RXYUlIuywRW8XbWsRVTURsNx6bGH2ouGE= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3 h1:tfzawK1gIIgRjVZeANXOr0Ziu+kqCIBuKMe0TXfl5Aw= github.com/ledgerwatch/erigon-snapshot v1.1.1-0.20230306083105-1391330d62a3/go.mod h1:3AuPxZc85jkehh/HA9h8gabv5MSi3kb/ddtzBsTVJFo= github.com/ledgerwatch/log/v3 v3.7.0 h1:aFPEZdwZx4jzA3+/Pf8wNDN5tCI0cIolq/kfvgcM+og= From f3f9980baf7a8bfaf36602c1285d70589d7cdcd3 Mon Sep 17 00:00:00 2001 From: "alex.sharov" Date: Wed, 15 Mar 2023 13:46:41 +0700 Subject: [PATCH 100/100] save --- eth/stagedsync/exec3.go | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/eth/stagedsync/exec3.go b/eth/stagedsync/exec3.go index e8f381ca0dd..5c30b737d7f 100644 --- a/eth/stagedsync/exec3.go +++ b/eth/stagedsync/exec3.go @@ -221,7 +221,6 @@ func ExecV3(ctx context.Context, applyWorker.ResetTx(tx) - notifyReceived := func() { rwsReceiveCond.Signal() } var t time.Time var lastBlockNum uint64 drainF := func(txTask *exec22.TxTask) (added int64) { @@ -259,7 +258,7 @@ func ExecV3(ctx context.Context, processedResultSize, processedTxNum, conflicts, processedBlockNum, err := func() (processedResultSize int64, processedTxNum, conflicts, processedBlockNum uint64, err error) { rwsLock.Lock() defer rwsLock.Unlock() - return processResultQueue(rws, outputTxNum.Load(), rs, agg, tx, triggerCount, notifyReceived, applyWorker) + return processResultQueue(rws, outputTxNum.Load(), rs, agg, tx, triggerCount, rwsReceiveCond, applyWorker) }() if err != nil { return err @@ -365,7 +364,7 @@ func ExecV3(ctx context.Context, } } applyWorker.ResetTx(tx) - processedResultSize, processedTxNum, conflicts, processedBlockNum, err := processResultQueue(rws, outputTxNum.Load(), rs, agg, tx, triggerCount, func() {}, applyWorker) + processedResultSize, processedTxNum, conflicts, processedBlockNum, err := processResultQueue(rws, outputTxNum.Load(), rs, agg, tx, triggerCount, nil, applyWorker) if err != nil { return err } @@ -769,7 +768,7 @@ func blockWithSenders(db kv.RoDB, tx kv.Tx, blockReader services.BlockReader, bl return b, nil } -func processResultQueue(rws *exec22.TxTaskQueue, outputTxNumIn uint64, rs *state.StateV3, agg *state2.AggregatorV3, applyTx kv.Tx, triggerCount *atomic2.Uint64, onSuccess func(), applyWorker *exec3.Worker) (resultSize int64, outputTxNum, conflicts, processedBlockNum uint64, err error) { +func processResultQueue(rws *exec22.TxTaskQueue, outputTxNumIn uint64, rs *state.StateV3, agg *state2.AggregatorV3, applyTx kv.Tx, triggerCount *atomic2.Uint64, rwsCond *sync.Cond, applyWorker *exec3.Worker) (resultSize int64, outputTxNum, conflicts, processedBlockNum uint64, err error) { var i int outputTxNum = outputTxNumIn for rws.Len() > 0 && (*rws)[0].TxNum == outputTxNum { @@ -797,7 +796,9 @@ func processResultQueue(rws *exec22.TxTaskQueue, outputTxNumIn uint64, rs *state } triggerCount.Add(rs.CommitTxNum(txTask.Sender, txTask.TxNum)) outputTxNum++ - onSuccess() + if rwsCond != nil { + rwsCond.Signal() + } if err := rs.ApplyHistory(txTask, agg); err != nil { return resultSize, outputTxNum, conflicts, processedBlockNum, fmt.Errorf("StateV3.Apply: %w", err) }