Local Replay feature

.dockerignore

@@ -11,4 +11,6 @@ vendor
 
 cache.db
 test
-.git
+.git
+
+mainnet

.github/workflows/ci_zkevm.yml

@@ -146,7 +146,7 @@ jobs:
           docker run --network=host  --volume "./tmp/:/config/" --env BRIDGE_TEST_CONFIG_FILE=/config/test.toml bridge-e2e-realnetwork-erc20
 
       - name: Upload logs
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: evm-rpc-tests-logs-${{ github.run_id }}
           path: ./cdk-erigon/logs/evm-rpc-tests.log
@@ -166,7 +166,7 @@ jobs:
 
       - name: Upload logs
         if: failure()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: logs_${{ github.run_id }}
           path: ./kurtosis-cdk/ci_logs
@@ -190,7 +190,7 @@ jobs:
         run: /usr/local/bin/polycli loadtest --rpc-url "$(kurtosis port print cdk-v1 cdk-erigon-rpc-001 rpc)" --private-key "0x12d7de8621a77640c9241b2595ba78ce443d05e94090365ab3bb5e19df82c625" --verbosity 700 --requests 500 --rate-limit 50  --mode uniswapv3 --legacy
 
       - name: Upload logs
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: evm-rpc-tests-logs-${{ github.run_id }}
           path: ./cdk-erigon/logs/evm-rpc-tests.log
@@ -206,7 +206,7 @@ jobs:
 
       - name: Upload logs
         if: failure()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: logs_${{ github.run_id }}
           path: ./kurtosis-cdk/ci_logs

.github/workflows/nightly-ansible.yml

@@ -37,7 +37,7 @@ jobs:
           cd zk/tests/ansible/evm-rpc-tests && docker compose build --quiet && docker compose up --exit-code-from ansible
 
       - name: Upload logs
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: evm-rpc-tests-logs-${{ matrix.nodes.name }}
           path: ./logs/evm-rpc-tests.log

.github/workflows/nightly-eth-bench.yml

@@ -99,7 +99,7 @@ jobs:
 
       - name: Upload benchmark log
         if: always()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: benchmark-log-${{ matrix.profile_name }}
           path: ./eth-bench/bench_${{ matrix.profile_name }}.log

.github/workflows/scheduler_xlayer.yml

@@ -103,7 +103,7 @@ jobs:
           docker run --network=host  --volume "./tmp/:/config/" --env BRIDGE_TEST_CONFIG_FILE=/config/test.toml bridge-e2e-realnetwork-erc20
 
       - name: Upload logs
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: evm-rpc-tests-logs-${{ github.run_id }}
           path: ./cdk-erigon/logs/evm-rpc-tests.log
@@ -122,7 +122,7 @@ jobs:
 
       - name: Upload logs
         if: failure()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: logs_${{ github.run_id }}
           path: ./kurtosis-cdk/ci_logs

.github/workflows/test-resequence.yml

@@ -32,7 +32,7 @@ jobs:
         run: .github/scripts/test_resequence.sh
 
       - name: Upload logs
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: evm-rpc-tests-logs-${{ github.run_id }}
           path: ./cdk-erigon/logs/evm-rpc-tests.log
@@ -51,7 +51,7 @@ jobs:
           kurtosis service logs cdk-v1 zkevm-bridge-service-001 --all > zkevm-bridge-service-001.log
       - name: Upload logs
         if: failure()
-        uses: actions/upload-artifact@v3
+        uses: actions/upload-artifact@v4
         with:
           name: logs_${{ github.run_id }}
           path: ./kurtosis-cdk/ci_logs

.gitignore

@@ -115,10 +115,12 @@ vendor
 data/
 test/sqlite
 start.sh
-xlayerconfig-mainnet.yaml
 xlayerconfig-testnet.yaml
 **/cover.out
 **/cover.html
+test/mainnet
+test/*.zip
+test/*.tar.gz
 
 datadir
 zk/tests/unwinds/datastream/hermez-dynamic-integration8-datastream

Dockerfile.local

@@ -15,7 +15,7 @@ ADD . .
 RUN --mount=type=cache,target=/root/.cache \
     --mount=type=cache,target=/tmp/go-build \
     --mount=type=cache,target=/go/pkg/mod \
-    make BUILD_TAGS=nosqlite,noboltdb,nosilkworm cdk-erigon
+    make BUILD_TAGS=nosqlite,noboltdb,nosilkworm cdk-erigon integration
 
 FROM docker.io/library/golang:1.21-alpine3.17 AS tools-builder
 RUN apk --no-cache add build-base linux-headers git bash ca-certificates libstdc++
@@ -46,6 +46,7 @@ WORKDIR /home/erigon
 
 ## then give each binary its own layer
 COPY --from=builder /app/build/bin/cdk-erigon /usr/local/bin/cdk-erigon
+COPY --from=builder /app/build/bin/integration /usr/local/bin/integration
 
 EXPOSE 8545 \
        8551 \

cmd/prometheus/prometheus.yml

@@ -11,6 +11,7 @@ scrape_configs:
       - targets:
           - xlayer-rpc:9095
           - xlayer-seq:9095
+          - xlayer-mainnet-lrp-resequence:9095
 
   - job_name: erigon
     metrics_path: /debug/metrics/prometheus

cmd/utils/flags.go

@@ -576,6 +576,11 @@ var (
 		Usage: "When enabled, the sequencer will automatically resequence unseen batches stored in data stream",
 		Value: false,
 	}
+	SequencerResequenceHaltOnBatchNumber = cli.Uint64Flag{
+		Name:  "zkevm.sequencer-resequence-halt-on-batch-number",
+		Usage: "Halt the sequencer on this batch number when resequencing",
+		Value: 0,
+	}
 	SequencerResequenceStrict = cli.BoolFlag{
 		Name:  "zkevm.sequencer-resequence-strict",
 		Usage: "Strictly resequence the rolledback batches",
@@ -1454,7 +1459,7 @@ var (
 	YieldSizeFlag = cli.Uint64Flag{
 		Name:  "yieldsize",
 		Usage: "transaction count fetched from txpool each time",
-		Value: 1000,
+		Value: 30,
 	}
 )
 
@@ -1945,7 +1950,7 @@ func setTxPool(ctx *cli.Context, fullCfg *ethconfig.Config) {
 
 	// For X Layer
 	setTxPoolXLayer(ctx, cfg)
-	
+
 	purgeEvery := ctx.Duration(TxpoolPurgeEveryFlag.Name)
 	purgeDistance := ctx.Duration(TxpoolPurgeDistanceFlag.Name)
 

eth/ethconfig/config_zkevm.go

@@ -43,6 +43,7 @@ type Zk struct {
 	SequencerTimeoutOnEmptyTxPool          time.Duration
 	SequencerHaltOnBatchNumber             uint64
 	SequencerResequence                    bool
+	SequencerResequenceHaltOnBatchNumber   uint64
 	SequencerResequenceStrict              bool
 	SequencerResequenceReuseL1InfoIndex    bool
 	ExecutorUrls                           []string
@@ -90,7 +91,7 @@ type Zk struct {
 
 	// For X Layer
 	XLayer XLayerConfig
-	
+
 	InitialBatchCfgFile            string
 	ACLPrintHistory                int
 	InfoTreeUpdateInterval         time.Duration

test/Makefile

@@ -145,3 +145,79 @@ test-1: stop all ## Runs group 1 e2e tests checking race conditions
 	sleep 3
 	docker ps -a
 	trap '$(STOP)' EXIT; MallocNanoZone=0 go test -count=1 -failfast -race -v -p 1 -timeout 600s ../ci/e2e-1/...
+
+DOCKER_COMPOSE_LRP := docker compose -f docker-compose-lrp.yml
+DOCKER_LRP_UNWIND := xlayer-lrp-unwind
+DOCKER_LRP_RESEQUENCE := xlayer-lrp-resequence
+DOCKER_MAINNET_LRP_UNWIND := xlayer-mainnet-lrp-unwind
+DOCKER_MAINNET_LRP_RESEQUENCE := xlayer-mainnet-lrp-resequence
+
+RUN_DOCKER_LRP_STATELESS_EXECUTOR := $(DOCKER_COMPOSE_LRP) up -d $(DOCKER_STATELESS_EXECUTOR)
+RUN_DOCKER_LRP_L1_NETWORK := $(DOCKER_COMPOSE_LRP) up -d $(DOCKER_L1_NETWORK)
+RUN_DOCKER_LRP_UNWIND := $(DOCKER_COMPOSE_LRP) up -d $(DOCKER_LRP_UNWIND)
+RUN_DOCKER_LRP_RESEQUENCE := $(DOCKER_COMPOSE_LRP) up -d $(DOCKER_LRP_RESEQUENCE)
+RUN_DOCKER_MAINNET_LRP_UNWIND := $(DOCKER_COMPOSE_LRP) up -d $(DOCKER_MAINNET_LRP_UNWIND)
+RUN_DOCKER_MAINNET_LRP_RESEQUENCE := $(DOCKER_COMPOSE_LRP) up -d $(DOCKER_MAINNET_LRP_RESEQUENCE)
+
+STOP_LRP := $(DOCKER_COMPOSE_LRP) down --remove-orphans; sleep 3; rm -rf data
+
+UNWIND_LOG_FILE := unwind.log
+RESEQUENCE_LOG_FILE := resequence.log
+RESEQUENCE_FILTERED_LOG_FILE := resequence_filtered.log
+
+.PHONY: configure-lrp
+configure-lrp:
+	@read -p "Enter unwind batch number: " UNWIND_BATCH_NO; \
+	read -p "Enter halt on batch number: " HALT_BATCH_NO; \
+	if [ "$(shell uname)" = "Darwin" ]; then \
+		sed -i '' "s/--unwind-batch-no=[0-9]*/--unwind-batch-no=$$UNWIND_BATCH_NO/" docker-compose-lrp.yml; \
+		sed -i '' "s/--zkevm.sequencer-resequence-halt-on-batch-number=[0-9]*/--zkevm.sequencer-resequence-halt-on-batch-number=$$HALT_BATCH_NO/" docker-compose-lrp.yml; \
+	else \
+		sed -i "s/--unwind-batch-no=[0-9]*/--unwind-batch-no=$$UNWIND_BATCH_NO/" docker-compose-lrp.yml; \
+		sed -i "s/--zkevm.sequencer-resequence-halt-on-batch-number=[0-9]*/--zkevm.sequencer-resequence-halt-on-batch-number=$$HALT_BATCH_NO/" docker-compose-lrp.yml; \
+	fi
+
+.PHONY: lrp-run
+lrp-run: build-docker configure-lrp
+	$(RUN_DOCKER_LRP_L1_NETWORK)
+	$(RUN_DOCKER_LRP_STATELESS_EXECUTOR)
+	sleep 3
+	$(RUN_DOCKER_LRP_UNWIND)
+	docker wait $(DOCKER_LRP_UNWIND)
+	docker logs $(DOCKER_LRP_UNWIND) > $(UNWIND_LOG_FILE) 2>&1
+	$(RUN_DOCKER_LRP_RESEQUENCE)
+
+.PHONY: lrp-check
+lrp-check: ## Filter logs for LRP & Compute TPS & Check stateroots (configure debugToolsConfig.yaml)
+	docker logs $(DOCKER_LRP_RESEQUENCE) > $(RESEQUENCE_LOG_FILE) 2>&1
+	docker logs $(DOCKER_LRP_RESEQUENCE) 2>&1 | grep "\[5/13 Execution\]" > $(RESEQUENCE_FILTERED_LOG_FILE)
+	python process_log.py $(RESEQUENCE_FILTERED_LOG_FILE)
+	go run ../zk/debug_tools/rpc-blockhashes-compare/main.go
+
+.PHONY: lrp-mainnet
+lrp-mainnet: build-docker configure-lrp
+	$(RUN_DOCKER_MAINNET_LRP_UNWIND)
+	docker wait $(DOCKER_MAINNET_LRP_UNWIND)
+	docker logs $(DOCKER_MAINNET_LRP_UNWIND) > $(UNWIND_LOG_FILE) 2>&1
+	$(RUN_DOCKER_MAINNET_LRP_RESEQUENCE)
+
+.PHONY: lrp-mainnet-unwind
+lrp-mainnet-unwind: build-docker configure-lrp
+	$(RUN_DOCKER_MAINNET_LRP_UNWIND)
+	docker wait $(DOCKER_MAINNET_LRP_UNWIND)
+	docker logs $(DOCKER_MAINNET_LRP_UNWIND) > $(UNWIND_LOG_FILE) 2>&1
+
+.PHONY: lrp-mainnet-post-unwind
+lrp-mainnet-post-unwind: build-docker
+	$(RUN_DOCKER_MAINNET_LRP_RESEQUENCE)
+
+.PHONY: lrp-mainnet-check
+lrp-mainnet-check: ## Filter logs for LRP & Compute TPS & Check stateroots (configure debugToolsConfig.yaml)
+	docker logs $(DOCKER_MAINNET_LRP_RESEQUENCE) > $(RESEQUENCE_LOG_FILE) 2>&1
+	docker logs $(DOCKER_MAINNET_LRP_RESEQUENCE) 2>&1 | grep "\[5/13 Execution\]" > $(RESEQUENCE_FILTERED_LOG_FILE)
+	python process_log.py $(RESEQUENCE_FILTERED_LOG_FILE)
+	go run ../zk/debug_tools/rpc-blockhashes-compare/main.go
+
+.PHONY: lrp-stop
+lrp-stop:
+	$(STOP_LRP)

test/config/xlayerconfig-mainnet.yaml

@@ -0,0 +1,66 @@
+datadir: /home/erigon/data/
+chain: xlayer-mainnet
+http: true
+private.api.addr: localhost:9091
+zkevm.l2-chain-id: 196
+zkevm.l2-sequencer-rpc-url: https://rpc.xlayer.tech
+zkevm.l2-datastreamer-url: stream.xlayer.tech:8800
+zkevm.l1-chain-id: 1
+zkevm.l1-rpc-url: https://rpc.ankr.com/eth/578c95407e7831f0ac1ef79cacae294dc9bf8307121ca9fffaf1e556a5cca662
+#zkevm.increment-tree-always: true
+
+#debug.limit: 8554110
+
+zkevm.executor-strict: false
+zkevm.witness-full: false
+
+zkevm.sequencer-block-seal-time: "3s"
+zkevm.sequencer-batch-seal-time: "10s"
+zkevm.sequencer-batch-sleep-duration: "0s"
+
+zkevm.data-stream-host: "localhost"
+zkevm.data-stream-port: 6900
+zkevm.default-gas-price: 1
+zkevm.max-gas-price: 0
+zkevm.gas-price-factor: 0.000001
+zkevm.allow-free-transactions: true
+
+txpool.packbatchspeciallist : ["0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266", "0x8f8E2d6cF621f30e9a11309D6A56A876281Fd534", "0xa03666Fb51Aa9aD2DE70e0434072A007b3C91A9E"]
+
+zkevm.address-sequencer: "0xAF9d27ffe4d51eD54AC8eEc78f2785D7E11E5ab1"
+zkevm.address-zkevm: "0x2B0ee28D4D51bC9aDde5E58E295873F61F4a0507"
+zkevm.address-rollup: "0x5132A183E9F3CB7C848b0AAC5Ae0c4f0491B7aB2"
+zkevm.address-ger-manager: "0x580bda1e7A0CFAe92Fa7F6c20A3794F169CE3CFb"
+
+zkevm.l1-rollup-id: 3
+zkevm.l1-first-block: 19218658
+zkevm.l1-block-range: 2000
+zkevm.l1-query-delay: 1000
+zkevm.datastream-version: 3
+
+http.api: [eth, debug, net, trace, web3, erigon, zkevm]
+http.addr: 0.0.0.0
+http.port: 8545
+
+gpo.type: "follower"
+gpo.update-period: "300s"
+gpo.factor: 0.01
+gpo.kafka-url: "0.0.0.0"
+gpo.topic: "explorer"
+gpo.group-id: "web3"
+gpo.blocks: 3
+gpo.default-l1-coin-price: 2000
+gpo.default-l2-coin-price: 50
+gpo.gas-price-usdt: 0.000000476190476
+gpo.congestion-threshold: 0
+
+pprof: true
+pprof.port: 6060
+pprof.addr: 0.0.0.0
+
+metrics: true
+metrics.addr: "0.0.0.0"
+metrics.port: 9095
+
+yieldsize: 35
+zkevm.seal-batch-immediately-on-overflow: true