From aef4682b517e25358c6f3793e445bff02b5ba9db Mon Sep 17 00:00:00 2001 From: KyrinCode Date: Tue, 21 Jan 2025 17:02:28 +0800 Subject: [PATCH 1/6] setup xlayer-recover (also as sequencer) --- test/Makefile | 3 +++ test/docker-compose.yml | 19 +++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/test/Makefile b/test/Makefile index bce4fe035bd..834d4014621 100644 --- a/test/Makefile +++ b/test/Makefile @@ -12,6 +12,7 @@ DOCKER_BRIDGE_UI := xlayer-bridge-ui DOCKER_PROVER := xlayer-prover DOCKER_APPROVE := xlayer-approve DOCKER_SEQ := xlayer-seq +DOCKER_RECOVER := xlayer-recover DOCKER_RPC := xlayer-rpc DOCKER_L1_NETWORK := xlayer-mock-l1-network DOCKER_DATA_AVAILABILITY := xlayer-da @@ -36,6 +37,7 @@ RUN_DOCKER_BRIDGE_UI := $(DOCKER_COMPOSE) up -d $(DOCKER_BRIDGE_UI) RUN_DOCKER_PROVER := $(DOCKER_COMPOSE) up -d $(DOCKER_PROVER) RUN_DOCKER_APPROVE := $(DOCKER_COMPOSE) up -d $(DOCKER_APPROVE) RUN_DOCKER_SEQ := $(DOCKER_COMPOSE) up -d $(DOCKER_SEQ) +RUN_DOCKER_RECOVER := $(DOCKER_COMPOSE) up -d $(DOCKER_RECOVER) RUN_DOCKER_RPC := $(DOCKER_COMPOSE) up -d $(DOCKER_RPC) RUN_DOCKER_L1_NETWORK := $(DOCKER_COMPOSE) up -d $(DOCKER_L1_NETWORK) RUN_DOCKER_DATA_AVAILABILITY := $(DOCKER_COMPOSE) up -d $(DOCKER_DATA_AVAILABILITY) @@ -69,6 +71,7 @@ run: build-docker ## Runs a full node sleep 3 $(RUN_DOCKER_SEQ) $(RUN_DOCKER_PROVER) + $(RUN_DOCKER_RECOVER) sleep 10 $(RUN_DOCKER_DS) $(RUN_DOCKER_SIGNER) diff --git a/test/docker-compose.yml b/test/docker-compose.yml index 914485ad82b..19340569c76 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -114,6 +114,25 @@ services: command: > cdk-erigon --http.vhosts=* --http.corsdomain=* --ws --config=/usr/src/app/config.yaml + xlayer-recover: + container_name: xlayer-recover + image: cdk-erigon + environment: + - CDK_ERIGON_SEQUENCER=1 + ports: + - 6059:6060 + - 8119:8545 + - 6899:6900 + - 9089:9095 + volumes: + - ./data/recover/:/home/erigon/data/ + - ./config/test.erigon.seq.config.yaml:/usr/src/app/config.yaml + - ./config/dynamic-mynetwork-allocs.json:/usr/src/app/dynamic-mynetwork-allocs.json + - ./config/dynamic-mynetwork-chainspec.json:/usr/src/app/dynamic-mynetwork-chainspec.json + - ./config/dynamic-mynetwork-conf.json:/usr/src/app/dynamic-mynetwork-conf.json + command: > + cdk-erigon --zkevm.l1-sync-start-block=353 --zkevm.da-url=http://xlayer-da:8444 --http.vhosts=* --http.corsdomain=* --ws --config=/usr/src/app/config.yaml + xlayer-rpc: container_name: xlayer-rpc image: cdk-erigon From d399fc859a57fac9e3de341f3f3b96f948ca71cd Mon Sep 17 00:00:00 2001 From: KyrinCode Date: Sun, 26 Jan 2025 18:09:29 +0800 Subject: [PATCH 2/6] setup xlayer-lrp --- Dockerfile.local | 3 +- cmd/utils/flags.go | 4 +- test/Makefile | 44 ++++++++++++- test/docker-compose-lrp.yml | 123 ++++++++++++++++++++++++++++++++++++ test/docker-compose.yml | 103 +++++++++++++++++++++++++++--- test/fetch_blocks.sh | 42 ++++++++++++ test/process_log.py | 75 ++++++++++++++++++++++ 7 files changed, 379 insertions(+), 15 deletions(-) create mode 100644 test/docker-compose-lrp.yml create mode 100755 test/fetch_blocks.sh create mode 100644 test/process_log.py diff --git a/Dockerfile.local b/Dockerfile.local index c834e7c3308..69eb76d7ea5 100644 --- a/Dockerfile.local +++ b/Dockerfile.local @@ -15,7 +15,7 @@ ADD . . RUN --mount=type=cache,target=/root/.cache \ --mount=type=cache,target=/tmp/go-build \ --mount=type=cache,target=/go/pkg/mod \ - make BUILD_TAGS=nosqlite,noboltdb,nosilkworm cdk-erigon + make BUILD_TAGS=nosqlite,noboltdb,nosilkworm cdk-erigon integration FROM docker.io/library/golang:1.21-alpine3.17 AS tools-builder RUN apk --no-cache add build-base linux-headers git bash ca-certificates libstdc++ @@ -46,6 +46,7 @@ WORKDIR /home/erigon ## then give each binary its own layer COPY --from=builder /app/build/bin/cdk-erigon /usr/local/bin/cdk-erigon +COPY --from=builder /app/build/bin/integration /usr/local/bin/integration EXPOSE 8545 \ 8551 \ diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index f4cb4aaa88f..82f887e45d8 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -1454,7 +1454,7 @@ var ( YieldSizeFlag = cli.Uint64Flag{ Name: "yieldsize", Usage: "transaction count fetched from txpool each time", - Value: 1000, + Value: 30, } ) @@ -1945,7 +1945,7 @@ func setTxPool(ctx *cli.Context, fullCfg *ethconfig.Config) { // For X Layer setTxPoolXLayer(ctx, cfg) - + purgeEvery := ctx.Duration(TxpoolPurgeEveryFlag.Name) purgeDistance := ctx.Duration(TxpoolPurgeDistanceFlag.Name) diff --git a/test/Makefile b/test/Makefile index 834d4014621..a2a23030b40 100644 --- a/test/Makefile +++ b/test/Makefile @@ -12,7 +12,6 @@ DOCKER_BRIDGE_UI := xlayer-bridge-ui DOCKER_PROVER := xlayer-prover DOCKER_APPROVE := xlayer-approve DOCKER_SEQ := xlayer-seq -DOCKER_RECOVER := xlayer-recover DOCKER_RPC := xlayer-rpc DOCKER_L1_NETWORK := xlayer-mock-l1-network DOCKER_DATA_AVAILABILITY := xlayer-da @@ -37,7 +36,6 @@ RUN_DOCKER_BRIDGE_UI := $(DOCKER_COMPOSE) up -d $(DOCKER_BRIDGE_UI) RUN_DOCKER_PROVER := $(DOCKER_COMPOSE) up -d $(DOCKER_PROVER) RUN_DOCKER_APPROVE := $(DOCKER_COMPOSE) up -d $(DOCKER_APPROVE) RUN_DOCKER_SEQ := $(DOCKER_COMPOSE) up -d $(DOCKER_SEQ) -RUN_DOCKER_RECOVER := $(DOCKER_COMPOSE) up -d $(DOCKER_RECOVER) RUN_DOCKER_RPC := $(DOCKER_COMPOSE) up -d $(DOCKER_RPC) RUN_DOCKER_L1_NETWORK := $(DOCKER_COMPOSE) up -d $(DOCKER_L1_NETWORK) RUN_DOCKER_DATA_AVAILABILITY := $(DOCKER_COMPOSE) up -d $(DOCKER_DATA_AVAILABILITY) @@ -71,7 +69,6 @@ run: build-docker ## Runs a full node sleep 3 $(RUN_DOCKER_SEQ) $(RUN_DOCKER_PROVER) - $(RUN_DOCKER_RECOVER) sleep 10 $(RUN_DOCKER_DS) $(RUN_DOCKER_SIGNER) @@ -148,3 +145,44 @@ test-1: stop all ## Runs group 1 e2e tests checking race conditions sleep 3 docker ps -a trap '$(STOP)' EXIT; MallocNanoZone=0 go test -count=1 -failfast -race -v -p 1 -timeout 600s ../ci/e2e-1/... + +DOCKER_COMPOSE_LRP := docker compose -f docker-compose-lrp.yml +DOCKER_LRP_UNWIND := xlayer-lrp-unwind +DOCKER_LRP_RESEQUENCE := xlayer-lrp-resequence + +RUN_DOCKER_LRP_STATELESS_EXECUTOR := $(DOCKER_COMPOSE_LRP) up -d $(DOCKER_STATELESS_EXECUTOR) +RUN_DOCKER_LRP_L1_NETWORK := $(DOCKER_COMPOSE_LRP) up -d $(DOCKER_L1_NETWORK) +RUN_DOCKER_LRP_UNWIND := $(DOCKER_COMPOSE_LRP) up -d $(DOCKER_LRP_UNWIND) +RUN_DOCKER_LRP_RESEQUENCE := $(DOCKER_COMPOSE_LRP) up -d $(DOCKER_LRP_RESEQUENCE) + +STOP_LRP := $(DOCKER_COMPOSE_LRP) down --remove-orphans; sleep 3; rm -rf data + +.PHONY: lrp-run +lrp-run: build-docker + $(RUN_DOCKER_LRP_L1_NETWORK) + $(RUN_DOCKER_LRP_STATELESS_EXECUTOR) + sleep 3 + @read -p "Enter unwind to batch number: " UNWIND_BATCH_NO; \ + if [ "$(shell uname)" = "Darwin" ]; then \ + sed -i '' "s/--unwind-batch-no=[0-9]*/--unwind-batch-no=$$UNWIND_BATCH_NO/" docker-compose-lrp.yml; \ + else \ + sed -i "s/--unwind-batch-no=[0-9]*/--unwind-batch-no=$$UNWIND_BATCH_NO/" docker-compose-lrp.yml; \ + fi + $(RUN_DOCKER_LRP_UNWIND) + sleep 20 + $(RUN_DOCKER_LRP_RESEQUENCE) + +.PHONY: lrp-stop +lrp-stop: + $(STOP_LRP) + + +TARGET_LOG_FILE := filtered.log + +.PHONY: lrp-check +lrp-check: ## Filter logs for LRP & Compute TPS & Check stateroots (configure debugToolsConfig.yaml) + docker logs $(DOCKER_LRP_RESEQUENCE) 2>&1 | grep "\[5/13 Execution\]" > $(TARGET_LOG_FILE) + sleep 3 + python process_log.py $(TARGET_LOG_FILE) + sleep 3 + go run ../zk/debug_tools/rpc-blockhashes-compare/main.go \ No newline at end of file diff --git a/test/docker-compose-lrp.yml b/test/docker-compose-lrp.yml new file mode 100644 index 00000000000..a078e287ea4 --- /dev/null +++ b/test/docker-compose-lrp.yml @@ -0,0 +1,123 @@ +version: "3.5" +networks: + default: + name: erigon-lrp +services: + # xlayer-lrp-sequence: + # container_name: xlayer-lrp-sequence + # image: cdk-erigon + # environment: + # - CDK_ERIGON_SEQUENCER=1 + # ports: + # - 6062:6060 + # - 8126:8545 + # - 6902:6900 + # - 9093:9095 + # volumes: + # - ./data/lrp/:/home/erigon/data/ + # - ./config/test.erigon.seq.config.yaml:/usr/src/app/config.yaml + # - ./config/dynamic-mynetwork-allocs.json:/usr/src/app/dynamic-mynetwork-allocs.json + # - ./config/dynamic-mynetwork-chainspec.json:/usr/src/app/dynamic-mynetwork-chainspec.json + # - ./config/dynamic-mynetwork-conf.json:/usr/src/app/dynamic-mynetwork-conf.json + # command: > + # cdk-erigon --zkevm.l1-cache-enabled=true --http.vhosts=* --http.corsdomain=* --ws --config=/usr/src/app/config.yaml + + xlayer-lrp: + container_name: xlayer-lrp + image: cdk-erigon + environment: + - CDK_ERIGON_SEQUENCER=0 + ports: + - 6062:6060 + - 8126:8545 + - 6902:6900 + - 9093:9095 + volumes: + - ./data/lrp/:/home/erigon/data/ + - ./config/test.erigon.rpc.config.yaml:/usr/src/app/config.yaml + - ./config/dynamic-mynetwork-allocs.json:/usr/src/app/dynamic-mynetwork-allocs.json + - ./config/dynamic-mynetwork-chainspec.json:/usr/src/app/dynamic-mynetwork-chainspec.json + - ./config/dynamic-mynetwork-conf.json:/usr/src/app/dynamic-mynetwork-conf.json + command: > + cdk-erigon --http.vhosts=* --http.corsdomain=* --ws --config=/usr/src/app/config.yaml + + xlayer-lrp-unwind: + container_name: xlayer-lrp-unwind + image: cdk-erigon + environment: + - CDK_ERIGON_SEQUENCER=0 + volumes: + - ./data/lrp/:/home/erigon/data/ + - ./config/test.erigon.rpc.config.yaml:/usr/src/app/config.yaml + - ./config/dynamic-mynetwork-allocs.json:/usr/src/app/dynamic-mynetwork-allocs.json + - ./config/dynamic-mynetwork-chainspec.json:/usr/src/app/dynamic-mynetwork-chainspec.json + - ./config/dynamic-mynetwork-conf.json:/usr/src/app/dynamic-mynetwork-conf.json + command: | + integration state_stages_zkevm \ + --config=/usr/src/app/config.yaml \ + --chain dynamic-mynetwork \ + --datadir /home/erigon/data/ \ + --unwind-batch-no=10 + + xlayer-lrp-resequence: + container_name: xlayer-lrp-resequence + image: cdk-erigon + environment: + - CDK_ERIGON_SEQUENCER=1 + ports: + - 6062:6060 + - 8126:8545 + - 6902:6900 + - 9093:9095 + volumes: + - ./data/lrp/:/home/erigon/data/ + - ./config/test.erigon.seq.config.yaml:/usr/src/app/config.yaml + - ./config/dynamic-mynetwork-allocs.json:/usr/src/app/dynamic-mynetwork-allocs.json + - ./config/dynamic-mynetwork-chainspec.json:/usr/src/app/dynamic-mynetwork-chainspec.json + - ./config/dynamic-mynetwork-conf.json:/usr/src/app/dynamic-mynetwork-conf.json + command: > + cdk-erigon --zkevm.l1-cache-enabled=true --zkevm.sequencer-resequence=true --http.vhosts=* --http.corsdomain=* --ws --config=/usr/src/app/config.yaml + + xlayer-executor: + container_name: xlayer-executor + image: hermeznetwork/zkevm-prover:v9.0.0-RC1-fork.13 + platform: linux/amd64 + environment: + - EXPERIMENTAL_DOCKER_DESKTOP_FORCE_QEMU=1 + ports: + - 0.0.0.0:50061:50061 # MT + - 0.0.0.0:50071:50071 # Executor + volumes: + - ./config/test.stateless_executor.config.json:/usr/src/app/config.json + command: > + zkProver -c /usr/src/app/config.json + + xlayer-mock-l1-network: + container_name: xlayer-mock-l1-network + image: zjg555543/geth:fork13-v0.0.3 + ports: + - 8545:8545 + - 8546:8546 + command: + - "--http" + - "--http.api" + - "admin,eth,debug,miner,net,txpool,personal,web3" + - "--http.addr" + - "0.0.0.0" + - "--http.corsdomain" + - "*" + - "--http.vhosts" + - "*" + - "--ws" + - "--ws.origins" + - "*" + - "--ws.addr" + - "0.0.0.0" + - "--dev" + - "--dev.period" + - "1" + - "--datadir" + - "/geth_data" + - "--syncmode" + - "full" + - "--rpc.allow-unprotected-txs" \ No newline at end of file diff --git a/test/docker-compose.yml b/test/docker-compose.yml index 19340569c76..567d33202b3 100644 --- a/test/docker-compose.yml +++ b/test/docker-compose.yml @@ -112,26 +112,59 @@ services: - ./config/dynamic-mynetwork-chainspec.json:/usr/src/app/dynamic-mynetwork-chainspec.json - ./config/dynamic-mynetwork-conf.json:/usr/src/app/dynamic-mynetwork-conf.json command: > - cdk-erigon --http.vhosts=* --http.corsdomain=* --ws --config=/usr/src/app/config.yaml + cdk-erigon --zkevm.l1-cache-enabled=true --http.vhosts=* --http.corsdomain=* --ws --config=/usr/src/app/config.yaml - xlayer-recover: - container_name: xlayer-recover + xlayer-seq-resequence: + container_name: xlayer-seq-resequence image: cdk-erigon environment: - CDK_ERIGON_SEQUENCER=1 ports: - - 6059:6060 - - 8119:8545 - - 6899:6900 - - 9089:9095 + - 6060:6060 + - 8123:8545 + - 6900:6900 + - 9092:9095 volumes: - - ./data/recover/:/home/erigon/data/ + - ./data/seq/:/home/erigon/data/ + - ./config/test.erigon.seq.config.yaml:/usr/src/app/config.yaml + - ./config/dynamic-mynetwork-allocs.json:/usr/src/app/dynamic-mynetwork-allocs.json + - ./config/dynamic-mynetwork-chainspec.json:/usr/src/app/dynamic-mynetwork-chainspec.json + - ./config/dynamic-mynetwork-conf.json:/usr/src/app/dynamic-mynetwork-conf.json + command: > + cdk-erigon --zkevm.l1-cache-enabled=true --zkevm.sequencer-resequence=true --http.vhosts=* --http.corsdomain=* --ws --config=/usr/src/app/config.yaml + + xlayer-seq-recover: + container_name: xlayer-seq-recover + image: cdk-erigon + environment: + - CDK_ERIGON_SEQUENCER=1 + ports: + - 6060:6060 + - 8123:8545 + - 6900:6900 + - 9092:9095 + volumes: + - ./data/seq/:/home/erigon/data/ + - ./config/test.erigon.seq.config.yaml:/usr/src/app/config.yaml + - ./config/dynamic-mynetwork-allocs.json:/usr/src/app/dynamic-mynetwork-allocs.json + - ./config/dynamic-mynetwork-chainspec.json:/usr/src/app/dynamic-mynetwork-chainspec.json + - ./config/dynamic-mynetwork-conf.json:/usr/src/app/dynamic-mynetwork-conf.json + command: > + cdk-erigon --zkevm.l1-cache-enabled=true --zkevm.l1-sync-start-block=353 --zkevm.da-url=http://xlayer-da:8444 --http.vhosts=* --http.corsdomain=* --ws --config=/usr/src/app/config.yaml + + xlayer-seq-unwind: + container_name: xlayer-seq-unwind + image: cdk-erigon + environment: + - CDK_ERIGON_SEQUENCER=1 + volumes: + - ./data/seq/:/home/erigon/data/ - ./config/test.erigon.seq.config.yaml:/usr/src/app/config.yaml - ./config/dynamic-mynetwork-allocs.json:/usr/src/app/dynamic-mynetwork-allocs.json - ./config/dynamic-mynetwork-chainspec.json:/usr/src/app/dynamic-mynetwork-chainspec.json - ./config/dynamic-mynetwork-conf.json:/usr/src/app/dynamic-mynetwork-conf.json command: > - cdk-erigon --zkevm.l1-sync-start-block=353 --zkevm.da-url=http://xlayer-da:8444 --http.vhosts=* --http.corsdomain=* --ws --config=/usr/src/app/config.yaml + integration state_stages_zkevm --config=/usr/src/app/config.yaml --chain dynamic-mynetwork --datadir /home/erigon/data/ --unwind-batch-no=1 xlayer-rpc: container_name: xlayer-rpc @@ -152,6 +185,58 @@ services: command: > cdk-erigon --http.vhosts=* --http.corsdomain=* --ws --config=/usr/src/app/config.yaml + xlayer-lrp: + container_name: xlayer-lrp + image: cdk-erigon + environment: + - CDK_ERIGON_SEQUENCER=0 + ports: + - 6062:6060 + - 8126:8545 + - 6902:6900 + - 9093:9095 + volumes: + - ./data/lrp/:/home/erigon/data/ + - ./config/test.erigon.rpc.config.yaml:/usr/src/app/config.yaml + - ./config/dynamic-mynetwork-allocs.json:/usr/src/app/dynamic-mynetwork-allocs.json + - ./config/dynamic-mynetwork-chainspec.json:/usr/src/app/dynamic-mynetwork-chainspec.json + - ./config/dynamic-mynetwork-conf.json:/usr/src/app/dynamic-mynetwork-conf.json + command: > + cdk-erigon --http.vhosts=* --http.corsdomain=* --ws --config=/usr/src/app/config.yaml + + xlayer-lrp-unwind: + container_name: xlayer-lrp-unwind + image: cdk-erigon + environment: + - CDK_ERIGON_SEQUENCER=0 + volumes: + - ./data/lrp/:/home/erigon/data/ + - ./config/test.erigon.rpc.config.yaml:/usr/src/app/config.yaml + - ./config/dynamic-mynetwork-allocs.json:/usr/src/app/dynamic-mynetwork-allocs.json + - ./config/dynamic-mynetwork-chainspec.json:/usr/src/app/dynamic-mynetwork-chainspec.json + - ./config/dynamic-mynetwork-conf.json:/usr/src/app/dynamic-mynetwork-conf.json + command: > + integration state_stages_zkevm --config=/usr/src/app/config.yaml --chain dynamic-mynetwork --datadir /home/erigon/data/ --unwind-batch-no=1 + + xlayer-lrp-resequence: + container_name: xlayer-lrp-resequence + image: cdk-erigon + environment: + - CDK_ERIGON_SEQUENCER=1 + ports: + - 6062:6060 + - 8126:8545 + - 6902:6900 + - 9093:9095 + volumes: + - ./data/lrp/:/home/erigon/data/ + - ./config/test.erigon.seq.config.yaml:/usr/src/app/config.yaml + - ./config/dynamic-mynetwork-allocs.json:/usr/src/app/dynamic-mynetwork-allocs.json + - ./config/dynamic-mynetwork-chainspec.json:/usr/src/app/dynamic-mynetwork-chainspec.json + - ./config/dynamic-mynetwork-conf.json:/usr/src/app/dynamic-mynetwork-conf.json + command: > + cdk-erigon --zkevm.l1-cache-enabled=true --zkevm.sequencer-resequence=true --http.vhosts=* --http.corsdomain=* --ws --config=/usr/src/app/config.yaml + xlayer-mock-l1-network: container_name: xlayer-mock-l1-network image: zjg555543/geth:fork13-v0.0.3 diff --git a/test/fetch_blocks.sh b/test/fetch_blocks.sh new file mode 100755 index 00000000000..af2c1f6865b --- /dev/null +++ b/test/fetch_blocks.sh @@ -0,0 +1,42 @@ +#!/bin/bash + +# 检查是否提供了所有必要的参数 +if [ -z "$1" ] || [ -z "$2" ] || [ -z "$3" ] || [ -z "$4" ]; then + echo "Usage: $0 --to --rpc-url [--from ] [-f ]" + exit 1 +fi + +# 设置默认目标文件名 +START_BLOCKNUMBER=0 +TARGET_FILE="blocks_info.txt" + +# 读取参数 +while [[ "$#" -gt 0 ]]; do + case $1 in + --from) START_BLOCKNUMBER="$2"; shift ;; + --to) END_BLOCKNUMBER="$2"; shift ;; + --rpc-url) RPC_URL="$2"; shift ;; + -f) TARGET_FILE="$2"; shift ;; + *) echo "Unknown parameter passed: $1"; exit 1 ;; + esac + shift +done + +# 检查是否所有参数都已设置 +if [ -z "$END_BLOCKNUMBER" ] || [ -z "$RPC_URL" ]; then + echo "Usage: $0 --to --rpc-url [--from ] [-f ]" + exit 1 +fi + +# 清空目标文件 +> $TARGET_FILE + +# 循环获取区块信息 +for blocknumber in $(seq $START_BLOCKNUMBER $END_BLOCKNUMBER) +do + echo "Fetching block $blocknumber..." + cast block $blocknumber --rpc-url $RPC_URL >> $TARGET_FILE + echo "" >> $TARGET_FILE # 添加一个空行以分隔每个区块的信息 +done + +echo "All blocks fetched and saved to $TARGET_FILE" \ No newline at end of file diff --git a/test/process_log.py b/test/process_log.py new file mode 100644 index 00000000000..61a764a691f --- /dev/null +++ b/test/process_log.py @@ -0,0 +1,75 @@ +import re +import argparse +import logging +from datetime import datetime + +# 配置日志记录 +logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(levelname)s - %(message)s') + +def parse_time(line): + # 假设时间戳在行的开头，格式为 [MM-DD|HH:MM:SS.SSS] + match = re.search(r'\[(\d{2}-\d{2}\|\d{2}:\d{2}:\d{2}\.\d{3})\]', line) + if match: + return datetime.strptime(match.group(1), '%m-%d|%H:%M:%S.%f') + return None + +def main(log_file): + last_batch = None + highest_batch_in_data_stream = None + start_time = None + end_time = None + tx_count = 0 + + with open(log_file, 'r') as file: + lines = file.readlines() + + if len(lines) < 5: + logging.error("Log file does not contain enough lines.") + return + + # 读取 lastBatch 和 highestBatchInDataStream + last_batch = int(re.search(r'Last batch (\d+)', lines[0]).group(1)) + highest_batch_in_data_stream = int(re.search(r'Highest batch in data stream (\d+)', lines[1]).group(1)) + + # 计算需要读取的批次数量 + batches_to_read = highest_batch_in_data_stream - last_batch + + # 找到 startTime + for i, line in enumerate(lines): + if f"Read {batches_to_read} batches from data stream" in line: + start_time = parse_time(line) + break + + if not start_time: + logging.error("Start time not found.") + return + + # 计算 txCount + for line in lines[i:]: + if "Finish block" in line: + match = re.search(r'Finish block \d+ with (\d+) transactions', line) + if match: + tx_count += int(match.group(1)) + if "Resequencing completed." in line: + end_time = parse_time(line) + break + + if not end_time: + logging.error("End time not found.") + return + + # 计算 TPS + duration = (end_time - start_time).total_seconds() + tps = tx_count / duration if duration > 0 else 0 + + logging.info(f"{'Start Time:':<20} {start_time}") + logging.info(f"{'End Time:':<20} {end_time}") + logging.info(f"{'Total Transactions:':<20} {tx_count}") + logging.info(f"{'Duration:':<20} {duration} seconds") + logging.info(f"{'TPS:':<20} {tps}") + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description='Process log file to calculate TPS.') + parser.add_argument('log_file', type=str, help='Path to the log file') + args = parser.parse_args() + main(args.log_file) \ No newline at end of file From 1a4ac07fe17e0dc2071f1c7d46e895246d835c18 Mon Sep 17 00:00:00 2001 From: KyrinCode Date: Thu, 6 Feb 2025 16:55:33 +0800 Subject: [PATCH 3/6] Add specifying toBatchNo for lrp feature by adding a new zkevm.sequencer-resequence-halt-on-batch-number flag to the system, which doesn't interfere with the rest functionalities. --- cmd/utils/flags.go | 5 ++++ eth/ethconfig/config_zkevm.go | 3 ++- test/Makefile | 16 ++++++----- test/docker-compose-lrp.yml | 4 +-- test/process_log.py | 27 +++++++++++++------ turbo/cli/default_flags.go | 2 +- turbo/cli/flags_zkevm.go | 1 + .../stage_sequence_execute_resequence.go | 20 +++++++++++--- 8 files changed, 56 insertions(+), 22 deletions(-) diff --git a/cmd/utils/flags.go b/cmd/utils/flags.go index 82f887e45d8..414dc085317 100644 --- a/cmd/utils/flags.go +++ b/cmd/utils/flags.go @@ -576,6 +576,11 @@ var ( Usage: "When enabled, the sequencer will automatically resequence unseen batches stored in data stream", Value: false, } + SequencerResequenceHaltOnBatchNumber = cli.Uint64Flag{ + Name: "zkevm.sequencer-resequence-halt-on-batch-number", + Usage: "Halt the sequencer on this batch number when resequencing", + Value: 0, + } SequencerResequenceStrict = cli.BoolFlag{ Name: "zkevm.sequencer-resequence-strict", Usage: "Strictly resequence the rolledback batches", diff --git a/eth/ethconfig/config_zkevm.go b/eth/ethconfig/config_zkevm.go index f1233d4ce5f..47731184e24 100644 --- a/eth/ethconfig/config_zkevm.go +++ b/eth/ethconfig/config_zkevm.go @@ -43,6 +43,7 @@ type Zk struct { SequencerTimeoutOnEmptyTxPool time.Duration SequencerHaltOnBatchNumber uint64 SequencerResequence bool + SequencerResequenceHaltOnBatchNumber uint64 SequencerResequenceStrict bool SequencerResequenceReuseL1InfoIndex bool ExecutorUrls []string @@ -90,7 +91,7 @@ type Zk struct { // For X Layer XLayer XLayerConfig - + InitialBatchCfgFile string ACLPrintHistory int InfoTreeUpdateInterval time.Duration diff --git a/test/Makefile b/test/Makefile index a2a23030b40..3e2037b12c8 100644 --- a/test/Makefile +++ b/test/Makefile @@ -162,12 +162,15 @@ lrp-run: build-docker $(RUN_DOCKER_LRP_L1_NETWORK) $(RUN_DOCKER_LRP_STATELESS_EXECUTOR) sleep 3 - @read -p "Enter unwind to batch number: " UNWIND_BATCH_NO; \ - if [ "$(shell uname)" = "Darwin" ]; then \ - sed -i '' "s/--unwind-batch-no=[0-9]*/--unwind-batch-no=$$UNWIND_BATCH_NO/" docker-compose-lrp.yml; \ - else \ - sed -i "s/--unwind-batch-no=[0-9]*/--unwind-batch-no=$$UNWIND_BATCH_NO/" docker-compose-lrp.yml; \ - fi + @read -p "Enter unwind batch number: " UNWIND_BATCH_NO; \ + read -p "Enter halt on batch number: " HALT_BATCH_NO; \ + if [ "$(shell uname)" = "Darwin" ]; then \ + sed -i '' "s/--unwind-batch-no=[0-9]*/--unwind-batch-no=$$UNWIND_BATCH_NO/" docker-compose-lrp.yml; \ + sed -i '' "s/--zkevm.sequencer-resequence-halt-on-batch-number=[0-9]*/--zkevm.sequencer-resequence-halt-on-batch-number=$$HALT_BATCH_NO/" docker-compose-lrp.yml; \ + else \ + sed -i "s/--unwind-batch-no=[0-9]*/--unwind-batch-no=$$UNWIND_BATCH_NO/" docker-compose-lrp.yml; \ + sed -i "s/--zkevm.sequencer-resequence-halt-on-batch-number=[0-9]*/--zkevm.sequencer-resequence-halt-on-batch-number=$$HALT_BATCH_NO/" docker-compose-lrp.yml; \ + fi $(RUN_DOCKER_LRP_UNWIND) sleep 20 $(RUN_DOCKER_LRP_RESEQUENCE) @@ -176,7 +179,6 @@ lrp-run: build-docker lrp-stop: $(STOP_LRP) - TARGET_LOG_FILE := filtered.log .PHONY: lrp-check diff --git a/test/docker-compose-lrp.yml b/test/docker-compose-lrp.yml index a078e287ea4..612c7711b20 100644 --- a/test/docker-compose-lrp.yml +++ b/test/docker-compose-lrp.yml @@ -57,7 +57,7 @@ services: --config=/usr/src/app/config.yaml \ --chain dynamic-mynetwork \ --datadir /home/erigon/data/ \ - --unwind-batch-no=10 + --unwind-batch-no=1200 xlayer-lrp-resequence: container_name: xlayer-lrp-resequence @@ -76,7 +76,7 @@ services: - ./config/dynamic-mynetwork-chainspec.json:/usr/src/app/dynamic-mynetwork-chainspec.json - ./config/dynamic-mynetwork-conf.json:/usr/src/app/dynamic-mynetwork-conf.json command: > - cdk-erigon --zkevm.l1-cache-enabled=true --zkevm.sequencer-resequence=true --http.vhosts=* --http.corsdomain=* --ws --config=/usr/src/app/config.yaml + cdk-erigon --zkevm.l1-cache-enabled=true --zkevm.sequencer-resequence=true --zkevm.sequencer-resequence-halt-on-batch-number=1250 --http.vhosts=* --http.corsdomain=* --ws --config=/usr/src/app/config.yaml xlayer-executor: container_name: xlayer-executor diff --git a/test/process_log.py b/test/process_log.py index 61a764a691f..fcf4cc2bf95 100644 --- a/test/process_log.py +++ b/test/process_log.py @@ -10,7 +10,9 @@ def parse_time(line): # 假设时间戳在行的开头，格式为 [MM-DD|HH:MM:SS.SSS] match = re.search(r'\[(\d{2}-\d{2}\|\d{2}:\d{2}:\d{2}\.\d{3})\]', line) if match: - return datetime.strptime(match.group(1), '%m-%d|%H:%M:%S.%f') + current_year = datetime.now().year + parsed_time = datetime.strptime(match.group(1), '%m-%d|%H:%M:%S.%f') + return parsed_time.replace(year=current_year) return None def main(log_file): @@ -29,10 +31,16 @@ def main(log_file): # 读取 lastBatch 和 highestBatchInDataStream last_batch = int(re.search(r'Last batch (\d+)', lines[0]).group(1)) - highest_batch_in_data_stream = int(re.search(r'Highest batch in data stream (\d+)', lines[1]).group(1)) + # highest_batch_in_data_stream = int(re.search(r'highest batch in datastream (\d+)', lines[0]).group(1)) + halt_batch = int(re.search(r'resequencing from batch \d+ to (\d+)', lines[0]).group(1)) # 计算需要读取的批次数量 - batches_to_read = highest_batch_in_data_stream - last_batch + batches_to_read = halt_batch - last_batch + + # 计算启动时间 + first_line_time = parse_time(lines[0]) + second_line_time = parse_time(lines[1]) + startup_duration = (second_line_time - first_line_time).total_seconds() # 找到 startTime for i, line in enumerate(lines): @@ -62,11 +70,14 @@ def main(log_file): duration = (end_time - start_time).total_seconds() tps = tx_count / duration if duration > 0 else 0 - logging.info(f"{'Start Time:':<20} {start_time}") - logging.info(f"{'End Time:':<20} {end_time}") - logging.info(f"{'Total Transactions:':<20} {tx_count}") - logging.info(f"{'Duration:':<20} {duration} seconds") - logging.info(f"{'TPS:':<20} {tps}") + logging.info(f"{'From Batch:':<25} {last_batch+1}") + logging.info(f"{'To Batch:':<25} {halt_batch}") + logging.info(f"{'Startup Duration:':<25} {startup_duration} seconds") + logging.info(f"{'Start Time:':<25} {start_time}") + logging.info(f"{'End Time:':<25} {end_time}") + logging.info(f"{'Total Transactions:':<25} {tx_count}") + logging.info(f"{'Re-sequencing Duration:':<25} {duration} seconds") + logging.info(f"{'TPS:':<25} {tps}") if __name__ == "__main__": parser = argparse.ArgumentParser(description='Process log file to calculate TPS.') diff --git a/turbo/cli/default_flags.go b/turbo/cli/default_flags.go index e3b06026a38..bfec3c6de40 100644 --- a/turbo/cli/default_flags.go +++ b/turbo/cli/default_flags.go @@ -211,6 +211,7 @@ var DefaultFlags = []cli.Flag{ &utils.SequencerTimeoutOnEmptyTxPool, &utils.SequencerHaltOnBatchNumber, &utils.SequencerResequence, + &utils.SequencerResequenceHaltOnBatchNumber, &utils.SequencerResequenceStrict, &utils.SequencerResequenceReuseL1InfoIndex, &utils.ExecutorUrls, @@ -287,7 +288,6 @@ var DefaultFlags = []cli.Flag{ &utils.BadBatches, &utils.InitialBatchCfgFile, - // X Layer Flags &utils.AllowInternalTransactions, &utils.TxPoolPackBatchSpecialList, diff --git a/turbo/cli/flags_zkevm.go b/turbo/cli/flags_zkevm.go index 20c09205fac..6fb8474c2d7 100644 --- a/turbo/cli/flags_zkevm.go +++ b/turbo/cli/flags_zkevm.go @@ -186,6 +186,7 @@ func ApplyFlagsForZkConfig(ctx *cli.Context, cfg *ethconfig.Config) { SequencerTimeoutOnEmptyTxPool: sequencerTimeoutOnEmptyTxPool, SequencerHaltOnBatchNumber: ctx.Uint64(utils.SequencerHaltOnBatchNumber.Name), SequencerResequence: ctx.Bool(utils.SequencerResequence.Name), + SequencerResequenceHaltOnBatchNumber: ctx.Uint64(utils.SequencerResequenceHaltOnBatchNumber.Name), SequencerResequenceStrict: ctx.Bool(utils.SequencerResequenceStrict.Name), SequencerResequenceReuseL1InfoIndex: ctx.Bool(utils.SequencerResequenceReuseL1InfoIndex.Name), ExecutorUrls: strings.Split(strings.ReplaceAll(ctx.String(utils.ExecutorUrls.Name), " ", ""), ","), diff --git a/zk/stages/stage_sequence_execute_resequence.go b/zk/stages/stage_sequence_execute_resequence.go index d7fa2e18ab7..0212a57d64b 100644 --- a/zk/stages/stage_sequence_execute_resequence.go +++ b/zk/stages/stage_sequence_execute_resequence.go @@ -21,18 +21,32 @@ func resequence( panic(fmt.Sprintf("[%s] The node need re-sequencing but this option is disabled.", s.LogPrefix())) } - log.Info(fmt.Sprintf("[%s] Last batch %d is lower than highest batch in datastream %d, resequencing...", s.LogPrefix(), lastBatch, highestBatchInDs)) + haltBatch := uint64(0) + if cfg.zk.SequencerResequenceHaltOnBatchNumber > 0 { + haltBatch = cfg.zk.SequencerResequenceHaltOnBatchNumber + if haltBatch <= lastBatch { + panic(fmt.Sprintf("[%s] The zkevm.sequencer-resequence-halt-on-batch-number is set lower than the last batch number.", s.LogPrefix())) + } else if haltBatch > highestBatchInDs { + panic(fmt.Sprintf("[%s] The zkevm.sequencer-resequence-halt-on-batch-number is set higher than the highest batch in datastream.", s.LogPrefix())) + } + } else { + haltBatch = highestBatchInDs + } - batches, err := cfg.dataStreamServer.ReadBatches(lastBatch+1, highestBatchInDs) + log.Info(fmt.Sprintf("[%s] Last batch %d is lower than highest batch in datastream %d, resequencing from batch %d to %d ...", s.LogPrefix(), lastBatch, highestBatchInDs, lastBatch+1, haltBatch)) + + batches, err := cfg.dataStreamServer.ReadBatches(lastBatch+1, haltBatch) if err != nil { return err } + log.Info(fmt.Sprintf("[%s] Read %d batches from data stream", s.LogPrefix(), len(batches))) + if err = cfg.dataStreamServer.UnwindToBatchStart(lastBatch + 1); err != nil { return err } - log.Info(fmt.Sprintf("[%s] Resequence from batch %d to %d in data stream", s.LogPrefix(), lastBatch+1, highestBatchInDs)) + log.Info(fmt.Sprintf("[%s] Resequence from batch %d to %d in data stream", s.LogPrefix(), lastBatch+1, haltBatch)) for _, batch := range batches { batchJob := NewResequenceBatchJob(batch) subBatchCount := 0 From e50a4ce2d1cfa4c038e25694c35b3e9564134e3a Mon Sep 17 00:00:00 2001 From: KyrinCode Date: Fri, 7 Feb 2025 21:47:03 +0800 Subject: [PATCH 4/6] update --- .github/workflows/ci_zkevm.yml | 8 ++++---- .github/workflows/nightly-ansible.yml | 2 +- .github/workflows/nightly-eth-bench.yml | 2 +- .github/workflows/scheduler_xlayer.yml | 4 ++-- .github/workflows/test-resequence.yml | 4 ++-- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci_zkevm.yml b/.github/workflows/ci_zkevm.yml index df2a8497e55..3dd24d3f2b5 100644 --- a/.github/workflows/ci_zkevm.yml +++ b/.github/workflows/ci_zkevm.yml @@ -146,7 +146,7 @@ jobs: docker run --network=host --volume "./tmp/:/config/" --env BRIDGE_TEST_CONFIG_FILE=/config/test.toml bridge-e2e-realnetwork-erc20 - name: Upload logs - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: evm-rpc-tests-logs-${{ github.run_id }} path: ./cdk-erigon/logs/evm-rpc-tests.log @@ -166,7 +166,7 @@ jobs: - name: Upload logs if: failure() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: logs_${{ github.run_id }} path: ./kurtosis-cdk/ci_logs @@ -190,7 +190,7 @@ jobs: run: /usr/local/bin/polycli loadtest --rpc-url "$(kurtosis port print cdk-v1 cdk-erigon-rpc-001 rpc)" --private-key "0x12d7de8621a77640c9241b2595ba78ce443d05e94090365ab3bb5e19df82c625" --verbosity 700 --requests 500 --rate-limit 50 --mode uniswapv3 --legacy - name: Upload logs - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: evm-rpc-tests-logs-${{ github.run_id }} path: ./cdk-erigon/logs/evm-rpc-tests.log @@ -206,7 +206,7 @@ jobs: - name: Upload logs if: failure() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: logs_${{ github.run_id }} path: ./kurtosis-cdk/ci_logs diff --git a/.github/workflows/nightly-ansible.yml b/.github/workflows/nightly-ansible.yml index d5b55d307a9..a7b4c1836c0 100644 --- a/.github/workflows/nightly-ansible.yml +++ b/.github/workflows/nightly-ansible.yml @@ -37,7 +37,7 @@ jobs: cd zk/tests/ansible/evm-rpc-tests && docker compose build --quiet && docker compose up --exit-code-from ansible - name: Upload logs - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: evm-rpc-tests-logs-${{ matrix.nodes.name }} path: ./logs/evm-rpc-tests.log diff --git a/.github/workflows/nightly-eth-bench.yml b/.github/workflows/nightly-eth-bench.yml index 708608e4b16..875c43a59fd 100644 --- a/.github/workflows/nightly-eth-bench.yml +++ b/.github/workflows/nightly-eth-bench.yml @@ -99,7 +99,7 @@ jobs: - name: Upload benchmark log if: always() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: benchmark-log-${{ matrix.profile_name }} path: ./eth-bench/bench_${{ matrix.profile_name }}.log diff --git a/.github/workflows/scheduler_xlayer.yml b/.github/workflows/scheduler_xlayer.yml index 3f0b170b8f1..763fa7f8720 100644 --- a/.github/workflows/scheduler_xlayer.yml +++ b/.github/workflows/scheduler_xlayer.yml @@ -103,7 +103,7 @@ jobs: docker run --network=host --volume "./tmp/:/config/" --env BRIDGE_TEST_CONFIG_FILE=/config/test.toml bridge-e2e-realnetwork-erc20 - name: Upload logs - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: evm-rpc-tests-logs-${{ github.run_id }} path: ./cdk-erigon/logs/evm-rpc-tests.log @@ -122,7 +122,7 @@ jobs: - name: Upload logs if: failure() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: logs_${{ github.run_id }} path: ./kurtosis-cdk/ci_logs diff --git a/.github/workflows/test-resequence.yml b/.github/workflows/test-resequence.yml index cc58f71d5d9..50be20d386b 100644 --- a/.github/workflows/test-resequence.yml +++ b/.github/workflows/test-resequence.yml @@ -32,7 +32,7 @@ jobs: run: .github/scripts/test_resequence.sh - name: Upload logs - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: evm-rpc-tests-logs-${{ github.run_id }} path: ./cdk-erigon/logs/evm-rpc-tests.log @@ -51,7 +51,7 @@ jobs: kurtosis service logs cdk-v1 zkevm-bridge-service-001 --all > zkevm-bridge-service-001.log - name: Upload logs if: failure() - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: name: logs_${{ github.run_id }} path: ./kurtosis-cdk/ci_logs \ No newline at end of file From bc42f8565b816ad05dcd99cf72e208d8dd108627 Mon Sep 17 00:00:00 2001 From: KyrinCode Date: Fri, 7 Feb 2025 21:48:23 +0800 Subject: [PATCH 5/6] mainnet unwind --- .dockerignore | 4 +- .gitignore | 4 +- cmd/prometheus/prometheus.yml | 1 + test/Makefile | 19 ++++++++ test/config/xlayerconfig-mainnet.yaml | 66 +++++++++++++++++++++++++++ test/docker-compose-lrp.yml | 54 +++++++++++++--------- 6 files changed, 125 insertions(+), 23 deletions(-) create mode 100644 test/config/xlayerconfig-mainnet.yaml diff --git a/.dockerignore b/.dockerignore index ef4c067d97e..946af808d50 100644 --- a/.dockerignore +++ b/.dockerignore @@ -11,4 +11,6 @@ vendor cache.db test -.git \ No newline at end of file +.git + +mainnet \ No newline at end of file diff --git a/.gitignore b/.gitignore index fbec92a780b..34a32c85f81 100644 --- a/.gitignore +++ b/.gitignore @@ -115,10 +115,12 @@ vendor data/ test/sqlite start.sh -xlayerconfig-mainnet.yaml xlayerconfig-testnet.yaml **/cover.out **/cover.html +test/mainnet +test/*.zip +test/*.tar.gz datadir zk/tests/unwinds/datastream/hermez-dynamic-integration8-datastream diff --git a/cmd/prometheus/prometheus.yml b/cmd/prometheus/prometheus.yml index db855a991dc..3fdb6470732 100644 --- a/cmd/prometheus/prometheus.yml +++ b/cmd/prometheus/prometheus.yml @@ -11,6 +11,7 @@ scrape_configs: - targets: - xlayer-rpc:9095 - xlayer-seq:9095 + - xlayer-mainnet-lrp-resequence:9095 - job_name: erigon metrics_path: /debug/metrics/prometheus diff --git a/test/Makefile b/test/Makefile index 3e2037b12c8..42e357f5d45 100644 --- a/test/Makefile +++ b/test/Makefile @@ -149,11 +149,15 @@ test-1: stop all ## Runs group 1 e2e tests checking race conditions DOCKER_COMPOSE_LRP := docker compose -f docker-compose-lrp.yml DOCKER_LRP_UNWIND := xlayer-lrp-unwind DOCKER_LRP_RESEQUENCE := xlayer-lrp-resequence +DOCKER_MAINNET_LRP_UNWIND := xlayer-mainnet-lrp-unwind +DOCKER_MAINNET_LRP_RESEQUENCE := xlayer-mainnet-lrp-resequence RUN_DOCKER_LRP_STATELESS_EXECUTOR := $(DOCKER_COMPOSE_LRP) up -d $(DOCKER_STATELESS_EXECUTOR) RUN_DOCKER_LRP_L1_NETWORK := $(DOCKER_COMPOSE_LRP) up -d $(DOCKER_L1_NETWORK) RUN_DOCKER_LRP_UNWIND := $(DOCKER_COMPOSE_LRP) up -d $(DOCKER_LRP_UNWIND) RUN_DOCKER_LRP_RESEQUENCE := $(DOCKER_COMPOSE_LRP) up -d $(DOCKER_LRP_RESEQUENCE) +RUN_DOCKER_MAINNET_LRP_UNWIND := $(DOCKER_COMPOSE_LRP) up -d $(DOCKER_MAINNET_LRP_UNWIND) +RUN_DOCKER_MAINNET_LRP_RESEQUENCE := $(DOCKER_COMPOSE_LRP) up -d $(DOCKER_MAINNET_LRP_RESEQUENCE) STOP_LRP := $(DOCKER_COMPOSE_LRP) down --remove-orphans; sleep 3; rm -rf data @@ -175,6 +179,21 @@ lrp-run: build-docker sleep 20 $(RUN_DOCKER_LRP_RESEQUENCE) +.PHONY: lrp-mainnet +lrp-mainnet: build-docker + @read -p "Enter unwind batch number: " UNWIND_BATCH_NO; \ + read -p "Enter halt on batch number: " HALT_BATCH_NO; \ + if [ "$(shell uname)" = "Darwin" ]; then \ + sed -i '' "s/--unwind-batch-no=[0-9]*/--unwind-batch-no=$$UNWIND_BATCH_NO/" docker-compose-lrp.yml; \ + sed -i '' "s/--zkevm.sequencer-resequence-halt-on-batch-number=[0-9]*/--zkevm.sequencer-resequence-halt-on-batch-number=$$HALT_BATCH_NO/" docker-compose-lrp.yml; \ + else \ + sed -i "s/--unwind-batch-no=[0-9]*/--unwind-batch-no=$$UNWIND_BATCH_NO/" docker-compose-lrp.yml; \ + sed -i "s/--zkevm.sequencer-resequence-halt-on-batch-number=[0-9]*/--zkevm.sequencer-resequence-halt-on-batch-number=$$HALT_BATCH_NO/" docker-compose-lrp.yml; \ + fi + $(RUN_DOCKER_MAINNET_LRP_UNWIND) + sleep 30 + $(RUN_DOCKER_MAINNET_LRP_RESEQUENCE) + .PHONY: lrp-stop lrp-stop: $(STOP_LRP) diff --git a/test/config/xlayerconfig-mainnet.yaml b/test/config/xlayerconfig-mainnet.yaml new file mode 100644 index 00000000000..9949be54402 --- /dev/null +++ b/test/config/xlayerconfig-mainnet.yaml @@ -0,0 +1,66 @@ +datadir: /home/erigon/data/ +chain: xlayer-mainnet +http: true +private.api.addr: localhost:9091 +zkevm.l2-chain-id: 196 +zkevm.l2-sequencer-rpc-url: https://rpc.xlayer.tech +zkevm.l2-datastreamer-url: stream.xlayer.tech:8800 +zkevm.l1-chain-id: 1 +zkevm.l1-rpc-url: https://rpc.ankr.com/eth/578c95407e7831f0ac1ef79cacae294dc9bf8307121ca9fffaf1e556a5cca662 +#zkevm.increment-tree-always: true + +#debug.limit: 8554110 + +zkevm.executor-strict: false +zkevm.witness-full: false + +zkevm.sequencer-block-seal-time: "3s" +zkevm.sequencer-batch-seal-time: "10s" +zkevm.sequencer-batch-sleep-duration: "0s" + +zkevm.data-stream-host: "localhost" +zkevm.data-stream-port: 6900 +zkevm.default-gas-price: 1 +zkevm.max-gas-price: 0 +zkevm.gas-price-factor: 0.000001 +zkevm.allow-free-transactions: true + +txpool.packbatchspeciallist : ["0xf39fd6e51aad88f6f4ce6ab8827279cfffb92266", "0x8f8E2d6cF621f30e9a11309D6A56A876281Fd534", "0xa03666Fb51Aa9aD2DE70e0434072A007b3C91A9E"] + +zkevm.address-sequencer: "0xAF9d27ffe4d51eD54AC8eEc78f2785D7E11E5ab1" +zkevm.address-zkevm: "0x2B0ee28D4D51bC9aDde5E58E295873F61F4a0507" +zkevm.address-rollup: "0x5132A183E9F3CB7C848b0AAC5Ae0c4f0491B7aB2" +zkevm.address-ger-manager: "0x580bda1e7A0CFAe92Fa7F6c20A3794F169CE3CFb" + +zkevm.l1-rollup-id: 3 +zkevm.l1-first-block: 19218658 +zkevm.l1-block-range: 2000 +zkevm.l1-query-delay: 1000 +zkevm.datastream-version: 3 + +http.api: [eth, debug, net, trace, web3, erigon, zkevm] +http.addr: 0.0.0.0 +http.port: 8545 + +gpo.type: "follower" +gpo.update-period: "300s" +gpo.factor: 0.01 +gpo.kafka-url: "0.0.0.0" +gpo.topic: "explorer" +gpo.group-id: "web3" +gpo.blocks: 3 +gpo.default-l1-coin-price: 2000 +gpo.default-l2-coin-price: 50 +gpo.gas-price-usdt: 0.000000476190476 +gpo.congestion-threshold: 0 + +pprof: true +pprof.port: 6060 +pprof.addr: 0.0.0.0 + +metrics: true +metrics.addr: "0.0.0.0" +metrics.port: 9095 + +yieldsize: 35 +zkevm.seal-batch-immediately-on-overflow: true \ No newline at end of file diff --git a/test/docker-compose-lrp.yml b/test/docker-compose-lrp.yml index 612c7711b20..13d495cdc00 100644 --- a/test/docker-compose-lrp.yml +++ b/test/docker-compose-lrp.yml @@ -3,25 +3,6 @@ networks: default: name: erigon-lrp services: - # xlayer-lrp-sequence: - # container_name: xlayer-lrp-sequence - # image: cdk-erigon - # environment: - # - CDK_ERIGON_SEQUENCER=1 - # ports: - # - 6062:6060 - # - 8126:8545 - # - 6902:6900 - # - 9093:9095 - # volumes: - # - ./data/lrp/:/home/erigon/data/ - # - ./config/test.erigon.seq.config.yaml:/usr/src/app/config.yaml - # - ./config/dynamic-mynetwork-allocs.json:/usr/src/app/dynamic-mynetwork-allocs.json - # - ./config/dynamic-mynetwork-chainspec.json:/usr/src/app/dynamic-mynetwork-chainspec.json - # - ./config/dynamic-mynetwork-conf.json:/usr/src/app/dynamic-mynetwork-conf.json - # command: > - # cdk-erigon --zkevm.l1-cache-enabled=true --http.vhosts=* --http.corsdomain=* --ws --config=/usr/src/app/config.yaml - xlayer-lrp: container_name: xlayer-lrp image: cdk-erigon @@ -57,7 +38,7 @@ services: --config=/usr/src/app/config.yaml \ --chain dynamic-mynetwork \ --datadir /home/erigon/data/ \ - --unwind-batch-no=1200 + --unwind-batch-no=136800 xlayer-lrp-resequence: container_name: xlayer-lrp-resequence @@ -76,7 +57,38 @@ services: - ./config/dynamic-mynetwork-chainspec.json:/usr/src/app/dynamic-mynetwork-chainspec.json - ./config/dynamic-mynetwork-conf.json:/usr/src/app/dynamic-mynetwork-conf.json command: > - cdk-erigon --zkevm.l1-cache-enabled=true --zkevm.sequencer-resequence=true --zkevm.sequencer-resequence-halt-on-batch-number=1250 --http.vhosts=* --http.corsdomain=* --ws --config=/usr/src/app/config.yaml + cdk-erigon --zkevm.l1-cache-enabled=true --zkevm.sequencer-resequence=true --zkevm.sequencer-resequence-halt-on-batch-number=136911 --http.vhosts=* --http.corsdomain=* --ws --config=/usr/src/app/config.yaml + + xlayer-mainnet-lrp-unwind: + container_name: xlayer-mainnet-lrp-unwind + image: cdk-erigon + environment: + - CDK_ERIGON_SEQUENCER=0 + volumes: + - ./mainnet/seq/:/home/erigon/data/ + - ./config/xlayerconfig-mainnet.yaml:/usr/src/app/config.yaml + command: | + integration state_stages_zkevm \ + --config=/usr/src/app/config.yaml \ + --chain xlayer-mainnet \ + --datadir /home/erigon/data/ \ + --unwind-batch-no=136800 + + xlayer-mainnet-lrp-resequence: + container_name: xlayer-mainnet-lrp-resequence + image: cdk-erigon + environment: + - CDK_ERIGON_SEQUENCER=1 + ports: + - 6060:6060 + - 8126:8545 + - 6902:6900 + - 9093:9095 + volumes: + - ./mainnet/seq:/home/erigon/data/ + - ./config/xlayerconfig-mainnet.yaml:/usr/src/app/config.yaml + command: > + cdk-erigon --zkevm.sequencer-resequence=true --zkevm.sequencer-resequence-halt-on-batch-number=136911 --http.vhosts=* --http.corsdomain=* --ws --config=/usr/src/app/config.yaml xlayer-executor: container_name: xlayer-executor From 07d2472f73c85b88d9a0ba58665c45c23ef883e3 Mon Sep 17 00:00:00 2001 From: KyrinCode Date: Sat, 8 Feb 2025 13:12:55 +0800 Subject: [PATCH 6/6] support mainnet lrp --- test/Makefile | 70 +++++++++++-------- test/docker-compose-lrp.yml | 4 +- test/process_log.py | 2 +- .../stage_sequence_execute_resequence.go | 36 +++++++++- 4 files changed, 79 insertions(+), 33 deletions(-) diff --git a/test/Makefile b/test/Makefile index 42e357f5d45..16e85c896d4 100644 --- a/test/Makefile +++ b/test/Makefile @@ -161,11 +161,12 @@ RUN_DOCKER_MAINNET_LRP_RESEQUENCE := $(DOCKER_COMPOSE_LRP) up -d $(DOCKER_MAINNE STOP_LRP := $(DOCKER_COMPOSE_LRP) down --remove-orphans; sleep 3; rm -rf data -.PHONY: lrp-run -lrp-run: build-docker - $(RUN_DOCKER_LRP_L1_NETWORK) - $(RUN_DOCKER_LRP_STATELESS_EXECUTOR) - sleep 3 +UNWIND_LOG_FILE := unwind.log +RESEQUENCE_LOG_FILE := resequence.log +RESEQUENCE_FILTERED_LOG_FILE := resequence_filtered.log + +.PHONY: configure-lrp +configure-lrp: @read -p "Enter unwind batch number: " UNWIND_BATCH_NO; \ read -p "Enter halt on batch number: " HALT_BATCH_NO; \ if [ "$(shell uname)" = "Darwin" ]; then \ @@ -175,35 +176,48 @@ lrp-run: build-docker sed -i "s/--unwind-batch-no=[0-9]*/--unwind-batch-no=$$UNWIND_BATCH_NO/" docker-compose-lrp.yml; \ sed -i "s/--zkevm.sequencer-resequence-halt-on-batch-number=[0-9]*/--zkevm.sequencer-resequence-halt-on-batch-number=$$HALT_BATCH_NO/" docker-compose-lrp.yml; \ fi + +.PHONY: lrp-run +lrp-run: build-docker configure-lrp + $(RUN_DOCKER_LRP_L1_NETWORK) + $(RUN_DOCKER_LRP_STATELESS_EXECUTOR) + sleep 3 $(RUN_DOCKER_LRP_UNWIND) - sleep 20 + docker wait $(DOCKER_LRP_UNWIND) + docker logs $(DOCKER_LRP_UNWIND) > $(UNWIND_LOG_FILE) 2>&1 $(RUN_DOCKER_LRP_RESEQUENCE) +.PHONY: lrp-check +lrp-check: ## Filter logs for LRP & Compute TPS & Check stateroots (configure debugToolsConfig.yaml) + docker logs $(DOCKER_LRP_RESEQUENCE) > $(RESEQUENCE_LOG_FILE) 2>&1 + docker logs $(DOCKER_LRP_RESEQUENCE) 2>&1 | grep "\[5/13 Execution\]" > $(RESEQUENCE_FILTERED_LOG_FILE) + python process_log.py $(RESEQUENCE_FILTERED_LOG_FILE) + go run ../zk/debug_tools/rpc-blockhashes-compare/main.go + .PHONY: lrp-mainnet -lrp-mainnet: build-docker - @read -p "Enter unwind batch number: " UNWIND_BATCH_NO; \ - read -p "Enter halt on batch number: " HALT_BATCH_NO; \ - if [ "$(shell uname)" = "Darwin" ]; then \ - sed -i '' "s/--unwind-batch-no=[0-9]*/--unwind-batch-no=$$UNWIND_BATCH_NO/" docker-compose-lrp.yml; \ - sed -i '' "s/--zkevm.sequencer-resequence-halt-on-batch-number=[0-9]*/--zkevm.sequencer-resequence-halt-on-batch-number=$$HALT_BATCH_NO/" docker-compose-lrp.yml; \ - else \ - sed -i "s/--unwind-batch-no=[0-9]*/--unwind-batch-no=$$UNWIND_BATCH_NO/" docker-compose-lrp.yml; \ - sed -i "s/--zkevm.sequencer-resequence-halt-on-batch-number=[0-9]*/--zkevm.sequencer-resequence-halt-on-batch-number=$$HALT_BATCH_NO/" docker-compose-lrp.yml; \ - fi +lrp-mainnet: build-docker configure-lrp $(RUN_DOCKER_MAINNET_LRP_UNWIND) - sleep 30 + docker wait $(DOCKER_MAINNET_LRP_UNWIND) + docker logs $(DOCKER_MAINNET_LRP_UNWIND) > $(UNWIND_LOG_FILE) 2>&1 $(RUN_DOCKER_MAINNET_LRP_RESEQUENCE) -.PHONY: lrp-stop -lrp-stop: - $(STOP_LRP) +.PHONY: lrp-mainnet-unwind +lrp-mainnet-unwind: build-docker configure-lrp + $(RUN_DOCKER_MAINNET_LRP_UNWIND) + docker wait $(DOCKER_MAINNET_LRP_UNWIND) + docker logs $(DOCKER_MAINNET_LRP_UNWIND) > $(UNWIND_LOG_FILE) 2>&1 -TARGET_LOG_FILE := filtered.log +.PHONY: lrp-mainnet-post-unwind +lrp-mainnet-post-unwind: build-docker + $(RUN_DOCKER_MAINNET_LRP_RESEQUENCE) -.PHONY: lrp-check -lrp-check: ## Filter logs for LRP & Compute TPS & Check stateroots (configure debugToolsConfig.yaml) - docker logs $(DOCKER_LRP_RESEQUENCE) 2>&1 | grep "\[5/13 Execution\]" > $(TARGET_LOG_FILE) - sleep 3 - python process_log.py $(TARGET_LOG_FILE) - sleep 3 - go run ../zk/debug_tools/rpc-blockhashes-compare/main.go \ No newline at end of file +.PHONY: lrp-mainnet-check +lrp-mainnet-check: ## Filter logs for LRP & Compute TPS & Check stateroots (configure debugToolsConfig.yaml) + docker logs $(DOCKER_MAINNET_LRP_RESEQUENCE) > $(RESEQUENCE_LOG_FILE) 2>&1 + docker logs $(DOCKER_MAINNET_LRP_RESEQUENCE) 2>&1 | grep "\[5/13 Execution\]" > $(RESEQUENCE_FILTERED_LOG_FILE) + python process_log.py $(RESEQUENCE_FILTERED_LOG_FILE) + go run ../zk/debug_tools/rpc-blockhashes-compare/main.go + +.PHONY: lrp-stop +lrp-stop: + $(STOP_LRP) \ No newline at end of file diff --git a/test/docker-compose-lrp.yml b/test/docker-compose-lrp.yml index 13d495cdc00..b660e4881d2 100644 --- a/test/docker-compose-lrp.yml +++ b/test/docker-compose-lrp.yml @@ -38,7 +38,7 @@ services: --config=/usr/src/app/config.yaml \ --chain dynamic-mynetwork \ --datadir /home/erigon/data/ \ - --unwind-batch-no=136800 + --unwind-batch-no=136799 xlayer-lrp-resequence: container_name: xlayer-lrp-resequence @@ -72,7 +72,7 @@ services: --config=/usr/src/app/config.yaml \ --chain xlayer-mainnet \ --datadir /home/erigon/data/ \ - --unwind-batch-no=136800 + --unwind-batch-no=136799 xlayer-mainnet-lrp-resequence: container_name: xlayer-mainnet-lrp-resequence diff --git a/test/process_log.py b/test/process_log.py index fcf4cc2bf95..39d060a829f 100644 --- a/test/process_log.py +++ b/test/process_log.py @@ -72,7 +72,7 @@ def main(log_file): logging.info(f"{'From Batch:':<25} {last_batch+1}") logging.info(f"{'To Batch:':<25} {halt_batch}") - logging.info(f"{'Startup Duration:':<25} {startup_duration} seconds") + logging.info(f"{'Data Stream Startup:':<25} {startup_duration} seconds") logging.info(f"{'Start Time:':<25} {start_time}") logging.info(f"{'End Time:':<25} {end_time}") logging.info(f"{'Total Transactions:':<25} {tx_count}") diff --git a/zk/stages/stage_sequence_execute_resequence.go b/zk/stages/stage_sequence_execute_resequence.go index 0212a57d64b..eb67a2c7a5e 100644 --- a/zk/stages/stage_sequence_execute_resequence.go +++ b/zk/stages/stage_sequence_execute_resequence.go @@ -4,9 +4,9 @@ import ( "context" "fmt" - "github.com/ledgerwatch/log/v3" - "github.com/ledgerwatch/erigon/eth/stagedsync" + "github.com/ledgerwatch/erigon/zk/hermez_db" + "github.com/ledgerwatch/log/v3" ) func resequence( @@ -42,6 +42,38 @@ func resequence( log.Info(fmt.Sprintf("[%s] Read %d batches from data stream", s.LogPrefix(), len(batches))) + tx, err := cfg.db.BeginRw(ctx) + if err != nil { + return err + } + defer tx.Rollback() + + // delete L1InfoTreeIndexesProgress + hermezDb := hermez_db.NewHermezDb(tx) + + // 从第一个 batch 中获取起始区块号 + if len(batches) == 0 { + return fmt.Errorf("no batches to process") + } + fromBlock := batches[0][0].L2BlockNumber + + latestBlockNumber, _, err := hermezDb.GetLatestBlockL1InfoTreeIndexProgress() + if err != nil { + return fmt.Errorf("get latest block l1 info tree index progress error: %v", err) + } + + // 使用实际的区块号范围进行删除 + if err = hermezDb.DeleteBlockL1InfoTreeIndexesProgress(fromBlock, latestBlockNumber); err != nil { + return fmt.Errorf("truncate block l1 info tree index progress error: %v", err) + } + + // commit + if err = tx.Commit(); err != nil { + return err + } + + log.Info(fmt.Sprintf("[%s] Deleted L1InfoTreeIndexesProgress from block %d to %d", s.LogPrefix(), fromBlock, latestBlockNumber)) + if err = cfg.dataStreamServer.UnwindToBatchStart(lastBatch + 1); err != nil { return err }