Andrew7234/e2e extension

.gitattributes

@@ -1,4 +1,4 @@
 # A hint to GitHub to show autogenerated files (*.gen.go) less prominently in diffs
 *.gen.go linguist-generated=true
-tests/e2e_regression/rpc-cache/**/*.pix filter=lfs diff=lfs merge=lfs -text
-tests/e2e_regression/rpc-cache/**/*.psg filter=lfs diff=lfs merge=lfs -text
+tests/e2e_regression/**/rpc-cache/**/*.pix filter=lfs diff=lfs merge=lfs -text
+tests/e2e_regression/**/rpc-cache/**/*.psg filter=lfs diff=lfs merge=lfs -text

.github/workflows/ci-test.yaml

@@ -125,7 +125,7 @@ jobs:
             /tmp/*.log
         if: success() || failure()  # but not if job is manually cancelled
 
-  test-e2e-regression:
+  test-e2e-regression-eden:
     runs-on: ubuntu-20.04
     steps:
       - name: Checkout code
@@ -148,10 +148,41 @@ jobs:
           make postgres
       - name: Run e2e regression tests
         run : |
-          make test-e2e-regression
+          make test-e2e-regression-eden
       - uses: actions/upload-artifact@v3
         with:
           name: Actual nexus api responses
           path: |
-            tests/e2e_regression/actual
+            tests/e2e_regression/eden/actual
+        if: failure()  # but not if success or job is manually cancelled
+
+  test-e2e-regression-damask:
+    runs-on: ubuntu-20.04
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v3
+        with:
+          lfs: true
+      - name: Set up Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.21
+      - name: Autogenerate Go code
+        run: |
+          go install github.com/deepmap/oapi-codegen/cmd/[email protected]
+          make codegen-go
+      - name: Build Go
+        run: |
+          make nexus
+      - name: Start db
+        run: |
+          make postgres
+      - name: Run e2e regression tests
+        run : |
+          make test-e2e-regression-damask
+      - uses: actions/upload-artifact@v3
+        with:
+          name: Actual nexus api responses
+          path: |
+            tests/e2e_regression/damask/actual
         if: failure()  # but not if success or job is manually cancelled

.gitignore

@@ -10,7 +10,7 @@ docker/node/etc/genesis.json
 
 # Testing data
 tests/e2e/testnet/net-runner
-tests/e2e_regression/actual/*
+tests/e2e_regression/*/actual/*
 
 # Log output.
 **/*.log

Makefile

@@ -60,31 +60,65 @@ test-e2e: export OASIS_INDEXER_E2E = true
 test-e2e:
 	@$(GO) test -race -coverpkg=./... -coverprofile=coverage.txt -covermode=atomic -v ./tests/e2e
 
-fill-cache-for-e2e-regression: nexus
+fill-cache-for-e2e-regression:
+	@$(MAKE) fill-cache-for-e2e-regression-eden
+	@$(MAKE) fill-cache-for-e2e-regression-damask
+
+fill-cache-for-e2e-regression-eden: nexus
+	@./tests/e2e_regression/ensure_consistent_config.sh
+	./nexus --config tests/e2e_regression/eden/e2e_config_1.yml analyze
+	./nexus --config tests/e2e_regression/eden/e2e_config_2.yml analyze
+
+fill-cache-for-e2e-regression-damask: nexus
 	@./tests/e2e_regression/ensure_consistent_config.sh
-	./nexus --config tests/e2e_regression/e2e_config_1.yml analyze
-	./nexus --config tests/e2e_regression/e2e_config_2.yml analyze
+	./nexus --config tests/e2e_regression/damask/e2e_config_1.yml analyze
+	./nexus --config tests/e2e_regression/damask/e2e_config_2.yml analyze
 
 # Run the api tests locally, assuming the environment is set up with an oasis-node that is
 # accessible as specified in the config file.
-test-e2e-regression: nexus
+test-e2e-regression:
+	@$(MAKE) test-e2e-regression-eden
+	@$(MAKE) test-e2e-regression-damask
+
+test-e2e-regression-eden: nexus
 	@./tests/e2e_regression/ensure_consistent_config.sh
-	./nexus --config tests/e2e_regression/e2e_config_1.yml analyze
-	./nexus --config tests/e2e_regression/e2e_config_2.yml analyze
+	./nexus --config tests/e2e_regression/eden/e2e_config_1.yml analyze
+	./nexus --config tests/e2e_regression/eden/e2e_config_2.yml analyze
 	@$(ECHO) "$(CYAN)*** Analyzers finished; starting api tests...$(OFF)"
-	./tests/e2e_regression/run.sh
+	./tests/e2e_regression/run.sh eden
+
+
+test-e2e-regression-damask: nexus
+	@./tests/e2e_regression/ensure_consistent_config.sh
+	./nexus --config tests/e2e_regression/damask/e2e_config_1.yml analyze
+	./nexus --config tests/e2e_regression/damask/e2e_config_2.yml analyze
+	@$(ECHO) "$(CYAN)*** Analyzers finished; starting api tests...$(OFF)"
+	./tests/e2e_regression/run.sh damask
 
 # Accept the outputs of the e2e tests as the new expected outputs.
 accept-e2e-regression:
+	@$(MAKE) accept-e2e-regression-eden
+	@$(MAKE) accept-e2e-regression-damask
+
+accept-e2e-regression-eden:
+	SUITE=eden $(MAKE) accept-e2e-regression-suite
+
+accept-e2e-regression-damask:
+	SUITE=damask $(MAKE) accept-e2e-regression-suite
+
+accept-e2e-regression-suite:
+ifndef SUITE
+	$(error SUITE is undefined)
+endif
 	@# Delete all old expected files first, in case any test cases were renamed or removed.
-	rm -rf ./tests/e2e_regression/expected
+	rm -rf ./tests/e2e_regression/$(SUITE)/expected
 	@# Copy the actual outputs to the expected outputs.
-	cp -r  ./tests/e2e_regression/{actual,expected}
+	cp -r  ./tests/e2e_regression/$(SUITE)/{actual,expected}
 	@# The result of the "spec" test is a special case. It should always match the
 	@# current openAPI spec file, so we symlink it to avoid having to update the expected
 	@# output every time the spec changes.
-	rm ./tests/e2e_regression/expected/spec.body
-	ln -s  ../../../api/spec/v1.yaml ./tests/e2e_regression/expected/spec.body
+	rm ./tests/e2e_regression/$(SUITE)/expected/spec.body
+	ln -s  ../../../../api/spec/v1.yaml ./tests/e2e_regression/$(SUITE)/expected/spec.body
 
 # Format code.
 fmt:

tests/e2e_regression/damask/e2e_config_1.yml

@@ -0,0 +1,46 @@
+# Config for phase 1 of 2 of e2e regression tests.
+# See also ensure_consistent_config.sh
+#
+# Phase 1 runs all block-based analyzers; phase 2 runs all others. This is a slightly hacky
+# but low-effort way to ensure that all non-block analyzers perform RPCs at a deterministic
+# height and therefore always hit the RPC cache. (Most EVM-related analyzers make RPCs at the
+# maximum height that's been reached by block analyzers by that point.)
+analysis:
+  source:
+    cache: { cache_dir: tests/e2e_regression/damask/rpc-cache }
+    chain_name: mainnet
+    nodes:
+      # Node will be accessed only when the filling up the cache.
+      # In CI, the cache is expected to have all the node responses as the node is not reachable.
+      damask: { default: { rpc: unix:/tmp/node.sock } }
+    ipfs:
+      gateway: https://ipfs.io
+    fast_startup: true
+  analyzers:
+    consensus:
+      from: 8_048_956  # Damask genesis
+      to: 8_049_955    # 1000 blocks; compromise between fast and comprehensive
+    emerald:
+      from: 1_003_298  # round at Damask genesis
+      to: 1_004_297    # 1000 blocks; compromise between fast and comprehensive
+    # sapphire: # Not tested. Does not use much code that is not covered by emerald.
+    #   from: 0  # first round in Damask
+  storage:
+    backend: postgres
+    endpoint: postgresql://rwuser:password@localhost:5432/indexer?sslmode=disable
+    DANGER__WIPE_STORAGE_ON_STARTUP: true
+    migrations: file://storage/migrations
+
+server:
+  chain_name: mainnet
+  endpoint: localhost:8008
+  storage:
+    endpoint: postgresql://api:password@localhost:5432/indexer?sslmode=disable
+    backend: postgres
+
+log:
+  level: debug
+  format: json
+
+metrics:
+  pull_endpoint: localhost:8009

tests/e2e_regression/damask/e2e_config_2.yml

@@ -0,0 +1,34 @@
+# Config for phase 2 of e2e regression tests. See e2e_config_1.yml for more info.
+
+analysis:
+  source:
+    cache: { cache_dir: tests/e2e_regression/damask/rpc-cache }
+    chain_name: mainnet
+    nodes:
+      damask: { default: { rpc: unix:/tmp/node.sock } }
+    ipfs:
+      gateway: https://ipfs.io
+    fast_startup: true
+  analyzers:
+    evm_tokens_emerald:         { stop_on_empty_queue: true }
+    evm_nfts_emerald:           { stop_on_empty_queue: true }
+    evm_token_balances_emerald: { stop_on_empty_queue: true }
+    evm_contract_code_emerald:  { stop_on_empty_queue: true }
+    evm_abi_emerald:            { stop_on_empty_queue: true }
+    # Some non-block analyzers are not tested in e2e regressions.
+    # They are largely not worth the trouble as they do not interact with rest of the system much.
+    # metadata_registry: {} # Awkward to inject mock registry responses.
+    # node_stats: {}        # Awkward to inject mock node response using the current paradigm (= reseponse caching).
+    # aggregate_stats: {}   # Awkward to make stop after a single run.
+  storage:
+    backend: postgres
+    endpoint: postgresql://rwuser:password@localhost:5432/indexer?sslmode=disable
+    # DANGER__WIPE_STORAGE_ON_STARTUP: true
+    migrations: file://storage/migrations
+
+log:
+  level: debug
+  format: json
+
+metrics:
+  pull_endpoint: localhost:8009