chore(end-to-end tests): refactor and fix flaky tests (ChainSafe#2470)

- make RPC helper functions not test aware
- `waitForNode` context based function
- Use `t.TempDir()` for nodes base paths
- Add `GenerateGenesisAuths` helper function
- Remove global variables
  - `GenesisDefault`, `GenesisDev`, `GenesisTwoAuthsSecondaryVRF0_9_10`
  - framework variable
  - configuration variables
  - `HOSTNAME` variable
  - `maxTries` in stress tests
  - `testTimeout` in stress tests
- Write node configurations to `t.TempDir()` per test
- Remove `DecodeRPC_NT` function
- Split `tests/utils` package
  - `tests/utils/rpc` package
  - `tests/utils/websocket` package
  - `tests/utils/config` package
- Rework start and stop of nodes
  - Init nodes in parallel
  - Remove unneeded logs
  - Log when node is ready
  - Prefix writer with node string for multiple nodes
  - Wait for nodes after starting all nodes
- Add test writer using `t.Logf`
- Refactor `getResponse`
  - not test aware
  - get passed target interface
  - push test assertion/skip to actual test
- Add helper `retry.UntilNoError` and `retry.UntilOK` functions
  - Use in `compareBlocksByNumber`
  - Use in `waitForNode`
  - Change `GetBlockHash` to not retry RPC
  - Remove `PostWithRetry`
  - Replace `compareFinalizedHeadsWithRetry` with `retry.UntilOK`
- Refactor and fix `TestSystemRPC`
  - Add empty skipped tests for missing cases
  - Split each subtest individually
  - Keep on retrying until main context is canceled
  - Fix `networkState` test case
  - Assert more fields
  - Fix ChainSafe#2161 and ChainSafe#807
- Refactor and fix `TestSync_SingleBlockProducer`
- Refactor and make faster `TestAuthorSubmitExtrinsic`
- Configure nodes using TOML Go struct only
- Pass `--no-telemetry` as flag since it's not in TOML config
- `dot`: remove `exportConfig` and `TestExportConfig`
- `dot`: remove `ExportTomlConfig` and its test
- Install NPM dependencies from Go code for polkdaot JS test
- Remove `websocket` package
  - Use `Decode` from rpc package instead
  - Add fields to RPC server response struct
- Do not use `reflect`
  - Add `fetchWithTimeout` helper function
  - Change test cases slices to individual subtests
  - Add `t.SkipNow()` to parent tests where all subtests are skipped
  - Make `TestChainRPC` event driven and increase testing depth
  - Add a few `TODO`s to increase testing depth
  - Run each query call in parallel
- Refactor `TestChainSubscriptionRPC`
  - Refactor websocket code
  - Assert things and increase test depth
  - Graceful shutdown of websocket
- Merge stable workflow in rpc workflow
  - Use Go to implement `integration-test-all.sh` directly in single 'stable' test
  - The test is fast and only does RPC calls, it should be part of the RPC workflow
  - Remove stable workflow from Makefile, github workflows directory and document
- Log out command, logs and configuration file content on runtime error
- Fixes directly related issues ChainSafe#2389, ChainSafe#2390, ChainSafe#2391
- Fixes flaky tests issues ChainSafe#807, ChainSafe#2161, ChainSafe#2165, ChainSafe#2167

Co-authored-by: Timothy Wu <[email protected]>

.gitignore

@@ -23,9 +23,6 @@ test_data
 trie_putandget_failed_test_data_*
 tmp
 
-tests/utils/config*
-tests/utils/genesis*
-
 # node_modules used by polkadot.js/api tests
 tests/polkadotjs_test/node_modules
 !tests/polkadotjs_test/test/*.wasm

Dockerfile

@@ -17,11 +17,6 @@ RUN wget -qO- https://deb.nodesource.com/setup_14.x | bash - && \
 RUN wget -O /usr/local/bin/subkey https://chainbridge.ams3.digitaloceanspaces.com/subkey-v2.0.0 && \
     chmod +x /usr/local/bin/subkey
 
-# Polkadot JS dependencies
-WORKDIR /go/src/github.com/ChainSafe/gossamer/tests/polkadotjs_test
-COPY tests/polkadotjs_test/package.json tests/polkadotjs_test/package-lock.json ./
-RUN npm install
-
 WORKDIR /go/src/github.com/ChainSafe/gossamer
 
 # Go dependencies

Makefile

@@ -36,32 +36,26 @@ test:
 	git lfs pull
 	go test -short -coverprofile c.out ./... -timeout=30m
 
-## it-stable: Runs Integration Tests Stable mode
-it-stable:
-	@echo "  >  \033[32mRunning Integration Tests...\033[0m "
-	@chmod +x scripts/integration-test-all.sh
-	./scripts/integration-test-all.sh -q 3 -s 10
-
 ## it-stress: Runs Integration Tests stress mode
 it-stress: build
 	@echo "  >  \033[32mRunning stress tests...\033[0m "
-	HOSTNAME=0.0.0.0 MODE=stress go test ./tests/stress/... -timeout=15m -v -short -run TestSync_
+	MODE=stress go test ./tests/stress/... -timeout=15m -v -short -run TestSync_
 
 it-grandpa: build
 	@echo "  >  \033[32mRunning GRANDPA stress tests...\033[0m "
-	HOSTNAME=0.0.0.0 MODE=stress go test ./tests/stress/... -timeout=12m -v -short -run TestStress_Grandpa_
+	MODE=stress go test ./tests/stress/... -timeout=12m -v -short -run TestStress_Grandpa_
 
 it-rpc: build
 	@echo "  >  \033[32mRunning Integration Tests RPC Specs mode...\033[0m "
-	HOSTNAME=0.0.0.0 MODE=rpc go test ./tests/rpc/... -timeout=10m -v
+	MODE=rpc go test ./tests/rpc/... -timeout=10m -v
 
 it-sync: build
 	@echo "  >  \033[32mRunning Integration Tests sync mode...\033[0m "
-	HOSTNAME=0.0.0.0 MODE=sync go test ./tests/sync/... -timeout=5m -v
+	MODE=sync go test ./tests/sync/... -timeout=5m -v
 
 it-polkadotjs: build
 	@echo "  >  \033[32mRunning Integration Tests polkadot.js/api mode...\033[0m "
-	HOSTNAME=0.0.0.0 MODE=polkadot go test ./tests/polkadotjs_test/... -timeout=5m -v
+	MODE=polkadot go test ./tests/polkadotjs_test/... -timeout=5m -v
 
 ## test: Runs `go test -race` on project test files.
 test-state-race:

docs/docs/testing-and-debugging/test-suite.md

@@ -25,13 +25,6 @@ Proceed to open `cover.html` in your preferred browser.
 ### Gossamer Integration Tests
 
 Running Gossamer's integration tests with the below commands will build a Gossamer binary, install required dependencies, and then proceeds to run the provided set of tests. Integration tests can also be run within a docker container.
-
-
-To run Gossamer integration tests in **stable** mode run the following command:
-
-```
-make it-stable
-```
 
 To run Gossamer integration tests in **stress** mode run the following command:
 

dot/utils.go

@@ -9,37 +9,9 @@ import (
 	"os"
 	"strings"
 
-	ctoml "github.com/ChainSafe/gossamer/dot/config/toml"
 	"github.com/cosmos/go-bip39"
-	"github.com/naoina/toml"
 )
 
-// exportConfig exports a dot configuration to a toml configuration file
-func exportConfig(cfg *Config, fp string) {
-	raw, err := toml.Marshal(*cfg)
-	if err != nil {
-		logger.Errorf("failed to marshal configuration: %s", err)
-		os.Exit(1)
-	}
-	if err := os.WriteFile(fp, raw, 0600); err != nil {
-		logger.Errorf("failed to write file: %s", err)
-		os.Exit(1)
-	}
-}
-
-// ExportTomlConfig exports a dot configuration to a toml configuration file
-func ExportTomlConfig(cfg *ctoml.Config, fp string) {
-	raw, err := toml.Marshal(*cfg)
-	if err != nil {
-		logger.Errorf("failed to marshal configuration: %s", err)
-		os.Exit(1)
-	}
-	if err := os.WriteFile(fp, raw, 0600); err != nil {
-		logger.Errorf("failed to write file: %s", err)
-		os.Exit(1)
-	}
-}
-
 // CreateJSONRawFile will generate a JSON genesis file with raw storage
 func CreateJSONRawFile(bs *BuildSpec, fp string) {
 	data, err := bs.ToJSONRaw()

dot/utils_test.go

@@ -12,7 +12,6 @@ import (
 	"path/filepath"
 	"testing"
 
-	ctoml "github.com/ChainSafe/gossamer/dot/config/toml"
 	"github.com/ChainSafe/gossamer/dot/types"
 	"github.com/ChainSafe/gossamer/internal/log"
 	"github.com/ChainSafe/gossamer/lib/genesis"
@@ -80,154 +79,6 @@ func TestCreateJSONRawFile(t *testing.T) {
 	}
 }
 
-func TestExportConfig(t *testing.T) {
-	filepath := filepath.Join(t.TempDir(), "test.json")
-	type args struct {
-		cfg *Config
-		fp  string
-	}
-	tests := []struct {
-		name          string
-		args          args
-		want          *os.File
-		wantedContent string
-	}{
-		{
-			name: "working example",
-			args: args{
-				cfg: &Config{},
-				fp:  filepath,
-			},
-			want: &os.File{},
-			wantedContent: `[global]
-name = ""
-id = ""
-base_path = ""
-log_lvl = 0
-publish_metrics = false
-metrics_address = ""
-no_telemetry = false
-telemetry_urls = []
-retain_blocks = 0
-pruning = ""
-
-[log]
-core_lvl = 0
-digest_lvl = 0
-sync_lvl = 0
-network_lvl = 0
-rpc_lvl = 0
-state_lvl = 0
-runtime_lvl = 0
-block_producer_lvl = 0
-finality_gadget_lvl = 0
-
-[init]
-genesis = ""
-
-[account]
-key = ""
-unlock = ""
-
-[core]
-roles = 0
-babe_authority = false
-b_a_b_e_lead = false
-grandpa_authority = false
-wasm_interpreter = ""
-grandpa_interval = 0
-
-[network]
-port = 0
-bootnodes = []
-protocol_id = ""
-no_bootstrap = false
-no_m_dns = false
-min_peers = 0
-max_peers = 0
-persistent_peers = []
-discovery_interval = 0
-public_ip = ""
-public_dns = ""
-
-[rpc]
-enabled = false
-external = false
-unsafe = false
-unsafe_external = false
-port = 0
-host = ""
-modules = []
-w_s_port = 0
-w_s = false
-w_s_external = false
-w_s_unsafe = false
-w_s_unsafe_external = false
-
-[system]
-system_name = ""
-system_version = ""
-
-[state]
-rewind = 0
-
-[pprof]
-enabled = false
-
-[pprof.settings]
-listening_address = ""
-block_profile_rate = 0
-mutex_profile_rate = 0
-`,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			exportConfig(tt.args.cfg, tt.args.fp)
-
-			content, err := ioutil.ReadFile(tt.args.fp)
-			require.NoError(t, err)
-			require.Equal(t, tt.wantedContent, string(content))
-
-		})
-	}
-}
-
-func TestExportTomlConfig(t *testing.T) {
-	filepath := filepath.Join(t.TempDir(), "test.json")
-	type args struct {
-		cfg *ctoml.Config
-		fp  string
-	}
-	tests := []struct {
-		name          string
-		args          args
-		wantedContent string
-	}{
-		{
-			name: "working example",
-			args: args{
-				cfg: &ctoml.Config{},
-				fp:  filepath,
-			},
-			wantedContent: `[core]
-babe-authority = false
-grandpa-authority = false
-`,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			ExportTomlConfig(tt.args.cfg, tt.args.fp)
-
-			content, err := ioutil.ReadFile(tt.args.fp)
-			require.NoError(t, err)
-			require.Equal(t, tt.wantedContent, string(content))
-
-		})
-	}
-}
-
 func TestNewTestConfig(t *testing.T) {
 	basePath := t.TempDir()
 	incBasePath := basePath[:len(basePath)-1] + "2"

lib/utils/utils.go

@@ -181,6 +181,11 @@ func GetDevGenesisPath(t *testing.T) string {
 	return filepath.Join(GetProjectRootPathTest(t), "./chain/dev/genesis.json")
 }
 
+// GetDevGenesisSpecPathTest gets the dev genesis spec path
+func GetDevGenesisSpecPathTest(t *testing.T) string {
+	return filepath.Join(GetProjectRootPathTest(t), "./chain/dev/genesis-spec.json")
+}
+
 // GetGssmrGenesisPath gets the gssmr genesis path
 // and returns an error if it cannot find it.
 func GetGssmrGenesisPath() (path string, err error) {