diff --git a/.github/workflows/e2e-fork.yml b/.github/workflows/e2e-fork.yml index d3717d2c35a..240c4322545 100644 --- a/.github/workflows/e2e-fork.yml +++ b/.github/workflows/e2e-fork.yml @@ -24,8 +24,8 @@ jobs: e2e: env: - SIMD_TAG: latest - SIMD_IMAGE: ibc-go-simd-e2e + CHAIN_A_SIMD_TAG: latest + CHAIN_A_SIMD_IMAGE: ibc-go-simd-e2e if: ${{ github.event.pull_request.head.repo.fork || github.actor == 'dependabot[bot]' || github.event_name == 'workflow_dispatch' }} needs: - build-test-matrix @@ -36,7 +36,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Docker Build - run: docker build . -t "${SIMD_IMAGE}:${SIMD_TAG}" + run: docker build . -t "${CHAIN_A_SIMD_IMAGE}:${CHAIN_A_SIMD_TAG}" - name: Setup Go uses: actions/setup-go@v3 with: diff --git a/.github/workflows/e2e-manual.yaml b/.github/workflows/e2e-manual.yaml new file mode 100644 index 00000000000..65be1c8f8e2 --- /dev/null +++ b/.github/workflows/e2e-manual.yaml @@ -0,0 +1,94 @@ +name: Manual E2E +on: + # when https://github.com/community/community/discussions/11795 is resolved + # we will be able to dynamically build up the list of valid inputs. + # for now this needs to be manual. + workflow_dispatch: + inputs: + test-entry-point: + description: 'Test entry point' + required: true + type: choice + options: + - TestFeeMiddlewareTestSuite + chain-a-image: + description: 'The image to use for chain A' + required: true + type: string + default: "ghcr.io/cosmos/ibc-go-simd-e2e" + chain-a-tag: + description: 'The tag to use for chain A' + required: true + type: choice + default: main + options: + - main + - v4.0.0-rc3 + - v3.0.0 + - v2.2.0 + - v2.1.0 + - v2.0.0 + chain-b-image: + description: 'The image to use for chain B' + required: true + type: string + default: "ghcr.io/cosmos/ibc-go-simd" + chain-b-tag: + default: v4.0.0-rc3 + description: 'The tag to use for chain B' + required: true + type: choice + options: + - v4.0.0-rc3 + - v3.0.0 + - v2.2.0 + - v2.1.0 + - v2.0.0 + relayer-tag: + description: 'The tag to use for the relayer' + required: true + default: "v2.0.0-rc2" + type: string + + +jobs: + # dynamically build a matrix of test/test suite pairs to run + build-test-matrix: + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-matrix.outputs.matrix }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: 1.18 + - id: set-matrix + run: echo "::set-output name=matrix::$(go run cmd/build_test_matrix/main.go)" + env: + TEST_ENTRYPOINT: "${{ github.event.inputs.test-entry-point }}" + + + + e2e-manual: + runs-on: ubuntu-latest + needs: + - build-test-matrix + env: + CHAIN_A_SIMD_TAG: "${{ github.event.inputs.chain-a-tag }}" + CHAIN_A_SIMD_IMAGE: "${{ github.event.inputs.chain-a-image }}" + CHAIN_B_SIMD_TAG: "${{ github.event.inputs.chain-b-tag }}" + CHAIN_B_SIMD_IMAGE: "${{ github.event.inputs.chain-b-image }}" + # see images here https://github.com/cosmos/relayer/pkgs/container/relayer/versions + RLY_TAG: "${{ github.event.inputs.relayer-tag }}" + strategy: + fail-fast: false + matrix: ${{ fromJSON(needs.build-test-matrix.outputs.matrix) }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-go@v3 + with: + go-version: 1.18 + - name: Run e2e Test + run: | + cd e2e + make e2e-test suite=${{ matrix.suite }} test=${{ matrix.test }} diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 071e76ffb33..627d2f3307d 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -81,8 +81,8 @@ jobs: - determine-image-tag - docker-build env: - SIMD_TAG: ${{ needs.determine-image-tag.outputs.simd-tag }} - SIMD_IMAGE: ghcr.io/cosmos/ibc-go-simd-e2e + CHAIN_A_SIMD_TAG: ${{ needs.determine-image-tag.outputs.simd-tag }} + CHAIN_A_SIMD_IMAGE: ghcr.io/cosmos/ibc-go-simd-e2e # see images here https://github.com/cosmos/relayer/pkgs/container/relayer/versions RLY_TAG: "v2.0.0-rc2" strategy: diff --git a/cmd/build_test_matrix/main.go b/cmd/build_test_matrix/main.go index 9841e6e9035..87a70c373bf 100644 --- a/cmd/build_test_matrix/main.go +++ b/cmd/build_test_matrix/main.go @@ -16,6 +16,7 @@ const ( testNamePrefix = "Test" testFileNameSuffix = "_test.go" e2eTestDirectory = "e2e" + testEntryPointEnv = "TEST_ENTRYPOINT" ) // GithubActionTestMatrix represents @@ -29,7 +30,7 @@ type TestSuitePair struct { } func main() { - githubActionMatrix, err := getGithubActionMatrixForTests(e2eTestDirectory) + githubActionMatrix, err := getGithubActionMatrixForTests(e2eTestDirectory, getTestFunctionToRun()) if err != nil { fmt.Printf("error generating github action json: %s", err) os.Exit(1) @@ -43,10 +44,20 @@ func main() { fmt.Println(string(ghBytes)) } +// getTestFunctionToRun returns the specified test function to run if present, otherwise +// it returns an empty string which will result in running all test suites. +func getTestFunctionToRun() string { + testSuite, ok := os.LookupEnv(testEntryPointEnv) + if !ok { + return "" + } + return testSuite +} + // getGithubActionMatrixForTests returns a json string representing the contents that should go in the matrix // field in a github action workflow. This string can be used with `fromJSON(str)` to dynamically build // the workflow matrix to include all E2E tests under the e2eRootDirectory directory. -func getGithubActionMatrixForTests(e2eRootDirectory string) (GithubActionTestMatrix, error) { +func getGithubActionMatrixForTests(e2eRootDirectory, suite string) (GithubActionTestMatrix, error) { testSuiteMapping := map[string][]string{} fset := token.NewFileSet() err := filepath.Walk(e2eRootDirectory, func(path string, info fs.FileInfo, err error) error { @@ -69,7 +80,9 @@ func getGithubActionMatrixForTests(e2eRootDirectory string) (GithubActionTestMat return fmt.Errorf("failed extracting test suite name and test cases: %s", err) } - testSuiteMapping[suiteNameForFile] = testCases + if suite == "" || suiteNameForFile == suite { + testSuiteMapping[suiteNameForFile] = testCases + } return nil }) diff --git a/cmd/build_test_matrix/main_test.go b/cmd/build_test_matrix/main_test.go index 8dd75b4e4c1..578cc5f2e3a 100644 --- a/cmd/build_test_matrix/main_test.go +++ b/cmd/build_test_matrix/main_test.go @@ -19,7 +19,7 @@ const ( func TestGetGithubActionMatrixForTests(t *testing.T) { t.Run("empty dir does not fail", func(t *testing.T) { testingDir := t.TempDir() - _, err := getGithubActionMatrixForTests(testingDir) + _, err := getGithubActionMatrixForTests(testingDir, "") assert.NoError(t, err) }) @@ -27,7 +27,7 @@ func TestGetGithubActionMatrixForTests(t *testing.T) { testingDir := t.TempDir() createFileWithTestSuiteAndTests(t, "FeeMiddlewareTestSuite", "TestA", "TestB", testingDir, goTestFileNameOne) - gh, err := getGithubActionMatrixForTests(testingDir) + gh, err := getGithubActionMatrixForTests(testingDir, "") assert.NoError(t, err) expected := GithubActionTestMatrix{ @@ -50,7 +50,7 @@ func TestGetGithubActionMatrixForTests(t *testing.T) { createFileWithTestSuiteAndTests(t, "FeeMiddlewareTestSuite", "TestA", "TestB", testingDir, goTestFileNameOne) createFileWithTestSuiteAndTests(t, "TransferTestSuite", "TestC", "TestD", testingDir, goTestFileNameTwo) - gh, err := getGithubActionMatrixForTests(testingDir) + gh, err := getGithubActionMatrixForTests(testingDir, "") assert.NoError(t, err) expected := GithubActionTestMatrix{ @@ -81,7 +81,7 @@ func TestGetGithubActionMatrixForTests(t *testing.T) { testingDir := t.TempDir() createFileWithTestSuiteAndTests(t, "FeeMiddlewareTestSuite", "TestA", "TestB", testingDir, nonTestFile) - gh, err := getGithubActionMatrixForTests(testingDir) + gh, err := getGithubActionMatrixForTests(testingDir, "") assert.NoError(t, err) assert.Empty(t, gh.Include) }) @@ -105,7 +105,7 @@ type FeeMiddlewareTestSuite struct {} err := os.WriteFile(path.Join(testingDir, goTestFileNameOne), []byte(fileWithTwoSuites), os.FileMode(777)) assert.NoError(t, err) - _, err = getGithubActionMatrixForTests(testingDir) + _, err = getGithubActionMatrixForTests(testingDir, "") assert.Error(t, err) }) } diff --git a/e2e/README.md b/e2e/README.md index 4475891db22..0605d507176 100644 --- a/e2e/README.md +++ b/e2e/README.md @@ -57,8 +57,14 @@ Every time changes are pushed to a branch or to `main`, a new `simd` image is bu #### Example Command: ```sh -export SIMD_IMAGE="ghcr.io/cosmos/ibc-go-simd-e2e" -export SIMD_TAG="pr-1650" +export CHAIN_A_SIMD_IMAGE="ghcr.io/cosmos/ibc-go-simd-e2e" +export CHAIN_A_SIMD_TAG="pr-1650" + +# We can also specify different values for the chains if needed. +# they will default to the same as chain a. +# export CHAIN_B_SIMD_IMAGE="ghcr.io/cosmos/ibc-go-simd-e2e" +# export CHAIN_B_SIMD_TAG="pr-1650" + export RLY_TAG="v2.0.0-rc2" make e2e-test suite=FeeMiddlewareTestSuite test=TestMultiMsg_MsgPayPacketFeeSingleSender ``` diff --git a/e2e/scripts/run-e2e.sh b/e2e/scripts/run-e2e.sh index 1e0875c099f..44996100990 100755 --- a/e2e/scripts/run-e2e.sh +++ b/e2e/scripts/run-e2e.sh @@ -4,8 +4,8 @@ set -euo pipefail SUITE="${1}" TEST="${2}" -export SIMD_TAG="${SIMD_TAG:-latest}" -export SIMD_IMAGE="${SIMD_IMAGE:-ibc-go-simd-e2e}" +export SIMD_TAG="${CHAIN_A_SIMD_TAG:-latest}" +export SIMD_IMAGE="${CHAIN_A_SIMD_IMAGE:-ibc-go-simd-e2e}" # In CI, the docker images will be built separately. # context for building the image is one directory up. diff --git a/e2e/testconfig/testconfig.go b/e2e/testconfig/testconfig.go index 46be84e1e58..eac0bd3f070 100644 --- a/e2e/testconfig/testconfig.go +++ b/e2e/testconfig/testconfig.go @@ -8,42 +8,71 @@ import ( ) const ( - DefaultSimdImage = "ghcr.io/cosmos/ibc-go-simd-e2e" - SimdImageEnv = "SIMD_IMAGE" - SimdTagEnv = "SIMD_TAG" - GoRelayerTag = "RLY_TAG" + // ChainASimdImageEnv specifies the image that Chain A will use. + ChainASimdImageEnv = "CHAIN_A_SIMD_IMAGE" + // ChainASimdTagEnv specifies the tag that Chain A will use. + ChainASimdTagEnv = "CHAIN_A_SIMD_TAG" + // ChainBSimdImageEnv specifies the image that Chain B will use. If unspecified + // the value will default to the same value as Chain A. + ChainBSimdImageEnv = "CHAIN_B_SIMD_IMAGE" + // ChainBSimdTagEnv specifies the tag that Chain B will use. If unspecified + // the value will default to the same value as Chain A. + ChainBSimdTagEnv = "CHAIN_B_SIMD_TAG" + GoRelayerTagEnv = "RLY_TAG" - defaultRlyTag = "main" + defaultSimdImage = "ghcr.io/cosmos/ibc-go-simd-e2e" + defaultRlyTag = "main" ) // TestConfig holds various fields used in the E2E tests. type TestConfig struct { - SimdImage string - SimdTag string - RlyTag string + ChainAConfig ChainConfig + ChainBConfig ChainConfig + RlyTag string +} + +type ChainConfig struct { + Image string + Tag string } // FromEnv returns a TestConfig constructed from environment variables. func FromEnv() TestConfig { - simdImage, ok := os.LookupEnv(SimdImageEnv) + chainASimdImage, ok := os.LookupEnv(ChainASimdImageEnv) + if !ok { + chainASimdImage = defaultSimdImage + } + + chainASimdTag, ok := os.LookupEnv(ChainASimdTagEnv) if !ok { - simdImage = DefaultSimdImage + panic(fmt.Sprintf("must specify simd version for test with environment variable [%s]", ChainASimdTagEnv)) } - simdTag, ok := os.LookupEnv(SimdTagEnv) + chainBSimdImage, ok := os.LookupEnv(ChainBSimdImageEnv) if !ok { - panic(fmt.Sprintf("must specify simd version for test with environment variable [%s]", SimdTagEnv)) + chainBSimdImage = chainASimdImage } - rlyTag, ok := os.LookupEnv(GoRelayerTag) + chainBSimdTag, ok := os.LookupEnv(ChainBSimdTagEnv) + if !ok { + chainBSimdTag = chainASimdTag + } + + rlyTag, ok := os.LookupEnv(GoRelayerTagEnv) if !ok { rlyTag = defaultRlyTag } return TestConfig{ - SimdImage: simdImage, - SimdTag: simdTag, - RlyTag: rlyTag, + ChainAConfig: ChainConfig{ + Image: chainASimdImage, + Tag: chainASimdTag, + }, + ChainBConfig: ChainConfig{ + Image: chainBSimdImage, + Tag: chainBSimdTag, + }, + RlyTag: rlyTag, } } @@ -62,8 +91,8 @@ type ChainOptionConfiguration func(options *ChainOptions) // These options can be configured by passing configuration functions to E2ETestSuite.GetChains. func DefaultChainOptions() ChainOptions { tc := FromEnv() - chainACfg := newDefaultSimappConfig(tc, "simapp-a", "chain-a", "atoma") - chainBCfg := newDefaultSimappConfig(tc, "simapp-b", "chain-b", "atomb") + chainACfg := newDefaultSimappConfig(tc.ChainAConfig, "simapp-a", "chain-a", "atoma") + chainBCfg := newDefaultSimappConfig(tc.ChainBConfig, "simapp-b", "chain-b", "atomb") return ChainOptions{ ChainAConfig: &chainACfg, ChainBConfig: &chainBCfg, @@ -71,15 +100,15 @@ func DefaultChainOptions() ChainOptions { } // newDefaultSimappConfig creates an ibc configuration for simd. -func newDefaultSimappConfig(tc TestConfig, name, chainID, denom string) ibc.ChainConfig { +func newDefaultSimappConfig(cc ChainConfig, name, chainID, denom string) ibc.ChainConfig { return ibc.ChainConfig{ Type: "cosmos", Name: name, ChainID: chainID, Images: []ibc.DockerImage{ { - Repository: tc.SimdImage, - Version: tc.SimdTag, + Repository: cc.Image, + Version: cc.Tag, }, }, Bin: "simd",