Multi-consumer capable e2e tests

tests/e2e/channel_init.go

@@ -8,6 +8,7 @@ import (
 
 	channeltypes "github.com/cosmos/ibc-go/v3/modules/core/04-channel/types"
 
+	icstestingutils "github.com/cosmos/interchain-security/testutil/ibc_testing"
 	ccv "github.com/cosmos/interchain-security/x/ccv/types"
 
 	abci "github.com/tendermint/tendermint/abci/types"
@@ -22,7 +23,9 @@ func (suite *CCVTestSuite) TestConsumerGenesis() {
 	genesis := consumerKeeper.ExportGenesis(suite.consumerChain.GetContext())
 
 	// Confirm that client and cons state are exported from consumer keeper properly
-	consumerEndpointClientState, consumerEndpointConsState := suite.GetConsumerEndpointClientAndConsState()
+	consumerEndpointClientState, consumerEndpointConsState := suite.GetConsumerEndpointClientAndConsState(
+		*suite.consumerBundles[icstestingutils.FirstConsumerChainID],
+	)
 	suite.Require().Equal(consumerEndpointClientState, genesis.ProviderClientState)
 	suite.Require().Equal(consumerEndpointConsState, genesis.ProviderConsensusState)
 

tests/e2e/common.go

@@ -7,6 +7,7 @@ import (
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types"
 	"github.com/cosmos/interchain-security/testutil/e2e"
+	icstestingutils "github.com/cosmos/interchain-security/testutil/ibc_testing"
 	consumertypes "github.com/cosmos/interchain-security/x/ccv/consumer/types"
 	providertypes "github.com/cosmos/interchain-security/x/ccv/provider/types"
 	ccv "github.com/cosmos/interchain-security/x/ccv/types"
@@ -33,6 +34,7 @@ func (s *CCVTestSuite) providerCtx() sdk.Context {
 	return s.providerChain.GetContext()
 }
 
+// consumerCtx returns the context of only the FIRST consumer chain
 func (s *CCVTestSuite) consumerCtx() sdk.Context {
 	return s.consumerChain.GetContext()
 }
@@ -407,17 +409,20 @@ func (suite *CCVTestSuite) CreateCustomClient(endpoint *ibctesting.Endpoint, unb
 	require.NoError(endpoint.Chain.T, err)
 }
 
-func (suite *CCVTestSuite) GetConsumerEndpointClientAndConsState() (exported.ClientState, exported.ConsensusState) {
+func (suite *CCVTestSuite) GetConsumerEndpointClientAndConsState(
+	consumerBundle icstestingutils.ConsumerBundle) (exported.ClientState, exported.ConsensusState) {
 
-	clientID, found := suite.consumerApp.GetConsumerKeeper().GetProviderClientID(suite.consumerCtx())
+	ctx := consumerBundle.GetCtx()
+	consumerKeeper := consumerBundle.GetKeeper()
+
+	clientID, found := consumerKeeper.GetProviderClientID(ctx)
 	suite.Require().True(found)
 
-	clientState, found := suite.consumerApp.GetIBCKeeper().ClientKeeper.GetClientState(
-		suite.consumerCtx(), clientID)
+	clientState, found := consumerBundle.App.GetIBCKeeper().ClientKeeper.GetClientState(ctx, clientID)
 	suite.Require().True(found)
 
-	consState, found := suite.consumerApp.GetIBCKeeper().ClientKeeper.GetClientConsensusState(
-		suite.consumerCtx(), clientID, clientState.GetLatestHeight())
+	consState, found := consumerBundle.App.GetIBCKeeper().ClientKeeper.GetClientConsensusState(
+		ctx, clientID, clientState.GetLatestHeight())
 	suite.Require().True(found)
 
 	return clientState, consState

tests/e2e/setup.go

@@ -6,8 +6,8 @@ import (
 
 	ibctmtypes "github.com/cosmos/ibc-go/v3/modules/light-clients/07-tendermint/types"
 	e2eutil "github.com/cosmos/interchain-security/testutil/e2e"
-	icstestingutils "github.com/cosmos/interchain-security/testutil/ibc_testing"
 
+	icstestingutils "github.com/cosmos/interchain-security/testutil/ibc_testing"
 	ccv "github.com/cosmos/interchain-security/x/ccv/types"
 	"github.com/cosmos/interchain-security/x/ccv/utils"
 
@@ -27,13 +27,23 @@ import (
 type CCVTestSuite struct {
 	suite.Suite
 	coordinator   *ibctesting.Coordinator
+	setupCallback SetupCallback
+
 	providerChain *ibctesting.TestChain
-	consumerChain *ibctesting.TestChain
 	providerApp   e2eutil.ProviderApp
-	consumerApp   e2eutil.ConsumerApp
-	path          *ibctesting.Path
-	transferPath  *ibctesting.Path
-	setupCallback SetupCallback
+
+	// The first consumer chain among multiple.
+	consumerChain *ibctesting.TestChain
+	// The first consumer app among multiple.
+	consumerApp e2eutil.ConsumerApp
+	// The ccv path to the first consumer among multiple.
+	path *ibctesting.Path
+	// The transfer path to the first consumer among multiple.
+	transferPath *ibctesting.Path
+
+	// A map from consumer chain ID to its consumer bundle.
+	// The preferred way to access chains, apps, and paths when designing tests around multiple consumers.
+	consumerBundles map[string]*icstestingutils.ConsumerBundle
 }
 
 // NewCCVTestSuite returns a new instance of CCVTestSuite, ready to be tested against using suite.Run().
@@ -42,12 +52,12 @@ func NewCCVTestSuite[Tp e2eutil.ProviderApp, Tc e2eutil.ConsumerApp](
 
 	ccvSuite := new(CCVTestSuite)
 
+	// Define callback called before each test.
 	ccvSuite.setupCallback = func(t *testing.T) (
 		*ibctesting.Coordinator,
 		*ibctesting.TestChain,
-		*ibctesting.TestChain,
 		e2eutil.ProviderApp,
-		e2eutil.ConsumerApp,
+		map[string]*icstestingutils.ConsumerBundle,
 	) {
 		// Instantiate the test coordinator.
 		coordinator := ibctesting.NewCoordinator(t, 0)
@@ -57,15 +67,17 @@ func NewCCVTestSuite[Tp e2eutil.ProviderApp, Tc e2eutil.ConsumerApp](
 		provider, providerApp := icstestingutils.AddProvider[Tp](
 			coordinator, t, providerAppIniter)
 
+		numConsumers := 5
+
 		// Add specified number of consumers to coordinator, store returned test chains and apps.
 		// Concrete consumer app type is passed to the generic function here.
-		consumers, consumerApps := icstestingutils.AddConsumers[Tc](
-			coordinator, t, 1, consumerAppIniter)
+		consumerBundles := icstestingutils.AddConsumers[Tc](
+			coordinator, t, numConsumers, consumerAppIniter)
 
 		// Pass variables to suite.
-		// TODO: accept multiple consumers here
-		return coordinator, provider, consumers[0], providerApp, consumerApps[0]
+		return coordinator, provider, providerApp, consumerBundles
 	}
+
 	return ccvSuite
 }
 
@@ -74,103 +86,122 @@ func NewCCVTestSuite[Tp e2eutil.ProviderApp, Tc e2eutil.ConsumerApp](
 type SetupCallback func(t *testing.T) (
 	coord *ibctesting.Coordinator,
 	providerChain *ibctesting.TestChain,
-	consumerChain *ibctesting.TestChain,
 	providerApp e2eutil.ProviderApp,
-	consumerApp e2eutil.ConsumerApp,
+	consumerBundles map[string]*icstestingutils.ConsumerBundle,
 )
 
 // SetupTest sets up in-mem state before every test
 func (suite *CCVTestSuite) SetupTest() {
 
 	// Instantiate new test utils using callback
 	suite.coordinator, suite.providerChain,
-		suite.consumerChain, suite.providerApp,
-		suite.consumerApp = suite.setupCallback(suite.T())
-
-	providerKeeper := suite.providerApp.GetProviderKeeper()
-	consumerKeeper := suite.consumerApp.GetConsumerKeeper()
-
-	// valsets must match
-	providerValUpdates := tmtypes.TM2PB.ValidatorUpdates(suite.providerChain.Vals)
-	consumerValUpdates := tmtypes.TM2PB.ValidatorUpdates(suite.consumerChain.Vals)
-	suite.Require().True(len(providerValUpdates) == len(consumerValUpdates), "initial valset not matching")
-	for i := 0; i < len(providerValUpdates); i++ {
-		addr1 := utils.GetChangePubKeyAddress(providerValUpdates[i])
-		addr2 := utils.GetChangePubKeyAddress(consumerValUpdates[i])
-		suite.Require().True(bytes.Equal(addr1, addr2), "validator mismatch")
+		suite.providerApp, suite.consumerBundles = suite.setupCallback(suite.T())
+
+	// valsets must match between provider and all consumers
+	for _, bundle := range suite.consumerBundles {
+
+		providerValUpdates := tmtypes.TM2PB.ValidatorUpdates(suite.providerChain.Vals)
+		consumerValUpdates := tmtypes.TM2PB.ValidatorUpdates(bundle.Chain.Vals)
+		suite.Require().True(len(providerValUpdates) == len(consumerValUpdates), "initial valset not matching")
+		for i := 0; i < len(providerValUpdates); i++ {
+			addr1 := utils.GetChangePubKeyAddress(providerValUpdates[i])
+			addr2 := utils.GetChangePubKeyAddress(consumerValUpdates[i])
+			suite.Require().True(bytes.Equal(addr1, addr2), "validator mismatch")
+		}
+		// Move each consumer to next block
+		bundle.Chain.NextBlock()
 	}
 
-	// move both chains to the next block
-	suite.providerChain.NextBlock()
-	suite.consumerChain.NextBlock()
-
-	// create consumer client on provider chain and set as consumer client for consumer chainID in provider keeper.
-	err := providerKeeper.CreateConsumerClient(
-		suite.providerCtx(),
-		suite.consumerChain.ChainID,
-		suite.consumerChain.LastHeader.GetHeight().(clienttypes.Height),
-		false,
-	)
-	suite.Require().NoError(err)
-	// move provider to next block to commit the state
+	// move provider to next block
 	suite.providerChain.NextBlock()
 
-	// initialize the consumer chain with the genesis state stored on the provider
-	consumerGenesisState, found := providerKeeper.GetConsumerGenesis(
-		suite.providerCtx(),
-		suite.consumerChain.ChainID,
-	)
-	suite.Require().True(found, "consumer genesis not found")
-	consumerKeeper.InitGenesis(suite.consumerCtx(), &consumerGenesisState)
+	providerKeeper := suite.providerApp.GetProviderKeeper()
 
-	// Confirm client and cons state for consumer were set correctly in InitGenesis
-	consumerEndpointClientState, consumerEndpointConsState := suite.GetConsumerEndpointClientAndConsState()
-	suite.Require().Equal(consumerGenesisState.ProviderClientState, consumerEndpointClientState)
-	suite.Require().Equal(consumerGenesisState.ProviderConsensusState, consumerEndpointConsState)
+	for chainID, bundle := range suite.consumerBundles {
+		// For each consumer, create client to that consumer on the provider chain.
+		err := providerKeeper.CreateConsumerClient(
+			suite.providerCtx(),
+			chainID,
+			bundle.Chain.LastHeader.GetHeight().(clienttypes.Height),
+			false,
+		)
+		suite.Require().NoError(err)
+	}
 
-	// create path for the CCV channel
-	suite.path = ibctesting.NewPath(suite.consumerChain, suite.providerChain)
+	// move provider to next block to commit the state
+	suite.providerChain.NextBlock()
 
-	// Set provider endpoint's clientID
-	providerEndpointClientID, found := providerKeeper.GetConsumerClientId(
-		suite.providerCtx(),
-		suite.consumerChain.ChainID,
-	)
-	suite.Require().True(found, "provider endpoint clientID not found")
-	suite.path.EndpointB.ClientID = providerEndpointClientID
-
-	// Set consumer endpoint's clientID
-	consumerEndpointClientID, found := consumerKeeper.GetProviderClientID(suite.consumerChain.GetContext())
-	suite.Require().True(found, "consumer endpoint clientID not found")
-	suite.path.EndpointA.ClientID = consumerEndpointClientID
-
-	// Note: suite.path.EndpointA.ClientConfig and suite.path.EndpointB.ClientConfig are not populated,
-	// since these IBC testing package fields are unused in our tests.
-
-	// Confirm client config is now correct
-	suite.ValidateEndpointsClientConfig()
-
-	// - channel config
-	suite.path.EndpointA.ChannelConfig.PortID = ccv.ConsumerPortID
-	suite.path.EndpointB.ChannelConfig.PortID = ccv.ProviderPortID
-	suite.path.EndpointA.ChannelConfig.Version = ccv.Version
-	suite.path.EndpointB.ChannelConfig.Version = ccv.Version
-	suite.path.EndpointA.ChannelConfig.Order = channeltypes.ORDERED
-	suite.path.EndpointB.ChannelConfig.Order = channeltypes.ORDERED
-
-	// set chains sender account number
-	// TODO: to be fixed in #151
-	err = suite.path.EndpointB.Chain.SenderAccount.SetAccountNumber(6)
-	suite.Require().NoError(err)
-	err = suite.path.EndpointA.Chain.SenderAccount.SetAccountNumber(1)
-	suite.Require().NoError(err)
+	// initialize each consumer chain with it's corresponding genesis state
+	// stored on the provider.
+	for chainID, bundle := range suite.consumerBundles {
+
+		consumerGenesisState, found := providerKeeper.GetConsumerGenesis(
+			suite.providerCtx(),
+			chainID,
+		)
+		suite.Require().True(found, "consumer genesis not found")
+
+		consumerKeeper := bundle.GetKeeper()
+		consumerKeeper.InitGenesis(bundle.GetCtx(), &consumerGenesisState)
+
+		// Confirm client and cons state for consumer were set correctly in InitGenesis
+		consumerEndpointClientState,
+			consumerEndpointConsState := suite.GetConsumerEndpointClientAndConsState(*bundle)
+		suite.Require().Equal(consumerGenesisState.ProviderClientState, consumerEndpointClientState)
+		suite.Require().Equal(consumerGenesisState.ProviderConsensusState, consumerEndpointConsState)
+
+		// create path for the CCV channel
+		bundle.Path = ibctesting.NewPath(bundle.Chain, suite.providerChain)
+
+		// Set provider endpoint's clientID for each consumer
+		providerEndpointClientID, found := providerKeeper.GetConsumerClientId(
+			suite.providerCtx(),
+			chainID,
+		)
+		suite.Require().True(found, "provider endpoint clientID not found")
+		bundle.Path.EndpointB.ClientID = providerEndpointClientID
+
+		// Set consumer endpoint's clientID
+		consumerKeeper = bundle.GetKeeper()
+		consumerEndpointClientID, found := consumerKeeper.GetProviderClientID(bundle.GetCtx())
+		suite.Require().True(found, "consumer endpoint clientID not found")
+		bundle.Path.EndpointA.ClientID = consumerEndpointClientID
+
+		// Note: suite.path.EndpointA.ClientConfig and suite.path.EndpointB.ClientConfig are not populated,
+		// since these IBC testing package fields are unused in our tests.
+
+		// Confirm client config is now correct
+		suite.ValidateEndpointsClientConfig(*bundle)
+
+		// - channel config
+		bundle.Path.EndpointA.ChannelConfig.PortID = ccv.ConsumerPortID
+		bundle.Path.EndpointB.ChannelConfig.PortID = ccv.ProviderPortID
+		bundle.Path.EndpointA.ChannelConfig.Version = ccv.Version
+		bundle.Path.EndpointB.ChannelConfig.Version = ccv.Version
+		bundle.Path.EndpointA.ChannelConfig.Order = channeltypes.ORDERED
+		bundle.Path.EndpointB.ChannelConfig.Order = channeltypes.ORDERED
+
+		// set chains sender account number
+		// TODO: to be fixed in #151
+		err := bundle.Path.EndpointB.Chain.SenderAccount.SetAccountNumber(6)
+		suite.Require().NoError(err)
+		err = bundle.Path.EndpointA.Chain.SenderAccount.SetAccountNumber(1)
+		suite.Require().NoError(err)
+
+		// create path for the transfer channel
+		bundle.TransferPath = ibctesting.NewPath(bundle.Chain, suite.providerChain)
+		bundle.TransferPath.EndpointA.ChannelConfig.PortID = transfertypes.PortID
+		bundle.TransferPath.EndpointB.ChannelConfig.PortID = transfertypes.PortID
+		bundle.TransferPath.EndpointA.ChannelConfig.Version = transfertypes.Version
+		bundle.TransferPath.EndpointB.ChannelConfig.Version = transfertypes.Version
+	}
 
-	// create path for the transfer channel
-	suite.transferPath = ibctesting.NewPath(suite.consumerChain, suite.providerChain)
-	suite.transferPath.EndpointA.ChannelConfig.PortID = transfertypes.PortID
-	suite.transferPath.EndpointB.ChannelConfig.PortID = transfertypes.PortID
-	suite.transferPath.EndpointA.ChannelConfig.Version = transfertypes.Version
-	suite.transferPath.EndpointB.ChannelConfig.Version = transfertypes.Version
+	// Support tests that were written before multiple consumers were supported.
+	firstBundle := suite.consumerBundles[icstestingutils.FirstConsumerChainID]
+	suite.consumerApp = firstBundle.App
+	suite.consumerChain = firstBundle.Chain
+	suite.path = firstBundle.Path
+	suite.transferPath = firstBundle.TransferPath
 }
 
 func (suite *CCVTestSuite) SetupCCVChannel() {
@@ -229,12 +260,13 @@ func (suite *CCVTestSuite) SetupTransferChannel() {
 	suite.Require().NoError(err)
 }
 
-func (s CCVTestSuite) ValidateEndpointsClientConfig() {
-	consumerKeeper := s.consumerApp.GetConsumerKeeper()
+func (s CCVTestSuite) ValidateEndpointsClientConfig(consumerBundle icstestingutils.ConsumerBundle) {
+	consumerKeeper := consumerBundle.GetKeeper()
 	providerStakingKeeper := s.providerApp.GetStakingKeeper()
 
-	consumerUnbondingPeriod := consumerKeeper.GetUnbondingPeriod(s.consumerCtx())
-	cs, ok := s.providerApp.GetIBCKeeper().ClientKeeper.GetClientState(s.providerCtx(), s.path.EndpointB.ClientID)
+	consumerUnbondingPeriod := consumerKeeper.GetUnbondingPeriod(consumerBundle.GetCtx())
+	cs, ok := s.providerApp.GetIBCKeeper().ClientKeeper.GetClientState(s.providerCtx(),
+		consumerBundle.Path.EndpointB.ClientID)
 	s.Require().True(ok)
 	s.Require().Equal(
 		consumerUnbondingPeriod,
@@ -243,7 +275,8 @@ func (s CCVTestSuite) ValidateEndpointsClientConfig() {
 	)
 
 	providerUnbondingPeriod := providerStakingKeeper.UnbondingTime(s.providerCtx())
-	cs, ok = s.consumerApp.GetIBCKeeper().ClientKeeper.GetClientState(s.consumerCtx(), s.path.EndpointA.ClientID)
+	cs, ok = consumerBundle.App.GetIBCKeeper().ClientKeeper.GetClientState(
+		consumerBundle.GetCtx(), consumerBundle.Path.EndpointA.ClientID)
 	s.Require().True(ok)
 	s.Require().Equal(
 		providerUnbondingPeriod,

tests/e2e/unbonding.go

@@ -9,8 +9,8 @@ import (
 	ccv "github.com/cosmos/interchain-security/x/ccv/types"
 )
 
-// TestUndelegationNormalOperation tests that undelegations complete after 
-// the unbonding period elapses on both the consumer and provider, without 
+// TestUndelegationNormalOperation tests that undelegations complete after
+// the unbonding period elapses on both the consumer and provider, without
 // VSC packets timing out.
 func (s *CCVTestSuite) TestUndelegationNormalOperation() {
 	unbondConsumer := func(expectedPackets int) {
@@ -304,8 +304,10 @@ func (s *CCVTestSuite) TestUnbondingNoConsumer() {
 	providerKeeper := s.providerApp.GetProviderKeeper()
 	providerStakingKeeper := s.providerApp.GetE2eStakingKeeper()
 
-	// remove the consumer chain, which was already registered during setup
-	providerKeeper.DeleteConsumerClientId(s.providerCtx(), s.consumerChain.ChainID)
+	// remove all consumer chains, which were already registered during setup
+	for chainID := range s.consumerBundles {
+		providerKeeper.DeleteConsumerClientId(s.providerCtx(), chainID)
+	}
 
 	// delegate bondAmt and undelegate 1/2 of it
 	bondAmt := sdk.NewInt(10000000)