diff --git a/.changeset/moody-swans-worry.md b/.changeset/moody-swans-worry.md new file mode 100644 index 00000000000..dc761e52c9e --- /dev/null +++ b/.changeset/moody-swans-worry.md @@ -0,0 +1,5 @@ +--- +"chainlink": major +--- + +Remove support for mercury diff --git a/core/services/chainlink/application.go b/core/services/chainlink/application.go index fffb22aee8a..21d58ba9327 100644 --- a/core/services/chainlink/application.go +++ b/core/services/chainlink/application.go @@ -66,7 +66,6 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/periodicbackup" "github.com/smartcontractkit/chainlink/v2/core/services/pipeline" "github.com/smartcontractkit/chainlink/v2/core/services/registrysyncer" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury" "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/wsrpc" "github.com/smartcontractkit/chainlink/v2/core/services/standardcapabilities" "github.com/smartcontractkit/chainlink/v2/core/services/streams" @@ -524,7 +523,6 @@ func NewApplication(opts ApplicationOpts) (Application, error) { var ( pipelineORM = pipeline.NewORM(opts.DS, globalLogger, cfg.JobPipeline().MaxSuccessfulRuns()) bridgeORM = bridges.NewORM(opts.DS) - mercuryORM = mercury.NewORM(opts.DS) pipelineRunner = pipeline.NewRunner(pipelineORM, bridgeORM, cfg.JobPipeline(), cfg.WebServer(), legacyEVMChains, keyStore.Eth(), keyStore.VRF(), globalLogger, restrictedHTTPClient, unrestrictedHTTPClient) jobORM = job.NewORM(opts.DS, pipelineORM, bridgeORM, keyStore, globalLogger) txmORM = txmgr.NewTxStore(opts.DS, globalLogger) @@ -684,7 +682,6 @@ func NewApplication(opts ApplicationOpts) (Application, error) { Ds: opts.DS, JobORM: jobORM, BridgeORM: bridgeORM, - MercuryORM: mercuryORM, PipelineRunner: pipelineRunner, StreamRegistry: streamRegistry, PeerWrapper: peerWrapper, diff --git a/core/services/job/job_orm_test.go b/core/services/job/job_orm_test.go index 9eb20d6dab2..42ddc7a695b 100644 --- a/core/services/job/job_orm_test.go +++ b/core/services/job/job_orm_test.go @@ -52,36 +52,6 @@ import ( "github.com/smartcontractkit/chainlink/v2/evm/utils/big" ) -const mercuryOracleTOML = `name = 'LINK / ETH | 0x0000000000000000000000000000000000000000000000000000000000000001 | verifier_proxy 0x0000000000000000000000000000000000000001' -type = 'offchainreporting2' -schemaVersion = 1 -externalJobID = '00000000-0000-0000-0000-000000000001' -contractID = '0x0000000000000000000000000000000000000006' -transmitterID = '%s' -feedID = '%s' -relay = 'evm' -pluginType = 'mercury' -observationSource = """ - ds [type=http method=GET url="https://chain.link/ETH-USD"]; - ds_parse [type=jsonparse path="data.price" separator="."]; - ds_multiply [type=multiply times=100]; - ds -> ds_parse -> ds_multiply; -""" - -[relayConfig] -chainID = 1 -fromBlock = 1000 - -[onchainSigningStrategy] -strategyName = 'single-chain' -[onchainSigningStrategy.config] -publicKey = '8fa807463ad73f9ee855cfd60ba406dcf98a2855b3dd8af613107b0f6890a707' - -[pluginConfig] -serverURL = 'wss://localhost:8080' -serverPubKey = '8fa807463ad73f9ee855cfd60ba406dcf98a2855b3dd8af613107b0f6890a707' -` - func TestORM(t *testing.T) { t.Parallel() @@ -1185,28 +1155,6 @@ func Test_FindJob(t *testing.T) { jobOCR2.OCR2OracleSpec.PluginConfig["juelsPerFeeCoinSource"] = juelsPerFeeCoinSource - ocr2WithFeedID1 := "0x0001000000000000000000000000000000000000000000000000000000000001" - ocr2WithFeedID2 := "0x0001000000000000000000000000000000000000000000000000000000000002" - jobOCR2WithFeedID1, err := ocr2validate.ValidatedOracleSpecToml( - testutils.Context(t), - config.OCR2(), - config.Insecure(), - fmt.Sprintf(mercuryOracleTOML, cltest.DefaultCSAKey.PublicKeyString(), ocr2WithFeedID1), - nil, - ) - require.NoError(t, err) - - jobOCR2WithFeedID2, err := ocr2validate.ValidatedOracleSpecToml( - testutils.Context(t), - config.OCR2(), - config.Insecure(), - fmt.Sprintf(mercuryOracleTOML, cltest.DefaultCSAKey.PublicKeyString(), ocr2WithFeedID2), - nil, - ) - jobOCR2WithFeedID2.ExternalJobID = uuid.New() - jobOCR2WithFeedID2.Name = null.StringFrom("new name") - require.NoError(t, err) - err = orm.CreateJob(ctx, &job) require.NoError(t, err) @@ -1216,13 +1164,6 @@ func Test_FindJob(t *testing.T) { err = orm.CreateJob(ctx, &jobOCR2) require.NoError(t, err) - err = orm.CreateJob(ctx, &jobOCR2WithFeedID1) - require.NoError(t, err) - - // second ocr2 job with same contract id but different feed id - err = orm.CreateJob(ctx, &jobOCR2WithFeedID2) - require.NoError(t, err) - t.Run("by id", func(t *testing.T) { ctx, cancel := context.WithTimeout(testutils.Context(t), 5*time.Second) defer cancel() @@ -1281,7 +1222,7 @@ func Test_FindJob(t *testing.T) { assert.Equal(t, job.ID, jbID) }) - t.Run("by contract id without feed id", func(t *testing.T) { + t.Run("by contract id", func(t *testing.T) { ctx := testutils.Context(t) contractID := "0x613a38AC1659769640aaE063C651F48E0250454C" @@ -1291,30 +1232,6 @@ func Test_FindJob(t *testing.T) { assert.Equal(t, jobOCR2.ID, jbID) }) - - t.Run("by contract id with valid feed id", func(t *testing.T) { - ctx := testutils.Context(t) - contractID := "0x0000000000000000000000000000000000000006" - feedID := common.HexToHash(ocr2WithFeedID1) - - // Find job ID for ocr2 job with feed ID - jbID, err2 := orm.FindOCR2JobIDByAddress(ctx, contractID, &feedID) - require.NoError(t, err2) - - assert.Equal(t, jobOCR2WithFeedID1.ID, jbID) - }) - - t.Run("with duplicate contract id but different feed id", func(t *testing.T) { - ctx := testutils.Context(t) - contractID := "0x0000000000000000000000000000000000000006" - feedID := common.HexToHash(ocr2WithFeedID2) - - // Find job ID for ocr2 job with feed ID - jbID, err2 := orm.FindOCR2JobIDByAddress(ctx, contractID, &feedID) - require.NoError(t, err2) - - assert.Equal(t, jobOCR2WithFeedID2.ID, jbID) - }) } func Test_FindJobsByPipelineSpecIDs(t *testing.T) { diff --git a/core/services/ocr2/delegate.go b/core/services/ocr2/delegate.go index f4d92ca079e..0c104d98842 100644 --- a/core/services/ocr2/delegate.go +++ b/core/services/ocr2/delegate.go @@ -59,7 +59,6 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/generic" lloconfig "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/llo/config" "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/median" - "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/mercury" "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper" "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/autotelemetry21" ocr2keeper21core "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ocr2keeper/evmregistry/v21/core" @@ -69,8 +68,6 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/relay" evmrelay "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm" functionsRelay "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/functions" - evmmercury "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury" - mercuryutils "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/utils" evmrelaytypes "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/types" "github.com/smartcontractkit/chainlink/v2/core/services/streams" "github.com/smartcontractkit/chainlink/v2/core/services/synchronization" @@ -109,7 +106,6 @@ type Delegate struct { ds sqlutil.DataSource jobORM job.ORM bridgeORM bridges.ORM - mercuryORM evmmercury.ORM pipelineRunner pipeline.Runner streamRegistry streams.Getter peerWrapper *ocrcommon.SingletonPeerWrapper @@ -218,7 +214,6 @@ type DelegateOpts struct { Ds sqlutil.DataSource JobORM job.ORM BridgeORM bridges.ORM - MercuryORM evmmercury.ORM PipelineRunner pipeline.Runner StreamRegistry streams.Getter PeerWrapper *ocrcommon.SingletonPeerWrapper @@ -241,7 +236,6 @@ func NewDelegate( ds: opts.Ds, jobORM: opts.JobORM, bridgeORM: opts.BridgeORM, - mercuryORM: opts.MercuryORM, pipelineRunner: opts.PipelineRunner, streamRegistry: opts.StreamRegistry, peerWrapper: opts.PeerWrapper, @@ -497,9 +491,6 @@ func (d *Delegate) ServicesForSpec(ctx context.Context, jb job.Job) ([]job.Servi ctx = lggrCtx.ContextWithValues(ctx) switch spec.PluginType { - case types.Mercury: - return d.newServicesMercury(ctx, lggr, jb, bootstrapPeers, kb, ocrDB, lc) - case types.LLO: return d.newServicesLLO(ctx, lggr, jb, bootstrapPeers, kb, ocrDB, lc) @@ -534,7 +525,7 @@ func (d *Delegate) ServicesForSpec(ctx context.Context, jb job.Job) ([]job.Servi func GetEVMEffectiveTransmitterID(ctx context.Context, jb *job.Job, chain legacyevm.Chain, lggr logger.SugaredLogger) (string, error) { spec := jb.OCR2OracleSpec - if spec.PluginType == types.Mercury || spec.PluginType == types.LLO { + if spec.PluginType == types.LLO { return spec.TransmitterID.String, nil } @@ -553,7 +544,6 @@ func GetEVMEffectiveTransmitterID(ctx context.Context, jb *job.Job, chain legacy // effectiveTransmitterID is the transmitter address registered on the ocr contract. This is by default the EOA account on the node. // In the case of forwarding, the transmitter address is the forwarder contract deployed onchain between EOA and OCR contract. - // ForwardingAllowed cannot be set with Mercury, so this should always be false for mercury jobs if jb.ForwardingAllowed { if chain == nil { return "", fmt.Errorf("job forwarding requires non-nil chain") @@ -828,115 +818,6 @@ func (d *Delegate) newServicesGenericPlugin( return srvs, nil } -func (d *Delegate) newServicesMercury( - ctx context.Context, - lggr logger.SugaredLogger, - jb job.Job, - bootstrapPeers []commontypes.BootstrapperLocator, - kb ocr2key.KeyBundle, - ocrDB *db, - lc ocrtypes.LocalConfig, -) ([]job.ServiceCtx, error) { - if jb.OCR2OracleSpec.FeedID == nil || (*jb.OCR2OracleSpec.FeedID == (common.Hash{})) { - return nil, errors.Errorf("ServicesForSpec: mercury job type requires feedID") - } - spec := jb.OCR2OracleSpec - transmitterID := spec.TransmitterID.String - if len(transmitterID) != 64 { - return nil, errors.Errorf("ServicesForSpec: mercury job type requires transmitter ID to be a 32-byte hex string, got: %q", transmitterID) - } - if _, err := hex.DecodeString(transmitterID); err != nil { - return nil, errors.Wrapf(err, "ServicesForSpec: mercury job type requires transmitter ID to be a 32-byte hex string, got: %q", transmitterID) - } - - rid, err := spec.RelayID() - if err != nil { - return nil, ErrJobSpecNoRelayer{Err: err, PluginName: "mercury"} - } - if rid.Network != relay.NetworkEVM { - return nil, fmt.Errorf("mercury services: expected EVM relayer got %q", rid.Network) - } - relayer, err := d.RelayGetter.Get(rid) - if err != nil { - return nil, ErrRelayNotEnabled{Err: err, Relay: spec.Relay, PluginName: "mercury"} - } - - provider, err2 := relayer.NewPluginProvider(ctx, - types.RelayArgs{ - ExternalJobID: jb.ExternalJobID, - JobID: jb.ID, - ContractID: spec.ContractID, - New: d.isNewlyCreatedJob, - RelayConfig: spec.RelayConfig.Bytes(), - ProviderType: string(spec.PluginType), - }, types.PluginArgs{ - TransmitterID: transmitterID, - PluginConfig: spec.PluginConfig.Bytes(), - }) - if err2 != nil { - return nil, err2 - } - - mercuryProvider, ok := provider.(types.MercuryProvider) - if !ok { - return nil, errors.New("could not coerce PluginProvider to MercuryProvider") - } - - lc.ContractConfigTrackerPollInterval = 1 * time.Second // This is the fastest that libocr supports. See: https://github.com/smartcontractkit/offchain-reporting/pull/520 - - // Disable OCR debug+info logging for legacy mercury jobs unless tracelogging is enabled, because its simply too verbose (150 jobs => ~50k logs per second) - ocrLogger := ocrcommon.NewOCRWrapper(llo.NewSuppressedLogger(lggr, d.cfg.OCR2().TraceLogging()), d.cfg.OCR2().TraceLogging(), func(ctx context.Context, msg string) { - lggr.ErrorIf(d.jobORM.RecordError(ctx, jb.ID, msg), "unable to record error") - }) - - var relayConfig evmrelaytypes.RelayConfig - err = json.Unmarshal(jb.OCR2OracleSpec.RelayConfig.Bytes(), &relayConfig) - if err != nil { - return nil, fmt.Errorf("error while unmarshalling relay config: %w", err) - } - - var telemetryType synchronization.TelemetryType - if relayConfig.EnableTriggerCapability && len(jb.OCR2OracleSpec.PluginConfig) == 0 { - telemetryType = synchronization.OCR3DataFeeds - // First use case for TriggerCapability transmission is Data Feeds, so telemetry should be routed accordingly. - // This is only true if TriggerCapability is the *only* transmission method (PluginConfig is empty). - } else { - telemetryType = synchronization.OCR3Mercury - } - - oracleArgsNoPlugin := libocr2.MercuryOracleArgs{ - BinaryNetworkEndpointFactory: d.peerWrapper.Peer2, - V2Bootstrappers: bootstrapPeers, - ContractTransmitter: mercuryProvider.ContractTransmitter(), - ContractConfigTracker: mercuryProvider.ContractConfigTracker(), - Database: ocrDB, - LocalConfig: lc, - Logger: ocrLogger, - MonitoringEndpoint: d.monitoringEndpointGen.GenMonitoringEndpoint(rid.Network, rid.ChainID, spec.FeedID.String(), telemetryType), - OffchainConfigDigester: mercuryProvider.OffchainConfigDigester(), - OffchainKeyring: kb, - OnchainKeyring: kb, - MetricsRegisterer: prometheus.WrapRegistererWith(map[string]string{"job_name": jb.Name.ValueOrZero()}, prometheus.DefaultRegisterer), - } - - chEnhancedTelem := make(chan ocrcommon.EnhancedTelemetryMercuryData, 100) - - mCfg := mercury.NewMercuryConfig(d.cfg.JobPipeline().MaxSuccessfulRuns(), d.cfg.JobPipeline().ResultWriteQueueDepth(), d.cfg) - - mercuryServices, err2 := mercury.NewServices(jb, mercuryProvider, d.pipelineRunner, lggr, oracleArgsNoPlugin, mCfg, chEnhancedTelem, d.mercuryORM, (mercuryutils.FeedID)(*spec.FeedID), relayConfig.EnableTriggerCapability) - - if ocrcommon.ShouldCollectEnhancedTelemetryMercury(jb) { - enhancedTelemService := ocrcommon.NewEnhancedTelemetryService(&jb, chEnhancedTelem, make(chan struct{}), d.monitoringEndpointGen.GenMonitoringEndpoint(rid.Network, rid.ChainID, spec.FeedID.String(), synchronization.EnhancedEAMercury), lggr.Named("EnhancedTelemetryMercury")) - mercuryServices = append(mercuryServices, enhancedTelemService) - } else { - lggr.Infow("Enhanced telemetry is disabled for mercury job", "job", jb.Name) - } - - mercuryServices = append(mercuryServices, ocrLogger) - - return mercuryServices, err2 -} - func (d *Delegate) newServicesLLO( ctx context.Context, lggr logger.SugaredLogger, diff --git a/core/services/ocr2/plugins/llo/config/config.go b/core/services/ocr2/plugins/llo/config/config.go index ca272b73d55..d9c32bfa994 100644 --- a/core/services/ocr2/plugins/llo/config/config.go +++ b/core/services/ocr2/plugins/llo/config/config.go @@ -16,10 +16,14 @@ import ( llotypes "github.com/smartcontractkit/chainlink-common/pkg/types/llo" "github.com/smartcontractkit/chainlink/v2/core/services/keystore/chaintype" - mercuryconfig "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/mercury/config" "github.com/smartcontractkit/chainlink/v2/core/utils" ) +type Server struct { + URL string + PubKey utils.PlainHexBytes +} + type PluginConfig struct { ChannelDefinitionsContractAddress common.Address `json:"channelDefinitionsContractAddress" toml:"channelDefinitionsContractAddress"` ChannelDefinitionsContractFromBlock int64 `json:"channelDefinitionsContractFromBlock" toml:"channelDefinitionsContractFromBlock"` @@ -49,9 +53,9 @@ func (p *PluginConfig) Unmarshal(data []byte) error { return json.Unmarshal(data, p) } -func (p PluginConfig) GetServers() (servers []mercuryconfig.Server) { +func (p PluginConfig) GetServers() (servers []Server) { for url, pubKey := range p.Servers { - servers = append(servers, mercuryconfig.Server{URL: wssRegexp.ReplaceAllString(url, ""), PubKey: pubKey}) + servers = append(servers, Server{URL: wssRegexp.ReplaceAllString(url, ""), PubKey: pubKey}) } sort.Slice(servers, func(i, j int) bool { return servers[i].URL < servers[j].URL diff --git a/core/services/ocr2/plugins/llo/integration_test.go b/core/services/ocr2/plugins/llo/integration_test.go index 3a23b5e28d2..6d976f43a7c 100644 --- a/core/services/ocr2/plugins/llo/integration_test.go +++ b/core/services/ocr2/plugins/llo/integration_test.go @@ -34,30 +34,24 @@ import ( llotypes "github.com/smartcontractkit/chainlink-common/pkg/types/llo" datastreamsllo "github.com/smartcontractkit/chainlink-data-streams/llo" "github.com/smartcontractkit/chainlink-data-streams/rpc" - evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils" + ubig "github.com/smartcontractkit/chainlink/v2/core/chains/evm/utils/big" "github.com/smartcontractkit/chainlink/v2/core/config" - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/link_token_interface" + + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/llo-feeds/generated/channel_config_store" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/llo-feeds/generated/configurator" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/llo-feeds/generated/destination_verifier" "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/llo-feeds/generated/destination_verifier_proxy" - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/llo-feeds/generated/fee_manager" - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/llo-feeds/generated/reward_manager" - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/llo-feeds/generated/verifier" - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/llo-feeds/generated/verifier_proxy" "github.com/smartcontractkit/chainlink/v2/core/internal/cltest" "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" "github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/csakey" lloevm "github.com/smartcontractkit/chainlink/v2/core/services/llo/evm" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm" "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/llo" "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury" reportcodecv3 "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v3/reportcodec" mercuryverifier "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/verifier" - "github.com/smartcontractkit/chainlink/v2/evm/assets" - "github.com/smartcontractkit/chainlink/v2/evm/utils" - ubig "github.com/smartcontractkit/chainlink/v2/evm/utils/big" ) var ( @@ -76,10 +70,6 @@ func setupBlockchain(t *testing.T) ( common.Address, *channel_config_store.ChannelConfigStore, common.Address, - *verifier.Verifier, - common.Address, - *verifier_proxy.VerifierProxy, - common.Address, ) { steve := testutils.MustNewSimTransactor(t) // config contract deployer and owner genesisData := gethtypes.GenesisAlloc{steve.From: {Balance: assets.Ether(1000).ToInt()}} @@ -105,53 +95,13 @@ func setupBlockchain(t *testing.T) ( require.NoError(t, err) backend.Commit() - // Legacy mercury verifier - legacyVerifier, legacyVerifierAddr, legacyVerifierProxy, legacyVerifierProxyAddr := setupLegacyMercuryVerifier(t, steve, backend) - // ChannelConfigStore configStoreAddress, _, configStore, err := channel_config_store.DeployChannelConfigStore(steve, backend.Client()) require.NoError(t, err) backend.Commit() - return steve, backend, configurator, configuratorAddress, destinationVerifier, destinationVerifierAddr, verifierProxy, destinationVerifierProxyAddr, configStore, configStoreAddress, legacyVerifier, legacyVerifierAddr, legacyVerifierProxy, legacyVerifierProxyAddr -} - -func setupLegacyMercuryVerifier(t *testing.T, steve *bind.TransactOpts, backend evmtypes.Backend) (*verifier.Verifier, common.Address, *verifier_proxy.VerifierProxy, common.Address) { - linkTokenAddress, _, linkToken, err := link_token_interface.DeployLinkToken(steve, backend.Client()) - require.NoError(t, err) - backend.Commit() - _, err = linkToken.Transfer(steve, steve.From, big.NewInt(1000)) - require.NoError(t, err) - backend.Commit() - nativeTokenAddress, _, nativeToken, err := link_token_interface.DeployLinkToken(steve, backend.Client()) - require.NoError(t, err) - backend.Commit() - _, err = nativeToken.Transfer(steve, steve.From, big.NewInt(1000)) - require.NoError(t, err) - backend.Commit() - verifierProxyAddr, _, verifierProxy, err := verifier_proxy.DeployVerifierProxy(steve, backend.Client(), common.Address{}) // zero address for access controller disables access control - require.NoError(t, err) - backend.Commit() - verifierAddress, _, verifier, err := verifier.DeployVerifier(steve, backend.Client(), verifierProxyAddr) - require.NoError(t, err) - backend.Commit() - _, err = verifierProxy.InitializeVerifier(steve, verifierAddress) - require.NoError(t, err) - backend.Commit() - rewardManagerAddr, _, rewardManager, err := reward_manager.DeployRewardManager(steve, backend.Client(), linkTokenAddress) - require.NoError(t, err) - backend.Commit() - feeManagerAddr, _, _, err := fee_manager.DeployFeeManager(steve, backend.Client(), linkTokenAddress, nativeTokenAddress, verifierProxyAddr, rewardManagerAddr) - require.NoError(t, err) - backend.Commit() - _, err = verifierProxy.SetFeeManager(steve, feeManagerAddr) - require.NoError(t, err) - backend.Commit() - _, err = rewardManager.SetFeeManager(steve, feeManagerAddr) - require.NoError(t, err) - backend.Commit() - return verifier, verifierAddress, verifierProxy, verifierProxyAddr + return steve, backend, configurator, configuratorAddress, destinationVerifier, destinationVerifierAddr, verifierProxy, destinationVerifierProxyAddr, configStore, configStoreAddress } type Stream struct { @@ -247,37 +197,6 @@ func generateConfig(t *testing.T, oracles []confighelper.OracleIdentityExtra, in return } -func setLegacyConfig(t *testing.T, donID uint32, steve *bind.TransactOpts, backend evmtypes.Backend, legacyVerifier *verifier.Verifier, legacyVerifierAddr common.Address, nodes []Node, oracles []confighelper.OracleIdentityExtra) ocr2types.ConfigDigest { - onchainConfig, err := (&datastreamsllo.EVMOnchainConfigCodec{}).Encode(datastreamsllo.OnchainConfig{ - Version: 1, - PredecessorConfigDigest: nil, - }) - require.NoError(t, err) - - signers, _, _, _, offchainConfigVersion, offchainConfig := generateConfig(t, oracles, onchainConfig) - - signerAddresses, err := evm.OnchainPublicKeyToAddress(signers) - require.NoError(t, err) - offchainTransmitters := make([][32]byte, nNodes) - for i := 0; i < nNodes; i++ { - offchainTransmitters[i] = nodes[i].ClientPubKey - } - donIDPadded := llo.DonIDToBytes32(donID) - _, err = legacyVerifier.SetConfig(steve, donIDPadded, signerAddresses, offchainTransmitters, fNodes, onchainConfig, offchainConfigVersion, offchainConfig, nil) - require.NoError(t, err) - - // libocr requires a few confirmations to accept the config - backend.Commit() - backend.Commit() - backend.Commit() - backend.Commit() - - l, err := legacyVerifier.LatestConfigDigestAndEpoch(&bind.CallOpts{}, donIDPadded) - require.NoError(t, err) - - return l.ConfigDigest -} - func setStagingConfig(t *testing.T, donID uint32, steve *bind.TransactOpts, backend evmtypes.Backend, configurator *configurator.Configurator, configuratorAddress common.Address, nodes []Node, oracles []confighelper.OracleIdentityExtra, predecessorConfigDigest ocr2types.ConfigDigest) ocr2types.ConfigDigest { return setBlueGreenConfig(t, donID, steve, backend, configurator, configuratorAddress, nodes, oracles, &predecessorConfigDigest) } @@ -323,7 +242,7 @@ func setBlueGreenConfig(t *testing.T, donID uint32, steve *bind.TransactOpts, ba require.NoError(t, err) require.GreaterOrEqual(t, len(logs), 1) - cfg, err := mercury.ConfigFromLog(logs[len(logs)-1].Data) + cfg, err := llo.DecodeProductionConfigSetLog(logs[len(logs)-1].Data) require.NoError(t, err) return cfg.ConfigDigest @@ -359,7 +278,7 @@ func TestIntegration_LLO_evm_premium_legacy(t *testing.T) { clientPubKeys[i] = key.PublicKey } - steve, backend, _, _, verifier, _, verifierProxy, _, configStore, configStoreAddress, legacyVerifier, legacyVerifierAddr, _, _ := setupBlockchain(t) + steve, backend, configurator, configuratorAddress, _, _, _, _, configStore, configStoreAddress := setupBlockchain(t) fromBlock := 1 // Setup bootstrap @@ -392,9 +311,9 @@ func TestIntegration_LLO_evm_premium_legacy(t *testing.T) { chainID = "%s" fromBlock = %d lloDonID = %d -lloConfigMode = "mercury" +lloConfigMode = "bluegreen" `, chainID, fromBlock, donID) - addBootstrapJob(t, bootstrapNode, legacyVerifierAddr, "job-2", relayType, relayConfig) + addBootstrapJob(t, bootstrapNode, configuratorAddress, "job-2", relayType, relayConfig) // Channel definitions channelDefinitions := llotypes.ChannelDefinitions{ @@ -447,25 +366,17 @@ lloConfigMode = "mercury" donID = %d channelDefinitionsContractAddress = "0x%x" channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, configStoreAddress, fromBlock) - addOCRJobsEVMPremiumLegacy(t, streams, serverPubKey, serverURL, legacyVerifierAddr, bootstrapPeerID, bootstrapNodePort, nodes, configStoreAddress, clientPubKeys, pluginConfig, relayType, relayConfig) + addOCRJobsEVMPremiumLegacy(t, streams, serverPubKey, serverURL, configuratorAddress, bootstrapPeerID, bootstrapNodePort, nodes, configStoreAddress, clientPubKeys, pluginConfig, relayType, relayConfig) // Set config on configurator - setLegacyConfig( - t, donID, steve, backend, legacyVerifier, legacyVerifierAddr, nodes, oracles, + setProductionConfig( + t, donID, steve, backend, configurator, configuratorAddress, nodes, oracles, ) - // Set config on the destination verifier signerAddresses := make([]common.Address, len(oracles)) for i, oracle := range oracles { signerAddresses[i] = common.BytesToAddress(oracle.OracleIdentity.OnchainPublicKey) } - { - recipientAddressesAndWeights := []destination_verifier.CommonAddressAndWeight{} - - _, err := verifier.SetConfig(steve, signerAddresses, fNodes, recipientAddressesAndWeights) - require.NoError(t, err) - backend.Commit() - } t.Run("receives at least one report per channel from each oracle when EAs are at 100% reliability", func(t *testing.T) { // Expect at least one report per feed from each oracle @@ -534,16 +445,6 @@ channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, confi assert.Subset(t, signerAddresses, reportSigners) } - // test on-chain verification - t.Run("on-chain verification", func(t *testing.T) { - t.Skip("SKIP - MERC-6637") - // Disabled because it flakes, sometimes returns "execution reverted" - // No idea why - // https://smartcontract-it.atlassian.net/browse/MERC-6637 - _, err = verifierProxy.Verify(steve, req.req.Payload, []byte{}) - require.NoError(t, err) - }) - t.Logf("oracle %x reported for 0x%x", req.pk[:], feedID[:]) seen[feedID][req.pk] = struct{}{} @@ -576,7 +477,7 @@ func TestIntegration_LLO_evm_abi_encode_unpacked(t *testing.T) { clientPubKeys[i] = key.PublicKey } - steve, backend, configurator, configuratorAddress, _, _, _, _, configStore, configStoreAddress, _, _, _, _ := setupBlockchain(t) + steve, backend, configurator, configuratorAddress, _, _, _, _, configStore, configStoreAddress := setupBlockchain(t) fromBlock := 1 // Setup bootstrap @@ -1056,7 +957,7 @@ func TestIntegration_LLO_blue_green_lifecycle(t *testing.T) { clientPubKeys[i] = key.PublicKey } - steve, backend, configurator, configuratorAddress, _, _, _, _, configStore, configStoreAddress, _, _, _, _ := setupBlockchain(t) + steve, backend, configurator, configuratorAddress, _, _, _, _, configStore, configStoreAddress := setupBlockchain(t) fromBlock := 1 // Setup bootstrap diff --git a/core/services/ocr2/plugins/llo/integration_test.go.orig b/core/services/ocr2/plugins/llo/integration_test.go.orig new file mode 100644 index 00000000000..43364f35c11 --- /dev/null +++ b/core/services/ocr2/plugins/llo/integration_test.go.orig @@ -0,0 +1,1310 @@ +package llo_test + +import ( + "crypto/ed25519" + "encoding/hex" + "encoding/json" + "fmt" + "math/big" + "net/http" + "net/http/httptest" + "sort" + "strings" + "testing" + "time" + + "github.com/ethereum/go-ethereum" + "github.com/ethereum/go-ethereum/accounts/abi" + "github.com/ethereum/go-ethereum/accounts/abi/bind" + "github.com/ethereum/go-ethereum/common" + gethtypes "github.com/ethereum/go-ethereum/core/types" + "github.com/ethereum/go-ethereum/eth/ethconfig" + "github.com/hashicorp/consul/sdk/freeport" + "github.com/shopspring/decimal" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "golang.org/x/crypto/sha3" + + "github.com/smartcontractkit/libocr/offchainreporting2/types" + "github.com/smartcontractkit/libocr/offchainreporting2plus/confighelper" + "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3confighelper" + ocr2types "github.com/smartcontractkit/libocr/offchainreporting2plus/types" + "github.com/smartcontractkit/wsrpc/credentials" + + llotypes "github.com/smartcontractkit/chainlink-common/pkg/types/llo" + datastreamsllo "github.com/smartcontractkit/chainlink-data-streams/llo" + "github.com/smartcontractkit/chainlink-data-streams/rpc" + +<<<<<<< HEAD + evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" + "github.com/smartcontractkit/chainlink/v2/core/config" + "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/link_token_interface" +======= + "github.com/smartcontractkit/chainlink/v2/core/chains/evm/assets" +>>>>>>> 3949e0ac22 (Remove legacy Mercury plugin and associated code) + "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/llo-feeds/generated/channel_config_store" + "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/llo-feeds/generated/configurator" + "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/llo-feeds/generated/destination_verifier" + "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/llo-feeds/generated/destination_verifier_proxy" + "github.com/smartcontractkit/chainlink/v2/core/internal/cltest" + "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" + "github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/csakey" + lloevm "github.com/smartcontractkit/chainlink/v2/core/services/llo/evm" + "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/llo" + "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury" + reportcodecv3 "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v3/reportcodec" + mercuryverifier "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/verifier" + "github.com/smartcontractkit/chainlink/v2/evm/assets" + "github.com/smartcontractkit/chainlink/v2/evm/utils" + ubig "github.com/smartcontractkit/chainlink/v2/evm/utils/big" +) + +var ( + fNodes = uint8(1) + nNodes = 4 // number of nodes (not including bootstrap) +) + +func setupBlockchain(t *testing.T) ( + *bind.TransactOpts, + evmtypes.Backend, + *configurator.Configurator, + common.Address, + *destination_verifier.DestinationVerifier, + common.Address, + *destination_verifier_proxy.DestinationVerifierProxy, + common.Address, + *channel_config_store.ChannelConfigStore, + common.Address, +) { + steve := testutils.MustNewSimTransactor(t) // config contract deployer and owner + genesisData := gethtypes.GenesisAlloc{steve.From: {Balance: assets.Ether(1000).ToInt()}} + backend := cltest.NewSimulatedBackend(t, genesisData, ethconfig.Defaults.Miner.GasCeil) + backend.Commit() + backend.Commit() // ensure starting block number at least 1 + + // Configurator + configuratorAddress, _, configurator, err := configurator.DeployConfigurator(steve, backend.Client()) + require.NoError(t, err) + backend.Commit() + + // DestinationVerifierProxy + destinationVerifierProxyAddr, _, verifierProxy, err := destination_verifier_proxy.DeployDestinationVerifierProxy(steve, backend.Client()) + require.NoError(t, err) + backend.Commit() + // DestinationVerifier + destinationVerifierAddr, _, destinationVerifier, err := destination_verifier.DeployDestinationVerifier(steve, backend.Client(), destinationVerifierProxyAddr) + require.NoError(t, err) + backend.Commit() + // AddVerifier + _, err = verifierProxy.SetVerifier(steve, destinationVerifierAddr) + require.NoError(t, err) + backend.Commit() + + // ChannelConfigStore + configStoreAddress, _, configStore, err := channel_config_store.DeployChannelConfigStore(steve, backend.Client()) + require.NoError(t, err) + + backend.Commit() + + return steve, backend, configurator, configuratorAddress, destinationVerifier, destinationVerifierAddr, verifierProxy, destinationVerifierProxyAddr, configStore, configStoreAddress +} + +type Stream struct { + id uint32 + baseBenchmarkPrice decimal.Decimal + baseBid decimal.Decimal + baseAsk decimal.Decimal +} + +const ( + ethStreamID = 52 + linkStreamID = 53 + quoteStreamID1 = 55 + quoteStreamID2 = 56 +) + +var ( + quoteStreamFeedID1 = common.HexToHash(`0x0003111111111111111111111111111111111111111111111111111111111111`) + quoteStreamFeedID2 = common.HexToHash(`0x0003222222222222222222222222222222222222222222222222222222222222`) + ethStream = Stream{ + id: 52, + baseBenchmarkPrice: decimal.NewFromFloat32(2_976.39), + } + linkStream = Stream{ + id: 53, + baseBenchmarkPrice: decimal.NewFromFloat32(13.25), + } + quoteStream1 = Stream{ + id: 55, + baseBenchmarkPrice: decimal.NewFromFloat32(1000.1212), + baseBid: decimal.NewFromFloat32(998.5431), + baseAsk: decimal.NewFromFloat32(1001.6999), + } + quoteStream2 = Stream{ + id: 56, + baseBenchmarkPrice: decimal.NewFromFloat32(500.1212), + baseBid: decimal.NewFromFloat32(499.5431), + baseAsk: decimal.NewFromFloat32(502.6999), + } +) + +func generateBlueGreenConfig(t *testing.T, oracles []confighelper.OracleIdentityExtra, predecessorConfigDigest *ocr2types.ConfigDigest) ( + signers []types.OnchainPublicKey, + transmitters []types.Account, + f uint8, + onchainConfig []byte, + offchainConfigVersion uint64, + offchainConfig []byte, +) { + onchainConfig, err := (&datastreamsllo.EVMOnchainConfigCodec{}).Encode(datastreamsllo.OnchainConfig{ + Version: 1, + PredecessorConfigDigest: predecessorConfigDigest, + }) + require.NoError(t, err) + return generateConfig(t, oracles, onchainConfig) +} + +func generateConfig(t *testing.T, oracles []confighelper.OracleIdentityExtra, inOnchainConfig []byte) ( + signers []types.OnchainPublicKey, + transmitters []types.Account, + f uint8, + outOnchainConfig []byte, + offchainConfigVersion uint64, + offchainConfig []byte, +) { + rawReportingPluginConfig := datastreamsllo.OffchainConfig{} + reportingPluginConfig, err := rawReportingPluginConfig.Encode() + require.NoError(t, err) + + signers, transmitters, f, outOnchainConfig, offchainConfigVersion, offchainConfig, err = ocr3confighelper.ContractSetConfigArgsForTests( + 2*time.Second, // DeltaProgress + 20*time.Second, // DeltaResend + 400*time.Millisecond, // DeltaInitial + 500*time.Millisecond, // DeltaRound + 250*time.Millisecond, // DeltaGrace + 300*time.Millisecond, // DeltaCertifiedCommitRequest + 1*time.Minute, // DeltaStage + 100, // rMax + []int{len(oracles)}, // S + oracles, + reportingPluginConfig, // reportingPluginConfig []byte, + nil, // maxDurationInitialization + 0, // maxDurationQuery + 250*time.Millisecond, // maxDurationObservation + 0, // maxDurationShouldAcceptAttestedReport + 0, // maxDurationShouldTransmitAcceptedReport + int(fNodes), // f + inOnchainConfig, // encoded onchain config + ) + + require.NoError(t, err) + + return +} + +func setStagingConfig(t *testing.T, donID uint32, steve *bind.TransactOpts, backend evmtypes.Backend, configurator *configurator.Configurator, configuratorAddress common.Address, nodes []Node, oracles []confighelper.OracleIdentityExtra, predecessorConfigDigest ocr2types.ConfigDigest) ocr2types.ConfigDigest { + return setBlueGreenConfig(t, donID, steve, backend, configurator, configuratorAddress, nodes, oracles, &predecessorConfigDigest) +} + +func setProductionConfig(t *testing.T, donID uint32, steve *bind.TransactOpts, backend evmtypes.Backend, configurator *configurator.Configurator, configuratorAddress common.Address, nodes []Node, oracles []confighelper.OracleIdentityExtra) ocr2types.ConfigDigest { + return setBlueGreenConfig(t, donID, steve, backend, configurator, configuratorAddress, nodes, oracles, nil) +} + +func setBlueGreenConfig(t *testing.T, donID uint32, steve *bind.TransactOpts, backend evmtypes.Backend, configurator *configurator.Configurator, configuratorAddress common.Address, nodes []Node, oracles []confighelper.OracleIdentityExtra, predecessorConfigDigest *ocr2types.ConfigDigest) ocr2types.ConfigDigest { + signers, _, _, onchainConfig, offchainConfigVersion, offchainConfig := generateBlueGreenConfig(t, oracles, predecessorConfigDigest) + + var onchainPubKeys [][]byte + for _, signer := range signers { + onchainPubKeys = append(onchainPubKeys, signer) + } + offchainTransmitters := make([][32]byte, nNodes) + for i := 0; i < nNodes; i++ { + offchainTransmitters[i] = nodes[i].ClientPubKey + } + donIDPadded := llo.DonIDToBytes32(donID) + isProduction := predecessorConfigDigest == nil + var err error + if isProduction { + _, err = configurator.SetProductionConfig(steve, donIDPadded, onchainPubKeys, offchainTransmitters, fNodes, onchainConfig, offchainConfigVersion, offchainConfig) + } else { + _, err = configurator.SetStagingConfig(steve, donIDPadded, onchainPubKeys, offchainTransmitters, fNodes, onchainConfig, offchainConfigVersion, offchainConfig) + } + require.NoError(t, err) + + // libocr requires a few confirmations to accept the config + backend.Commit() + backend.Commit() + backend.Commit() + backend.Commit() + + var topic common.Hash + if isProduction { + topic = llo.ProductionConfigSet + } else { + topic = llo.StagingConfigSet + } + logs, err := backend.Client().FilterLogs(testutils.Context(t), ethereum.FilterQuery{Addresses: []common.Address{configuratorAddress}, Topics: [][]common.Hash{[]common.Hash{topic, donIDPadded}}}) + require.NoError(t, err) + require.GreaterOrEqual(t, len(logs), 1) + + cfg, err := llo.DecodeProductionConfigSetLog(logs[len(logs)-1].Data) + require.NoError(t, err) + + return cfg.ConfigDigest +} + +func promoteStagingConfig(t *testing.T, donID uint32, steve *bind.TransactOpts, backend evmtypes.Backend, configurator *configurator.Configurator, configuratorAddress common.Address, isGreenProduction bool) { + donIDPadded := llo.DonIDToBytes32(donID) + _, err := configurator.PromoteStagingConfig(steve, donIDPadded, isGreenProduction) + require.NoError(t, err) + + // libocr requires a few confirmations to accept the config + backend.Commit() + backend.Commit() + backend.Commit() + backend.Commit() +} + +func TestIntegration_LLO_evm_premium_legacy(t *testing.T) { + t.Parallel() + + testStartTimeStamp := time.Now() + multiplier := decimal.New(1, 18) + expirationWindow := time.Hour / time.Second + + const salt = 100 + + clientCSAKeys := make([]csakey.KeyV2, nNodes) + clientPubKeys := make([]ed25519.PublicKey, nNodes) + for i := 0; i < nNodes; i++ { + k := big.NewInt(int64(salt + i)) + key := csakey.MustNewV2XXXTestingOnly(k) + clientCSAKeys[i] = key + clientPubKeys[i] = key.PublicKey + } + + steve, backend, configurator, configuratorAddress, _, _, _, _, configStore, configStoreAddress := setupBlockchain(t) + fromBlock := 1 + + // Setup bootstrap + bootstrapCSAKey := csakey.MustNewV2XXXTestingOnly(big.NewInt(salt - 1)) + bootstrapNodePort := freeport.GetOne(t) + appBootstrap, bootstrapPeerID, _, bootstrapKb, _ := setupNode(t, bootstrapNodePort, "bootstrap_llo", backend, bootstrapCSAKey, "") + bootstrapNode := Node{App: appBootstrap, KeyBundle: bootstrapKb} + + t.Run("using legacy verifier configuration contract, produces reports in v0.3 format", func(t *testing.T) { + reqs := make(chan wsrpcRequest, 100000) + serverKey := csakey.MustNewV2XXXTestingOnly(big.NewInt(salt - 2)) + serverPubKey := serverKey.PublicKey + srv := NewWSRPCMercuryServer(t, ed25519.PrivateKey(serverKey.Raw()), reqs) + + serverURL := startWSRPCMercuryServer(t, srv, clientPubKeys) + + donID := uint32(995544) + streams := []Stream{ethStream, linkStream, quoteStream1, quoteStream2} + streamMap := make(map[uint32]Stream) + for _, strm := range streams { + streamMap[strm.id] = strm + } + + // Setup oracle nodes + oracles, nodes := setupNodes(t, nNodes, backend, clientCSAKeys, streams, config.MercuryTransmitterProtocolWSRPC) + + chainID := testutils.SimulatedChainID + relayType := "evm" + relayConfig := fmt.Sprintf(` +chainID = "%s" +fromBlock = %d +lloDonID = %d +lloConfigMode = "bluegreen" +`, chainID, fromBlock, donID) + addBootstrapJob(t, bootstrapNode, configuratorAddress, "job-2", relayType, relayConfig) + + // Channel definitions + channelDefinitions := llotypes.ChannelDefinitions{ + 1: { + ReportFormat: llotypes.ReportFormatEVMPremiumLegacy, + Streams: []llotypes.Stream{ + { + StreamID: ethStreamID, + Aggregator: llotypes.AggregatorMedian, + }, + { + StreamID: linkStreamID, + Aggregator: llotypes.AggregatorMedian, + }, + { + StreamID: quoteStreamID1, + Aggregator: llotypes.AggregatorQuote, + }, + }, + Opts: llotypes.ChannelOpts([]byte(fmt.Sprintf(`{"baseUSDFee":"0.1","expirationWindow":%d,"feedId":"0x%x","multiplier":"%s"}`, expirationWindow, quoteStreamFeedID1, multiplier.String()))), + }, + 2: { + ReportFormat: llotypes.ReportFormatEVMPremiumLegacy, + Streams: []llotypes.Stream{ + { + StreamID: ethStreamID, + Aggregator: llotypes.AggregatorMedian, + }, + { + StreamID: linkStreamID, + Aggregator: llotypes.AggregatorMedian, + }, + { + StreamID: quoteStreamID2, + Aggregator: llotypes.AggregatorQuote, + }, + }, + Opts: llotypes.ChannelOpts([]byte(fmt.Sprintf(`{"baseUSDFee":"0.1","expirationWindow":%d,"feedId":"0x%x","multiplier":"%s"}`, expirationWindow, quoteStreamFeedID2, multiplier.String()))), + }, + } + + url, sha := newChannelDefinitionsServer(t, channelDefinitions) + + // Set channel definitions + _, err := configStore.SetChannelDefinitions(steve, donID, url, sha) + require.NoError(t, err) + backend.Commit() + + pluginConfig := fmt.Sprintf(`servers = { "%s" = "%x" } +donID = %d +channelDefinitionsContractAddress = "0x%x" +channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, configStoreAddress, fromBlock) + addOCRJobsEVMPremiumLegacy(t, streams, serverPubKey, serverURL, configuratorAddress, bootstrapPeerID, bootstrapNodePort, nodes, configStoreAddress, clientPubKeys, pluginConfig, relayType, relayConfig) + + // Set config on configurator + setProductionConfig( + t, donID, steve, backend, configurator, configuratorAddress, nodes, oracles, + ) + + signerAddresses := make([]common.Address, len(oracles)) + for i, oracle := range oracles { + signerAddresses[i] = common.BytesToAddress(oracle.OracleIdentity.OnchainPublicKey) + } + + t.Run("receives at least one report per channel from each oracle when EAs are at 100% reliability", func(t *testing.T) { + // Expect at least one report per feed from each oracle + seen := make(map[[32]byte]map[credentials.StaticSizedPublicKey]struct{}) + for _, cd := range channelDefinitions { + var opts lloevm.ReportFormatEVMPremiumLegacyOpts + err := json.Unmarshal(cd.Opts, &opts) + require.NoError(t, err) + // feedID will be deleted when all n oracles have reported + seen[opts.FeedID] = make(map[credentials.StaticSizedPublicKey]struct{}, nNodes) + } + for req := range reqs { + assert.Equal(t, uint32(llotypes.ReportFormatEVMPremiumLegacy), req.req.ReportFormat) + v := make(map[string]interface{}) + err := mercury.PayloadTypes.UnpackIntoMap(v, req.req.Payload) + require.NoError(t, err) + report, exists := v["report"] + if !exists { + t.Fatalf("expected payload %#v to contain 'report'", v) + } + reportElems := make(map[string]interface{}) + err = reportcodecv3.ReportTypes.UnpackIntoMap(reportElems, report.([]byte)) + require.NoError(t, err) + + feedID := reportElems["feedId"].([32]uint8) + + if _, exists := seen[feedID]; !exists { + continue // already saw all oracles for this feed + } + + var expectedBm, expectedBid, expectedAsk *big.Int + if feedID == quoteStreamFeedID1 { + expectedBm = quoteStream1.baseBenchmarkPrice.Mul(multiplier).BigInt() + expectedBid = quoteStream1.baseBid.Mul(multiplier).BigInt() + expectedAsk = quoteStream1.baseAsk.Mul(multiplier).BigInt() + } else if feedID == quoteStreamFeedID2 { + expectedBm = quoteStream2.baseBenchmarkPrice.Mul(multiplier).BigInt() + expectedBid = quoteStream2.baseBid.Mul(multiplier).BigInt() + expectedAsk = quoteStream2.baseAsk.Mul(multiplier).BigInt() + } else { + t.Fatalf("unrecognized feedID: 0x%x", feedID) + } + + assert.GreaterOrEqual(t, reportElems["validFromTimestamp"].(uint32), uint32(testStartTimeStamp.Unix())) + assert.GreaterOrEqual(t, int(reportElems["observationsTimestamp"].(uint32)), int(testStartTimeStamp.Unix())) + assert.Equal(t, "33597747607000", reportElems["nativeFee"].(*big.Int).String()) + assert.Equal(t, "7547169811320755", reportElems["linkFee"].(*big.Int).String()) + assert.Equal(t, reportElems["observationsTimestamp"].(uint32)+uint32(expirationWindow), reportElems["expiresAt"].(uint32)) + assert.Equal(t, expectedBm.String(), reportElems["benchmarkPrice"].(*big.Int).String()) + assert.Equal(t, expectedBid.String(), reportElems["bid"].(*big.Int).String()) + assert.Equal(t, expectedAsk.String(), reportElems["ask"].(*big.Int).String()) + + // emulate mercury server verifying report (local verification) + { + rv := mercuryverifier.NewVerifier() + + reportSigners, err := rv.Verify(mercuryverifier.SignedReport{ + RawRs: v["rawRs"].([][32]byte), + RawSs: v["rawSs"].([][32]byte), + RawVs: v["rawVs"].([32]byte), + ReportContext: v["reportContext"].([3][32]byte), + Report: v["report"].([]byte), + }, fNodes, signerAddresses) + require.NoError(t, err) + assert.GreaterOrEqual(t, len(reportSigners), int(fNodes+1)) + assert.Subset(t, signerAddresses, reportSigners) + } + + t.Logf("oracle %x reported for 0x%x", req.pk[:], feedID[:]) + + seen[feedID][req.pk] = struct{}{} + if len(seen[feedID]) == nNodes { + t.Logf("all oracles reported for 0x%x", feedID[:]) + delete(seen, feedID) + if len(seen) == 0 { + break // saw all oracles; success! + } + } + } + }) + }) +} + +func TestIntegration_LLO_evm_abi_encode_unpacked(t *testing.T) { + t.Parallel() + + testStartTimeStamp := time.Now() + expirationWindow := uint32(3600) + + const salt = 200 + + clientCSAKeys := make([]csakey.KeyV2, nNodes) + clientPubKeys := make([]ed25519.PublicKey, nNodes) + for i := 0; i < nNodes; i++ { + k := big.NewInt(int64(salt + i)) + key := csakey.MustNewV2XXXTestingOnly(k) + clientCSAKeys[i] = key + clientPubKeys[i] = key.PublicKey + } + + steve, backend, configurator, configuratorAddress, _, _, _, _, configStore, configStoreAddress := setupBlockchain(t) + fromBlock := 1 + + // Setup bootstrap + bootstrapCSAKey := csakey.MustNewV2XXXTestingOnly(big.NewInt(salt - 1)) + bootstrapNodePort := freeport.GetOne(t) + appBootstrap, bootstrapPeerID, _, bootstrapKb, _ := setupNode(t, bootstrapNodePort, "bootstrap_llo", backend, bootstrapCSAKey, "") + bootstrapNode := Node{App: appBootstrap, KeyBundle: bootstrapKb} + + t.Run("generates reports using go ReportFormatEVMABIEncodeUnpacked format", func(t *testing.T) { + reqs := make(chan *rpc.TransmitRequest, 100000) + serverKey := csakey.MustNewV2XXXTestingOnly(big.NewInt(salt - 2)) + serverPubKey := serverKey.PublicKey + srv := NewMercuryServer(t, ed25519.PrivateKey(serverKey.Raw()), reqs) + + serverURL := startMercuryServer(t, srv, clientPubKeys) + + donID := uint32(888333) + streams := []Stream{ethStream, linkStream} + streamMap := make(map[uint32]Stream) + for _, strm := range streams { + streamMap[strm.id] = strm + } + + // Setup oracle nodes + oracles, nodes := setupNodes(t, nNodes, backend, clientCSAKeys, streams, config.MercuryTransmitterProtocolGRPC) + + chainID := testutils.SimulatedChainID + relayType := "evm" + relayConfig := fmt.Sprintf(` +chainID = "%s" +fromBlock = %d +lloDonID = %d +lloConfigMode = "bluegreen" +`, chainID, fromBlock, donID) + addBootstrapJob(t, bootstrapNode, configuratorAddress, "job-4", relayType, relayConfig) + + dexBasedAssetPriceStreamID := uint32(1) + marketStatusStreamID := uint32(2) + baseMarketDepthStreamID := uint32(3) + quoteMarketDepthStreamID := uint32(4) + benchmarkPriceStreamID := uint32(5) + binanceFundingRateStreamID := uint32(6) + binanceFundingTimeStreamID := uint32(7) + binanceFundingIntervalHoursStreamID := uint32(8) + deribitFundingRateStreamID := uint32(9) + deribitFundingTimeStreamID := uint32(10) + deribitFundingIntervalHoursStreamID := uint32(11) + + mustEncodeOpts := func(opts *lloevm.ReportFormatEVMABIEncodeOpts) []byte { + encoded, err := json.Marshal(opts) + require.NoError(t, err) + return encoded + } + + standardMultiplier := ubig.NewI(1e18) + + dexBasedAssetFeedID := utils.NewHash() + rwaFeedID := utils.NewHash() + benchmarkPriceFeedID := utils.NewHash() + fundingRateFeedID := utils.NewHash() + // Channel definitions + channelDefinitions := llotypes.ChannelDefinitions{ + // Sample DEX-based asset schema + 1: { + ReportFormat: llotypes.ReportFormatEVMABIEncodeUnpacked, + Streams: []llotypes.Stream{ + { + StreamID: ethStreamID, + Aggregator: llotypes.AggregatorMedian, + }, + { + StreamID: linkStreamID, + Aggregator: llotypes.AggregatorMedian, + }, + { + StreamID: dexBasedAssetPriceStreamID, + Aggregator: llotypes.AggregatorMedian, + }, + { + StreamID: baseMarketDepthStreamID, + Aggregator: llotypes.AggregatorMedian, + }, + { + StreamID: quoteMarketDepthStreamID, + Aggregator: llotypes.AggregatorMedian, + }, + }, + Opts: mustEncodeOpts(&lloevm.ReportFormatEVMABIEncodeOpts{ + BaseUSDFee: decimal.NewFromFloat32(0.1), + ExpirationWindow: expirationWindow, + FeedID: dexBasedAssetFeedID, + ABI: []lloevm.ABIEncoder{ + lloevm.ABIEncoder{ + StreamID: dexBasedAssetPriceStreamID, + Type: "int192", + Multiplier: standardMultiplier, + }, + lloevm.ABIEncoder{ + StreamID: baseMarketDepthStreamID, + Type: "int192", + }, + lloevm.ABIEncoder{ + StreamID: quoteMarketDepthStreamID, + Type: "int192", + }, + }, + }), + }, + // Sample RWA schema + 2: { + ReportFormat: llotypes.ReportFormatEVMABIEncodeUnpacked, + Streams: []llotypes.Stream{ + { + StreamID: ethStreamID, + Aggregator: llotypes.AggregatorMedian, + }, + { + StreamID: linkStreamID, + Aggregator: llotypes.AggregatorMedian, + }, + { + StreamID: marketStatusStreamID, + Aggregator: llotypes.AggregatorMode, + }, + }, + Opts: mustEncodeOpts(&lloevm.ReportFormatEVMABIEncodeOpts{ + BaseUSDFee: decimal.NewFromFloat32(0.1), + ExpirationWindow: expirationWindow, + FeedID: rwaFeedID, + ABI: []lloevm.ABIEncoder{ + { + StreamID: marketStatusStreamID, + Type: "uint32", + }, + }, + }), + }, + // Sample Benchmark price schema + 3: { + ReportFormat: llotypes.ReportFormatEVMABIEncodeUnpacked, + Streams: []llotypes.Stream{ + { + StreamID: ethStreamID, + Aggregator: llotypes.AggregatorMedian, + }, + { + StreamID: linkStreamID, + Aggregator: llotypes.AggregatorMedian, + }, + { + StreamID: benchmarkPriceStreamID, + Aggregator: llotypes.AggregatorMedian, + }, + }, + Opts: mustEncodeOpts(&lloevm.ReportFormatEVMABIEncodeOpts{ + BaseUSDFee: decimal.NewFromFloat32(0.1), + ExpirationWindow: expirationWindow, + FeedID: benchmarkPriceFeedID, + ABI: []lloevm.ABIEncoder{ + { + StreamID: benchmarkPriceStreamID, + Type: "int192", + Multiplier: standardMultiplier, + }, + }, + }), + }, + // Sample funding rate scheam + 4: { + ReportFormat: llotypes.ReportFormatEVMABIEncodeUnpacked, + Streams: []llotypes.Stream{ + { + StreamID: ethStreamID, + Aggregator: llotypes.AggregatorMedian, + }, + { + StreamID: linkStreamID, + Aggregator: llotypes.AggregatorMedian, + }, + { + StreamID: binanceFundingRateStreamID, + Aggregator: llotypes.AggregatorMedian, + }, + { + StreamID: binanceFundingTimeStreamID, + Aggregator: llotypes.AggregatorMedian, + }, + { + StreamID: binanceFundingIntervalHoursStreamID, + Aggregator: llotypes.AggregatorMedian, + }, + { + StreamID: deribitFundingRateStreamID, + Aggregator: llotypes.AggregatorMedian, + }, + { + StreamID: deribitFundingTimeStreamID, + Aggregator: llotypes.AggregatorMedian, + }, + { + StreamID: deribitFundingIntervalHoursStreamID, + Aggregator: llotypes.AggregatorMedian, + }, + }, + Opts: mustEncodeOpts(&lloevm.ReportFormatEVMABIEncodeOpts{ + BaseUSDFee: decimal.NewFromFloat32(0.1), + ExpirationWindow: expirationWindow, + FeedID: fundingRateFeedID, + ABI: []lloevm.ABIEncoder{ + { + StreamID: binanceFundingRateStreamID, + Type: "int192", + }, + { + StreamID: binanceFundingTimeStreamID, + Type: "int192", + }, + { + StreamID: binanceFundingIntervalHoursStreamID, + Type: "int192", + }, + { + StreamID: deribitFundingRateStreamID, + Type: "int192", + }, + { + StreamID: deribitFundingTimeStreamID, + Type: "int192", + }, + { + StreamID: deribitFundingIntervalHoursStreamID, + Type: "int192", + }, + }, + }), + }, + } + url, sha := newChannelDefinitionsServer(t, channelDefinitions) + + // Set channel definitions + _, err := configStore.SetChannelDefinitions(steve, donID, url, sha) + require.NoError(t, err) + backend.Commit() + + pluginConfig := fmt.Sprintf(`servers = { "%s" = "%x" } +donID = %d +channelDefinitionsContractAddress = "0x%x" +channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, configStoreAddress, fromBlock) + + bridgeName := "superbridge" + + resultJSON := `{ + "benchmarkPrice": "2976.39", + "baseMarketDepth": "1000.1212", + "quoteMarketDepth": "998.5431", + "marketStatus": "1", + "binanceFundingRate": "1234.5678", + "binanceFundingTime": "1630000000", + "binanceFundingIntervalHours": "8", + "deribitFundingRate": "5432.2345", + "deribitFundingTime": "1630000000", + "deribitFundingIntervalHours": "8" +}` + + dexBasedAssetPipeline := fmt.Sprintf(` +dp [type=bridge name="%s" requestData="{\\"data\\":{\\"data\\":\\"foo\\"}}"]; + +bp_parse [type=jsonparse path="result,benchmarkPrice"]; +base_market_depth_parse [type=jsonparse path="result,baseMarketDepth"]; +quote_market_depth_parse [type=jsonparse path="result,quoteMarketDepth"]; + +bp_decimal [type=multiply times=1 streamID=%d]; +base_market_depth_decimal [type=multiply times=1 streamID=%d]; +quote_market_depth_decimal [type=multiply times=1 streamID=%d]; + +dp -> bp_parse -> bp_decimal; +dp -> base_market_depth_parse -> base_market_depth_decimal; +dp -> quote_market_depth_parse -> quote_market_depth_decimal; +`, bridgeName, dexBasedAssetPriceStreamID, baseMarketDepthStreamID, quoteMarketDepthStreamID) + + rwaPipeline := fmt.Sprintf(` +dp [type=bridge name="%s" requestData="{\\"data\\":{\\"data\\":\\"foo\\"}}"]; + +market_status_parse [type=jsonparse path="result,marketStatus"]; +market_status_decimal [type=multiply times=1 streamID=%d]; + +dp -> market_status_parse -> market_status_decimal; +`, bridgeName, marketStatusStreamID) + + benchmarkPricePipeline := fmt.Sprintf(` +dp [type=bridge name="%s" requestData="{\\"data\\":{\\"data\\":\\"foo\\"}}"]; + +bp_parse [type=jsonparse path="result,benchmarkPrice"]; +bp_decimal [type=multiply times=1 streamID=%d]; + +dp -> bp_parse -> bp_decimal; +`, bridgeName, benchmarkPriceStreamID) + + fundingRatePipeline := fmt.Sprintf(` +dp [type=bridge name="%s" requestData="{\\"data\\":{\\"data\\":\\"foo\\"}}"]; + +binance_funding_rate_parse [type=jsonparse path="result,binanceFundingRate"]; +binance_funding_rate_decimal [type=multiply times=1 streamID=%d]; + +binance_funding_time_parse [type=jsonparse path="result,binanceFundingTime"]; +binance_funding_time_decimal [type=multiply times=1 streamID=%d]; + +binance_funding_interval_hours_parse [type=jsonparse path="result,binanceFundingIntervalHours"]; +binance_funding_interval_hours_decimal [type=multiply times=1 streamID=%d]; + +deribit_funding_rate_parse [type=jsonparse path="result,deribitFundingRate"]; +deribit_funding_rate_decimal [type=multiply times=1 streamID=%d]; + +deribit_funding_time_parse [type=jsonparse path="result,deribitFundingTime"]; +deribit_funding_time_decimal [type=multiply times=1 streamID=%d]; + +deribit_funding_interval_hours_parse [type=jsonparse path="result,deribitFundingIntervalHours"]; +deribit_funding_interval_hours_decimal [type=multiply times=1 streamID=%d]; + +dp -> binance_funding_rate_parse -> binance_funding_rate_decimal; +dp -> binance_funding_time_parse -> binance_funding_time_decimal; +dp -> binance_funding_interval_hours_parse -> binance_funding_interval_hours_decimal; +dp -> deribit_funding_rate_parse -> deribit_funding_rate_decimal; +dp -> deribit_funding_time_parse -> deribit_funding_time_decimal; +dp -> deribit_funding_interval_hours_parse -> deribit_funding_interval_hours_decimal; +`, bridgeName, binanceFundingRateStreamID, binanceFundingTimeStreamID, binanceFundingIntervalHoursStreamID, deribitFundingRateStreamID, deribitFundingTimeStreamID, deribitFundingIntervalHoursStreamID) + + for i, node := range nodes { + // superBridge returns a JSON with everything you want in it, + // stream specs can just pick the individual fields they need + createBridge(t, bridgeName, resultJSON, node.App.BridgeORM()) + addStreamSpec(t, node, "dexBasedAssetPipeline", nil, dexBasedAssetPipeline) + addStreamSpec(t, node, "rwaPipeline", nil, rwaPipeline) + addStreamSpec(t, node, "benchmarkPricePipeline", nil, benchmarkPricePipeline) + addStreamSpec(t, node, "fundingRatePipeline", nil, fundingRatePipeline) + addLLOJob( + t, + node, + configuratorAddress, + bootstrapPeerID, + bootstrapNodePort, + clientPubKeys[i], + "llo-evm-abi-encode-unpacked-test", + pluginConfig, + relayType, + relayConfig, + ) + } + + // Set config on configurator + digest := setProductionConfig( + t, donID, steve, backend, configurator, configuratorAddress, nodes, oracles, + ) + + // NOTE: Wait for one of each type of report + feedIDs := map[[32]byte]struct{}{ + dexBasedAssetFeedID: {}, + rwaFeedID: {}, + benchmarkPriceFeedID: {}, + fundingRateFeedID: {}, + } + + for req := range reqs { + assert.Equal(t, uint32(llotypes.ReportFormatEVMABIEncodeUnpacked), req.ReportFormat) + v := make(map[string]interface{}) + err := mercury.PayloadTypes.UnpackIntoMap(v, req.Payload) + require.NoError(t, err) + report, exists := v["report"] + if !exists { + t.Fatalf("expected payload %#v to contain 'report'", v) + } + reportCtx, exists := v["reportContext"] + if !exists { + t.Fatalf("expected payload %#v to contain 'reportContext'", v) + } + + // Check the report context + assert.Equal(t, [32]byte(digest), reportCtx.([3][32]uint8)[0]) // config digest + assert.Equal(t, "000000000000000000000000000000000000000000000000000d8e0d00000001", fmt.Sprintf("%x", reportCtx.([3][32]uint8)[2])) // extra hash + + reportElems := make(map[string]interface{}) + err = lloevm.BaseSchema.UnpackIntoMap(reportElems, report.([]byte)) + require.NoError(t, err) + + feedID := reportElems["feedId"].([32]uint8) + delete(feedIDs, feedID) + + // Check headers + assert.GreaterOrEqual(t, reportElems["validFromTimestamp"].(uint32), uint32(testStartTimeStamp.Unix())) //nolint:gosec // G115 + assert.GreaterOrEqual(t, int(reportElems["observationsTimestamp"].(uint32)), int(testStartTimeStamp.Unix())) + // Zero fees since both eth/link stream specs are missing, don't + // care about billing for purposes of this test + assert.Equal(t, "0", reportElems["nativeFee"].(*big.Int).String()) + assert.Equal(t, "0", reportElems["linkFee"].(*big.Int).String()) + assert.Equal(t, reportElems["observationsTimestamp"].(uint32)+expirationWindow, reportElems["expiresAt"].(uint32)) + + // Check payload values + payload := report.([]byte)[192:] + switch hex.EncodeToString(feedID[:]) { + case hex.EncodeToString(dexBasedAssetFeedID[:]): + require.Len(t, payload, 96) + args := abi.Arguments([]abi.Argument{ + {Name: "benchmarkPrice", Type: mustNewType("int192")}, + {Name: "baseMarketDepth", Type: mustNewType("int192")}, + {Name: "quoteMarketDepth", Type: mustNewType("int192")}, + }) + v := make(map[string]interface{}) + err := args.UnpackIntoMap(v, payload) + require.NoError(t, err) + + assert.Equal(t, "2976390000000000000000", v["benchmarkPrice"].(*big.Int).String()) + assert.Equal(t, "1000", v["baseMarketDepth"].(*big.Int).String()) + assert.Equal(t, "998", v["quoteMarketDepth"].(*big.Int).String()) + case hex.EncodeToString(rwaFeedID[:]): + require.Len(t, payload, 32) + args := abi.Arguments([]abi.Argument{ + {Name: "marketStatus", Type: mustNewType("uint32")}, + }) + v := make(map[string]interface{}) + err := args.UnpackIntoMap(v, payload) + require.NoError(t, err) + + assert.Equal(t, uint32(1), v["marketStatus"].(uint32)) + case hex.EncodeToString(benchmarkPriceFeedID[:]): + require.Len(t, payload, 32) + args := abi.Arguments([]abi.Argument{ + {Name: "benchmarkPrice", Type: mustNewType("int192")}, + }) + v := make(map[string]interface{}) + err := args.UnpackIntoMap(v, payload) + require.NoError(t, err) + + assert.Equal(t, "2976390000000000000000", v["benchmarkPrice"].(*big.Int).String()) + case hex.EncodeToString(fundingRateFeedID[:]): + require.Len(t, payload, 192) + args := abi.Arguments([]abi.Argument{ + {Name: "binanceFundingRate", Type: mustNewType("int192")}, + {Name: "binanceFundingTime", Type: mustNewType("int192")}, + {Name: "binanceFundingIntervalHours", Type: mustNewType("int192")}, + {Name: "deribitFundingRate", Type: mustNewType("int192")}, + {Name: "deribitFundingTime", Type: mustNewType("int192")}, + {Name: "deribitFundingIntervalHours", Type: mustNewType("int192")}, + }) + v := make(map[string]interface{}) + err := args.UnpackIntoMap(v, payload) + require.NoError(t, err) + + assert.Equal(t, "1234", v["binanceFundingRate"].(*big.Int).String()) + assert.Equal(t, "1630000000", v["binanceFundingTime"].(*big.Int).String()) + assert.Equal(t, "8", v["binanceFundingIntervalHours"].(*big.Int).String()) + assert.Equal(t, "5432", v["deribitFundingRate"].(*big.Int).String()) + assert.Equal(t, "1630000000", v["deribitFundingTime"].(*big.Int).String()) + assert.Equal(t, "8", v["deribitFundingIntervalHours"].(*big.Int).String()) + default: + t.Fatalf("unexpected feedID: %x", feedID) + } + + if len(feedIDs) == 0 { + break + } + } + }) +} + +func TestIntegration_LLO_blue_green_lifecycle(t *testing.T) { + t.Parallel() + + clientCSAKeys := make([]csakey.KeyV2, nNodes) + clientPubKeys := make([]ed25519.PublicKey, nNodes) + + const salt = 300 + + for i := 0; i < nNodes; i++ { + k := big.NewInt(int64(salt + i)) + key := csakey.MustNewV2XXXTestingOnly(k) + clientCSAKeys[i] = key + clientPubKeys[i] = key.PublicKey + } + + steve, backend, configurator, configuratorAddress, _, _, _, _, configStore, configStoreAddress := setupBlockchain(t) + fromBlock := 1 + + // Setup bootstrap + bootstrapCSAKey := csakey.MustNewV2XXXTestingOnly(big.NewInt(salt - 1)) + bootstrapNodePort := freeport.GetOne(t) + appBootstrap, bootstrapPeerID, _, bootstrapKb, _ := setupNode(t, bootstrapNodePort, "bootstrap_llo", backend, bootstrapCSAKey, "") + bootstrapNode := Node{App: appBootstrap, KeyBundle: bootstrapKb} + + t.Run("Blue/Green lifecycle (using JSON report format)", func(t *testing.T) { + reqs := make(chan *rpc.TransmitRequest, 100000) + serverKey := csakey.MustNewV2XXXTestingOnly(big.NewInt(salt - 2)) + serverPubKey := serverKey.PublicKey + srv := NewMercuryServer(t, ed25519.PrivateKey(serverKey.Raw()), reqs) + + serverURL := startMercuryServer(t, srv, clientPubKeys) + + donID := uint32(888333) + streams := []Stream{ethStream, linkStream} + streamMap := make(map[uint32]Stream) + for _, strm := range streams { + streamMap[strm.id] = strm + } + + // Setup oracle nodes + oracles, nodes := setupNodes(t, nNodes, backend, clientCSAKeys, streams, config.MercuryTransmitterProtocolGRPC) + + chainID := testutils.SimulatedChainID + relayType := "evm" + relayConfig := fmt.Sprintf(` +chainID = "%s" +fromBlock = %d +lloDonID = %d +lloConfigMode = "bluegreen" +`, chainID, fromBlock, donID) + addBootstrapJob(t, bootstrapNode, configuratorAddress, "job-3", relayType, relayConfig) + + // Channel definitions + channelDefinitions := llotypes.ChannelDefinitions{ + 1: { + ReportFormat: llotypes.ReportFormatJSON, + Streams: []llotypes.Stream{ + { + StreamID: ethStreamID, + Aggregator: llotypes.AggregatorMedian, + }, + }, + }, + } + url, sha := newChannelDefinitionsServer(t, channelDefinitions) + + // Set channel definitions + _, err := configStore.SetChannelDefinitions(steve, donID, url, sha) + require.NoError(t, err) + backend.Commit() + + pluginConfig := fmt.Sprintf(`servers = { "%s" = "%x" } +donID = %d +channelDefinitionsContractAddress = "0x%x" +channelDefinitionsContractFromBlock = %d`, serverURL, serverPubKey, donID, configStoreAddress, fromBlock) + addOCRJobsEVMPremiumLegacy(t, streams, serverPubKey, serverURL, configuratorAddress, bootstrapPeerID, bootstrapNodePort, nodes, configStoreAddress, clientPubKeys, pluginConfig, relayType, relayConfig) + + var blueDigest ocr2types.ConfigDigest + var greenDigest ocr2types.ConfigDigest + + allReports := make(map[types.ConfigDigest][]datastreamsllo.Report) + // start off with blue=production, green=staging (specimen reports) + { + // Set config on configurator + blueDigest = setProductionConfig( + t, donID, steve, backend, configurator, configuratorAddress, nodes, oracles, + ) + + // NOTE: Wait until blue produces a report + + for req := range reqs { + assert.Equal(t, uint32(llotypes.ReportFormatJSON), req.ReportFormat) + _, _, r, _, err := (datastreamsllo.JSONReportCodec{}).UnpackDecode(req.Payload) + require.NoError(t, err) + + allReports[r.ConfigDigest] = append(allReports[r.ConfigDigest], r) + + assert.Equal(t, blueDigest, r.ConfigDigest) + assert.False(t, r.Specimen) + assert.Len(t, r.Values, 1) + assert.Equal(t, "2976.39", r.Values[0].(*datastreamsllo.Decimal).String()) + break + } + } + // setStagingConfig does not affect production + { + greenDigest = setStagingConfig( + t, donID, steve, backend, configurator, configuratorAddress, nodes, oracles, blueDigest, + ) + + // NOTE: Wait until green produces the first "specimen" report + + for req := range reqs { + assert.Equal(t, uint32(llotypes.ReportFormatJSON), req.ReportFormat) + _, _, r, _, err := (datastreamsllo.JSONReportCodec{}).UnpackDecode(req.Payload) + require.NoError(t, err) + + allReports[r.ConfigDigest] = append(allReports[r.ConfigDigest], r) + if r.Specimen { + assert.Len(t, r.Values, 1) + assert.Equal(t, "2976.39", r.Values[0].(*datastreamsllo.Decimal).String()) + + assert.Equal(t, greenDigest, r.ConfigDigest) + break + } + assert.Equal(t, blueDigest, r.ConfigDigest) + } + } + // promoteStagingConfig flow has clean and gapless hand off from old production to newly promoted staging instance, leaving old production instance in 'retired' state + { + promoteStagingConfig(t, donID, steve, backend, configurator, configuratorAddress, false) + + // NOTE: Wait for first non-specimen report for the newly promoted (green) instance + + for req := range reqs { + assert.Equal(t, uint32(llotypes.ReportFormatJSON), req.ReportFormat) + _, _, r, _, err := (datastreamsllo.JSONReportCodec{}).UnpackDecode(req.Payload) + require.NoError(t, err) + + allReports[r.ConfigDigest] = append(allReports[r.ConfigDigest], r) + + if !r.Specimen && r.ConfigDigest == greenDigest { + break + } + } + + initialPromotedGreenReport := allReports[greenDigest][len(allReports[greenDigest])-1] + finalBlueReport := allReports[blueDigest][len(allReports[blueDigest])-1] + + for _, digest := range []ocr2types.ConfigDigest{blueDigest, greenDigest} { + // Transmissions are not guaranteed to be in order + sort.Slice(allReports[digest], func(i, j int) bool { + return allReports[digest][i].SeqNr < allReports[digest][j].SeqNr + }) + seenSeqNr := uint64(0) + highestObservationTs := uint32(0) + highestValidAfterSeconds := uint32(0) + for i := 0; i < len(allReports[digest]); i++ { + r := allReports[digest][i] + switch digest { + case greenDigest: + if i == len(allReports[digest])-1 { + assert.False(t, r.Specimen) + } else { + assert.True(t, r.Specimen) + } + case blueDigest: + assert.False(t, r.Specimen) + } + if r.SeqNr > seenSeqNr { + // skip first one + if highestObservationTs > 0 { + if digest == greenDigest && i == len(allReports[digest])-1 { + // NOTE: This actually CHANGES on the staging + // handover and can go backwards - the gapless + // handover test is handled below + break + } + assert.Equal(t, highestObservationTs, r.ValidAfterSeconds, "%d: (n-1)ObservationsTimestampSeconds->(n)ValidAfterSeconds should be gapless, got: %d vs %d", i, highestObservationTs, r.ValidAfterSeconds) + assert.Greater(t, r.ObservationTimestampSeconds, highestObservationTs, "%d: overlapping/duplicate report ObservationTimestampSeconds, got: %d vs %d", i, r.ObservationTimestampSeconds, highestObservationTs) + assert.Greater(t, r.ValidAfterSeconds, highestValidAfterSeconds, "%d: overlapping/duplicate report ValidAfterSeconds, got: %d vs %d", i, r.ValidAfterSeconds, highestValidAfterSeconds) + assert.Less(t, r.ValidAfterSeconds, r.ObservationTimestampSeconds) + } + seenSeqNr = r.SeqNr + highestObservationTs = r.ObservationTimestampSeconds + highestValidAfterSeconds = r.ValidAfterSeconds + } + } + } + + // Gapless handover + assert.Less(t, finalBlueReport.ValidAfterSeconds, finalBlueReport.ObservationTimestampSeconds) + assert.Equal(t, finalBlueReport.ObservationTimestampSeconds, initialPromotedGreenReport.ValidAfterSeconds) + assert.Less(t, initialPromotedGreenReport.ValidAfterSeconds, initialPromotedGreenReport.ObservationTimestampSeconds) + } + // retired instance does not produce reports + { + // NOTE: Wait for five "green" reports to be produced and assert no "blue" reports + + i := 0 + for req := range reqs { + i++ + if i == 5 { + break + } + assert.Equal(t, uint32(llotypes.ReportFormatJSON), req.ReportFormat) + _, _, r, _, err := (datastreamsllo.JSONReportCodec{}).UnpackDecode(req.Payload) + require.NoError(t, err) + + allReports[r.ConfigDigest] = append(allReports[r.ConfigDigest], r) + assert.False(t, r.Specimen) + assert.Equal(t, greenDigest, r.ConfigDigest) + } + } + // setStagingConfig replaces 'retired' instance with new config and starts producing specimen reports again + { + blueDigest = setStagingConfig( + t, donID, steve, backend, configurator, configuratorAddress, nodes, oracles, greenDigest, + ) + + // NOTE: Wait until blue produces the first "specimen" report + + for req := range reqs { + assert.Equal(t, uint32(llotypes.ReportFormatJSON), req.ReportFormat) + _, _, r, _, err := (datastreamsllo.JSONReportCodec{}).UnpackDecode(req.Payload) + require.NoError(t, err) + + allReports[r.ConfigDigest] = append(allReports[r.ConfigDigest], r) + if r.Specimen { + assert.Equal(t, blueDigest, r.ConfigDigest) + break + } + assert.Equal(t, greenDigest, r.ConfigDigest) + } + } + // promoteStagingConfig swaps the instances again + { + // TODO: Check that once an instance enters 'retired' state, it + // doesn't produce reports or bother making observations + promoteStagingConfig(t, donID, steve, backend, configurator, configuratorAddress, true) + + // NOTE: Wait for first non-specimen report for the newly promoted (blue) instance + + for req := range reqs { + assert.Equal(t, uint32(llotypes.ReportFormatJSON), req.ReportFormat) + _, _, r, _, err := (datastreamsllo.JSONReportCodec{}).UnpackDecode(req.Payload) + require.NoError(t, err) + + allReports[r.ConfigDigest] = append(allReports[r.ConfigDigest], r) + + if !r.Specimen && r.ConfigDigest == blueDigest { + break + } + } + + initialPromotedBlueReport := allReports[blueDigest][len(allReports[blueDigest])-1] + finalGreenReport := allReports[greenDigest][len(allReports[greenDigest])-1] + + // Gapless handover + assert.Less(t, finalGreenReport.ValidAfterSeconds, finalGreenReport.ObservationTimestampSeconds) + assert.Equal(t, finalGreenReport.ObservationTimestampSeconds, initialPromotedBlueReport.ValidAfterSeconds) + assert.Less(t, initialPromotedBlueReport.ValidAfterSeconds, initialPromotedBlueReport.ObservationTimestampSeconds) + } + // adding a new channel definition is picked up on the fly + { + channelDefinitions[2] = llotypes.ChannelDefinition{ + ReportFormat: llotypes.ReportFormatJSON, + Streams: []llotypes.Stream{ + { + StreamID: linkStreamID, + Aggregator: llotypes.AggregatorMedian, + }, + }, + } + + url, sha := newChannelDefinitionsServer(t, channelDefinitions) + + // Set channel definitions + _, err := configStore.SetChannelDefinitions(steve, donID, url, sha) + require.NoError(t, err) + backend.Commit() + + // NOTE: Wait until the first report for the new channel definition is produced + + for req := range reqs { + assert.Equal(t, uint32(llotypes.ReportFormatJSON), req.ReportFormat) + _, _, r, _, err := (datastreamsllo.JSONReportCodec{}).UnpackDecode(req.Payload) + require.NoError(t, err) + + allReports[r.ConfigDigest] = append(allReports[r.ConfigDigest], r) + + // Green is retired, it shouldn't be producing anything + assert.Equal(t, blueDigest, r.ConfigDigest) + assert.False(t, r.Specimen) + + if r.ChannelID == 2 { + assert.Len(t, r.Values, 1) + assert.Equal(t, "13.25", r.Values[0].(*datastreamsllo.Decimal).String()) + break + } + assert.Len(t, r.Values, 1) + assert.Equal(t, "2976.39", r.Values[0].(*datastreamsllo.Decimal).String()) + } + } + t.Run("deleting the jobs turns off oracles and cleans up resources", func(t *testing.T) { + t.Skip("TODO - MERC-3524") + }) + t.Run("adding new jobs again picks up the correct configs", func(t *testing.T) { + t.Skip("TODO - MERC-3524") + }) + }) +} + +func setupNodes(t *testing.T, nNodes int, backend evmtypes.Backend, clientCSAKeys []csakey.KeyV2, streams []Stream, transmitterProtocol config.MercuryTransmitterProtocol) (oracles []confighelper.OracleIdentityExtra, nodes []Node) { + ports := freeport.GetN(t, nNodes) + for i := 0; i < nNodes; i++ { + app, peerID, transmitter, kb, observedLogs := setupNode(t, ports[i], fmt.Sprintf("oracle_streams_%d", i), backend, clientCSAKeys[i], transmitterProtocol) + + nodes = append(nodes, Node{ + app, transmitter, kb, observedLogs, + }) + offchainPublicKey, err := hex.DecodeString(strings.TrimPrefix(kb.OnChainPublicKey(), "0x")) + require.NoError(t, err) + oracles = append(oracles, confighelper.OracleIdentityExtra{ + OracleIdentity: confighelper.OracleIdentity{ + OnchainPublicKey: offchainPublicKey, + TransmitAccount: ocr2types.Account(fmt.Sprintf("%x", transmitter[:])), + OffchainPublicKey: kb.OffchainPublicKey(), + PeerID: peerID, + }, + ConfigEncryptionPublicKey: kb.ConfigEncryptionPublicKey(), + }) + } + return +} + +func newChannelDefinitionsServer(t *testing.T, channelDefinitions llotypes.ChannelDefinitions) (url string, sha [32]byte) { + channelDefinitionsJSON, err := json.MarshalIndent(channelDefinitions, "", " ") + require.NoError(t, err) + channelDefinitionsSHA := sha3.Sum256(channelDefinitionsJSON) + + // Set up channel definitions server + channelDefinitionsServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + assert.Equal(t, "GET", r.Method) + assert.Equal(t, "application/json", r.Header.Get("Content-Type")) + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(http.StatusOK) + _, err := w.Write(channelDefinitionsJSON) + require.NoError(t, err) + })) + t.Cleanup(channelDefinitionsServer.Close) + return channelDefinitionsServer.URL, channelDefinitionsSHA +} + +func mustNewType(t string) abi.Type { + result, err := abi.NewType(t, "", []abi.ArgumentMarshaling{}) + if err != nil { + panic(fmt.Sprintf("Unexpected error during abi.NewType: %s", err)) + } + return result +} diff --git a/core/services/ocr2/plugins/mercury/config/config.go b/core/services/ocr2/plugins/mercury/config/config.go deleted file mode 100644 index 40854bd8c0a..00000000000 --- a/core/services/ocr2/plugins/mercury/config/config.go +++ /dev/null @@ -1,129 +0,0 @@ -// config is a separate package so that we can validate -// the config in other packages, for example in job at job create time. - -package config - -import ( - "errors" - "fmt" - "net/url" - "regexp" - "sort" - - pkgerrors "github.com/pkg/errors" - - "github.com/smartcontractkit/chainlink/v2/core/null" - mercuryutils "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/utils" - "github.com/smartcontractkit/chainlink/v2/core/utils" -) - -type PluginConfig struct { - // Must either specify details for single server OR multiple servers. - // Specifying both is not valid. - - // Single mercury server - // LEGACY: This is the old way of specifying a mercury server - RawServerURL string `json:"serverURL" toml:"serverURL"` - ServerPubKey utils.PlainHexBytes `json:"serverPubKey" toml:"serverPubKey"` - - // Multi mercury servers - // This is the preferred way to specify mercury server(s) - Servers map[string]utils.PlainHexBytes `json:"servers" toml:"servers"` - - // InitialBlockNumber allows to set a custom "validFromBlockNumber" for - // the first ever report in the case of a brand new feed, where the mercury - // server does not have any previous reports. For a brand new feed, this - // effectively sets the "first" validFromBlockNumber. - InitialBlockNumber null.Int64 `json:"initialBlockNumber" toml:"initialBlockNumber"` - - LinkFeedID *mercuryutils.FeedID `json:"linkFeedID" toml:"linkFeedID"` - NativeFeedID *mercuryutils.FeedID `json:"nativeFeedID" toml:"nativeFeedID"` -} - -func validateURL(rawServerURL string) error { - var normalizedURI string - if schemeRegexp.MatchString(rawServerURL) { - normalizedURI = rawServerURL - } else { - normalizedURI = fmt.Sprintf("wss://%s", rawServerURL) - } - uri, err := url.ParseRequestURI(normalizedURI) - if err != nil { - return pkgerrors.Errorf(`Mercury: invalid value for ServerURL, got: %q`, rawServerURL) - } - if uri.Scheme != "wss" { - return pkgerrors.Errorf(`Mercury: invalid scheme specified for MercuryServer, got: %q (scheme: %q) but expected a websocket url e.g. "192.0.2.2:4242" or "wss://192.0.2.2:4242"`, rawServerURL, uri.Scheme) - } - return nil -} - -type Server struct { - URL string - PubKey utils.PlainHexBytes -} - -func (p PluginConfig) GetServers() (servers []Server) { - if p.RawServerURL != "" { - return []Server{{URL: wssRegexp.ReplaceAllString(p.RawServerURL, ""), PubKey: p.ServerPubKey}} - } - for url, pubKey := range p.Servers { - servers = append(servers, Server{URL: wssRegexp.ReplaceAllString(url, ""), PubKey: pubKey}) - } - sort.Slice(servers, func(i, j int) bool { - return servers[i].URL < servers[j].URL - }) - return -} - -func ValidatePluginConfig(config PluginConfig, feedID mercuryutils.FeedID) (merr error) { - if len(config.Servers) > 0 { - if config.RawServerURL != "" || len(config.ServerPubKey) != 0 { - merr = errors.Join(merr, errors.New("Mercury: Servers and RawServerURL/ServerPubKey may not be specified together")) - } else { - for serverName, serverPubKey := range config.Servers { - if err := validateURL(serverName); err != nil { - merr = errors.Join(merr, pkgerrors.Wrap(err, "Mercury: invalid value for ServerURL")) - } - if len(serverPubKey) != 32 { - merr = errors.Join(merr, errors.New("Mercury: ServerPubKey must be a 32-byte hex string")) - } - } - } - } else if config.RawServerURL == "" { - merr = errors.Join(merr, errors.New("Mercury: Servers must be specified")) - } else { - if err := validateURL(config.RawServerURL); err != nil { - merr = errors.Join(merr, pkgerrors.Wrap(err, "Mercury: invalid value for ServerURL")) - } - if len(config.ServerPubKey) != 32 { - merr = errors.Join(merr, errors.New("Mercury: If RawServerURL is specified, ServerPubKey is also required and must be a 32-byte hex string")) - } - } - - switch feedID.Version() { - case 1: - if config.LinkFeedID != nil { - merr = errors.Join(merr, errors.New("linkFeedID may not be specified for v1 jobs")) - } - if config.NativeFeedID != nil { - merr = errors.Join(merr, errors.New("nativeFeedID may not be specified for v1 jobs")) - } - case 2, 3, 4: - if config.LinkFeedID == nil { - merr = errors.Join(merr, fmt.Errorf("linkFeedID must be specified for v%d jobs", feedID.Version())) - } - if config.NativeFeedID == nil { - merr = errors.Join(merr, fmt.Errorf("nativeFeedID must be specified for v%d jobs", feedID.Version())) - } - if config.InitialBlockNumber.Valid { - merr = errors.Join(merr, fmt.Errorf("initialBlockNumber may not be specified for v%d jobs", feedID.Version())) - } - default: - merr = errors.Join(merr, fmt.Errorf("got unsupported schema version %d; supported versions are 1,2,3,4", feedID.Version())) - } - - return merr -} - -var schemeRegexp = regexp.MustCompile(`^[a-zA-Z][a-zA-Z0-9+.-]*://`) -var wssRegexp = regexp.MustCompile(`^wss://`) diff --git a/core/services/ocr2/plugins/mercury/config/config_test.go b/core/services/ocr2/plugins/mercury/config/config_test.go deleted file mode 100644 index 5beba287133..00000000000 --- a/core/services/ocr2/plugins/mercury/config/config_test.go +++ /dev/null @@ -1,216 +0,0 @@ -package config - -import ( - "testing" - - "github.com/pelletier/go-toml/v2" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/smartcontractkit/chainlink/v2/core/utils" -) - -var v1FeedId = [32]uint8{00, 01, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114} -var v2FeedId = [32]uint8{00, 02, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114} - -func Test_PluginConfig(t *testing.T) { - t.Run("Mercury v1", func(t *testing.T) { - t.Run("with valid values", func(t *testing.T) { - rawToml := ` - ServerURL = "example.com:80" - ServerPubKey = "724ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93" - InitialBlockNumber = 1234 - ` - - var mc PluginConfig - err := toml.Unmarshal([]byte(rawToml), &mc) - require.NoError(t, err) - - assert.Equal(t, "example.com:80", mc.RawServerURL) - assert.Equal(t, "724ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93", mc.ServerPubKey.String()) - assert.Equal(t, int64(1234), mc.InitialBlockNumber.Int64) - - err = ValidatePluginConfig(mc, v1FeedId) - require.NoError(t, err) - }) - t.Run("with multiple server URLs", func(t *testing.T) { - t.Run("if no ServerURL/ServerPubKey is specified", func(t *testing.T) { - rawToml := ` - Servers = { "example.com:80" = "724ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93", "example2.invalid:1234" = "524ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93" } - ` - - var mc PluginConfig - err := toml.Unmarshal([]byte(rawToml), &mc) - require.NoError(t, err) - - assert.Len(t, mc.Servers, 2) - assert.Equal(t, "724ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93", mc.Servers["example.com:80"].String()) - assert.Equal(t, "524ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93", mc.Servers["example2.invalid:1234"].String()) - - err = ValidatePluginConfig(mc, v1FeedId) - require.NoError(t, err) - }) - t.Run("if ServerURL or ServerPubKey is specified", func(t *testing.T) { - rawToml := ` - Servers = { "example.com:80" = "724ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93", "example2.invalid:1234" = "524ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93" } - ServerURL = "example.com:80" - ` - var mc PluginConfig - err := toml.Unmarshal([]byte(rawToml), &mc) - require.NoError(t, err) - - err = ValidatePluginConfig(mc, v1FeedId) - require.EqualError(t, err, "Mercury: Servers and RawServerURL/ServerPubKey may not be specified together") - - rawToml = ` - Servers = { "example.com:80" = "724ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93", "example2.invalid:1234" = "524ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93" } - ServerPubKey = "724ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93" - ` - err = toml.Unmarshal([]byte(rawToml), &mc) - require.NoError(t, err) - - err = ValidatePluginConfig(mc, v1FeedId) - require.EqualError(t, err, "Mercury: Servers and RawServerURL/ServerPubKey may not be specified together") - }) - }) - - t.Run("with invalid values", func(t *testing.T) { - rawToml := ` - InitialBlockNumber = "invalid" - ` - - var mc PluginConfig - err := toml.Unmarshal([]byte(rawToml), &mc) - require.Error(t, err) - assert.EqualError(t, err, `toml: strconv.ParseInt: parsing "invalid": invalid syntax`) - - rawToml = ` - ServerURL = "http://example.com" - ServerPubKey = "4242" - ` - - err = toml.Unmarshal([]byte(rawToml), &mc) - require.NoError(t, err) - - err = ValidatePluginConfig(mc, v1FeedId) - require.Error(t, err) - assert.Contains(t, err.Error(), `Mercury: invalid scheme specified for MercuryServer, got: "http://example.com" (scheme: "http") but expected a websocket url e.g. "192.0.2.2:4242" or "wss://192.0.2.2:4242"`) - assert.Contains(t, err.Error(), `If RawServerURL is specified, ServerPubKey is also required and must be a 32-byte hex string`) - }) - - t.Run("with unnecessary values", func(t *testing.T) { - rawToml := ` - ServerURL = "example.com:80" - ServerPubKey = "724ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93" - LinkFeedID = "0x00026b4aa7e57ca7b68ae1bf45653f56b656fd3aa335ef7fae696b663f1b8472" - ` - - var mc PluginConfig - err := toml.Unmarshal([]byte(rawToml), &mc) - require.NoError(t, err) - - err = ValidatePluginConfig(mc, v1FeedId) - assert.Contains(t, err.Error(), `linkFeedID may not be specified for v1 jobs`) - }) - }) - - t.Run("Mercury v2/v3", func(t *testing.T) { - t.Run("with valid values", func(t *testing.T) { - rawToml := ` - ServerURL = "example.com:80" - ServerPubKey = "724ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93" - LinkFeedID = "0x00026b4aa7e57ca7b68ae1bf45653f56b656fd3aa335ef7fae696b663f1b8472" - NativeFeedID = "0x00036b4aa7e57ca7b68ae1bf45653f56b656fd3aa335ef7fae696b663f1b8472" - ` - - var mc PluginConfig - err := toml.Unmarshal([]byte(rawToml), &mc) - require.NoError(t, err) - - err = ValidatePluginConfig(mc, v2FeedId) - require.NoError(t, err) - - require.NotNil(t, mc.LinkFeedID) - require.NotNil(t, mc.NativeFeedID) - assert.Equal(t, "0x00026b4aa7e57ca7b68ae1bf45653f56b656fd3aa335ef7fae696b663f1b8472", (*mc.LinkFeedID).String()) - assert.Equal(t, "0x00036b4aa7e57ca7b68ae1bf45653f56b656fd3aa335ef7fae696b663f1b8472", (*mc.NativeFeedID).String()) - }) - - t.Run("with invalid values", func(t *testing.T) { - var mc PluginConfig - - rawToml := `LinkFeedID = "test"` - err := toml.Unmarshal([]byte(rawToml), &mc) - assert.Contains(t, err.Error(), "toml: hash: expected a hex string starting with '0x'") - - rawToml = `LinkFeedID = "0xtest"` - err = toml.Unmarshal([]byte(rawToml), &mc) - assert.Contains(t, err.Error(), `toml: hash: UnmarshalText failed: encoding/hex: invalid byte: U+0074 't'`) - - rawToml = ` - ServerURL = "example.com:80" - ServerPubKey = "724ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93" - LinkFeedID = "0x00026b4aa7e57ca7b68ae1bf45653f56b656fd3aa335ef7fae696b663f1b8472" - ` - err = toml.Unmarshal([]byte(rawToml), &mc) - require.NoError(t, err) - - err = ValidatePluginConfig(mc, v2FeedId) - assert.Contains(t, err.Error(), "nativeFeedID must be specified for v2 jobs") - }) - - t.Run("with unnecessary values", func(t *testing.T) { - rawToml := ` - ServerURL = "example.com:80" - ServerPubKey = "724ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93" - InitialBlockNumber = 1234 - ` - - var mc PluginConfig - err := toml.Unmarshal([]byte(rawToml), &mc) - require.NoError(t, err) - - err = ValidatePluginConfig(mc, v2FeedId) - assert.Contains(t, err.Error(), `initialBlockNumber may not be specified for v2 jobs`) - }) - }) -} - -func Test_PluginConfig_GetServers(t *testing.T) { - t.Run("with single server", func(t *testing.T) { - pubKey := utils.PlainHexBytes([]byte{1, 2, 3}) - pc := PluginConfig{RawServerURL: "example.com", ServerPubKey: pubKey} - require.Len(t, pc.GetServers(), 1) - assert.Equal(t, "example.com", pc.GetServers()[0].URL) - assert.Equal(t, pubKey, pc.GetServers()[0].PubKey) - - pc = PluginConfig{RawServerURL: "wss://example.com", ServerPubKey: pubKey} - require.Len(t, pc.GetServers(), 1) - assert.Equal(t, "example.com", pc.GetServers()[0].URL) - assert.Equal(t, pubKey, pc.GetServers()[0].PubKey) - - pc = PluginConfig{RawServerURL: "example.com:1234/foo", ServerPubKey: pubKey} - require.Len(t, pc.GetServers(), 1) - assert.Equal(t, "example.com:1234/foo", pc.GetServers()[0].URL) - assert.Equal(t, pubKey, pc.GetServers()[0].PubKey) - - pc = PluginConfig{RawServerURL: "wss://example.com:1234/foo", ServerPubKey: pubKey} - require.Len(t, pc.GetServers(), 1) - assert.Equal(t, "example.com:1234/foo", pc.GetServers()[0].URL) - assert.Equal(t, pubKey, pc.GetServers()[0].PubKey) - }) - - t.Run("with multiple servers", func(t *testing.T) { - servers := map[string]utils.PlainHexBytes{ - "example.com:80": utils.PlainHexBytes([]byte{1, 2, 3}), - "mercuryserver.invalid:1234/foo": utils.PlainHexBytes([]byte{4, 5, 6}), - } - pc := PluginConfig{Servers: servers} - - require.Len(t, pc.GetServers(), 2) - assert.Equal(t, "example.com:80", pc.GetServers()[0].URL) - assert.Equal(t, utils.PlainHexBytes{1, 2, 3}, pc.GetServers()[0].PubKey) - assert.Equal(t, "mercuryserver.invalid:1234/foo", pc.GetServers()[1].URL) - assert.Equal(t, utils.PlainHexBytes{4, 5, 6}, pc.GetServers()[1].PubKey) - }) -} diff --git a/core/services/ocr2/plugins/mercury/helpers_test.go b/core/services/ocr2/plugins/mercury/helpers_test.go deleted file mode 100644 index 2b67242db64..00000000000 --- a/core/services/ocr2/plugins/mercury/helpers_test.go +++ /dev/null @@ -1,547 +0,0 @@ -package mercury_test - -import ( - "context" - "crypto/ed25519" - "encoding/binary" - "errors" - "fmt" - "math/big" - "net" - "strings" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/zap/zapcore" - "go.uber.org/zap/zaptest/observer" - - "github.com/smartcontractkit/wsrpc" - "github.com/smartcontractkit/wsrpc/credentials" - "github.com/smartcontractkit/wsrpc/peer" - - "github.com/smartcontractkit/libocr/offchainreporting2/chains/evmutil" - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - - commonconfig "github.com/smartcontractkit/chainlink-common/pkg/config" - evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" - "github.com/smartcontractkit/chainlink/v2/evm/utils" - - "github.com/smartcontractkit/chainlink/v2/core/internal/cltest" - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/keystest" - "github.com/smartcontractkit/chainlink/v2/core/logger" - "github.com/smartcontractkit/chainlink/v2/core/services/chainlink" - "github.com/smartcontractkit/chainlink/v2/core/services/keystore/chaintype" - "github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/csakey" - "github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/ocr2key" - "github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/p2pkey" - "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/validate" - "github.com/smartcontractkit/chainlink/v2/core/services/ocrbootstrap" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/wsrpc/pb" - "github.com/smartcontractkit/chainlink/v2/core/utils/testutils/heavyweight" -) - -var _ pb.MercuryServer = &mercuryServer{} - -type request struct { - pk credentials.StaticSizedPublicKey - req *pb.TransmitRequest -} - -type mercuryServer struct { - privKey ed25519.PrivateKey - reqsCh chan request - t *testing.T - buildReport func() []byte -} - -func NewMercuryServer(t *testing.T, privKey ed25519.PrivateKey, reqsCh chan request, buildReport func() []byte) *mercuryServer { - return &mercuryServer{privKey, reqsCh, t, buildReport} -} - -func (s *mercuryServer) Transmit(ctx context.Context, req *pb.TransmitRequest) (*pb.TransmitResponse, error) { - p, ok := peer.FromContext(ctx) - if !ok { - return nil, errors.New("could not extract public key") - } - r := request{p.PublicKey, req} - s.reqsCh <- r - - return &pb.TransmitResponse{ - Code: 1, - Error: "", - }, nil -} - -func (s *mercuryServer) LatestReport(ctx context.Context, lrr *pb.LatestReportRequest) (*pb.LatestReportResponse, error) { - p, ok := peer.FromContext(ctx) - if !ok { - return nil, errors.New("could not extract public key") - } - s.t.Logf("mercury server got latest report from %x for feed id 0x%x", p.PublicKey, lrr.FeedId) - - out := new(pb.LatestReportResponse) - out.Report = new(pb.Report) - out.Report.FeedId = lrr.FeedId - - report := s.buildReport() - payload, err := mercury.PayloadTypes.Pack(evmutil.RawReportContext(ocrtypes.ReportContext{}), report, [][32]byte{}, [][32]byte{}, [32]byte{}) - if err != nil { - panic(err) - } - out.Report.Payload = payload - return out, nil -} - -func startMercuryServer(t *testing.T, srv *mercuryServer, pubKeys []ed25519.PublicKey) (serverURL string) { - // Set up the wsrpc server - lis, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatalf("[MAIN] failed to listen: %v", err) - } - serverURL = lis.Addr().String() - s := wsrpc.NewServer(wsrpc.WithCreds(srv.privKey, pubKeys)) - - // Register mercury implementation with the wsrpc server - pb.RegisterMercuryServer(s, srv) - - // Start serving - go s.Serve(lis) - t.Cleanup(s.Stop) - - return -} - -type Feed struct { - name string - id [32]byte - baseBenchmarkPrice *big.Int - baseBid *big.Int - baseAsk *big.Int - baseMarketStatus uint32 -} - -func randomFeedID(version uint16) [32]byte { - id := [32]byte(utils.NewHash()) - binary.BigEndian.PutUint16(id[:2], version) - return id -} - -type Node struct { - App chainlink.Application - ClientPubKey credentials.StaticSizedPublicKey - KeyBundle ocr2key.KeyBundle -} - -func (node *Node) AddJob(t *testing.T, spec string) { - c := node.App.GetConfig() - job, err := validate.ValidatedOracleSpecToml(testutils.Context(t), c.OCR2(), c.Insecure(), spec, nil) - require.NoError(t, err) - err = node.App.AddJobV2(testutils.Context(t), &job) - require.NoError(t, err) -} - -func (node *Node) AddBootstrapJob(t *testing.T, spec string) { - job, err := ocrbootstrap.ValidatedBootstrapSpecToml(spec) - require.NoError(t, err) - err = node.App.AddJobV2(testutils.Context(t), &job) - require.NoError(t, err) -} - -func setupNode( - t *testing.T, - port int, - dbName string, - backend evmtypes.Backend, - csaKey csakey.KeyV2, -) (app chainlink.Application, peerID string, clientPubKey credentials.StaticSizedPublicKey, ocr2kb ocr2key.KeyBundle, observedLogs *observer.ObservedLogs) { - k := big.NewInt(int64(port)) // keys unique to port - p2pKey := p2pkey.MustNewV2XXXTestingOnly(k) - rdr := keystest.NewRandReaderFromSeed(int64(port)) - ocr2kb = ocr2key.MustNewInsecure(rdr, chaintype.EVM) - - p2paddresses := []string{fmt.Sprintf("127.0.0.1:%d", port)} - - config, _ := heavyweight.FullTestDBV2(t, func(c *chainlink.Config, s *chainlink.Secrets) { - // [JobPipeline] - // MaxSuccessfulRuns = 0 - c.JobPipeline.MaxSuccessfulRuns = ptr(uint64(0)) - c.JobPipeline.VerboseLogging = ptr(true) - - // [Feature] - // UICSAKeys=true - // LogPoller = true - // FeedsManager = false - c.Feature.UICSAKeys = ptr(true) - c.Feature.LogPoller = ptr(true) - c.Feature.FeedsManager = ptr(false) - - // [OCR] - // Enabled = false - c.OCR.Enabled = ptr(false) - - // [OCR2] - // Enabled = true - c.OCR2.Enabled = ptr(true) - - // [P2P] - // PeerID = '$PEERID' - // TraceLogging = true - c.P2P.PeerID = ptr(p2pKey.PeerID()) - c.P2P.TraceLogging = ptr(true) - - // [P2P.V2] - // Enabled = true - // AnnounceAddresses = ['$EXT_IP:17775'] - // ListenAddresses = ['127.0.0.1:17775'] - // DeltaDial = 500ms - // DeltaReconcile = 5s - c.P2P.V2.Enabled = ptr(true) - c.P2P.V2.AnnounceAddresses = &p2paddresses - c.P2P.V2.ListenAddresses = &p2paddresses - c.P2P.V2.DeltaDial = commonconfig.MustNewDuration(500 * time.Millisecond) - c.P2P.V2.DeltaReconcile = commonconfig.MustNewDuration(5 * time.Second) - }) - - lggr, observedLogs := logger.TestLoggerObserved(t, zapcore.DebugLevel) - app = cltest.NewApplicationWithConfigV2OnSimulatedBlockchain(t, config, backend, p2pKey, ocr2kb, csaKey, lggr.Named(dbName)) - err := app.Start(testutils.Context(t)) - require.NoError(t, err) - - t.Cleanup(func() { - assert.NoError(t, app.Stop()) - }) - - return app, p2pKey.PeerID().Raw(), csaKey.StaticSizedPublicKey(), ocr2kb, observedLogs -} - -func ptr[T any](t T) *T { return &t } - -func addBootstrapJob(t *testing.T, bootstrapNode Node, chainID *big.Int, verifierAddress common.Address, feedName string, feedID [32]byte) { - bootstrapNode.AddBootstrapJob(t, fmt.Sprintf(` -type = "bootstrap" -relay = "evm" -schemaVersion = 1 -name = "boot-%s" -contractID = "%s" -feedID = "0x%x" -contractConfigTrackerPollInterval = "1s" - -[relayConfig] -chainID = %d - `, feedName, verifierAddress, feedID, chainID)) -} - -func addV1MercuryJob( - t *testing.T, - node Node, - i int, - verifierAddress common.Address, - bootstrapPeerID string, - bootstrapNodePort int, - bmBridge, - bidBridge, - askBridge, - serverURL string, - serverPubKey, - clientPubKey ed25519.PublicKey, - feedName string, - feedID [32]byte, - chainID *big.Int, - fromBlock int, -) { - node.AddJob(t, fmt.Sprintf(` -type = "offchainreporting2" -schemaVersion = 1 -name = "mercury-%[1]d-%[14]s" -forwardingAllowed = false -maxTaskDuration = "1s" -contractID = "%[2]s" -feedID = "0x%[11]x" -contractConfigTrackerPollInterval = "1s" -ocrKeyBundleID = "%[3]s" -p2pv2Bootstrappers = [ - "%[4]s" -] -relay = "evm" -pluginType = "mercury" -transmitterID = "%[10]x" -observationSource = """ - // Benchmark Price - price1 [type=bridge name="%[5]s" timeout="50ms" requestData="{\\"data\\":{\\"from\\":\\"ETH\\",\\"to\\":\\"USD\\"}}"]; - price1_parse [type=jsonparse path="result"]; - price1_multiply [type=multiply times=100000000 index=0]; - - price1 -> price1_parse -> price1_multiply; - - // Bid - bid [type=bridge name="%[6]s" timeout="50ms" requestData="{\\"data\\":{\\"from\\":\\"ETH\\",\\"to\\":\\"USD\\"}}"]; - bid_parse [type=jsonparse path="result"]; - bid_multiply [type=multiply times=100000000 index=1]; - - bid -> bid_parse -> bid_multiply; - - // Ask - ask [type=bridge name="%[7]s" timeout="50ms" requestData="{\\"data\\":{\\"from\\":\\"ETH\\",\\"to\\":\\"USD\\"}}"]; - ask_parse [type=jsonparse path="result"]; - ask_multiply [type=multiply times=100000000 index=2]; - - ask -> ask_parse -> ask_multiply; -""" - -[pluginConfig] -serverURL = "%[8]s" -serverPubKey = "%[9]x" -initialBlockNumber = %[13]d - -[relayConfig] -chainID = %[12]d - - `, - i, - verifierAddress, - node.KeyBundle.ID(), - fmt.Sprintf("%s@127.0.0.1:%d", bootstrapPeerID, bootstrapNodePort), - bmBridge, - bidBridge, - askBridge, - serverURL, - serverPubKey, - clientPubKey, - feedID, - chainID, - fromBlock, - feedName, - )) -} - -func addV2MercuryJob( - t *testing.T, - node Node, - i int, - verifierAddress common.Address, - bootstrapPeerID string, - bootstrapNodePort int, - bmBridge, - serverURL string, - serverPubKey, - clientPubKey ed25519.PublicKey, - feedName string, - feedID [32]byte, - linkFeedID [32]byte, - nativeFeedID [32]byte, -) { - node.AddJob(t, fmt.Sprintf(` -type = "offchainreporting2" -schemaVersion = 1 -name = "mercury-%[1]d-%[10]s" -forwardingAllowed = false -maxTaskDuration = "1s" -contractID = "%[2]s" -feedID = "0x%[9]x" -contractConfigTrackerPollInterval = "1s" -ocrKeyBundleID = "%[3]s" -p2pv2Bootstrappers = [ - "%[4]s" -] -relay = "evm" -pluginType = "mercury" -transmitterID = "%[8]x" -observationSource = """ - // Benchmark Price - price1 [type=bridge name="%[5]s" timeout="50ms" requestData="{\\"data\\":{\\"from\\":\\"ETH\\",\\"to\\":\\"USD\\"}}"]; - price1_parse [type=jsonparse path="result"]; - price1_multiply [type=multiply times=100000000 index=0]; - - price1 -> price1_parse -> price1_multiply; -""" - -[pluginConfig] -serverURL = "%[6]s" -serverPubKey = "%[7]x" -linkFeedID = "0x%[11]x" -nativeFeedID = "0x%[12]x" - -[relayConfig] -chainID = 1337 - `, - i, - verifierAddress, - node.KeyBundle.ID(), - fmt.Sprintf("%s@127.0.0.1:%d", bootstrapPeerID, bootstrapNodePort), - bmBridge, - serverURL, - serverPubKey, - clientPubKey, - feedID, - feedName, - linkFeedID, - nativeFeedID, - )) -} - -func addV3MercuryJob( - t *testing.T, - node Node, - i int, - verifierAddress common.Address, - bootstrapPeerID string, - bootstrapNodePort int, - bmBridge, - bidBridge, - askBridge string, - servers map[string]string, - clientPubKey ed25519.PublicKey, - feedName string, - feedID [32]byte, - linkFeedID [32]byte, - nativeFeedID [32]byte, -) { - srvs := make([]string, 0, len(servers)) - for u, k := range servers { - srvs = append(srvs, fmt.Sprintf("%q = %q", u, k)) - } - serversStr := fmt.Sprintf("{ %s }", strings.Join(srvs, ", ")) - - node.AddJob(t, fmt.Sprintf(` -type = "offchainreporting2" -schemaVersion = 1 -name = "mercury-%[1]d-%[11]s" -forwardingAllowed = false -maxTaskDuration = "1s" -contractID = "%[2]s" -feedID = "0x%[10]x" -contractConfigTrackerPollInterval = "1s" -ocrKeyBundleID = "%[3]s" -p2pv2Bootstrappers = [ - "%[4]s" -] -relay = "evm" -pluginType = "mercury" -transmitterID = "%[9]x" -observationSource = """ - // Benchmark Price - price1 [type=bridge name="%[5]s" timeout="50ms" requestData="{\\"data\\":{\\"from\\":\\"ETH\\",\\"to\\":\\"USD\\"}}"]; - price1_parse [type=jsonparse path="result"]; - price1_multiply [type=multiply times=100000000 index=0]; - - price1 -> price1_parse -> price1_multiply; - - // Bid - bid [type=bridge name="%[6]s" timeout="50ms" requestData="{\\"data\\":{\\"from\\":\\"ETH\\",\\"to\\":\\"USD\\"}}"]; - bid_parse [type=jsonparse path="result"]; - bid_multiply [type=multiply times=100000000 index=1]; - - bid -> bid_parse -> bid_multiply; - - // Ask - ask [type=bridge name="%[7]s" timeout="50ms" requestData="{\\"data\\":{\\"from\\":\\"ETH\\",\\"to\\":\\"USD\\"}}"]; - ask_parse [type=jsonparse path="result"]; - ask_multiply [type=multiply times=100000000 index=2]; - - ask -> ask_parse -> ask_multiply; -""" - -[pluginConfig] -servers = %[8]s -linkFeedID = "0x%[12]x" -nativeFeedID = "0x%[13]x" - -[relayConfig] -chainID = 1337 - `, - i, - verifierAddress, - node.KeyBundle.ID(), - fmt.Sprintf("%s@127.0.0.1:%d", bootstrapPeerID, bootstrapNodePort), - bmBridge, - bidBridge, - askBridge, - serversStr, - clientPubKey, - feedID, - feedName, - linkFeedID, - nativeFeedID, - )) -} - -func addV4MercuryJob( - t *testing.T, - node Node, - i int, - verifierAddress common.Address, - bootstrapPeerID string, - bootstrapNodePort int, - bmBridge, - marketStatusBridge string, - servers map[string]string, - clientPubKey ed25519.PublicKey, - feedName string, - feedID [32]byte, - linkFeedID [32]byte, - nativeFeedID [32]byte, -) { - srvs := make([]string, 0, len(servers)) - for u, k := range servers { - srvs = append(srvs, fmt.Sprintf("%q = %q", u, k)) - } - serversStr := fmt.Sprintf("{ %s }", strings.Join(srvs, ", ")) - - node.AddJob(t, fmt.Sprintf(` -type = "offchainreporting2" -schemaVersion = 1 -name = "mercury-%[1]d-%[9]s" -forwardingAllowed = false -maxTaskDuration = "1s" -contractID = "%[2]s" -feedID = "0x%[8]x" -contractConfigTrackerPollInterval = "1s" -ocrKeyBundleID = "%[3]s" -p2pv2Bootstrappers = [ - "%[4]s" -] -relay = "evm" -pluginType = "mercury" -transmitterID = "%[7]x" -observationSource = """ - // Benchmark Price - price1 [type=bridge name="%[5]s" timeout="50ms" requestData="{\\"data\\":{\\"from\\":\\"ETH\\",\\"to\\":\\"USD\\"}}"]; - price1_parse [type=jsonparse path="result"]; - price1_multiply [type=multiply times=100000000 index=0]; - - price1 -> price1_parse -> price1_multiply; - - // Market Status - marketstatus [type=bridge name="%[12]s" timeout="50ms" requestData="{\\"data\\":{\\"from\\":\\"ETH\\",\\"to\\":\\"USD\\"}}"]; - marketstatus_parse [type=jsonparse path="result" index=1]; - - marketstatus -> marketstatus_parse; -""" - -[pluginConfig] -servers = %[6]s -linkFeedID = "0x%[10]x" -nativeFeedID = "0x%[11]x" - -[relayConfig] -chainID = 1337 - `, - i, - verifierAddress, - node.KeyBundle.ID(), - fmt.Sprintf("%s@127.0.0.1:%d", bootstrapPeerID, bootstrapNodePort), - bmBridge, - serversStr, - clientPubKey, - feedID, - feedName, - linkFeedID, - nativeFeedID, - marketStatusBridge, - )) -} diff --git a/core/services/ocr2/plugins/mercury/integration_plugin_test.go b/core/services/ocr2/plugins/mercury/integration_plugin_test.go deleted file mode 100644 index 1dedaadcb54..00000000000 --- a/core/services/ocr2/plugins/mercury/integration_plugin_test.go +++ /dev/null @@ -1,24 +0,0 @@ -//go:build integration - -package mercury_test - -import ( - "testing" - - "github.com/smartcontractkit/chainlink/v2/core/config/env" -) - -func TestIntegration_MercuryV1_Plugin(t *testing.T) { - t.Setenv(string(env.MercuryPlugin.Cmd), "chainlink-mercury") - integration_MercuryV1(t) -} - -func TestIntegration_MercuryV2_Plugin(t *testing.T) { - t.Setenv(string(env.MercuryPlugin.Cmd), "chainlink-mercury") - integration_MercuryV2(t) -} - -func TestIntegration_MercuryV3_Plugin(t *testing.T) { - t.Setenv(string(env.MercuryPlugin.Cmd), "chainlink-mercury") - integration_MercuryV3(t) -} diff --git a/core/services/ocr2/plugins/mercury/integration_test.go b/core/services/ocr2/plugins/mercury/integration_test.go deleted file mode 100644 index 70bedbdec3d..00000000000 --- a/core/services/ocr2/plugins/mercury/integration_test.go +++ /dev/null @@ -1,1355 +0,0 @@ -package mercury_test - -import ( - "crypto/ed25519" - "encoding/hex" - "encoding/json" - "fmt" - "io" - "math" - "math/big" - "math/rand" - "net/http" - "net/http/httptest" - "net/url" - "strings" - "sync/atomic" - "testing" - "time" - - "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/eth/ethconfig" - "github.com/hashicorp/consul/sdk/freeport" - "github.com/shopspring/decimal" - "github.com/smartcontractkit/libocr/offchainreporting2plus/confighelper" - "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3confighelper" - ocr2types "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - "github.com/smartcontractkit/wsrpc/credentials" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/zap/zapcore" - "go.uber.org/zap/zaptest/observer" - - mercurytypes "github.com/smartcontractkit/chainlink-common/pkg/types/mercury" - v1 "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v1" - v2 "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v2" - v3 "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v3" - v4 "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v4" - datastreamsmercury "github.com/smartcontractkit/chainlink-data-streams/mercury" - evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" - - "github.com/smartcontractkit/chainlink/v2/core/bridges" - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/generated/link_token_interface" - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/llo-feeds/generated/fee_manager" - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/llo-feeds/generated/reward_manager" - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/llo-feeds/generated/verifier" - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/llo-feeds/generated/verifier_proxy" - "github.com/smartcontractkit/chainlink/v2/core/internal/cltest" - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" - "github.com/smartcontractkit/chainlink/v2/core/logger" - "github.com/smartcontractkit/chainlink/v2/core/services/keystore/keys/csakey" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury" - reportcodecv1 "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v1/reportcodec" - reportcodecv2 "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v2/reportcodec" - reportcodecv3 "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v3/reportcodec" - reportcodecv4 "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v4/reportcodec" - "github.com/smartcontractkit/chainlink/v2/core/store/models" - "github.com/smartcontractkit/chainlink/v2/evm/assets" -) - -var ( - f = uint8(1) - n = 4 // number of nodes - multiplier int64 = 100000000 - rawOnchainConfig = mercurytypes.OnchainConfig{ - Min: big.NewInt(0), - Max: big.NewInt(math.MaxInt64), - } - rawReportingPluginConfig = datastreamsmercury.OffchainConfig{ - ExpirationWindow: 1, - BaseUSDFee: decimal.NewFromInt(100), - } -) - -func detectPanicLogs(t *testing.T, logObservers []*observer.ObservedLogs) { - var panicLines []string - for _, observedLogs := range logObservers { - panicLogs := observedLogs.Filter(func(e observer.LoggedEntry) bool { - return e.Level >= zapcore.DPanicLevel - }) - for _, log := range panicLogs.All() { - line := fmt.Sprintf("%v\t%s\t%s\t%s\t%s", log.Time.Format(time.RFC3339), log.Level.CapitalString(), log.LoggerName, log.Caller.TrimmedPath(), log.Message) - panicLines = append(panicLines, line) - } - } - if len(panicLines) > 0 { - t.Errorf("Found logs with DPANIC or higher level:\n%s", strings.Join(panicLines, "\n")) - } -} - -func setupBlockchain(t *testing.T) (*bind.TransactOpts, evmtypes.Backend, *verifier.Verifier, common.Address, func() common.Hash) { - steve := testutils.MustNewSimTransactor(t) // config contract deployer and owner - genesisData := types.GenesisAlloc{steve.From: {Balance: assets.Ether(1000).ToInt()}} - backend := cltest.NewSimulatedBackend(t, genesisData, ethconfig.Defaults.Miner.GasCeil) - backend.Commit() // ensure starting block number at least 1 - commit, stopMining := cltest.Mine(backend, 1*time.Second) // Should be greater than deltaRound since we cannot access old blocks on simulated blockchain - t.Cleanup(stopMining) - - // Deploy contracts - linkTokenAddress, _, linkToken, err := link_token_interface.DeployLinkToken(steve, backend.Client()) - require.NoError(t, err) - commit() - _, err = linkToken.Transfer(steve, steve.From, big.NewInt(1000)) - require.NoError(t, err) - commit() - nativeTokenAddress, _, nativeToken, err := link_token_interface.DeployLinkToken(steve, backend.Client()) - require.NoError(t, err) - commit() - - _, err = nativeToken.Transfer(steve, steve.From, big.NewInt(1000)) - require.NoError(t, err) - commit() - verifierProxyAddr, _, verifierProxy, err := verifier_proxy.DeployVerifierProxy(steve, backend.Client(), common.Address{}) // zero address for access controller disables access control - require.NoError(t, err) - commit() - verifierAddress, _, verifier, err := verifier.DeployVerifier(steve, backend.Client(), verifierProxyAddr) - require.NoError(t, err) - commit() - _, err = verifierProxy.InitializeVerifier(steve, verifierAddress) - require.NoError(t, err) - commit() - rewardManagerAddr, _, rewardManager, err := reward_manager.DeployRewardManager(steve, backend.Client(), linkTokenAddress) - require.NoError(t, err) - commit() - feeManagerAddr, _, _, err := fee_manager.DeployFeeManager(steve, backend.Client(), linkTokenAddress, nativeTokenAddress, verifierProxyAddr, rewardManagerAddr) - require.NoError(t, err) - commit() - _, err = verifierProxy.SetFeeManager(steve, feeManagerAddr) - require.NoError(t, err) - commit() - _, err = rewardManager.SetFeeManager(steve, feeManagerAddr) - require.NoError(t, err) - commit() - - return steve, backend, verifier, verifierAddress, commit -} - -func TestIntegration_MercuryV1(t *testing.T) { - t.Parallel() - - integration_MercuryV1(t) -} - -func integration_MercuryV1(t *testing.T) { - ctx := testutils.Context(t) - var logObservers []*observer.ObservedLogs - t.Cleanup(func() { - detectPanicLogs(t, logObservers) - }) - lggr := logger.TestLogger(t) - testStartTimeStamp := uint32(time.Now().Unix()) - - // test vars - // pError is the probability that an EA will return an error instead of a result, as integer percentage - // pError = 0 means it will never return error - pError := atomic.Int64{} - - // feeds - btcFeed := Feed{"BTC/USD", randomFeedID(1), big.NewInt(20_000 * multiplier), big.NewInt(19_997 * multiplier), big.NewInt(20_004 * multiplier), 0} - ethFeed := Feed{"ETH/USD", randomFeedID(1), big.NewInt(1_568 * multiplier), big.NewInt(1_566 * multiplier), big.NewInt(1_569 * multiplier), 0} - linkFeed := Feed{"LINK/USD", randomFeedID(1), big.NewInt(7150 * multiplier / 1000), big.NewInt(7123 * multiplier / 1000), big.NewInt(7177 * multiplier / 1000), 0} - feeds := []Feed{btcFeed, ethFeed, linkFeed} - feedM := make(map[[32]byte]Feed, len(feeds)) - for i := range feeds { - feedM[feeds[i].id] = feeds[i] - } - - reqs := make(chan request) - serverKey := csakey.MustNewV2XXXTestingOnly(big.NewInt(-1)) - serverPubKey := serverKey.PublicKey - srv := NewMercuryServer(t, ed25519.PrivateKey(serverKey.Raw()), reqs, func() []byte { - report, err := (&reportcodecv1.ReportCodec{}).BuildReport(ctx, v1.ReportFields{BenchmarkPrice: big.NewInt(234567), Bid: big.NewInt(1), Ask: big.NewInt(1), CurrentBlockHash: make([]byte, 32)}) - if err != nil { - panic(err) - } - return report - }) - clientCSAKeys := make([]csakey.KeyV2, n+1) - clientPubKeys := make([]ed25519.PublicKey, n+1) - for i := 0; i < n+1; i++ { - k := big.NewInt(int64(i)) - key := csakey.MustNewV2XXXTestingOnly(k) - clientCSAKeys[i] = key - clientPubKeys[i] = key.PublicKey - } - serverURL := startMercuryServer(t, srv, clientPubKeys) - chainID := testutils.SimulatedChainID - - steve, backend, verifier, verifierAddress, commit := setupBlockchain(t) - - // Setup bootstrap + oracle nodes - bootstrapNodePort := freeport.GetOne(t) - appBootstrap, bootstrapPeerID, _, bootstrapKb, observedLogs := setupNode(t, bootstrapNodePort, "bootstrap_mercury", backend, clientCSAKeys[n]) - bootstrapNode := Node{App: appBootstrap, KeyBundle: bootstrapKb} - logObservers = append(logObservers, observedLogs) - - // cannot use zero, start from finality depth - fromBlock := func() int { - // Commit blocks to finality depth to ensure LogPoller has finalized blocks to read from - ch, err := bootstrapNode.App.GetRelayers().LegacyEVMChains().Get(testutils.SimulatedChainID.String()) - require.NoError(t, err) - finalityDepth := ch.Config().EVM().FinalityDepth() - for i := 0; i < int(finalityDepth); i++ { - commit() - } - return int(finalityDepth) - }() - - // Set up n oracles - var ( - oracles []confighelper.OracleIdentityExtra - nodes []Node - ) - ports := freeport.GetN(t, n) - for i := 0; i < n; i++ { - app, peerID, transmitter, kb, observedLogs := setupNode(t, ports[i], fmt.Sprintf("oracle_mercury%d", i), backend, clientCSAKeys[i]) - - nodes = append(nodes, Node{ - app, transmitter, kb, - }) - offchainPublicKey, _ := hex.DecodeString(strings.TrimPrefix(kb.OnChainPublicKey(), "0x")) - oracles = append(oracles, confighelper.OracleIdentityExtra{ - OracleIdentity: confighelper.OracleIdentity{ - OnchainPublicKey: offchainPublicKey, - TransmitAccount: ocr2types.Account(fmt.Sprintf("%x", transmitter[:])), - OffchainPublicKey: kb.OffchainPublicKey(), - PeerID: peerID, - }, - ConfigEncryptionPublicKey: kb.ConfigEncryptionPublicKey(), - }) - logObservers = append(logObservers, observedLogs) - } - - for _, feed := range feeds { - addBootstrapJob(t, bootstrapNode, chainID, verifierAddress, feed.name, feed.id) - } - - createBridge := func(name string, i int, p *big.Int, borm bridges.ORM) (bridgeName string) { - bridge := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { - b, herr := io.ReadAll(req.Body) - require.NoError(t, herr) - require.Equal(t, `{"data":{"from":"ETH","to":"USD"}}`, string(b)) - - r := rand.Int63n(101) - if r > pError.Load() { - res.WriteHeader(http.StatusOK) - val := decimal.NewFromBigInt(p, 0).Div(decimal.NewFromInt(multiplier)).Add(decimal.NewFromInt(int64(i)).Div(decimal.NewFromInt(100))).String() - resp := fmt.Sprintf(`{"result": %s}`, val) - _, herr = res.Write([]byte(resp)) - require.NoError(t, herr) - } else { - res.WriteHeader(http.StatusInternalServerError) - resp := `{"error": "pError test error"}` - _, herr = res.Write([]byte(resp)) - require.NoError(t, herr) - } - })) - t.Cleanup(bridge.Close) - u, _ := url.Parse(bridge.URL) - bridgeName = fmt.Sprintf("bridge-%s-%d", name, i) - require.NoError(t, borm.CreateBridgeType(ctx, &bridges.BridgeType{ - Name: bridges.BridgeName(bridgeName), - URL: models.WebURL(*u), - })) - - return bridgeName - } - - // Add OCR jobs - one per feed on each node - for i, node := range nodes { - for j, feed := range feeds { - bmBridge := createBridge(fmt.Sprintf("benchmarkprice-%d", j), i, feed.baseBenchmarkPrice, node.App.BridgeORM()) - askBridge := createBridge(fmt.Sprintf("ask-%d", j), i, feed.baseAsk, node.App.BridgeORM()) - bidBridge := createBridge(fmt.Sprintf("bid-%d", j), i, feed.baseBid, node.App.BridgeORM()) - - addV1MercuryJob( - t, - node, - i, - verifierAddress, - bootstrapPeerID, - bootstrapNodePort, - bmBridge, - bidBridge, - askBridge, - serverURL, - serverPubKey, - clientPubKeys[i], - feed.name, - feed.id, - chainID, - fromBlock, - ) - } - } - // Setup config on contract - onchainConfig, err := (datastreamsmercury.StandardOnchainConfigCodec{}).Encode(ctx, rawOnchainConfig) - require.NoError(t, err) - - reportingPluginConfig, err := json.Marshal(rawReportingPluginConfig) - require.NoError(t, err) - - signers, _, _, onchainConfig, offchainConfigVersion, offchainConfig, err := ocr3confighelper.ContractSetConfigArgsForTestsMercuryV02( - 2*time.Second, // DeltaProgress - 20*time.Second, // DeltaResend - 400*time.Millisecond, // DeltaInitial - 200*time.Millisecond, // DeltaRound - 100*time.Millisecond, // DeltaGrace - 300*time.Millisecond, // DeltaCertifiedCommitRequest - 1*time.Minute, // DeltaStage - 100, // rMax - []int{len(nodes)}, // S - oracles, - reportingPluginConfig, // reportingPluginConfig []byte, - nil, - 250*time.Millisecond, // Max duration observation - int(f), // f - onchainConfig, - ) - - require.NoError(t, err) - signerAddresses, err := evm.OnchainPublicKeyToAddress(signers) - require.NoError(t, err) - - offchainTransmitters := make([][32]byte, n) - for i := 0; i < n; i++ { - offchainTransmitters[i] = nodes[i].ClientPubKey - } - - for i, feed := range feeds { - lggr.Infow("Setting Config on Oracle Contract", - "i", i, - "feedID", feed.id, - "feedName", feed.name, - "signerAddresses", signerAddresses, - "offchainTransmitters", offchainTransmitters, - "f", f, - "onchainConfig", onchainConfig, - "offchainConfigVersion", offchainConfigVersion, - "offchainConfig", offchainConfig, - ) - - _, ferr := verifier.SetConfig( - steve, - feed.id, - signerAddresses, - offchainTransmitters, - f, - onchainConfig, - offchainConfigVersion, - offchainConfig, - nil, - ) - require.NoError(t, ferr) - commit() - } - - t.Run("receives at least one report per feed from each oracle when EAs are at 100% reliability", func(t *testing.T) { - ctx := testutils.Context(t) - // Expect at least one report per feed from each oracle - seen := make(map[[32]byte]map[credentials.StaticSizedPublicKey]struct{}) - for i := range feeds { - // feedID will be deleted when all n oracles have reported - seen[feeds[i].id] = make(map[credentials.StaticSizedPublicKey]struct{}, n) - } - - for req := range reqs { - v := make(map[string]interface{}) - err := mercury.PayloadTypes.UnpackIntoMap(v, req.req.Payload) - require.NoError(t, err) - report, exists := v["report"] - if !exists { - t.Fatalf("expected payload %#v to contain 'report'", v) - } - reportElems := make(map[string]interface{}) - err = reportcodecv1.ReportTypes.UnpackIntoMap(reportElems, report.([]byte)) - require.NoError(t, err) - - feedID := reportElems["feedId"].([32]uint8) - feed, exists := feedM[feedID] - require.True(t, exists) - - if _, exists := seen[feedID]; !exists { - continue // already saw all oracles for this feed - } - - num, err := (&reportcodecv1.ReportCodec{}).CurrentBlockNumFromReport(ctx, ocr2types.Report(report.([]byte))) - require.NoError(t, err) - currentBlock, err := backend.Client().BlockByNumber(ctx, nil) - require.NoError(t, err) - - assert.GreaterOrEqual(t, currentBlock.Number().Int64(), num) - - expectedBm := feed.baseBenchmarkPrice - expectedBid := feed.baseBid - expectedAsk := feed.baseAsk - - assert.GreaterOrEqual(t, int(reportElems["observationsTimestamp"].(uint32)), int(testStartTimeStamp)) - assert.InDelta(t, expectedBm.Int64(), reportElems["benchmarkPrice"].(*big.Int).Int64(), 5000000) - assert.InDelta(t, expectedBid.Int64(), reportElems["bid"].(*big.Int).Int64(), 5000000) - assert.InDelta(t, expectedAsk.Int64(), reportElems["ask"].(*big.Int).Int64(), 5000000) - assert.GreaterOrEqual(t, int(currentBlock.Number().Int64()), int(reportElems["currentBlockNum"].(uint64))) - assert.GreaterOrEqual(t, currentBlock.Time(), reportElems["currentBlockTimestamp"].(uint64)) - assert.NotEqual(t, common.Hash{}, common.Hash(reportElems["currentBlockHash"].([32]uint8))) - assert.LessOrEqual(t, int(reportElems["validFromBlockNum"].(uint64)), int(reportElems["currentBlockNum"].(uint64))) - assert.Less(t, int64(0), int64(reportElems["validFromBlockNum"].(uint64))) - - t.Logf("oracle %x reported for feed %s (0x%x)", req.pk, feed.name, feed.id) - - seen[feedID][req.pk] = struct{}{} - if len(seen[feedID]) == n { - t.Logf("all oracles reported for feed %s (0x%x)", feed.name, feed.id) - delete(seen, feedID) - if len(seen) == 0 { - break // saw all oracles; success! - } - } - } - }) - - t.Run("receives at least one report per feed from each oracle when EAs are at 80% reliability", func(t *testing.T) { - ctx := testutils.Context(t) - pError.Store(20) // 20% chance of EA error - - // Expect at least one report per feed from each oracle - seen := make(map[[32]byte]map[credentials.StaticSizedPublicKey]struct{}) - for i := range feeds { - // feedID will be deleted when all n oracles have reported - seen[feeds[i].id] = make(map[credentials.StaticSizedPublicKey]struct{}, n) - } - - for req := range reqs { - v := make(map[string]interface{}) - err := mercury.PayloadTypes.UnpackIntoMap(v, req.req.Payload) - require.NoError(t, err) - report, exists := v["report"] - if !exists { - t.Fatalf("expected payload %#v to contain 'report'", v) - } - reportElems := make(map[string]interface{}) - err = reportcodecv1.ReportTypes.UnpackIntoMap(reportElems, report.([]byte)) - require.NoError(t, err) - - feedID := reportElems["feedId"].([32]uint8) - feed, exists := feedM[feedID] - require.True(t, exists) - - if _, exists := seen[feedID]; !exists { - continue // already saw all oracles for this feed - } - - num, err := (&reportcodecv1.ReportCodec{}).CurrentBlockNumFromReport(ctx, report.([]byte)) - require.NoError(t, err) - currentBlock, err := backend.Client().BlockByNumber(testutils.Context(t), nil) - require.NoError(t, err) - - assert.GreaterOrEqual(t, currentBlock.Number().Int64(), num) - - expectedBm := feed.baseBenchmarkPrice - expectedBid := feed.baseBid - expectedAsk := feed.baseAsk - - assert.GreaterOrEqual(t, int(reportElems["observationsTimestamp"].(uint32)), int(testStartTimeStamp)) - assert.InDelta(t, expectedBm.Int64(), reportElems["benchmarkPrice"].(*big.Int).Int64(), 5000000) - assert.InDelta(t, expectedBid.Int64(), reportElems["bid"].(*big.Int).Int64(), 5000000) - assert.InDelta(t, expectedAsk.Int64(), reportElems["ask"].(*big.Int).Int64(), 5000000) - assert.GreaterOrEqual(t, int(currentBlock.Number().Int64()), int(reportElems["currentBlockNum"].(uint64))) - assert.GreaterOrEqual(t, currentBlock.Time(), reportElems["currentBlockTimestamp"].(uint64)) - assert.NotEqual(t, common.Hash{}, common.Hash(reportElems["currentBlockHash"].([32]uint8))) - assert.LessOrEqual(t, int(reportElems["validFromBlockNum"].(uint64)), int(reportElems["currentBlockNum"].(uint64))) - - t.Logf("oracle %x reported for feed %s (0x%x)", req.pk, feed.name, feed.id) - - seen[feedID][req.pk] = struct{}{} - if len(seen[feedID]) == n { - t.Logf("all oracles reported for feed %s (0x%x)", feed.name, feed.id) - delete(seen, feedID) - if len(seen) == 0 { - break // saw all oracles; success! - } - } - } - }) -} - -func TestIntegration_MercuryV2(t *testing.T) { - t.Parallel() - - integration_MercuryV2(t) -} - -func integration_MercuryV2(t *testing.T) { - ctx := testutils.Context(t) - var logObservers []*observer.ObservedLogs - t.Cleanup(func() { - detectPanicLogs(t, logObservers) - }) - - testStartTimeStamp := uint32(time.Now().Unix()) - - // test vars - // pError is the probability that an EA will return an error instead of a result, as integer percentage - // pError = 0 means it will never return error - pError := atomic.Int64{} - - // feeds - btcFeed := Feed{ - name: "BTC/USD", - id: randomFeedID(2), - baseBenchmarkPrice: big.NewInt(20_000 * multiplier), - } - ethFeed := Feed{ - name: "ETH/USD", - id: randomFeedID(2), - baseBenchmarkPrice: big.NewInt(1_568 * multiplier), - } - linkFeed := Feed{ - name: "LINK/USD", - id: randomFeedID(2), - baseBenchmarkPrice: big.NewInt(7150 * multiplier / 1000), - } - feeds := []Feed{btcFeed, ethFeed, linkFeed} - feedM := make(map[[32]byte]Feed, len(feeds)) - for i := range feeds { - feedM[feeds[i].id] = feeds[i] - } - - reqs := make(chan request) - serverKey := csakey.MustNewV2XXXTestingOnly(big.NewInt(-1)) - serverPubKey := serverKey.PublicKey - srv := NewMercuryServer(t, ed25519.PrivateKey(serverKey.Raw()), reqs, func() []byte { - report, err := (&reportcodecv2.ReportCodec{}).BuildReport(ctx, v2.ReportFields{BenchmarkPrice: big.NewInt(234567), LinkFee: big.NewInt(1), NativeFee: big.NewInt(1)}) - if err != nil { - panic(err) - } - return report - }) - clientCSAKeys := make([]csakey.KeyV2, n+1) - clientPubKeys := make([]ed25519.PublicKey, n+1) - for i := 0; i < n+1; i++ { - k := big.NewInt(int64(i)) - key := csakey.MustNewV2XXXTestingOnly(k) - clientCSAKeys[i] = key - clientPubKeys[i] = key.PublicKey - } - serverURL := startMercuryServer(t, srv, clientPubKeys) - chainID := testutils.SimulatedChainID - - steve, backend, verifier, verifierAddress, commit := setupBlockchain(t) - - // Setup bootstrap + oracle nodes - bootstrapNodePort := freeport.GetOne(t) - appBootstrap, bootstrapPeerID, _, bootstrapKb, observedLogs := setupNode(t, bootstrapNodePort, "bootstrap_mercury", backend, clientCSAKeys[n]) - bootstrapNode := Node{App: appBootstrap, KeyBundle: bootstrapKb} - logObservers = append(logObservers, observedLogs) - - // Commit blocks to finality depth to ensure LogPoller has finalized blocks to read from - ch, err := bootstrapNode.App.GetRelayers().LegacyEVMChains().Get(testutils.SimulatedChainID.String()) - require.NoError(t, err) - finalityDepth := ch.Config().EVM().FinalityDepth() - for i := 0; i < int(finalityDepth); i++ { - commit() - } - - // Set up n oracles - var ( - oracles []confighelper.OracleIdentityExtra - nodes []Node - ) - ports := freeport.GetN(t, n) - for i := 0; i < n; i++ { - app, peerID, transmitter, kb, observedLogs := setupNode(t, ports[i], fmt.Sprintf("oracle_mercury%d", i), backend, clientCSAKeys[i]) - - nodes = append(nodes, Node{ - app, transmitter, kb, - }) - - offchainPublicKey, _ := hex.DecodeString(strings.TrimPrefix(kb.OnChainPublicKey(), "0x")) - oracles = append(oracles, confighelper.OracleIdentityExtra{ - OracleIdentity: confighelper.OracleIdentity{ - OnchainPublicKey: offchainPublicKey, - TransmitAccount: ocr2types.Account(fmt.Sprintf("%x", transmitter[:])), - OffchainPublicKey: kb.OffchainPublicKey(), - PeerID: peerID, - }, - ConfigEncryptionPublicKey: kb.ConfigEncryptionPublicKey(), - }) - logObservers = append(logObservers, observedLogs) - } - - for _, feed := range feeds { - addBootstrapJob(t, bootstrapNode, chainID, verifierAddress, feed.name, feed.id) - } - - createBridge := func(name string, i int, p *big.Int, borm bridges.ORM) (bridgeName string) { - bridge := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { - b, herr := io.ReadAll(req.Body) - require.NoError(t, herr) - require.Equal(t, `{"data":{"from":"ETH","to":"USD"}}`, string(b)) - - r := rand.Int63n(101) - if r > pError.Load() { - res.WriteHeader(http.StatusOK) - val := decimal.NewFromBigInt(p, 0).Div(decimal.NewFromInt(multiplier)).Add(decimal.NewFromInt(int64(i)).Div(decimal.NewFromInt(100))).String() - resp := fmt.Sprintf(`{"result": %s}`, val) - _, herr = res.Write([]byte(resp)) - require.NoError(t, herr) - } else { - res.WriteHeader(http.StatusInternalServerError) - resp := `{"error": "pError test error"}` - _, herr = res.Write([]byte(resp)) - require.NoError(t, herr) - } - })) - t.Cleanup(bridge.Close) - u, _ := url.Parse(bridge.URL) - bridgeName = fmt.Sprintf("bridge-%s-%d", name, i) - require.NoError(t, borm.CreateBridgeType(ctx, &bridges.BridgeType{ - Name: bridges.BridgeName(bridgeName), - URL: models.WebURL(*u), - })) - - return bridgeName - } - - // Add OCR jobs - one per feed on each node - for i, node := range nodes { - for j, feed := range feeds { - bmBridge := createBridge(fmt.Sprintf("benchmarkprice-%d", j), i, feed.baseBenchmarkPrice, node.App.BridgeORM()) - - addV2MercuryJob( - t, - node, - i, - verifierAddress, - bootstrapPeerID, - bootstrapNodePort, - bmBridge, - serverURL, - serverPubKey, - clientPubKeys[i], - feed.name, - feed.id, - randomFeedID(2), - randomFeedID(2), - ) - } - } - - // Setup config on contract - onchainConfig, err := (datastreamsmercury.StandardOnchainConfigCodec{}).Encode(ctx, rawOnchainConfig) - require.NoError(t, err) - - reportingPluginConfig, err := json.Marshal(rawReportingPluginConfig) - require.NoError(t, err) - - signers, _, _, onchainConfig, offchainConfigVersion, offchainConfig, err := ocr3confighelper.ContractSetConfigArgsForTestsMercuryV02( - 2*time.Second, // DeltaProgress - 20*time.Second, // DeltaResend - 400*time.Millisecond, // DeltaInitial - 100*time.Millisecond, // DeltaRound - 0, // DeltaGrace - 300*time.Millisecond, // DeltaCertifiedCommitRequest - 1*time.Minute, // DeltaStage - 100, // rMax - []int{len(nodes)}, // S - oracles, - reportingPluginConfig, // reportingPluginConfig []byte, - nil, - 250*time.Millisecond, // Max duration observation - int(f), // f - onchainConfig, - ) - - require.NoError(t, err) - signerAddresses, err := evm.OnchainPublicKeyToAddress(signers) - require.NoError(t, err) - - offchainTransmitters := make([][32]byte, n) - for i := 0; i < n; i++ { - offchainTransmitters[i] = nodes[i].ClientPubKey - } - - for _, feed := range feeds { - _, ferr := verifier.SetConfig( - steve, - feed.id, - signerAddresses, - offchainTransmitters, - f, - onchainConfig, - offchainConfigVersion, - offchainConfig, - nil, - ) - require.NoError(t, ferr) - commit() - } - - runTestSetup := func() { - // Expect at least one report per feed from each oracle - seen := make(map[[32]byte]map[credentials.StaticSizedPublicKey]struct{}) - for i := range feeds { - // feedID will be deleted when all n oracles have reported - seen[feeds[i].id] = make(map[credentials.StaticSizedPublicKey]struct{}, n) - } - - for req := range reqs { - v := make(map[string]interface{}) - err := mercury.PayloadTypes.UnpackIntoMap(v, req.req.Payload) - require.NoError(t, err) - report, exists := v["report"] - if !exists { - t.Fatalf("expected payload %#v to contain 'report'", v) - } - reportElems := make(map[string]interface{}) - err = reportcodecv2.ReportTypes.UnpackIntoMap(reportElems, report.([]byte)) - require.NoError(t, err) - - feedID := reportElems["feedId"].([32]uint8) - feed, exists := feedM[feedID] - require.True(t, exists) - - if _, exists := seen[feedID]; !exists { - continue // already saw all oracles for this feed - } - - expectedFee := datastreamsmercury.CalculateFee(big.NewInt(234567), rawReportingPluginConfig.BaseUSDFee) - expectedExpiresAt := reportElems["observationsTimestamp"].(uint32) + rawReportingPluginConfig.ExpirationWindow - - assert.GreaterOrEqual(t, int(reportElems["observationsTimestamp"].(uint32)), int(testStartTimeStamp)) - assert.InDelta(t, feed.baseBenchmarkPrice.Int64(), reportElems["benchmarkPrice"].(*big.Int).Int64(), 5000000) - assert.NotZero(t, reportElems["validFromTimestamp"].(uint32)) - assert.GreaterOrEqual(t, reportElems["observationsTimestamp"].(uint32), reportElems["validFromTimestamp"].(uint32)) - assert.Equal(t, expectedExpiresAt, reportElems["expiresAt"].(uint32)) - assert.Equal(t, expectedFee, reportElems["linkFee"].(*big.Int)) - assert.Equal(t, expectedFee, reportElems["nativeFee"].(*big.Int)) - - t.Logf("oracle %x reported for feed %s (0x%x)", req.pk, feed.name, feed.id) - - seen[feedID][req.pk] = struct{}{} - if len(seen[feedID]) == n { - t.Logf("all oracles reported for feed %s (0x%x)", feed.name, feed.id) - delete(seen, feedID) - if len(seen) == 0 { - break // saw all oracles; success! - } - } - } - } - - t.Run("receives at least one report per feed from each oracle when EAs are at 100% reliability", func(t *testing.T) { - runTestSetup() - }) - - t.Run("receives at least one report per feed from each oracle when EAs are at 80% reliability", func(t *testing.T) { - pError.Store(20) - runTestSetup() - }) -} - -func TestIntegration_MercuryV3(t *testing.T) { - t.Parallel() - - integration_MercuryV3(t) -} - -func integration_MercuryV3(t *testing.T) { - ctx := testutils.Context(t) - var logObservers []*observer.ObservedLogs - t.Cleanup(func() { - detectPanicLogs(t, logObservers) - }) - - testStartTimeStamp := uint32(time.Now().Unix()) - - // test vars - // pError is the probability that an EA will return an error instead of a result, as integer percentage - // pError = 0 means it will never return error - pError := atomic.Int64{} - - // feeds - btcFeed := Feed{ - name: "BTC/USD", - id: randomFeedID(3), - baseBenchmarkPrice: big.NewInt(20_000 * multiplier), - baseBid: big.NewInt(19_997 * multiplier), - baseAsk: big.NewInt(20_004 * multiplier), - } - ethFeed := Feed{ - name: "ETH/USD", - id: randomFeedID(3), - baseBenchmarkPrice: big.NewInt(1_568 * multiplier), - baseBid: big.NewInt(1_566 * multiplier), - baseAsk: big.NewInt(1_569 * multiplier), - } - linkFeed := Feed{ - name: "LINK/USD", - id: randomFeedID(3), - baseBenchmarkPrice: big.NewInt(7150 * multiplier / 1000), - baseBid: big.NewInt(7123 * multiplier / 1000), - baseAsk: big.NewInt(7177 * multiplier / 1000), - } - feeds := []Feed{btcFeed, ethFeed, linkFeed} - feedM := make(map[[32]byte]Feed, len(feeds)) - for i := range feeds { - feedM[feeds[i].id] = feeds[i] - } - - clientCSAKeys := make([]csakey.KeyV2, n+1) - clientPubKeys := make([]ed25519.PublicKey, n+1) - for i := 0; i < n+1; i++ { - k := big.NewInt(int64(i)) - key := csakey.MustNewV2XXXTestingOnly(k) - clientCSAKeys[i] = key - clientPubKeys[i] = key.PublicKey - } - - // Test multi-send to three servers - const nSrvs = 3 - reqChs := make([]chan request, nSrvs) - servers := make(map[string]string) - for i := 0; i < nSrvs; i++ { - k := csakey.MustNewV2XXXTestingOnly(big.NewInt(int64(-(i + 1)))) - reqs := make(chan request, 100) - srv := NewMercuryServer(t, ed25519.PrivateKey(k.Raw()), reqs, func() []byte { - report, err := (&reportcodecv3.ReportCodec{}).BuildReport(ctx, v3.ReportFields{BenchmarkPrice: big.NewInt(234567), Bid: big.NewInt(1), Ask: big.NewInt(1), LinkFee: big.NewInt(1), NativeFee: big.NewInt(1)}) - if err != nil { - panic(err) - } - return report - }) - serverURL := startMercuryServer(t, srv, clientPubKeys) - reqChs[i] = reqs - servers[serverURL] = fmt.Sprintf("%x", k.PublicKey) - } - chainID := testutils.SimulatedChainID - - steve, backend, verifier, verifierAddress, commit := setupBlockchain(t) - - // Setup bootstrap + oracle nodes - bootstrapNodePort := freeport.GetOne(t) - appBootstrap, bootstrapPeerID, _, bootstrapKb, observedLogs := setupNode(t, bootstrapNodePort, "bootstrap_mercury", backend, clientCSAKeys[n]) - bootstrapNode := Node{App: appBootstrap, KeyBundle: bootstrapKb} - logObservers = append(logObservers, observedLogs) - - // Commit blocks to finality depth to ensure LogPoller has finalized blocks to read from - ch, err := bootstrapNode.App.GetRelayers().LegacyEVMChains().Get(testutils.SimulatedChainID.String()) - require.NoError(t, err) - finalityDepth := ch.Config().EVM().FinalityDepth() - for i := 0; i < int(finalityDepth); i++ { - commit() - } - - // Set up n oracles - var ( - oracles []confighelper.OracleIdentityExtra - nodes []Node - ) - ports := freeport.GetN(t, n) - for i := 0; i < n; i++ { - app, peerID, transmitter, kb, observedLogs := setupNode(t, ports[i], fmt.Sprintf("oracle_mercury%d", i), backend, clientCSAKeys[i]) - - nodes = append(nodes, Node{ - app, transmitter, kb, - }) - - offchainPublicKey, _ := hex.DecodeString(strings.TrimPrefix(kb.OnChainPublicKey(), "0x")) - oracles = append(oracles, confighelper.OracleIdentityExtra{ - OracleIdentity: confighelper.OracleIdentity{ - OnchainPublicKey: offchainPublicKey, - TransmitAccount: ocr2types.Account(fmt.Sprintf("%x", transmitter[:])), - OffchainPublicKey: kb.OffchainPublicKey(), - PeerID: peerID, - }, - ConfigEncryptionPublicKey: kb.ConfigEncryptionPublicKey(), - }) - logObservers = append(logObservers, observedLogs) - } - - for _, feed := range feeds { - addBootstrapJob(t, bootstrapNode, chainID, verifierAddress, feed.name, feed.id) - } - - createBridge := func(name string, i int, p *big.Int, borm bridges.ORM) (bridgeName string) { - bridge := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { - b, herr := io.ReadAll(req.Body) - require.NoError(t, herr) - require.Equal(t, `{"data":{"from":"ETH","to":"USD"}}`, string(b)) - - r := rand.Int63n(101) - if r > pError.Load() { - res.WriteHeader(http.StatusOK) - val := decimal.NewFromBigInt(p, 0).Div(decimal.NewFromInt(multiplier)).Add(decimal.NewFromInt(int64(i)).Div(decimal.NewFromInt(100))).String() - resp := fmt.Sprintf(`{"result": %s}`, val) - _, herr = res.Write([]byte(resp)) - require.NoError(t, herr) - } else { - res.WriteHeader(http.StatusInternalServerError) - resp := `{"error": "pError test error"}` - _, herr = res.Write([]byte(resp)) - require.NoError(t, herr) - } - })) - t.Cleanup(bridge.Close) - u, _ := url.Parse(bridge.URL) - bridgeName = fmt.Sprintf("bridge-%s-%d", name, i) - require.NoError(t, borm.CreateBridgeType(ctx, &bridges.BridgeType{ - Name: bridges.BridgeName(bridgeName), - URL: models.WebURL(*u), - })) - - return bridgeName - } - - // Add OCR jobs - one per feed on each node - for i, node := range nodes { - for j, feed := range feeds { - bmBridge := createBridge(fmt.Sprintf("benchmarkprice-%d", j), i, feed.baseBenchmarkPrice, node.App.BridgeORM()) - bidBridge := createBridge(fmt.Sprintf("bid-%d", j), i, feed.baseBid, node.App.BridgeORM()) - askBridge := createBridge(fmt.Sprintf("ask-%d", j), i, feed.baseAsk, node.App.BridgeORM()) - - addV3MercuryJob( - t, - node, - i, - verifierAddress, - bootstrapPeerID, - bootstrapNodePort, - bmBridge, - bidBridge, - askBridge, - servers, - clientPubKeys[i], - feed.name, - feed.id, - randomFeedID(2), - randomFeedID(2), - ) - } - } - - // Setup config on contract - onchainConfig, err := (datastreamsmercury.StandardOnchainConfigCodec{}).Encode(ctx, rawOnchainConfig) - require.NoError(t, err) - - reportingPluginConfig, err := json.Marshal(rawReportingPluginConfig) - require.NoError(t, err) - - signers, _, _, onchainConfig, offchainConfigVersion, offchainConfig, err := ocr3confighelper.ContractSetConfigArgsForTestsMercuryV02( - 2*time.Second, // DeltaProgress - 20*time.Second, // DeltaResend - 400*time.Millisecond, // DeltaInitial - 100*time.Millisecond, // DeltaRound - 0, // DeltaGrace - 300*time.Millisecond, // DeltaCertifiedCommitRequest - 1*time.Minute, // DeltaStage - 100, // rMax - []int{len(nodes)}, // S - oracles, - reportingPluginConfig, // reportingPluginConfig []byte, - nil, - 250*time.Millisecond, // Max duration observation - int(f), // f - onchainConfig, - ) - - require.NoError(t, err) - signerAddresses, err := evm.OnchainPublicKeyToAddress(signers) - require.NoError(t, err) - - offchainTransmitters := make([][32]byte, n) - for i := 0; i < n; i++ { - offchainTransmitters[i] = nodes[i].ClientPubKey - } - - for _, feed := range feeds { - _, ferr := verifier.SetConfig( - steve, - feed.id, - signerAddresses, - offchainTransmitters, - f, - onchainConfig, - offchainConfigVersion, - offchainConfig, - nil, - ) - require.NoError(t, ferr) - commit() - } - - runTestSetup := func(reqs chan request) { - // Expect at least one report per feed from each oracle, per server - seen := make(map[[32]byte]map[credentials.StaticSizedPublicKey]struct{}) - for i := range feeds { - // feedID will be deleted when all n oracles have reported - seen[feeds[i].id] = make(map[credentials.StaticSizedPublicKey]struct{}, n) - } - - for req := range reqs { - v := make(map[string]interface{}) - err := mercury.PayloadTypes.UnpackIntoMap(v, req.req.Payload) - require.NoError(t, err) - report, exists := v["report"] - if !exists { - t.Fatalf("expected payload %#v to contain 'report'", v) - } - reportElems := make(map[string]interface{}) - err = reportcodecv3.ReportTypes.UnpackIntoMap(reportElems, report.([]byte)) - require.NoError(t, err) - - feedID := reportElems["feedId"].([32]uint8) - feed, exists := feedM[feedID] - require.True(t, exists) - - if _, exists := seen[feedID]; !exists { - continue // already saw all oracles for this feed - } - - expectedFee := datastreamsmercury.CalculateFee(big.NewInt(234567), rawReportingPluginConfig.BaseUSDFee) - expectedExpiresAt := reportElems["observationsTimestamp"].(uint32) + rawReportingPluginConfig.ExpirationWindow - - assert.GreaterOrEqual(t, int(reportElems["observationsTimestamp"].(uint32)), int(testStartTimeStamp)) - assert.InDelta(t, feed.baseBenchmarkPrice.Int64(), reportElems["benchmarkPrice"].(*big.Int).Int64(), 5000000) - assert.InDelta(t, feed.baseBid.Int64(), reportElems["bid"].(*big.Int).Int64(), 5000000) - assert.InDelta(t, feed.baseAsk.Int64(), reportElems["ask"].(*big.Int).Int64(), 5000000) - assert.NotZero(t, reportElems["validFromTimestamp"].(uint32)) - assert.GreaterOrEqual(t, reportElems["observationsTimestamp"].(uint32), reportElems["validFromTimestamp"].(uint32)) - assert.Equal(t, expectedExpiresAt, reportElems["expiresAt"].(uint32)) - assert.Equal(t, expectedFee, reportElems["linkFee"].(*big.Int)) - assert.Equal(t, expectedFee, reportElems["nativeFee"].(*big.Int)) - - t.Logf("oracle %x reported for feed %s (0x%x)", req.pk, feed.name, feed.id) - - seen[feedID][req.pk] = struct{}{} - if len(seen[feedID]) == n { - t.Logf("all oracles reported for feed %s (0x%x)", feed.name, feed.id) - delete(seen, feedID) - if len(seen) == 0 { - break // saw all oracles; success! - } - } - } - } - - t.Run("receives at least one report per feed for every server from each oracle when EAs are at 100% reliability", func(t *testing.T) { - for i := 0; i < nSrvs; i++ { - reqs := reqChs[i] - runTestSetup(reqs) - } - }) -} - -func TestIntegration_MercuryV4(t *testing.T) { - t.Parallel() - - integration_MercuryV4(t) -} - -func integration_MercuryV4(t *testing.T) { - ctx := testutils.Context(t) - var logObservers []*observer.ObservedLogs - t.Cleanup(func() { - detectPanicLogs(t, logObservers) - }) - - testStartTimeStamp := uint32(time.Now().Unix()) - - // test vars - // pError is the probability that an EA will return an error instead of a result, as integer percentage - // pError = 0 means it will never return error - pError := atomic.Int64{} - - // feeds - btcFeed := Feed{ - name: "BTC/USD", - id: randomFeedID(4), - baseBenchmarkPrice: big.NewInt(20_000 * multiplier), - baseBid: big.NewInt(19_997 * multiplier), - baseAsk: big.NewInt(20_004 * multiplier), - baseMarketStatus: 1, - } - ethFeed := Feed{ - name: "ETH/USD", - id: randomFeedID(4), - baseBenchmarkPrice: big.NewInt(1_568 * multiplier), - baseBid: big.NewInt(1_566 * multiplier), - baseAsk: big.NewInt(1_569 * multiplier), - baseMarketStatus: 2, - } - linkFeed := Feed{ - name: "LINK/USD", - id: randomFeedID(4), - baseBenchmarkPrice: big.NewInt(7150 * multiplier / 1000), - baseBid: big.NewInt(7123 * multiplier / 1000), - baseAsk: big.NewInt(7177 * multiplier / 1000), - baseMarketStatus: 3, - } - feeds := []Feed{btcFeed, ethFeed, linkFeed} - feedM := make(map[[32]byte]Feed, len(feeds)) - for i := range feeds { - feedM[feeds[i].id] = feeds[i] - } - - clientCSAKeys := make([]csakey.KeyV2, n+1) - clientPubKeys := make([]ed25519.PublicKey, n+1) - for i := 0; i < n+1; i++ { - k := big.NewInt(int64(i)) - key := csakey.MustNewV2XXXTestingOnly(k) - clientCSAKeys[i] = key - clientPubKeys[i] = key.PublicKey - } - - // Test multi-send to three servers - const nSrvs = 3 - reqChs := make([]chan request, nSrvs) - servers := make(map[string]string) - for i := 0; i < nSrvs; i++ { - k := csakey.MustNewV2XXXTestingOnly(big.NewInt(int64(-(i + 1)))) - reqs := make(chan request, 100) - srv := NewMercuryServer(t, ed25519.PrivateKey(k.Raw()), reqs, func() []byte { - report, err := (&reportcodecv4.ReportCodec{}).BuildReport(ctx, v4.ReportFields{BenchmarkPrice: big.NewInt(234567), LinkFee: big.NewInt(1), NativeFee: big.NewInt(1), MarketStatus: 1}) - if err != nil { - panic(err) - } - return report - }) - serverURL := startMercuryServer(t, srv, clientPubKeys) - reqChs[i] = reqs - servers[serverURL] = fmt.Sprintf("%x", k.PublicKey) - } - chainID := testutils.SimulatedChainID - - steve, backend, verifier, verifierAddress, commit := setupBlockchain(t) - - // Setup bootstrap + oracle nodes - bootstrapNodePort := freeport.GetOne(t) - appBootstrap, bootstrapPeerID, _, bootstrapKb, observedLogs := setupNode(t, bootstrapNodePort, "bootstrap_mercury", backend, clientCSAKeys[n]) - bootstrapNode := Node{App: appBootstrap, KeyBundle: bootstrapKb} - logObservers = append(logObservers, observedLogs) - - // Commit blocks to finality depth to ensure LogPoller has finalized blocks to read from - ch, err := bootstrapNode.App.GetRelayers().LegacyEVMChains().Get(testutils.SimulatedChainID.String()) - require.NoError(t, err) - finalityDepth := ch.Config().EVM().FinalityDepth() - for i := 0; i < int(finalityDepth); i++ { - commit() - } - - // Set up n oracles - var ( - oracles []confighelper.OracleIdentityExtra - nodes []Node - ) - ports := freeport.GetN(t, n) - for i := 0; i < n; i++ { - app, peerID, transmitter, kb, observedLogs := setupNode(t, ports[i], fmt.Sprintf("oracle_mercury%d", i), backend, clientCSAKeys[i]) - - nodes = append(nodes, Node{ - app, transmitter, kb, - }) - - offchainPublicKey, _ := hex.DecodeString(strings.TrimPrefix(kb.OnChainPublicKey(), "0x")) - oracles = append(oracles, confighelper.OracleIdentityExtra{ - OracleIdentity: confighelper.OracleIdentity{ - OnchainPublicKey: offchainPublicKey, - TransmitAccount: ocr2types.Account(fmt.Sprintf("%x", transmitter[:])), - OffchainPublicKey: kb.OffchainPublicKey(), - PeerID: peerID, - }, - ConfigEncryptionPublicKey: kb.ConfigEncryptionPublicKey(), - }) - logObservers = append(logObservers, observedLogs) - } - - for _, feed := range feeds { - addBootstrapJob(t, bootstrapNode, chainID, verifierAddress, feed.name, feed.id) - } - - createBridge := func(name string, i int, p *big.Int, marketStatus uint32, borm bridges.ORM) (bridgeName string) { - bridge := httptest.NewServer(http.HandlerFunc(func(res http.ResponseWriter, req *http.Request) { - b, herr := io.ReadAll(req.Body) - require.NoError(t, herr) - require.Equal(t, `{"data":{"from":"ETH","to":"USD"}}`, string(b)) - - r := rand.Int63n(101) - if r > pError.Load() { - res.WriteHeader(http.StatusOK) - - var val string - if p != nil { - val = decimal.NewFromBigInt(p, 0).Div(decimal.NewFromInt(multiplier)).Add(decimal.NewFromInt(int64(i)).Div(decimal.NewFromInt(100))).String() - } else { - val = fmt.Sprintf("%d", marketStatus) - } - - resp := fmt.Sprintf(`{"result": %s}`, val) - _, herr = res.Write([]byte(resp)) - require.NoError(t, herr) - } else { - res.WriteHeader(http.StatusInternalServerError) - resp := `{"error": "pError test error"}` - _, herr = res.Write([]byte(resp)) - require.NoError(t, herr) - } - })) - t.Cleanup(bridge.Close) - u, _ := url.Parse(bridge.URL) - bridgeName = fmt.Sprintf("bridge-%s-%d", name, i) - require.NoError(t, borm.CreateBridgeType(ctx, &bridges.BridgeType{ - Name: bridges.BridgeName(bridgeName), - URL: models.WebURL(*u), - })) - - return bridgeName - } - - // Add OCR jobs - one per feed on each node - for i, node := range nodes { - for j, feed := range feeds { - bmBridge := createBridge(fmt.Sprintf("benchmarkprice-%d", j), i, feed.baseBenchmarkPrice, 0, node.App.BridgeORM()) - marketStatusBridge := createBridge(fmt.Sprintf("marketstatus-%d", j), i, nil, feed.baseMarketStatus, node.App.BridgeORM()) - - addV4MercuryJob( - t, - node, - i, - verifierAddress, - bootstrapPeerID, - bootstrapNodePort, - bmBridge, - marketStatusBridge, - servers, - clientPubKeys[i], - feed.name, - feed.id, - randomFeedID(2), - randomFeedID(2), - ) - } - } - - // Setup config on contract - onchainConfig, err := (datastreamsmercury.StandardOnchainConfigCodec{}).Encode(ctx, rawOnchainConfig) - require.NoError(t, err) - - reportingPluginConfig, err := json.Marshal(rawReportingPluginConfig) - require.NoError(t, err) - - signers, _, _, onchainConfig, offchainConfigVersion, offchainConfig, err := ocr3confighelper.ContractSetConfigArgsForTestsMercuryV02( - 2*time.Second, // DeltaProgress - 20*time.Second, // DeltaResend - 400*time.Millisecond, // DeltaInitial - 100*time.Millisecond, // DeltaRound - 0, // DeltaGrace - 300*time.Millisecond, // DeltaCertifiedCommitRequest - 1*time.Minute, // DeltaStage - 100, // rMax - []int{len(nodes)}, // S - oracles, - reportingPluginConfig, // reportingPluginConfig []byte, - nil, - 250*time.Millisecond, // Max duration observation - int(f), // f - onchainConfig, - ) - - require.NoError(t, err) - signerAddresses, err := evm.OnchainPublicKeyToAddress(signers) - require.NoError(t, err) - - offchainTransmitters := make([][32]byte, n) - for i := 0; i < n; i++ { - offchainTransmitters[i] = nodes[i].ClientPubKey - } - - for _, feed := range feeds { - _, ferr := verifier.SetConfig( - steve, - feed.id, - signerAddresses, - offchainTransmitters, - f, - onchainConfig, - offchainConfigVersion, - offchainConfig, - nil, - ) - require.NoError(t, ferr) - commit() - } - - runTestSetup := func(reqs chan request) { - // Expect at least one report per feed from each oracle, per server - seen := make(map[[32]byte]map[credentials.StaticSizedPublicKey]struct{}) - for i := range feeds { - // feedID will be deleted when all n oracles have reported - seen[feeds[i].id] = make(map[credentials.StaticSizedPublicKey]struct{}, n) - } - - for req := range reqs { - v := make(map[string]interface{}) - err := mercury.PayloadTypes.UnpackIntoMap(v, req.req.Payload) - require.NoError(t, err) - report, exists := v["report"] - if !exists { - t.Fatalf("expected payload %#v to contain 'report'", v) - } - reportElems := make(map[string]interface{}) - err = reportcodecv4.ReportTypes.UnpackIntoMap(reportElems, report.([]byte)) - require.NoError(t, err) - - feedID := reportElems["feedId"].([32]uint8) - feed, exists := feedM[feedID] - require.True(t, exists) - - if _, exists := seen[feedID]; !exists { - continue // already saw all oracles for this feed - } - - expectedFee := datastreamsmercury.CalculateFee(big.NewInt(234567), rawReportingPluginConfig.BaseUSDFee) - expectedExpiresAt := reportElems["observationsTimestamp"].(uint32) + rawReportingPluginConfig.ExpirationWindow - - assert.GreaterOrEqual(t, int(reportElems["observationsTimestamp"].(uint32)), int(testStartTimeStamp)) - assert.InDelta(t, feed.baseBenchmarkPrice.Int64(), reportElems["benchmarkPrice"].(*big.Int).Int64(), 5000000) - assert.NotZero(t, reportElems["validFromTimestamp"].(uint32)) - assert.GreaterOrEqual(t, reportElems["observationsTimestamp"].(uint32), reportElems["validFromTimestamp"].(uint32)) - assert.Equal(t, expectedExpiresAt, reportElems["expiresAt"].(uint32)) - assert.Equal(t, expectedFee, reportElems["linkFee"].(*big.Int)) - assert.Equal(t, expectedFee, reportElems["nativeFee"].(*big.Int)) - assert.Equal(t, feed.baseMarketStatus, reportElems["marketStatus"].(uint32)) - - t.Logf("oracle %x reported for feed %s (0x%x)", req.pk, feed.name, feed.id) - - seen[feedID][req.pk] = struct{}{} - if len(seen[feedID]) == n { - t.Logf("all oracles reported for feed %s (0x%x)", feed.name, feed.id) - delete(seen, feedID) - if len(seen) == 0 { - break // saw all oracles; success! - } - } - } - } - - t.Run("receives at least one report per feed for every server from each oracle when EAs are at 100% reliability", func(t *testing.T) { - for i := 0; i < nSrvs; i++ { - reqs := reqChs[i] - runTestSetup(reqs) - } - }) -} diff --git a/core/services/ocr2/plugins/mercury/plugin.go b/core/services/ocr2/plugins/mercury/plugin.go deleted file mode 100644 index b0983e55c89..00000000000 --- a/core/services/ocr2/plugins/mercury/plugin.go +++ /dev/null @@ -1,394 +0,0 @@ -package mercury - -import ( - "context" - "encoding/json" - "fmt" - "os/exec" - - "github.com/pkg/errors" - - libocr2 "github.com/smartcontractkit/libocr/offchainreporting2plus" - "github.com/smartcontractkit/libocr/offchainreporting2plus/ocr3types" - - relaymercuryv1 "github.com/smartcontractkit/chainlink-data-streams/mercury/v1" - relaymercuryv2 "github.com/smartcontractkit/chainlink-data-streams/mercury/v2" - relaymercuryv3 "github.com/smartcontractkit/chainlink-data-streams/mercury/v3" - relaymercuryv4 "github.com/smartcontractkit/chainlink-data-streams/mercury/v4" - - "github.com/smartcontractkit/chainlink-common/pkg/loop" - commontypes "github.com/smartcontractkit/chainlink-common/pkg/types" - - "github.com/smartcontractkit/chainlink/v2/core/config/env" - "github.com/smartcontractkit/chainlink/v2/core/logger" - "github.com/smartcontractkit/chainlink/v2/core/services" - "github.com/smartcontractkit/chainlink/v2/core/services/job" - "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/mercury/config" - "github.com/smartcontractkit/chainlink/v2/core/services/ocrcommon" - "github.com/smartcontractkit/chainlink/v2/core/services/pipeline" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/types" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/utils" - mercuryv1 "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v1" - mercuryv2 "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v2" - mercuryv3 "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v3" - mercuryv4 "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v4" - "github.com/smartcontractkit/chainlink/v2/plugins" -) - -type Config interface { - MaxSuccessfulRuns() uint64 - ResultWriteQueueDepth() uint64 - plugins.RegistrarConfig -} - -// concrete implementation of MercuryConfig -type mercuryConfig struct { - jobPipelineMaxSuccessfulRuns uint64 - jobPipelineResultWriteQueueDepth uint64 - plugins.RegistrarConfig -} - -func NewMercuryConfig(jobPipelineMaxSuccessfulRuns uint64, jobPipelineResultWriteQueueDepth uint64, pluginProcessCfg plugins.RegistrarConfig) Config { - return &mercuryConfig{ - jobPipelineMaxSuccessfulRuns: jobPipelineMaxSuccessfulRuns, - jobPipelineResultWriteQueueDepth: jobPipelineResultWriteQueueDepth, - RegistrarConfig: pluginProcessCfg, - } -} - -func (m *mercuryConfig) MaxSuccessfulRuns() uint64 { - return m.jobPipelineMaxSuccessfulRuns -} - -func (m *mercuryConfig) ResultWriteQueueDepth() uint64 { - return m.jobPipelineResultWriteQueueDepth -} - -func NewServices( - jb job.Job, - ocr2Provider commontypes.MercuryProvider, - pipelineRunner pipeline.Runner, - lggr logger.Logger, - argsNoPlugin libocr2.MercuryOracleArgs, - cfg Config, - chEnhancedTelem chan ocrcommon.EnhancedTelemetryMercuryData, - orm types.DataSourceORM, - feedID utils.FeedID, - enableTriggerCapability bool, -) ([]job.ServiceCtx, error) { - if jb.PipelineSpec == nil { - return nil, errors.New("expected job to have a non-nil PipelineSpec") - } - - var pluginConfig config.PluginConfig - if len(jb.OCR2OracleSpec.PluginConfig) == 0 { - if !enableTriggerCapability { - return nil, fmt.Errorf("at least one transmission option must be configured") - } - } else { - err := json.Unmarshal(jb.OCR2OracleSpec.PluginConfig.Bytes(), &pluginConfig) - if err != nil { - return nil, errors.WithStack(err) - } - err = config.ValidatePluginConfig(pluginConfig, feedID) - if err != nil { - return nil, err - } - } - - lggr = lggr.Named("MercuryPlugin").With("jobID", jb.ID, "jobName", jb.Name.ValueOrZero()) - - // encapsulate all the subservices and ensure we close them all if any fail to start - srvs := []job.ServiceCtx{ocr2Provider} - abort := func() { - if cerr := services.MultiCloser(srvs).Close(); cerr != nil { - lggr.Errorw("Error closing unused services", "err", cerr) - } - } - saver := ocrcommon.NewResultRunSaver(pipelineRunner, lggr, cfg.MaxSuccessfulRuns(), cfg.ResultWriteQueueDepth()) - srvs = append(srvs, saver) - - // this is the factory that will be used to create the mercury plugin - var ( - factory ocr3types.MercuryPluginFactory - factoryServices []job.ServiceCtx - fErr error - ) - fCfg := factoryCfg{ - orm: orm, - pipelineRunner: pipelineRunner, - jb: jb, - lggr: lggr, - saver: saver, - chEnhancedTelem: chEnhancedTelem, - ocr2Provider: ocr2Provider, - reportingPluginConfig: pluginConfig, - cfg: cfg, - feedID: feedID, - } - switch feedID.Version() { - case 1: - factory, factoryServices, fErr = newv1factory(fCfg) - if fErr != nil { - abort() - return nil, fmt.Errorf("failed to create mercury v1 factory: %w", fErr) - } - srvs = append(srvs, factoryServices...) - case 2: - factory, factoryServices, fErr = newv2factory(fCfg) - if fErr != nil { - abort() - return nil, fmt.Errorf("failed to create mercury v2 factory: %w", fErr) - } - srvs = append(srvs, factoryServices...) - case 3: - factory, factoryServices, fErr = newv3factory(fCfg) - if fErr != nil { - abort() - return nil, fmt.Errorf("failed to create mercury v3 factory: %w", fErr) - } - srvs = append(srvs, factoryServices...) - case 4: - factory, factoryServices, fErr = newv4factory(fCfg) - if fErr != nil { - abort() - return nil, fmt.Errorf("failed to create mercury v4 factory: %w", fErr) - } - srvs = append(srvs, factoryServices...) - default: - return nil, errors.Errorf("unknown Mercury report schema version: %d", feedID.Version()) - } - argsNoPlugin.MercuryPluginFactory = factory - oracle, err := libocr2.NewOracle(argsNoPlugin) - if err != nil { - abort() - return nil, errors.WithStack(err) - } - srvs = append(srvs, job.NewServiceAdapter(oracle)) - return srvs, nil -} - -type factoryCfg struct { - orm types.DataSourceORM - pipelineRunner pipeline.Runner - jb job.Job - lggr logger.Logger - saver *ocrcommon.RunResultSaver - chEnhancedTelem chan ocrcommon.EnhancedTelemetryMercuryData - ocr2Provider commontypes.MercuryProvider - reportingPluginConfig config.PluginConfig - cfg Config - feedID utils.FeedID -} - -func getPluginFeedIDs(pluginConfig config.PluginConfig) (linkFeedID utils.FeedID, nativeFeedID utils.FeedID) { - if pluginConfig.LinkFeedID != nil { - linkFeedID = *pluginConfig.LinkFeedID - } - if pluginConfig.NativeFeedID != nil { - nativeFeedID = *pluginConfig.NativeFeedID - } - return linkFeedID, nativeFeedID -} - -func newv4factory(factoryCfg factoryCfg) (ocr3types.MercuryPluginFactory, []job.ServiceCtx, error) { - var factory ocr3types.MercuryPluginFactory - srvs := make([]job.ServiceCtx, 0) - - linkFeedID, nativeFeedID := getPluginFeedIDs(factoryCfg.reportingPluginConfig) - - ds := mercuryv4.NewDataSource( - factoryCfg.orm, - factoryCfg.pipelineRunner, - factoryCfg.jb, - *factoryCfg.jb.PipelineSpec, - factoryCfg.feedID, - factoryCfg.lggr, - factoryCfg.saver, - factoryCfg.chEnhancedTelem, - factoryCfg.ocr2Provider.MercuryServerFetcher(), - linkFeedID, - nativeFeedID, - ) - - loopCmd := env.MercuryPlugin.Cmd.Get() - loopEnabled := loopCmd != "" - - if loopEnabled { - cmdFn, unregisterer, opts, mercuryLggr, err := initLoop(loopCmd, factoryCfg.cfg, factoryCfg.feedID, factoryCfg.lggr) - if err != nil { - return nil, nil, fmt.Errorf("failed to init loop for feed %s: %w", factoryCfg.feedID, err) - } - // in loop mode, the factory is grpc server, and we need to handle the server lifecycle - // and unregistration of the loop - factoryServer := loop.NewMercuryV4Service(mercuryLggr, opts, cmdFn, factoryCfg.ocr2Provider, ds) - srvs = append(srvs, factoryServer, unregisterer) - // adapt the grpc server to the vanilla mercury plugin factory interface used by the oracle - factory = factoryServer - } else { - factory = relaymercuryv4.NewFactory(ds, factoryCfg.lggr, factoryCfg.ocr2Provider.OnchainConfigCodec(), factoryCfg.ocr2Provider.ReportCodecV4()) - } - return factory, srvs, nil -} - -func newv3factory(factoryCfg factoryCfg) (ocr3types.MercuryPluginFactory, []job.ServiceCtx, error) { - var factory ocr3types.MercuryPluginFactory - srvs := make([]job.ServiceCtx, 0) - - linkFeedID, nativeFeedID := getPluginFeedIDs(factoryCfg.reportingPluginConfig) - - ds := mercuryv3.NewDataSource( - factoryCfg.orm, - factoryCfg.pipelineRunner, - factoryCfg.jb, - *factoryCfg.jb.PipelineSpec, - factoryCfg.feedID, - factoryCfg.lggr, - factoryCfg.saver, - factoryCfg.chEnhancedTelem, - factoryCfg.ocr2Provider.MercuryServerFetcher(), - linkFeedID, - nativeFeedID, - ) - - loopCmd := env.MercuryPlugin.Cmd.Get() - loopEnabled := loopCmd != "" - - if loopEnabled { - cmdFn, unregisterer, opts, mercuryLggr, err := initLoop(loopCmd, factoryCfg.cfg, factoryCfg.feedID, factoryCfg.lggr) - if err != nil { - return nil, nil, fmt.Errorf("failed to init loop for feed %s: %w", factoryCfg.feedID, err) - } - // in loopp mode, the factory is grpc server, and we need to handle the server lifecycle - // and unregistration of the loop - factoryServer := loop.NewMercuryV3Service(mercuryLggr, opts, cmdFn, factoryCfg.ocr2Provider, ds) - srvs = append(srvs, factoryServer, unregisterer) - // adapt the grpc server to the vanilla mercury plugin factory interface used by the oracle - factory = factoryServer - } else { - factory = relaymercuryv3.NewFactory(ds, factoryCfg.lggr, factoryCfg.ocr2Provider.OnchainConfigCodec(), factoryCfg.ocr2Provider.ReportCodecV3()) - } - return factory, srvs, nil -} - -func newv2factory(factoryCfg factoryCfg) (ocr3types.MercuryPluginFactory, []job.ServiceCtx, error) { - var factory ocr3types.MercuryPluginFactory - srvs := make([]job.ServiceCtx, 0) - - linkFeedID, nativeFeedID := getPluginFeedIDs(factoryCfg.reportingPluginConfig) - - ds := mercuryv2.NewDataSource( - factoryCfg.orm, - factoryCfg.pipelineRunner, - factoryCfg.jb, - *factoryCfg.jb.PipelineSpec, - factoryCfg.feedID, - factoryCfg.lggr, - factoryCfg.saver, - factoryCfg.chEnhancedTelem, - factoryCfg.ocr2Provider.MercuryServerFetcher(), - linkFeedID, - nativeFeedID, - ) - - loopCmd := env.MercuryPlugin.Cmd.Get() - loopEnabled := loopCmd != "" - - if loopEnabled { - cmdFn, unregisterer, opts, mercuryLggr, err := initLoop(loopCmd, factoryCfg.cfg, factoryCfg.feedID, factoryCfg.lggr) - if err != nil { - return nil, nil, fmt.Errorf("failed to init loop for feed %s: %w", factoryCfg.feedID, err) - } - // in loopp mode, the factory is grpc server, and we need to handle the server lifecycle - // and unregistration of the loop - factoryServer := loop.NewMercuryV2Service(mercuryLggr, opts, cmdFn, factoryCfg.ocr2Provider, ds) - srvs = append(srvs, factoryServer, unregisterer) - // adapt the grpc server to the vanilla mercury plugin factory interface used by the oracle - factory = factoryServer - } else { - factory = relaymercuryv2.NewFactory(ds, factoryCfg.lggr, factoryCfg.ocr2Provider.OnchainConfigCodec(), factoryCfg.ocr2Provider.ReportCodecV2()) - } - return factory, srvs, nil -} - -func newv1factory(factoryCfg factoryCfg) (ocr3types.MercuryPluginFactory, []job.ServiceCtx, error) { - var factory ocr3types.MercuryPluginFactory - srvs := make([]job.ServiceCtx, 0) - - ds := mercuryv1.NewDataSource( - factoryCfg.orm, - factoryCfg.pipelineRunner, - factoryCfg.jb, - *factoryCfg.jb.PipelineSpec, - factoryCfg.lggr, - factoryCfg.saver, - factoryCfg.chEnhancedTelem, - factoryCfg.ocr2Provider.MercuryChainReader(), - factoryCfg.ocr2Provider.MercuryServerFetcher(), - factoryCfg.reportingPluginConfig.InitialBlockNumber.Ptr(), - factoryCfg.feedID, - ) - - loopCmd := env.MercuryPlugin.Cmd.Get() - loopEnabled := loopCmd != "" - - if loopEnabled { - cmdFn, unregisterer, opts, mercuryLggr, err := initLoop(loopCmd, factoryCfg.cfg, factoryCfg.feedID, factoryCfg.lggr) - if err != nil { - return nil, nil, fmt.Errorf("failed to init loop for feed %s: %w", factoryCfg.feedID, err) - } - // in loopp mode, the factory is grpc server, and we need to handle the server lifecycle - // and unregistration of the loop - factoryServer := loop.NewMercuryV1Service(mercuryLggr, opts, cmdFn, factoryCfg.ocr2Provider, ds) - srvs = append(srvs, factoryServer, unregisterer) - // adapt the grpc server to the vanilla mercury plugin factory interface used by the oracle - factory = factoryServer - } else { - factory = relaymercuryv1.NewFactory(ds, factoryCfg.lggr, factoryCfg.ocr2Provider.OnchainConfigCodec(), factoryCfg.ocr2Provider.ReportCodecV1()) - } - return factory, srvs, nil -} - -func initLoop(cmd string, cfg Config, feedID utils.FeedID, lggr logger.Logger) (func() *exec.Cmd, *loopUnregisterCloser, loop.GRPCOpts, logger.Logger, error) { - lggr.Debugw("Initializing Mercury loop", "command", cmd) - mercuryLggr := lggr.Named(fmt.Sprintf("MercuryV%d", feedID.Version())).Named(feedID.String()) - envVars, err := plugins.ParseEnvFile(env.MercuryPlugin.Env.Get()) - if err != nil { - return nil, nil, loop.GRPCOpts{}, nil, fmt.Errorf("failed to parse mercury env file: %w", err) - } - loopID := mercuryLggr.Name() - cmdFn, opts, err := cfg.RegisterLOOP(plugins.CmdConfig{ - ID: loopID, - Cmd: cmd, - Env: envVars, - }) - if err != nil { - return nil, nil, loop.GRPCOpts{}, nil, fmt.Errorf("failed to register loop: %w", err) - } - return cmdFn, newLoopUnregister(cfg, loopID), opts, mercuryLggr, nil -} - -// loopUnregisterCloser is a helper to unregister a loop -// as a service -// TODO BCF-3451 all other jobs that use custom plugin providers that should be refactored to use this pattern -// perhaps it can be implemented in the delegate on job delete. -type loopUnregisterCloser struct { - r plugins.RegistrarConfig - id string -} - -func (l *loopUnregisterCloser) Close() error { - l.r.UnregisterLOOP(l.id) - return nil -} - -func (l *loopUnregisterCloser) Start(ctx context.Context) error { - return nil -} - -func newLoopUnregister(r plugins.RegistrarConfig, id string) *loopUnregisterCloser { - return &loopUnregisterCloser{ - r: r, - id: id, - } -} diff --git a/core/services/ocr2/plugins/mercury/plugin_test.go b/core/services/ocr2/plugins/mercury/plugin_test.go deleted file mode 100644 index 71cfabce303..00000000000 --- a/core/services/ocr2/plugins/mercury/plugin_test.go +++ /dev/null @@ -1,432 +0,0 @@ -package mercury_test - -import ( - "context" - "errors" - "os/exec" - "reflect" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/google/uuid" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/smartcontractkit/chainlink/v2/core/config/env" - "github.com/smartcontractkit/chainlink/v2/core/logger" - "github.com/smartcontractkit/chainlink/v2/core/services/job" - "github.com/smartcontractkit/chainlink/v2/core/services/relay" - - "github.com/smartcontractkit/chainlink-common/pkg/loop" - commontypes "github.com/smartcontractkit/chainlink-common/pkg/types" - "github.com/smartcontractkit/chainlink-common/pkg/types/mercury" - v1 "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v1" - v2 "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v2" - v3 "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v3" - v4 "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v4" - "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" - - mercuryocr2 "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/mercury" - - libocr2 "github.com/smartcontractkit/libocr/offchainreporting2plus" - libocr2types "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - - "github.com/smartcontractkit/chainlink/v2/core/services/pipeline" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/types" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/utils" - "github.com/smartcontractkit/chainlink/v2/plugins" -) - -var ( - v1FeedId = [32]uint8{00, 01, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114} - v2FeedId = [32]uint8{00, 02, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114} - v3FeedId = [32]uint8{00, 03, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114} - v4FeedId = [32]uint8{00, 04, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114} - - testArgsNoPlugin = libocr2.MercuryOracleArgs{ - LocalConfig: libocr2types.LocalConfig{ - DevelopmentMode: libocr2types.EnableDangerousDevelopmentMode, - }, - } - - testCfg = mercuryocr2.NewMercuryConfig(1, 1, &testRegistrarConfig{}) - - v1jsonCfg = job.JSONConfig{ - "serverURL": "example.com:80", - "serverPubKey": "724ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93", - "initialBlockNumber": 1234, - } - - v2jsonCfg = job.JSONConfig{ - "serverURL": "example.com:80", - "serverPubKey": "724ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93", - "linkFeedID": "0x00026b4aa7e57ca7b68ae1bf45653f56b656fd3aa335ef7fae696b663f1b8472", - "nativeFeedID": "0x00036b4aa7e57ca7b68ae1bf45653f56b656fd3aa335ef7fae696b663f1b8472", - } - - v3jsonCfg = job.JSONConfig{ - "serverURL": "example.com:80", - "serverPubKey": "724ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93", - "linkFeedID": "0x00026b4aa7e57ca7b68ae1bf45653f56b656fd3aa335ef7fae696b663f1b8472", - "nativeFeedID": "0x00036b4aa7e57ca7b68ae1bf45653f56b656fd3aa335ef7fae696b663f1b8472", - } - - v4jsonCfg = job.JSONConfig{ - "serverURL": "example.com:80", - "serverPubKey": "724ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93", - "linkFeedID": "0x00026b4aa7e57ca7b68ae1bf45653f56b656fd3aa335ef7fae696b663f1b8472", - "nativeFeedID": "0x00036b4aa7e57ca7b68ae1bf45653f56b656fd3aa335ef7fae696b663f1b8472", - } - - testJob = job.Job{ - ID: 1, - ExternalJobID: uuid.Must(uuid.NewRandom()), - OCR2OracleSpecID: ptr(int32(7)), - OCR2OracleSpec: &job.OCR2OracleSpec{ - ID: 7, - ContractID: "phony", - FeedID: ptr(common.BytesToHash([]byte{1, 2, 3})), - Relay: relay.NetworkEVM, - ChainID: "1", - }, - PipelineSpec: &pipeline.Spec{}, - PipelineSpecID: int32(1), - } - - // this is kind of gross, but it's the best way to test return values of the services - expectedEmbeddedServiceCnt = 3 - expectedLoopServiceCnt = expectedEmbeddedServiceCnt + 2 // factory server and loop unregisterer -) - -func TestNewServices(t *testing.T) { - type args struct { - pluginConfig job.JSONConfig - feedID utils.FeedID - cfg mercuryocr2.Config - } - testCases := []struct { - name string - args args - loopMode bool - wantLoopFactory any - wantServiceCnt int - wantErr bool - wantErrStr string - }{ - { - name: "no plugin config error ", - args: args{ - feedID: v1FeedId, - }, - wantServiceCnt: 0, - wantErr: true, - }, - - { - name: "v1 legacy", - args: args{ - pluginConfig: v1jsonCfg, - feedID: v1FeedId, - }, - wantServiceCnt: expectedEmbeddedServiceCnt, - wantErr: false, - }, - { - name: "v2 legacy", - args: args{ - pluginConfig: v2jsonCfg, - feedID: v2FeedId, - }, - wantServiceCnt: expectedEmbeddedServiceCnt, - wantErr: false, - }, - { - name: "v3 legacy", - args: args{ - pluginConfig: v3jsonCfg, - feedID: v3FeedId, - }, - wantServiceCnt: expectedEmbeddedServiceCnt, - wantErr: false, - }, - { - name: "v4 legacy", - args: args{ - pluginConfig: v4jsonCfg, - feedID: v4FeedId, - }, - wantServiceCnt: expectedEmbeddedServiceCnt, - wantErr: false, - }, - { - name: "v1 loop", - loopMode: true, - args: args{ - pluginConfig: v1jsonCfg, - feedID: v1FeedId, - }, - wantServiceCnt: expectedLoopServiceCnt, - wantErr: false, - wantLoopFactory: &loop.MercuryV1Service{}, - }, - { - name: "v2 loop", - loopMode: true, - args: args{ - pluginConfig: v2jsonCfg, - feedID: v2FeedId, - }, - wantServiceCnt: expectedLoopServiceCnt, - wantErr: false, - wantLoopFactory: &loop.MercuryV2Service{}, - }, - { - name: "v3 loop", - loopMode: true, - args: args{ - pluginConfig: v3jsonCfg, - feedID: v3FeedId, - }, - wantServiceCnt: expectedLoopServiceCnt, - wantErr: false, - wantLoopFactory: &loop.MercuryV3Service{}, - }, - { - name: "v3 loop err", - loopMode: true, - args: args{ - pluginConfig: v3jsonCfg, - feedID: v3FeedId, - cfg: mercuryocr2.NewMercuryConfig(1, 1, &testRegistrarConfig{failRegister: true}), - }, - wantServiceCnt: expectedLoopServiceCnt, - wantErr: true, - wantLoopFactory: &loop.MercuryV3Service{}, - wantErrStr: "failed to init loop for feed", - }, - { - name: "v4 loop", - loopMode: true, - args: args{ - pluginConfig: v4jsonCfg, - feedID: v4FeedId, - }, - wantServiceCnt: expectedLoopServiceCnt, - wantErr: false, - wantLoopFactory: &loop.MercuryV4Service{}, - }, - } - for _, tt := range testCases { - t.Run(tt.name, func(t *testing.T) { - if tt.loopMode { - t.Setenv(string(env.MercuryPlugin.Cmd), "fake_cmd") - assert.NotEmpty(t, env.MercuryPlugin.Cmd.Get()) - } - // use default config if not provided - if tt.args.cfg == nil { - tt.args.cfg = testCfg - } - got, err := newServicesTestWrapper(t, tt.args.pluginConfig, tt.args.feedID, tt.args.cfg) - if (err != nil) != tt.wantErr { - t.Errorf("NewServices() error = %v, wantErr %v", err, tt.wantErr) - return - } - if err != nil { - if tt.wantErrStr != "" { - assert.Contains(t, err.Error(), tt.wantErrStr) - } - return - } - assert.Len(t, got, tt.wantServiceCnt) - if tt.loopMode { - foundLoopFactory := false - for i := 0; i < len(got); i++ { - if reflect.TypeOf(got[i]) == reflect.TypeOf(tt.wantLoopFactory) { - foundLoopFactory = true - break - } - } - assert.True(t, foundLoopFactory) - } - }) - } - - t.Run("restartable loop", func(t *testing.T) { - // setup a real loop registry to test restartability - registry := plugins.NewTestLoopRegistry(logger.TestLogger(t)) - loopRegistrarConfig := plugins.NewRegistrarConfig(loop.GRPCOpts{}, registry.Register, registry.Unregister) - prodCfg := mercuryocr2.NewMercuryConfig(1, 1, loopRegistrarConfig) - type args struct { - pluginConfig job.JSONConfig - feedID utils.FeedID - cfg mercuryocr2.Config - } - testCases := []struct { - name string - args args - wantErr bool - }{ - { - name: "v1 loop", - args: args{ - pluginConfig: v1jsonCfg, - feedID: v1FeedId, - cfg: prodCfg, - }, - wantErr: false, - }, - { - name: "v2 loop", - args: args{ - pluginConfig: v2jsonCfg, - feedID: v2FeedId, - cfg: prodCfg, - }, - wantErr: false, - }, - { - name: "v3 loop", - args: args{ - pluginConfig: v3jsonCfg, - feedID: v3FeedId, - cfg: prodCfg, - }, - wantErr: false, - }, - { - name: "v4 loop", - args: args{ - pluginConfig: v4jsonCfg, - feedID: v4FeedId, - cfg: prodCfg, - }, - wantErr: false, - }, - } - - for _, tt := range testCases { - t.Run(tt.name, func(t *testing.T) { - t.Setenv(string(env.MercuryPlugin.Cmd), "fake_cmd") - assert.NotEmpty(t, env.MercuryPlugin.Cmd.Get()) - - got, err := newServicesTestWrapper(t, tt.args.pluginConfig, tt.args.feedID, tt.args.cfg) - if (err != nil) != tt.wantErr { - t.Errorf("NewServices() error = %v, wantErr %v", err, tt.wantErr) - return - } - // hack to simulate a restart. we don't have enough boilerplate to start the oracle service - // only care about the subservices so we start all except the oracle, which happens to be the last one - for i := 0; i < len(got)-1; i++ { - require.NoError(t, got[i].Start(tests.Context(t))) - } - // if we don't close the services, we get conflicts with the loop registry - _, err = newServicesTestWrapper(t, tt.args.pluginConfig, tt.args.feedID, tt.args.cfg) - require.ErrorContains(t, err, "plugin already registered") - - // close all services and try again - for i := len(got) - 2; i >= 0; i-- { - require.NoError(t, got[i].Close()) - } - _, err = newServicesTestWrapper(t, tt.args.pluginConfig, tt.args.feedID, tt.args.cfg) - require.NoError(t, err) - }) - } - }) -} - -// we are only varying the version via feedID (and the plugin config) -// this wrapper supplies dummy values for the rest of the arguments -func newServicesTestWrapper(t *testing.T, pluginConfig job.JSONConfig, feedID utils.FeedID, cfg mercuryocr2.Config) ([]job.ServiceCtx, error) { - t.Helper() - jb := testJob - jb.OCR2OracleSpec.PluginConfig = pluginConfig - return mercuryocr2.NewServices(jb, &testProvider{}, nil, logger.TestLogger(t), testArgsNoPlugin, cfg, nil, &testDataSourceORM{}, feedID, false) -} - -type testProvider struct{} - -// ChainReader implements types.MercuryProvider. -func (*testProvider) ContractReader() commontypes.ContractReader { panic("unimplemented") } - -// Close implements types.MercuryProvider. -func (*testProvider) Close() error { return nil } - -// Codec implements types.MercuryProvider. -func (*testProvider) Codec() commontypes.Codec { panic("unimplemented") } - -// ContractConfigTracker implements types.MercuryProvider. -func (*testProvider) ContractConfigTracker() libocr2types.ContractConfigTracker { - panic("unimplemented") -} - -// ContractTransmitter implements types.MercuryProvider. -func (*testProvider) ContractTransmitter() libocr2types.ContractTransmitter { - panic("unimplemented") -} - -// HealthReport implements types.MercuryProvider. -func (*testProvider) HealthReport() map[string]error { panic("unimplemented") } - -// MercuryChainReader implements types.MercuryProvider. -func (*testProvider) MercuryChainReader() mercury.ChainReader { return nil } - -// MercuryServerFetcher implements types.MercuryProvider. -func (*testProvider) MercuryServerFetcher() mercury.ServerFetcher { return nil } - -// Name implements types.MercuryProvider. -func (*testProvider) Name() string { panic("unimplemented") } - -// OffchainConfigDigester implements types.MercuryProvider. -func (*testProvider) OffchainConfigDigester() libocr2types.OffchainConfigDigester { - panic("unimplemented") -} - -// OnchainConfigCodec implements types.MercuryProvider. -func (*testProvider) OnchainConfigCodec() mercury.OnchainConfigCodec { - return nil -} - -// Ready implements types.MercuryProvider. -func (*testProvider) Ready() error { panic("unimplemented") } - -// ReportCodecV1 implements types.MercuryProvider. -func (*testProvider) ReportCodecV1() v1.ReportCodec { return nil } - -// ReportCodecV2 implements types.MercuryProvider. -func (*testProvider) ReportCodecV2() v2.ReportCodec { return nil } - -// ReportCodecV3 implements types.MercuryProvider. -func (*testProvider) ReportCodecV3() v3.ReportCodec { return nil } - -// ReportCodecV4 implements types.MercuryProvider. -func (*testProvider) ReportCodecV4() v4.ReportCodec { return nil } - -// Start implements types.MercuryProvider. -func (*testProvider) Start(context.Context) error { return nil } - -var _ commontypes.MercuryProvider = (*testProvider)(nil) - -type testRegistrarConfig struct { - failRegister bool -} - -func (c *testRegistrarConfig) UnregisterLOOP(ID string) {} - -// RegisterLOOP implements plugins.RegistrarConfig. -func (c *testRegistrarConfig) RegisterLOOP(config plugins.CmdConfig) (func() *exec.Cmd, loop.GRPCOpts, error) { - if c.failRegister { - return nil, loop.GRPCOpts{}, errors.New("failed to register") - } - return nil, loop.GRPCOpts{}, nil -} - -var _ plugins.RegistrarConfig = (*testRegistrarConfig)(nil) - -type testDataSourceORM struct{} - -// LatestReport implements types.DataSourceORM. -func (*testDataSourceORM) LatestReport(ctx context.Context, feedID [32]byte) (report []byte, err error) { - return []byte{1, 2, 3}, nil -} - -var _ types.DataSourceORM = (*testDataSourceORM)(nil) diff --git a/core/services/ocr2/validate/validate.go b/core/services/ocr2/validate/validate.go index 27a5a885369..acb16777f06 100644 --- a/core/services/ocr2/validate/validate.go +++ b/core/services/ocr2/validate/validate.go @@ -22,11 +22,9 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/job" "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ccip/config" lloconfig "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/llo/config" - mercuryconfig "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/mercury/config" "github.com/smartcontractkit/chainlink/v2/core/services/ocrcommon" "github.com/smartcontractkit/chainlink/v2/core/services/pipeline" "github.com/smartcontractkit/chainlink/v2/core/services/relay" - evmtypes "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/types" "github.com/smartcontractkit/chainlink/v2/plugins" ) @@ -116,8 +114,6 @@ func validateSpec(ctx context.Context, tree *toml.Tree, spec job.Job, rc plugins case types.Functions: // TODO validator for DR-OCR spec: https://smartcontract-it.atlassian.net/browse/FUN-112 return nil - case types.Mercury: - return validateOCR2MercurySpec(spec.OCR2OracleSpec, *spec.OCR2OracleSpec.FeedID) case types.CCIPExecution: return validateOCR2CCIPExecutionSpec(spec.OCR2OracleSpec.PluginConfig) case types.CCIPCommit: @@ -297,28 +293,6 @@ func validateOCR2KeeperSpec(jsonConfig job.JSONConfig) error { return nil } -func validateOCR2MercurySpec(spec *job.OCR2OracleSpec, feedID [32]byte) error { - var relayConfig evmtypes.RelayConfig - err := json.Unmarshal(spec.RelayConfig.Bytes(), &relayConfig) - if err != nil { - return pkgerrors.Wrap(err, "error while unmarshalling relay config") - } - - if len(spec.PluginConfig) == 0 { - if !relayConfig.EnableTriggerCapability { - return pkgerrors.Wrap(err, "at least one transmission option must be configured") - } - return nil - } - - var pluginConfig mercuryconfig.PluginConfig - err = json.Unmarshal(spec.PluginConfig.Bytes(), &pluginConfig) - if err != nil { - return pkgerrors.Wrap(err, "error while unmarshalling plugin config") - } - return pkgerrors.Wrap(mercuryconfig.ValidatePluginConfig(pluginConfig, feedID), "Mercury PluginConfig is invalid") -} - func validateOCR2CCIPExecutionSpec(jsonConfig job.JSONConfig) error { if jsonConfig == nil { return errors.New("pluginConfig is empty") diff --git a/core/services/pg/connection.go b/core/services/pg/connection.go index b099bbb2f35..93c161b4c0d 100644 --- a/core/services/pg/connection.go +++ b/core/services/pg/connection.go @@ -2,13 +2,11 @@ package pg import ( "context" - "errors" "fmt" "log" "os" "time" - "github.com/jackc/pgconn" _ "github.com/jackc/pgx/v4/stdlib" // need to make sure pgx driver is registered before opening connection "github.com/jmoiron/sqlx" @@ -56,7 +54,6 @@ func NewConnection(ctx context.Context, uri string, driverName string, config Co if err != nil { return nil, err } - setMaxMercuryConns(db, config) if os.Getenv("SKIP_PG_VERSION_CHECK") != "true" { if err = checkVersion(db, MinRequiredPGVersion); err != nil { @@ -67,38 +64,6 @@ func NewConnection(ctx context.Context, uri string, driverName string, config Co return db, nil } -func setMaxMercuryConns(db *sqlx.DB, config ConnectionConfig) { - // HACK: In the case of mercury jobs, one conn is needed per job for good - // performance. Most nops will forget to increase the defaults to account - // for this so we detect it here instead. - // - // This problem will be solved by replacing mercury with parallel - // compositions (llo plugin). - // - // See: https://smartcontract-it.atlassian.net/browse/MERC-3654 - var cnt int - if err := db.Get(&cnt, `SELECT COUNT(*) FROM ocr2_oracle_specs WHERE plugin_type = 'mercury'`); err != nil { - const errUndefinedTable = "42P01" - var pqerr *pgconn.PgError - if errors.As(err, &pqerr) { - if pqerr.Code == errUndefinedTable { - // no mercury jobs defined - return - } - } - log.Printf("Error checking mercury jobs: %s", err.Error()) - return - } - if cnt > config.MaxOpenConns() { - log.Printf("Detected %d mercury jobs, increasing max open connections from %d to %d", cnt, config.MaxOpenConns(), cnt) - db.SetMaxOpenConns(cnt) - } - if cnt > config.MaxIdleConns() { - log.Printf("Detected %d mercury jobs, increasing max idle connections from %d to %d", cnt, config.MaxIdleConns(), cnt) - db.SetMaxIdleConns(cnt) - } -} - type Getter interface { Get(dest interface{}, query string, args ...interface{}) error } diff --git a/core/services/relay/evm/evm.go b/core/services/relay/evm/evm.go index 20011ce0414..104cb3a2755 100644 --- a/core/services/relay/evm/evm.go +++ b/core/services/relay/evm/evm.go @@ -13,7 +13,6 @@ import ( "net/http" "strings" "sync" - "time" "github.com/ethereum/go-ethereum/accounts/abi" "github.com/ethereum/go-ethereum/common" @@ -30,7 +29,6 @@ import ( chainselectors "github.com/smartcontractkit/chain-selectors" ocr3capability "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/ocr3" - "github.com/smartcontractkit/chainlink-common/pkg/capabilities/triggers" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/sqlutil" @@ -56,17 +54,10 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ccip/estimatorconfig" cciptransmitter "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/ccip/transmitter" lloconfig "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/llo/config" - mercuryconfig "github.com/smartcontractkit/chainlink/v2/core/services/ocr2/plugins/mercury/config" "github.com/smartcontractkit/chainlink/v2/core/services/ocrcommon" "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/codec" "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/functions" "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/interceptors/mantle" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury" - mercuryutils "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/utils" - reportcodecv1 "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v1/reportcodec" - reportcodecv2 "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v2/reportcodec" - reportcodecv3 "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v3/reportcodec" - reportcodecv4 "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v4/reportcodec" "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/wsrpc" "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/types" ) @@ -156,12 +147,8 @@ type Relayer struct { codec commontypes.Codec capabilitiesRegistry coretypes.CapabilitiesRegistry - // Mercury - mercuryORM mercury.ORM - mercuryCfg MercuryConfig - triggerCapability *triggers.MercuryTriggerService - // LLO/data streams + mercuryCfg MercuryConfig cdcFactory func() (llo.ChannelDefinitionCacheFactory, error) retirementReportCache llo.RetirementReportCache } @@ -210,7 +197,6 @@ func NewRelayer(ctx context.Context, lggr logger.Logger, chain legacyevm.Chain, return nil, fmt.Errorf("cannot create evm relayer: %w", err) } sugared := logger.Sugared(lggr).Named("Relayer") - mercuryORM := mercury.NewORM(opts.DS) cdcFactory := sync.OnceValues(func() (llo.ChannelDefinitionCacheFactory, error) { chainSelector, err := chainselectors.SelectorFromChainId(chain.ID().Uint64()) if err != nil { @@ -228,7 +214,6 @@ func NewRelayer(ctx context.Context, lggr logger.Logger, chain legacyevm.Chain, mercuryPool: opts.MercuryPool, cdcFactory: cdcFactory, retirementReportCache: opts.RetirementReportCache, - mercuryORM: mercuryORM, mercuryCfg: opts.MercuryConfig, capabilitiesRegistry: opts.CapabilitiesRegistry, } @@ -263,17 +248,6 @@ func (r *Relayer) Start(ctx context.Context) error { func (r *Relayer) Close() error { cs := make([]io.Closer, 0, 2) - if r.triggerCapability != nil { - cs = append(cs, r.triggerCapability) - - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - - err := r.capabilitiesRegistry.Remove(ctx, r.triggerCapability.ID) - if err != nil { - return err - } - } cs = append(cs, r.chain) return services.MultiCloser(cs).Close() } @@ -410,104 +384,7 @@ func (r *Relayer) NewPluginProvider(ctx context.Context, rargs commontypes.Relay } func (r *Relayer) NewMercuryProvider(ctx context.Context, rargs commontypes.RelayArgs, pargs commontypes.PluginArgs) (commontypes.MercuryProvider, error) { - lggr := logger.Sugared(r.lggr).Named("MercuryProvider").Named(rargs.ExternalJobID.String()) - relayOpts := types.NewRelayOpts(rargs) - relayConfig, err := relayOpts.RelayConfig() - if err != nil { - return nil, fmt.Errorf("failed to get relay config: %w", err) - } - - var mercuryConfig mercuryconfig.PluginConfig - if err = json.Unmarshal(pargs.PluginConfig, &mercuryConfig); err != nil { - return nil, pkgerrors.WithStack(err) - } - - if relayConfig.FeedID == nil { - return nil, pkgerrors.New("FeedID must be specified") - } - - if relayConfig.ChainID.String() != r.chain.ID().String() { - return nil, fmt.Errorf("internal error: chain id in spec does not match this relayer's chain: have %s expected %s", relayConfig.ChainID.String(), r.chain.ID().String()) - } - cp, err := newMercuryConfigProvider(ctx, lggr, r.chain, relayOpts) - if err != nil { - return nil, pkgerrors.WithStack(err) - } - - if !relayConfig.EffectiveTransmitterID.Valid { - return nil, pkgerrors.New("EffectiveTransmitterID must be specified") - } - privKey, err := r.ks.CSA().Get(relayConfig.EffectiveTransmitterID.String) - if err != nil { - return nil, pkgerrors.Wrap(err, "failed to get CSA key for mercury connection") - } - - clients := make(map[string]wsrpc.Client) - for _, server := range mercuryConfig.GetServers() { - clients[server.URL], err = r.mercuryPool.Checkout(ctx, privKey, server.PubKey, server.URL) - if err != nil { - return nil, err - } - } - - // initialize trigger capability service lazily - if relayConfig.EnableTriggerCapability && r.triggerCapability == nil { - if r.capabilitiesRegistry == nil { - lggr.Errorw("trigger capability is enabled but capabilities registry is not set") - } else { - var err2 error - r.triggerCapability, err2 = triggers.NewMercuryTriggerService(0, relayConfig.TriggerCapabilityName, relayConfig.TriggerCapabilityVersion, lggr) - if err2 != nil { - return nil, fmt.Errorf("failed to start required trigger service: %w", err2) - } - if err2 = r.triggerCapability.Start(ctx); err2 != nil { - return nil, err2 - } - if err2 = r.capabilitiesRegistry.Add(ctx, r.triggerCapability); err2 != nil { - return nil, err2 - } - lggr.Infow("successfully added trigger service to the Registry") - } - } - - reportCodecV1 := reportcodecv1.NewReportCodec(*relayConfig.FeedID, lggr.Named("ReportCodecV1")) - reportCodecV2 := reportcodecv2.NewReportCodec(*relayConfig.FeedID, lggr.Named("ReportCodecV2")) - reportCodecV3 := reportcodecv3.NewReportCodec(*relayConfig.FeedID, lggr.Named("ReportCodecV3")) - reportCodecV4 := reportcodecv4.NewReportCodec(*relayConfig.FeedID, lggr.Named("ReportCodecV4")) - - getCodecForFeed := func(feedID mercuryutils.FeedID) (mercury.TransmitterReportDecoder, error) { - var transmitterCodec mercury.TransmitterReportDecoder - switch feedID.Version() { - case 1: - transmitterCodec = reportCodecV1 - case 2: - transmitterCodec = reportCodecV2 - case 3: - transmitterCodec = reportCodecV3 - case 4: - transmitterCodec = reportCodecV4 - default: - return nil, fmt.Errorf("invalid feed version %d", feedID.Version()) - } - return transmitterCodec, nil - } - - benchmarkPriceDecoder := func(ctx context.Context, feedID mercuryutils.FeedID, report ocrtypes.Report) (*big.Int, error) { - benchmarkPriceCodec, benchmarkPriceErr := getCodecForFeed(feedID) - if benchmarkPriceErr != nil { - return nil, benchmarkPriceErr - } - return benchmarkPriceCodec.BenchmarkPriceFromReport(ctx, report) - } - - transmitterCodec, err := getCodecForFeed(mercuryutils.FeedID(*relayConfig.FeedID)) - if err != nil { - return nil, err - } - - transmitter := mercury.NewTransmitter(lggr, r.mercuryCfg.Transmitter(), clients, privKey.PublicKey, rargs.JobID, *relayConfig.FeedID, r.mercuryORM, transmitterCodec, benchmarkPriceDecoder, r.triggerCapability) - - return NewMercuryProvider(cp, r.codec, NewMercuryChainReader(r.chain.HeadTracker()), transmitter, reportCodecV1, reportCodecV2, reportCodecV3, reportCodecV4, lggr), nil + return nil, errors.New("mercury jobs are no longer supported") } func chainToUUID(chainID *big.Int) uuid.UUID { @@ -702,11 +579,6 @@ func (r *Relayer) NewLLOProvider(ctx context.Context, rargs commontypes.RelayArg if relayConfig.LLODONID == 0 { return nil, errors.New("donID must be specified in relayConfig for LLO jobs") } - switch relayConfig.LLOConfigMode { - case types.LLOConfigModeMercury, types.LLOConfigModeBlueGreen: - default: - return nil, fmt.Errorf("LLOConfigMode must be specified in relayConfig for LLO jobs (only %q or %q is currently supported)", types.LLOConfigModeMercury, types.LLOConfigModeBlueGreen) - } var lloCfg lloconfig.PluginConfig if err := json.Unmarshal(pargs.PluginConfig, &lloCfg); err != nil { @@ -722,9 +594,6 @@ func (r *Relayer) NewLLOProvider(ctx context.Context, rargs commontypes.RelayArg if relayConfig.LLODONID == 0 { return nil, errors.New("donID must be specified in relayConfig for LLO jobs") } - if relayConfig.LLOConfigMode == "" { - return nil, fmt.Errorf("LLOConfigMode must be specified in relayConfig for LLO jobs (can be either: %q or %q)", types.LLOConfigModeMercury, types.LLOConfigModeBlueGreen) - } if relayConfig.ChainID.String() != r.chain.ID().String() { return nil, fmt.Errorf("internal error: chain id in spec does not match this relayer's chain: have %s expected %s", relayConfig.ChainID.String(), r.chain.ID().String()) } @@ -813,23 +682,17 @@ func (r *Relayer) NewConfigProvider(ctx context.Context, args commontypes.RelayA return nil, fmt.Errorf("internal error: chain id in spec does not match this relayer's chain: have %s expected %s", relayConfig.ChainID.String(), r.chain.ID().String()) } - // Handle legacy jobs which did not yet specify provider type and - // switched between median/mercury based on presence of feed ID if args.ProviderType == "" { - if relayConfig.FeedID == nil { - args.ProviderType = "median" - } else if relayConfig.LLODONID > 0 { + if relayConfig.LLODONID > 0 { args.ProviderType = "llo" } else { - args.ProviderType = "mercury" + args.ProviderType = "median" } } switch args.ProviderType { case "median": configProvider, err = newStandardConfigProvider(ctx, lggr, r.chain, relayOpts) - case "mercury": - configProvider, err = newMercuryConfigProvider(ctx, lggr, r.chain, relayOpts) case "llo": // Use NullRetirementReportCache since we never run LLO jobs on // bootstrap nodes, and there's no need to introduce a failure mode or @@ -858,11 +721,7 @@ func FilterNamesFromRelayArgs(args commontypes.RelayArgs) (filterNames []string, return nil, pkgerrors.WithStack(err) } - if relayConfig.FeedID != nil { - filterNames = []string{mercury.FilterName(addr.Address(), *relayConfig.FeedID)} - } else { - filterNames = []string{configPollerFilterName(addr.Address()), transmitterFilterName(addr.Address())} - } + filterNames = []string{configPollerFilterName(addr.Address()), transmitterFilterName(addr.Address())} return filterNames, err } diff --git a/core/services/relay/evm/llo_provider.go b/core/services/relay/evm/llo_provider.go index ab7cac6da0c..b56fbe199ec 100644 --- a/core/services/relay/evm/llo_provider.go +++ b/core/services/relay/evm/llo_provider.go @@ -12,7 +12,6 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/chains/evm/logpoller" "github.com/smartcontractkit/chainlink/v2/core/chains/legacyevm" "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/llo" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury" "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/types" "github.com/smartcontractkit/chainlink-common/pkg/logger" @@ -180,64 +179,10 @@ func (p *lloProvider) ShouldRetireCache() llotypes.ShouldRetireCache { return p.shouldRetireCache } -// wrapper is needed to turn mercury config poller into a service -type mercuryConfigPollerWrapper struct { - *mercury.ConfigPoller - services.Service - eng *services.Engine - - runReplay bool - fromBlock uint64 -} - -func newMercuryConfigPollerWrapper(lggr logger.Logger, cp *mercury.ConfigPoller, fromBlock uint64, runReplay bool) *mercuryConfigPollerWrapper { - w := &mercuryConfigPollerWrapper{cp, nil, nil, runReplay, fromBlock} - w.Service, w.eng = services.Config{ - Name: "LLOMercuryConfigWrapper", - Start: w.start, - Close: w.close, - }.NewServiceEngine(lggr) - return w -} - -func (w *mercuryConfigPollerWrapper) Start(ctx context.Context) error { - return w.Service.Start(ctx) -} - -func (w *mercuryConfigPollerWrapper) start(ctx context.Context) error { - w.ConfigPoller.Start() - return nil -} - -func (w *mercuryConfigPollerWrapper) Close() error { - return w.Service.Close() -} - -func (w *mercuryConfigPollerWrapper) close() error { - return w.ConfigPoller.Close() -} - func newLLOConfigPollers(ctx context.Context, lggr logger.Logger, cc llo.ConfigCache, lp logpoller.LogPoller, chainID *big.Int, configuratorAddress common.Address, relayConfig types.RelayConfig) (cps []llo.ConfigPollerService, configDigester ocrtypes.OffchainConfigDigester, err error) { donID := relayConfig.LLODONID donIDHash := llo.DonIDToBytes32(donID) switch relayConfig.LLOConfigMode { - case types.LLOConfigModeMercury: - // NOTE: This uses the old config digest prefix for compatibility with legacy contracts - configDigester = mercury.NewOffchainConfigDigester(donIDHash, chainID, configuratorAddress, ocrtypes.ConfigDigestPrefixMercuryV02) - // Mercury config poller will register its own filter - mcp, err := mercury.NewConfigPoller( - ctx, - lggr, - lp, - configuratorAddress, - llo.DonIDToBytes32(donID), - ) - if err != nil { - return nil, nil, err - } - // don't need to replay in the wrapper since the provider will handle it - w := newMercuryConfigPollerWrapper(lggr, mcp, relayConfig.FromBlock, false) - cps = []llo.ConfigPollerService{w} case types.LLOConfigModeBlueGreen: // NOTE: Register filter here because the config poller doesn't do it on its own err := lp.RegisterFilter(ctx, logpoller.Filter{Name: lloProviderConfiguratorFilterName(configuratorAddress, donID), EventSigs: []common.Hash{llo.ProductionConfigSet, llo.StagingConfigSet, llo.PromoteStagingConfig}, Topic2: []common.Hash{donIDHash}, Addresses: []common.Address{configuratorAddress}}) diff --git a/core/services/relay/evm/mercury/config_digest.go b/core/services/relay/evm/mercury/config_digest.go deleted file mode 100644 index c7a6076c886..00000000000 --- a/core/services/relay/evm/mercury/config_digest.go +++ /dev/null @@ -1,66 +0,0 @@ -package mercury - -import ( - "encoding/binary" - "fmt" - "math/big" - "strings" - - "github.com/ethereum/go-ethereum/accounts/abi" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/crypto" - "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - "github.com/smartcontractkit/wsrpc/credentials" - - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/llo-feeds/generated/exposed_verifier" -) - -func makeConfigDigestArgs() abi.Arguments { - abi, err := abi.JSON(strings.NewReader(exposed_verifier.ExposedVerifierABI)) - if err != nil { - // assertion - panic(fmt.Sprintf("could not parse aggregator ABI: %s", err.Error())) - } - return abi.Methods["exposedConfigDigestFromConfigData"].Inputs -} - -var configDigestArgs = makeConfigDigestArgs() - -func configDigest( - feedID common.Hash, - chainID *big.Int, - contractAddress common.Address, - configCount uint64, - oracles []common.Address, - transmitters []credentials.StaticSizedPublicKey, - f uint8, - onchainConfig []byte, - offchainConfigVersion uint64, - offchainConfig []byte, - prefix types.ConfigDigestPrefix, -) types.ConfigDigest { - msg, err := configDigestArgs.Pack( - feedID, - chainID, - contractAddress, - configCount, - oracles, - transmitters, - f, - onchainConfig, - offchainConfigVersion, - offchainConfig, - ) - if err != nil { - // assertion - panic(err) - } - rawHash := crypto.Keccak256(msg) - configDigest := types.ConfigDigest{} - if n := copy(configDigest[:], rawHash); n != len(configDigest) { - // assertion - panic("copy too little data") - } - binary.BigEndian.PutUint16(configDigest[:2], uint16(prefix)) - return configDigest -} diff --git a/core/services/relay/evm/mercury/config_digest_test.go b/core/services/relay/evm/mercury/config_digest_test.go deleted file mode 100644 index 600eb8c88d5..00000000000 --- a/core/services/relay/evm/mercury/config_digest_test.go +++ /dev/null @@ -1,212 +0,0 @@ -package mercury - -import ( - "math/big" - "reflect" - "testing" - "unsafe" - - "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth/ethconfig" - "github.com/ethereum/go-ethereum/ethclient/simulated" - "github.com/leanovate/gopter" - "github.com/leanovate/gopter/gen" - "github.com/leanovate/gopter/prop" - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - "github.com/smartcontractkit/wsrpc/credentials" - "github.com/stretchr/testify/require" - - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/llo-feeds/generated/exposed_verifier" -) - -// Adapted from: https://github.com/smartcontractkit/offchain-reporting/blob/991ebe1462fd56826a1ddfb34287d542acb2baee/lib/offchainreporting2/chains/evmutil/config_digest_test.go - -func TestConfigCalculationMatches(t *testing.T) { - key, err := crypto.GenerateKey() - require.NoError(t, err, "could not make private key for EOA owner") - owner, err := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - require.NoError(t, err) - backend := simulated.NewBackend( - types.GenesisAlloc{owner.From: {Balance: new(big.Int).Lsh(big.NewInt(1), 60)}}, - simulated.WithBlockGasLimit(ethconfig.Defaults.Miner.GasCeil), - ) - _, _, eoa, err := exposed_verifier.DeployExposedVerifier( - owner, backend.Client(), - ) - backend.Commit() - require.NoError(t, err, "could not deploy test EOA") - p := gopter.NewProperties(nil) - p.Property("onchain/offchain config digests match", prop.ForAll( - func( - feedID [32]byte, - chainID uint64, - contractAddress common.Address, - configCount uint64, - oracles []common.Address, - transmitters [][32]byte, - f uint8, - onchainConfig []byte, - offchainConfigVersion uint64, - offchainConfig []byte, - ) bool { - chainIDBig := new(big.Int).SetUint64(chainID) - golangDigest := configDigest( - feedID, - chainIDBig, - contractAddress, - configCount, - oracles, - *(*[]credentials.StaticSizedPublicKey)(unsafe.Pointer(&transmitters)), - f, - onchainConfig, - offchainConfigVersion, - offchainConfig, - ocrtypes.ConfigDigestPrefixMercuryV02, - ) - - bigChainID := new(big.Int) - bigChainID.SetUint64(chainID) - - solidityDigest, err := eoa.ExposedConfigDigestFromConfigData(nil, - feedID, - bigChainID, - contractAddress, - configCount, - oracles, - transmitters, - f, - onchainConfig, - offchainConfigVersion, - offchainConfig, - ) - require.NoError(t, err, "could not compute solidity version of config digest") - return golangDigest == solidityDigest - }, - GenHash(t), - gen.UInt64(), - GenAddress(t), - gen.UInt64(), - GenAddressArray(t), - GenClientPubKeyArray(t), - gen.UInt8(), - GenBytes(t), - gen.UInt64(), - GenBytes(t), - )) - p.TestingRun(t) -} - -func GenHash(t *testing.T) gopter.Gen { - var byteGens []gopter.Gen - for i := 0; i < 32; i++ { - byteGens = append(byteGens, gen.UInt8()) - } - return gopter.CombineGens(byteGens...).Map( - func(byteArray interface{}) (rv common.Hash) { - array, ok := byteArray.(*gopter.GenResult).Retrieve() - require.True(t, ok, "failed to retrieve gen result") - for i, byteVal := range array.([]interface{}) { - rv[i] = byteVal.(uint8) - } - return rv - }, - ) -} - -func GenHashArray(t *testing.T) gopter.Gen { - return gen.UInt8Range(0, 31).FlatMap( - func(length interface{}) gopter.Gen { - var hashGens []gopter.Gen - for i := uint8(0); i < length.(uint8); i++ { - hashGens = append(hashGens, GenHash(t)) - } - return gopter.CombineGens(hashGens...).Map( - func(hashArray interface{}) (rv []common.Hash) { - array, ok := hashArray.(*gopter.GenResult).Retrieve() - require.True(t, ok, "could not extract hash array") - for _, hashVal := range array.([]interface{}) { - rv = append(rv, hashVal.(common.Hash)) - } - return rv - }, - ) - }, - reflect.ValueOf([]common.Hash{}).Type(), - ) -} - -func GenAddress(t *testing.T) gopter.Gen { - return GenHash(t).Map( - func(hash interface{}) common.Address { - iHash, ok := hash.(*gopter.GenResult).Retrieve() - require.True(t, ok, "failed to retrieve hash") - return common.BytesToAddress(iHash.(common.Hash).Bytes()) - }, - ) -} - -func GenAddressArray(t *testing.T) gopter.Gen { - return GenHashArray(t).Map( - func(hashes interface{}) (rv []common.Address) { - hashArray, ok := hashes.(*gopter.GenResult).Retrieve() - require.True(t, ok, "failed to retrieve hashes") - for _, hash := range hashArray.([]common.Hash) { - rv = append(rv, common.BytesToAddress(hash.Bytes())) - } - return rv - }, - ) -} - -func GenClientPubKey(t *testing.T) gopter.Gen { - return GenHash(t).Map( - func(hash interface{}) (pk [32]byte) { - iHash, ok := hash.(*gopter.GenResult).Retrieve() - require.True(t, ok, "failed to retrieve hash") - copy(pk[:], (iHash.(common.Hash).Bytes())) - return - }, - ) -} - -func GenClientPubKeyArray(t *testing.T) gopter.Gen { - return GenHashArray(t).Map( - func(hashes interface{}) (rv [][32]byte) { - hashArray, ok := hashes.(*gopter.GenResult).Retrieve() - require.True(t, ok, "failed to retrieve hashes") - for _, hash := range hashArray.([]common.Hash) { - pk := [32]byte{} - copy(pk[:], hash.Bytes()) - rv = append(rv, pk) - } - return rv - }, - ) -} - -func GenBytes(t *testing.T) gopter.Gen { - return gen.UInt16Range(0, 2000).FlatMap( - func(length interface{}) gopter.Gen { - var byteGens []gopter.Gen - for i := uint16(0); i < length.(uint16); i++ { - byteGens = append(byteGens, gen.UInt8()) - } - return gopter.CombineGens(byteGens...).Map( - func(byteArray interface{}) []byte { - array, ok := byteArray.(*gopter.GenResult).Retrieve() - require.True(t, ok, "failed to retrieve gen result") - iArray := array.([]interface{}) - rv := make([]byte, len(iArray)) - for i, byteVal := range iArray { - rv[i] = byteVal.(uint8) - } - return rv - }, - ) - }, - reflect.ValueOf([]byte{}).Type(), - ) -} diff --git a/core/services/relay/evm/mercury/config_poller.go b/core/services/relay/evm/mercury/config_poller.go deleted file mode 100644 index 7bb5d2af7b5..00000000000 --- a/core/services/relay/evm/mercury/config_poller.go +++ /dev/null @@ -1,177 +0,0 @@ -package mercury - -import ( - "context" - "database/sql" - "fmt" - "strings" - - "github.com/ethereum/go-ethereum/accounts/abi" - "github.com/ethereum/go-ethereum/common" - "github.com/pkg/errors" - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - - "github.com/smartcontractkit/chainlink-common/pkg/logger" - - "github.com/smartcontractkit/chainlink/v2/core/chains/evm/logpoller" - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/llo-feeds/generated/verifier" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/utils" -) - -// FeedScopedConfigSet ConfigSet with FeedID for use with mercury (and multi-config DON) -var FeedScopedConfigSet common.Hash - -var verifierABI abi.ABI - -const ( - configSetEventName = "ConfigSet" - feedIdTopicIndex = 1 -) - -func init() { - var err error - verifierABI, err = abi.JSON(strings.NewReader(verifier.VerifierABI)) - if err != nil { - panic(err) - } - FeedScopedConfigSet = verifierABI.Events[configSetEventName].ID -} - -// FullConfigFromLog defines the contract config with the feedID -type FullConfigFromLog struct { - ocrtypes.ContractConfig - feedID utils.FeedID -} - -func unpackLogData(d []byte) (*verifier.VerifierConfigSet, error) { - unpacked := new(verifier.VerifierConfigSet) - - err := verifierABI.UnpackIntoInterface(unpacked, configSetEventName, d) - if err != nil { - return nil, errors.Wrap(err, "failed to unpack log data") - } - - return unpacked, nil -} - -func ConfigFromLog(logData []byte) (FullConfigFromLog, error) { - unpacked, err := unpackLogData(logData) - if err != nil { - return FullConfigFromLog{}, err - } - - var transmitAccounts []ocrtypes.Account - for _, addr := range unpacked.OffchainTransmitters { - transmitAccounts = append(transmitAccounts, ocrtypes.Account(fmt.Sprintf("%x", addr))) - } - var signers []ocrtypes.OnchainPublicKey - for _, addr := range unpacked.Signers { - addr := addr - signers = append(signers, addr[:]) - } - - return FullConfigFromLog{ - feedID: unpacked.FeedId, - ContractConfig: ocrtypes.ContractConfig{ - ConfigDigest: unpacked.ConfigDigest, - ConfigCount: unpacked.ConfigCount, - Signers: signers, - Transmitters: transmitAccounts, - F: unpacked.F, - OnchainConfig: unpacked.OnchainConfig, - OffchainConfigVersion: unpacked.OffchainConfigVersion, - OffchainConfig: unpacked.OffchainConfig, - }, - }, nil -} - -// ConfigPoller defines the Mercury Config Poller -type ConfigPoller struct { - lggr logger.Logger - destChainLogPoller logpoller.LogPoller - addr common.Address - feedId common.Hash -} - -func FilterName(addr common.Address, feedID common.Hash) string { - return logpoller.FilterName("OCR3 Mercury ConfigPoller", addr.String(), feedID.Hex()) -} - -// NewConfigPoller creates a new Mercury ConfigPoller -func NewConfigPoller(ctx context.Context, lggr logger.Logger, destChainPoller logpoller.LogPoller, addr common.Address, feedId common.Hash) (*ConfigPoller, error) { - err := destChainPoller.RegisterFilter(ctx, logpoller.Filter{Name: FilterName(addr, feedId), EventSigs: []common.Hash{FeedScopedConfigSet}, Addresses: []common.Address{addr}}) - if err != nil { - return nil, err - } - - cp := &ConfigPoller{ - lggr: lggr, - destChainLogPoller: destChainPoller, - addr: addr, - feedId: feedId, - } - - return cp, nil -} - -func (cp *ConfigPoller) Start() {} - -func (cp *ConfigPoller) Close() error { - return nil -} - -func (cp *ConfigPoller) Notify() <-chan struct{} { - return nil // rely on libocr's builtin config polling -} - -// Replay abstracts the logpoller.LogPoller Replay() implementation -func (cp *ConfigPoller) Replay(ctx context.Context, fromBlock int64) error { - return cp.destChainLogPoller.Replay(ctx, fromBlock) -} - -// LatestConfigDetails returns the latest config details from the logs -func (cp *ConfigPoller) LatestConfigDetails(ctx context.Context) (changedInBlock uint64, configDigest ocrtypes.ConfigDigest, err error) { - cp.lggr.Debugw("LatestConfigDetails", "eventSig", FeedScopedConfigSet, "addr", cp.addr, "topicIndex", feedIdTopicIndex, "feedID", cp.feedId) - logs, err := cp.destChainLogPoller.IndexedLogs(ctx, FeedScopedConfigSet, cp.addr, feedIdTopicIndex, []common.Hash{cp.feedId}, 1) - if err != nil { - return 0, ocrtypes.ConfigDigest{}, err - } - if len(logs) == 0 { - return 0, ocrtypes.ConfigDigest{}, nil - } - latest := logs[len(logs)-1] - latestConfigSet, err := ConfigFromLog(latest.Data) - if err != nil { - return 0, ocrtypes.ConfigDigest{}, err - } - return uint64(latest.BlockNumber), latestConfigSet.ConfigDigest, nil -} - -// LatestConfig returns the latest config from the logs on a certain block -func (cp *ConfigPoller) LatestConfig(ctx context.Context, changedInBlock uint64) (ocrtypes.ContractConfig, error) { - lgs, err := cp.destChainLogPoller.IndexedLogsByBlockRange(ctx, int64(changedInBlock), int64(changedInBlock), FeedScopedConfigSet, cp.addr, feedIdTopicIndex, []common.Hash{cp.feedId}) - if err != nil { - return ocrtypes.ContractConfig{}, err - } - if len(lgs) == 0 { - return ocrtypes.ContractConfig{}, nil - } - latestConfigSet, err := ConfigFromLog(lgs[len(lgs)-1].Data) - if err != nil { - return ocrtypes.ContractConfig{}, err - } - cp.lggr.Infow("LatestConfig", "latestConfig", latestConfigSet) - return latestConfigSet.ContractConfig, nil -} - -// LatestBlockHeight returns the latest block height from the logs -func (cp *ConfigPoller) LatestBlockHeight(ctx context.Context) (blockHeight uint64, err error) { - latest, err := cp.destChainLogPoller.LatestBlock(ctx) - if err != nil { - if errors.Is(err, sql.ErrNoRows) { - return 0, nil - } - return 0, err - } - return uint64(latest.BlockNumber), nil -} diff --git a/core/services/relay/evm/mercury/config_poller_test.go b/core/services/relay/evm/mercury/config_poller_test.go deleted file mode 100644 index d0e0e0892bd..00000000000 --- a/core/services/relay/evm/mercury/config_poller_test.go +++ /dev/null @@ -1,124 +0,0 @@ -package mercury - -import ( - "fmt" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/onsi/gomega" - "github.com/pkg/errors" - confighelper2 "github.com/smartcontractkit/libocr/offchainreporting2plus/confighelper" - "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - ocrtypes2 "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "github.com/umbracle/ethgo/abi" - - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" - "github.com/smartcontractkit/chainlink/v2/core/utils" - evmutils "github.com/smartcontractkit/chainlink/v2/evm/utils" -) - -func TestMercuryConfigPoller(t *testing.T) { - feedID := evmutils.NewHash() - feedIDBytes := [32]byte(feedID) - - th := SetupTH(t, feedID) - - notify := th.configPoller.Notify() - assert.Empty(t, notify) - - // Should have no config to begin with. - _, config, err := th.configPoller.LatestConfigDetails(testutils.Context(t)) - require.NoError(t, err) - require.Equal(t, ocrtypes2.ConfigDigest{}, config) - - // Create minimum number of nodes. - n := 4 - var oracles []confighelper2.OracleIdentityExtra - for i := 0; i < n; i++ { - oracles = append(oracles, confighelper2.OracleIdentityExtra{ - OracleIdentity: confighelper2.OracleIdentity{ - OnchainPublicKey: evmutils.RandomAddress().Bytes(), - TransmitAccount: ocrtypes2.Account(evmutils.RandomAddress().String()), - OffchainPublicKey: evmutils.RandomBytes32(), - PeerID: utils.MustNewPeerID(), - }, - ConfigEncryptionPublicKey: evmutils.RandomBytes32(), - }) - } - f := uint8(1) - // Setup config on contract - configType := abi.MustNewType("tuple()") - onchainConfigVal, err := abi.Encode(map[string]interface{}{}, configType) - require.NoError(t, err) - signers, _, threshold, onchainConfig, offchainConfigVersion, offchainConfig, err := confighelper2.ContractSetConfigArgsForTests( - 2*time.Second, // DeltaProgress - 20*time.Second, // DeltaResend - 100*time.Millisecond, // DeltaRound - 0, // DeltaGrace - 1*time.Minute, // DeltaStage - 100, // rMax - []int{len(oracles)}, // S - oracles, - []byte{}, // reportingPluginConfig []byte, - nil, - 0, // Max duration query - 250*time.Millisecond, // Max duration observation - 250*time.Millisecond, // MaxDurationReport - 250*time.Millisecond, // MaxDurationShouldAcceptFinalizedReport - 250*time.Millisecond, // MaxDurationShouldTransmitAcceptedReport - int(f), // f - onchainConfigVal, - ) - require.NoError(t, err) - signerAddresses, err := onchainPublicKeyToAddress(signers) - require.NoError(t, err) - offchainTransmitters := make([][32]byte, n) - encodedTransmitter := make([]ocrtypes2.Account, n) - for i := 0; i < n; i++ { - offchainTransmitters[i] = oracles[i].OffchainPublicKey - encodedTransmitter[i] = ocrtypes2.Account(fmt.Sprintf("%x", oracles[i].OffchainPublicKey[:])) - } - - _, err = th.verifierContract.SetConfig(th.user, feedIDBytes, signerAddresses, offchainTransmitters, f, onchainConfig, offchainConfigVersion, offchainConfig, nil) - require.NoError(t, err, "failed to setConfig with feed ID") - th.backend.Commit() - - latest, err := th.backend.Client().BlockByNumber(testutils.Context(t), nil) - require.NoError(t, err) - // Ensure we capture this config set log. - require.NoError(t, th.logPoller.Replay(testutils.Context(t), latest.Number().Int64()-1)) - - // Send blocks until we see the config updated. - var configBlock uint64 - var digest [32]byte - gomega.NewGomegaWithT(t).Eventually(func() bool { - th.backend.Commit() - configBlock, digest, err = th.configPoller.LatestConfigDetails(testutils.Context(t)) - require.NoError(t, err) - return ocrtypes2.ConfigDigest{} != digest - }, testutils.WaitTimeout(t), 100*time.Millisecond).Should(gomega.BeTrue()) - - // Assert the config returned is the one we configured. - newConfig, err := th.configPoller.LatestConfig(testutils.Context(t), configBlock) - require.NoError(t, err) - // Note we don't check onchainConfig, as that is populated in the contract itself. - assert.Equal(t, digest, [32]byte(newConfig.ConfigDigest)) - assert.Equal(t, signers, newConfig.Signers) - assert.Equal(t, threshold, newConfig.F) - assert.Equal(t, encodedTransmitter, newConfig.Transmitters) - assert.Equal(t, offchainConfigVersion, newConfig.OffchainConfigVersion) - assert.Equal(t, offchainConfig, newConfig.OffchainConfig) -} - -func onchainPublicKeyToAddress(publicKeys []types.OnchainPublicKey) (addresses []common.Address, err error) { - for _, signer := range publicKeys { - if len(signer) != 20 { - return []common.Address{}, errors.Errorf("address is not 20 bytes %s", signer) - } - addresses = append(addresses, common.BytesToAddress(signer)) - } - return addresses, nil -} diff --git a/core/services/relay/evm/mercury/helpers_test.go b/core/services/relay/evm/mercury/helpers_test.go deleted file mode 100644 index 3ee95521412..00000000000 --- a/core/services/relay/evm/mercury/helpers_test.go +++ /dev/null @@ -1,201 +0,0 @@ -package mercury - -import ( - "math/big" - "testing" - "time" - - "github.com/ethereum/go-ethereum/accounts/abi/bind" - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/ethereum/go-ethereum/core/types" - "github.com/ethereum/go-ethereum/crypto" - "github.com/ethereum/go-ethereum/eth/ethconfig" - "github.com/ethereum/go-ethereum/ethclient/simulated" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/smartcontractkit/libocr/offchainreporting2plus/chains/evmutil" - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - - "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" - - evmclient "github.com/smartcontractkit/chainlink/v2/core/chains/evm/client" - "github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker" - "github.com/smartcontractkit/chainlink/v2/core/chains/evm/logpoller" - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/llo-feeds/generated/verifier" - "github.com/smartcontractkit/chainlink/v2/core/gethwrappers/llo-feeds/generated/verifier_proxy" - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/pgtest" - "github.com/smartcontractkit/chainlink/v2/core/logger" - reportcodecv1 "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v1/reportcodec" - reportcodecv2 "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v2/reportcodec" - reportcodecv3 "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v3/reportcodec" - "github.com/smartcontractkit/chainlink/v2/evm/utils" -) - -var ( - sampleFeedID = [32]uint8{28, 145, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114} - sampleClientPubKey = hexutil.MustDecode("0x724ff6eae9e900270edfff233e16322a70ec06e1a6e62a81ef13921f398f6c93") -) - -var sampleReports [][]byte - -var ( - sampleV1Report = buildSampleV1Report(242) - sampleV2Report = buildSampleV2Report(242) - sampleV3Report = buildSampleV3Report(242) - sig2 = ocrtypes.AttributedOnchainSignature{Signature: testutils.MustDecodeBase64("kbeuRczizOJCxBzj7MUAFpz3yl2WRM6K/f0ieEBvA+oTFUaKslbQey10krumVjzAvlvKxMfyZo0WkOgNyfF6xwE="), Signer: 2} - sig3 = ocrtypes.AttributedOnchainSignature{Signature: testutils.MustDecodeBase64("9jz4b6Dh2WhXxQ97a6/S9UNjSfrEi9016XKTrfN0mLQFDiNuws23x7Z4n+6g0sqKH/hnxx1VukWUH/ohtw83/wE="), Signer: 3} - sampleSigs = []ocrtypes.AttributedOnchainSignature{sig2, sig3} - sampleReportContext = ocrtypes.ReportContext{ - ReportTimestamp: ocrtypes.ReportTimestamp{ - ConfigDigest: MustHexToConfigDigest("0x0006fc30092226b37f6924b464e16a54a7978a9a524519a73403af64d487dc45"), - Epoch: 6, - Round: 28, - }, - ExtraHash: [32]uint8{27, 144, 106, 73, 166, 228, 123, 166, 179, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114}, - } -) - -func init() { - sampleReports = make([][]byte, 4) - for i := 0; i < len(sampleReports); i++ { - sampleReports[i] = buildSampleV1Report(int64(i)) - } -} - -func buildSampleV1Report(p int64) []byte { - feedID := sampleFeedID - timestamp := uint32(42) - bp := big.NewInt(p) - bid := big.NewInt(243) - ask := big.NewInt(244) - currentBlockNumber := uint64(143) - currentBlockHash := utils.NewHash() - currentBlockTimestamp := uint64(123) - validFromBlockNum := uint64(142) - - b, err := reportcodecv1.ReportTypes.Pack(feedID, timestamp, bp, bid, ask, currentBlockNumber, currentBlockHash, currentBlockTimestamp, validFromBlockNum) - if err != nil { - panic(err) - } - return b -} - -func buildSampleV2Report(ts int64) []byte { - feedID := sampleFeedID - timestamp := uint32(ts) - bp := big.NewInt(242) - validFromTimestamp := uint32(123) - expiresAt := uint32(456) - linkFee := big.NewInt(3334455) - nativeFee := big.NewInt(556677) - - b, err := reportcodecv2.ReportTypes.Pack(feedID, validFromTimestamp, timestamp, nativeFee, linkFee, expiresAt, bp) - if err != nil { - panic(err) - } - return b -} - -func buildSampleV3Report(ts int64) []byte { - feedID := sampleFeedID - timestamp := uint32(ts) - bp := big.NewInt(242) - bid := big.NewInt(243) - ask := big.NewInt(244) - validFromTimestamp := uint32(123) - expiresAt := uint32(456) - linkFee := big.NewInt(3334455) - nativeFee := big.NewInt(556677) - - b, err := reportcodecv3.ReportTypes.Pack(feedID, validFromTimestamp, timestamp, nativeFee, linkFee, expiresAt, bp, bid, ask) - if err != nil { - panic(err) - } - return b -} - -func buildSamplePayload(report []byte) []byte { - var rs [][32]byte - var ss [][32]byte - var vs [32]byte - for i, as := range sampleSigs { - r, s, v, err := evmutil.SplitSignature(as.Signature) - if err != nil { - panic("eventTransmit(ev): error in SplitSignature") - } - rs = append(rs, r) - ss = append(ss, s) - vs[i] = v - } - rawReportCtx := evmutil.RawReportContext(sampleReportContext) - payload, err := PayloadTypes.Pack(rawReportCtx, report, rs, ss, vs) - if err != nil { - panic(err) - } - return payload -} - -type TestHarness struct { - configPoller *ConfigPoller - user *bind.TransactOpts - backend *simulated.Backend - verifierAddress common.Address - verifierContract *verifier.Verifier - logPoller logpoller.LogPoller -} - -func SetupTH(t *testing.T, feedID common.Hash) TestHarness { - key, err := crypto.GenerateKey() - require.NoError(t, err) - user, err := bind.NewKeyedTransactorWithChainID(key, big.NewInt(1337)) - require.NoError(t, err) - b := simulated.NewBackend(types.GenesisAlloc{ - user.From: {Balance: big.NewInt(1000000000000000000)}}, - simulated.WithBlockGasLimit(5*ethconfig.Defaults.Miner.GasCeil)) - - proxyAddress, _, verifierProxy, err := verifier_proxy.DeployVerifierProxy(user, b.Client(), common.Address{}) - require.NoError(t, err, "failed to deploy test mercury verifier proxy contract") - b.Commit() - verifierAddress, _, verifierContract, err := verifier.DeployVerifier(user, b.Client(), proxyAddress) - require.NoError(t, err, "failed to deploy test mercury verifier contract") - b.Commit() - _, err = verifierProxy.InitializeVerifier(user, verifierAddress) - require.NoError(t, err) - b.Commit() - - db := pgtest.NewSqlxDB(t) - ethClient := evmclient.NewSimulatedBackendClient(t, b, big.NewInt(1337)) - lggr := logger.TestLogger(t) - lorm := logpoller.NewORM(big.NewInt(1337), db, lggr) - - lpOpts := logpoller.Opts{ - PollPeriod: 100 * time.Millisecond, - FinalityDepth: 1, - BackfillBatchSize: 2, - RpcBatchSize: 2, - KeepFinalizedBlocksDepth: 1000, - } - ht := headtracker.NewSimulatedHeadTracker(ethClient, lpOpts.UseFinalityTag, lpOpts.FinalityDepth) - lp := logpoller.NewLogPoller(lorm, ethClient, lggr, ht, lpOpts) - servicetest.Run(t, lp) - - configPoller, err := NewConfigPoller(testutils.Context(t), lggr, lp, verifierAddress, feedID) - require.NoError(t, err) - - configPoller.Start() - t.Cleanup(func() { - assert.NoError(t, configPoller.Close()) - }) - - return TestHarness{ - configPoller: configPoller, - user: user, - backend: b, - verifierAddress: verifierAddress, - verifierContract: verifierContract, - logPoller: lp, - } -} diff --git a/core/services/relay/evm/mercury/mocks/async_deleter.go b/core/services/relay/evm/mercury/mocks/async_deleter.go deleted file mode 100644 index ce9dee690e5..00000000000 --- a/core/services/relay/evm/mercury/mocks/async_deleter.go +++ /dev/null @@ -1,68 +0,0 @@ -// Code generated by mockery v2.50.0. DO NOT EDIT. - -package mocks - -import ( - pb "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/wsrpc/pb" - mock "github.com/stretchr/testify/mock" -) - -// AsyncDeleter is an autogenerated mock type for the asyncDeleter type -type AsyncDeleter struct { - mock.Mock -} - -type AsyncDeleter_Expecter struct { - mock *mock.Mock -} - -func (_m *AsyncDeleter) EXPECT() *AsyncDeleter_Expecter { - return &AsyncDeleter_Expecter{mock: &_m.Mock} -} - -// AsyncDelete provides a mock function with given fields: req -func (_m *AsyncDeleter) AsyncDelete(req *pb.TransmitRequest) { - _m.Called(req) -} - -// AsyncDeleter_AsyncDelete_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'AsyncDelete' -type AsyncDeleter_AsyncDelete_Call struct { - *mock.Call -} - -// AsyncDelete is a helper method to define mock.On call -// - req *pb.TransmitRequest -func (_e *AsyncDeleter_Expecter) AsyncDelete(req interface{}) *AsyncDeleter_AsyncDelete_Call { - return &AsyncDeleter_AsyncDelete_Call{Call: _e.mock.On("AsyncDelete", req)} -} - -func (_c *AsyncDeleter_AsyncDelete_Call) Run(run func(req *pb.TransmitRequest)) *AsyncDeleter_AsyncDelete_Call { - _c.Call.Run(func(args mock.Arguments) { - run(args[0].(*pb.TransmitRequest)) - }) - return _c -} - -func (_c *AsyncDeleter_AsyncDelete_Call) Return() *AsyncDeleter_AsyncDelete_Call { - _c.Call.Return() - return _c -} - -func (_c *AsyncDeleter_AsyncDelete_Call) RunAndReturn(run func(*pb.TransmitRequest)) *AsyncDeleter_AsyncDelete_Call { - _c.Run(run) - return _c -} - -// NewAsyncDeleter creates a new instance of AsyncDeleter. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. -// The first argument is typically a *testing.T value. -func NewAsyncDeleter(t interface { - mock.TestingT - Cleanup(func()) -}) *AsyncDeleter { - mock := &AsyncDeleter{} - mock.Mock.Test(t) - - t.Cleanup(func() { mock.AssertExpectations(t) }) - - return mock -} diff --git a/core/services/relay/evm/mercury/mocks/pipeline.go b/core/services/relay/evm/mercury/mocks/pipeline.go deleted file mode 100644 index 429eba66674..00000000000 --- a/core/services/relay/evm/mercury/mocks/pipeline.go +++ /dev/null @@ -1,44 +0,0 @@ -package mocks - -import ( - "context" - "time" - - "github.com/smartcontractkit/chainlink/v2/core/logger" - "github.com/smartcontractkit/chainlink/v2/core/services/pipeline" -) - -type MockRunner struct { - Trrs pipeline.TaskRunResults - Err error -} - -func (m *MockRunner) ExecuteRun(ctx context.Context, spec pipeline.Spec, vars pipeline.Vars) (run *pipeline.Run, trrs pipeline.TaskRunResults, err error) { - return &pipeline.Run{ID: 42}, m.Trrs, m.Err -} - -var _ pipeline.Task = &MockTask{} - -type MockTask struct { - result pipeline.Result -} - -func (m *MockTask) GetDescendantTasks() []pipeline.Task { return nil } - -func (m *MockTask) TaskTags() string { return "{\"anything\": \"here\"}" } - -func (m *MockTask) Type() pipeline.TaskType { return "MockTask" } -func (m *MockTask) ID() int { return 0 } -func (m *MockTask) DotID() string { return "" } -func (m *MockTask) Run(ctx context.Context, lggr logger.Logger, vars pipeline.Vars, inputs []pipeline.Result) (pipeline.Result, pipeline.RunInfo) { - return m.result, pipeline.RunInfo{} -} -func (m *MockTask) Base() *pipeline.BaseTask { return nil } -func (m *MockTask) Outputs() []pipeline.Task { return nil } -func (m *MockTask) Inputs() []pipeline.TaskDependency { return nil } -func (m *MockTask) OutputIndex() int32 { return 0 } -func (m *MockTask) TaskTimeout() (time.Duration, bool) { return 0, false } -func (m *MockTask) TaskRetries() uint32 { return 0 } -func (m *MockTask) TaskMinBackoff() time.Duration { return 0 } -func (m *MockTask) TaskMaxBackoff() time.Duration { return 0 } -func (m *MockTask) TaskStreamID() *uint32 { return nil } diff --git a/core/services/relay/evm/mercury/offchain_config_digester.go b/core/services/relay/evm/mercury/offchain_config_digester.go deleted file mode 100644 index e771053c37b..00000000000 --- a/core/services/relay/evm/mercury/offchain_config_digester.go +++ /dev/null @@ -1,74 +0,0 @@ -package mercury - -import ( - "context" - "crypto/ed25519" - "encoding/hex" - "math/big" - - "github.com/ethereum/go-ethereum/common" - "github.com/pkg/errors" - - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - "github.com/smartcontractkit/wsrpc/credentials" - - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/utils" -) - -// Originally sourced from: https://github.com/smartcontractkit/offchain-reporting/blob/991ebe1462fd56826a1ddfb34287d542acb2baee/lib/offchainreporting2/chains/evmutil/offchain_config_digester.go - -var _ ocrtypes.OffchainConfigDigester = OffchainConfigDigester{} - -func NewOffchainConfigDigester(feedID [32]byte, chainID *big.Int, contractAddress common.Address, prefix ocrtypes.ConfigDigestPrefix) OffchainConfigDigester { - return OffchainConfigDigester{feedID, chainID, contractAddress, prefix} -} - -type OffchainConfigDigester struct { - FeedID utils.FeedID - ChainID *big.Int - ContractAddress common.Address - Prefix ocrtypes.ConfigDigestPrefix -} - -func (d OffchainConfigDigester) ConfigDigest(ctx context.Context, cc ocrtypes.ContractConfig) (ocrtypes.ConfigDigest, error) { - signers := []common.Address{} - for i, signer := range cc.Signers { - if len(signer) != 20 { - return ocrtypes.ConfigDigest{}, errors.Errorf("%v-th evm signer should be a 20 byte address, but got %x", i, signer) - } - a := common.BytesToAddress(signer) - signers = append(signers, a) - } - transmitters := []credentials.StaticSizedPublicKey{} - for i, transmitter := range cc.Transmitters { - if len(transmitter) != 2*ed25519.PublicKeySize { - return ocrtypes.ConfigDigest{}, errors.Errorf("%v-th evm transmitter should be a 64 character hex-encoded ed25519 public key, but got '%v' (%d chars)", i, transmitter, len(transmitter)) - } - var t credentials.StaticSizedPublicKey - b, err := hex.DecodeString(string(transmitter)) - if err != nil { - return ocrtypes.ConfigDigest{}, errors.Wrapf(err, "%v-th evm transmitter is not valid hex, got: %q", i, transmitter) - } - copy(t[:], b) - - transmitters = append(transmitters, t) - } - - return configDigest( - common.Hash(d.FeedID), - d.ChainID, - d.ContractAddress, - cc.ConfigCount, - signers, - transmitters, - cc.F, - cc.OnchainConfig, - cc.OffchainConfigVersion, - cc.OffchainConfig, - d.Prefix, - ), nil -} - -func (d OffchainConfigDigester) ConfigDigestPrefix(ctx context.Context) (ocrtypes.ConfigDigestPrefix, error) { - return d.Prefix, nil -} diff --git a/core/services/relay/evm/mercury/offchain_config_digester_test.go b/core/services/relay/evm/mercury/offchain_config_digester_test.go deleted file mode 100644 index 62869cf6f3d..00000000000 --- a/core/services/relay/evm/mercury/offchain_config_digester_test.go +++ /dev/null @@ -1,58 +0,0 @@ -package mercury - -import ( - "math/big" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - "github.com/stretchr/testify/require" - - "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" -) - -func Test_OffchainConfigDigester_ConfigDigest(t *testing.T) { - ctx := tests.Context(t) - // ChainID and ContractAddress are taken into account for computation - cd1, err := OffchainConfigDigester{ChainID: big.NewInt(0)}.ConfigDigest(ctx, types.ContractConfig{}) - require.NoError(t, err) - cd2, err := OffchainConfigDigester{ChainID: big.NewInt(0)}.ConfigDigest(ctx, types.ContractConfig{}) - require.NoError(t, err) - cd3, err := OffchainConfigDigester{ChainID: big.NewInt(1)}.ConfigDigest(ctx, types.ContractConfig{}) - require.NoError(t, err) - cd4, err := OffchainConfigDigester{ChainID: big.NewInt(1), ContractAddress: common.Address{1}}.ConfigDigest(ctx, types.ContractConfig{}) - require.NoError(t, err) - - require.Equal(t, cd1, cd2) - require.NotEqual(t, cd2, cd3) - require.NotEqual(t, cd2, cd4) - require.NotEqual(t, cd3, cd4) - - // malformed signers - _, err = OffchainConfigDigester{}.ConfigDigest(ctx, types.ContractConfig{ - Signers: []types.OnchainPublicKey{{1, 2}}, - }) - require.Error(t, err) - - // malformed transmitters - _, err = OffchainConfigDigester{}.ConfigDigest(ctx, types.ContractConfig{ - Transmitters: []types.Account{"0x"}, - }) - require.Error(t, err) - - _, err = OffchainConfigDigester{}.ConfigDigest(ctx, types.ContractConfig{ - Transmitters: []types.Account{"7343581f55146951b0f678dc6cfa8fd360e2f353"}, - }) - require.Error(t, err) - - _, err = OffchainConfigDigester{}.ConfigDigest(ctx, types.ContractConfig{ - Transmitters: []types.Account{"7343581f55146951b0f678dc6cfa8fd360e2f353aabbccddeeffaaccddeeffaz"}, - }) - require.Error(t, err) - - // well-formed transmitters - _, err = OffchainConfigDigester{ChainID: big.NewInt(0)}.ConfigDigest(ctx, types.ContractConfig{ - Transmitters: []types.Account{"7343581f55146951b0f678dc6cfa8fd360e2f353aabbccddeeffaaccddeeffaa"}, - }) - require.NoError(t, err) -} diff --git a/core/services/relay/evm/mercury/orm.go b/core/services/relay/evm/mercury/orm.go deleted file mode 100644 index 65df9ab4cc6..00000000000 --- a/core/services/relay/evm/mercury/orm.go +++ /dev/null @@ -1,191 +0,0 @@ -package mercury - -import ( - "context" - "crypto/sha256" - "database/sql" - "errors" - "fmt" - "strings" - "sync" - - "github.com/ethereum/go-ethereum/common" - "github.com/lib/pq" - pkgerrors "github.com/pkg/errors" - - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - - "github.com/smartcontractkit/chainlink-common/pkg/sqlutil" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/utils" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/wsrpc/pb" -) - -type ORM interface { - InsertTransmitRequest(ctx context.Context, serverURLs []string, req *pb.TransmitRequest, jobID int32, reportCtx ocrtypes.ReportContext) error - DeleteTransmitRequests(ctx context.Context, serverURL string, reqs []*pb.TransmitRequest) error - GetTransmitRequests(ctx context.Context, serverURL string, jobID int32) ([]*Transmission, error) - PruneTransmitRequests(ctx context.Context, serverURL string, jobID int32, maxSize int) error - LatestReport(ctx context.Context, feedID [32]byte) (report []byte, err error) -} - -func FeedIDFromReport(report ocrtypes.Report) (feedID utils.FeedID, err error) { - if n := copy(feedID[:], report); n != 32 { - return feedID, pkgerrors.Errorf("invalid length for report: %d", len(report)) - } - return feedID, nil -} - -type orm struct { - ds sqlutil.DataSource -} - -func NewORM(ds sqlutil.DataSource) ORM { - return &orm{ds: ds} -} - -// InsertTransmitRequest inserts one transmit request if the payload does not exist already. -func (o *orm) InsertTransmitRequest(ctx context.Context, serverURLs []string, req *pb.TransmitRequest, jobID int32, reportCtx ocrtypes.ReportContext) error { - feedID, err := FeedIDFromReport(req.Payload) - if err != nil { - return err - } - if len(serverURLs) == 0 { - return errors.New("no server URLs provided") - } - - var wg sync.WaitGroup - wg.Add(2) - var err1, err2 error - - go func() { - defer wg.Done() - - values := make([]string, len(serverURLs)) - args := []interface{}{ - req.Payload, - hashPayload(req.Payload), - reportCtx.ConfigDigest[:], - reportCtx.Epoch, - reportCtx.Round, - reportCtx.ExtraHash[:], - jobID, - feedID[:], - } - for i, serverURL := range serverURLs { - // server url is the only thing that changes, might as well re-use - // the same parameters for each insert - values[i] = fmt.Sprintf("($1, $2, $3, $4, $5, $6, $7, $8, $%d)", i+9) - args = append(args, serverURL) - } - - _, err1 = o.ds.ExecContext(ctx, fmt.Sprintf(` - INSERT INTO mercury_transmit_requests (payload, payload_hash, config_digest, epoch, round, extra_hash, job_id, feed_id, server_url) - VALUES %s - ON CONFLICT (server_url, payload_hash) DO NOTHING - `, strings.Join(values, ",")), args...) - }() - - go func() { - defer wg.Done() - _, err2 = o.ds.ExecContext(ctx, ` - INSERT INTO feed_latest_reports (feed_id, report, epoch, round, updated_at, job_id) - VALUES ($1, $2, $3, $4, NOW(), $5) - ON CONFLICT (feed_id) DO UPDATE - SET feed_id=$1, report=$2, epoch=$3, round=$4, updated_at=NOW() - WHERE excluded.epoch > feed_latest_reports.epoch OR (excluded.epoch = feed_latest_reports.epoch AND excluded.round > feed_latest_reports.round) - `, feedID[:], req.Payload, reportCtx.Epoch, reportCtx.Round, jobID) - }() - wg.Wait() - return errors.Join(err1, err2) -} - -// DeleteTransmitRequest deletes the given transmit requests if they exist. -func (o *orm) DeleteTransmitRequests(ctx context.Context, serverURL string, reqs []*pb.TransmitRequest) error { - if len(reqs) == 0 { - return nil - } - - var hashes pq.ByteaArray - for _, req := range reqs { - hashes = append(hashes, hashPayload(req.Payload)) - } - - _, err := o.ds.ExecContext(ctx, ` - DELETE FROM mercury_transmit_requests - WHERE server_url = $1 AND payload_hash = ANY($2) - `, serverURL, hashes) - return err -} - -// GetTransmitRequests returns all transmit requests in chronologically descending order. -func (o *orm) GetTransmitRequests(ctx context.Context, serverURL string, jobID int32) ([]*Transmission, error) { - // The priority queue uses epoch and round to sort transmissions so order by - // the same fields here for optimal insertion into the pq. - rows, err := o.ds.QueryContext(ctx, ` - SELECT payload, config_digest, epoch, round, extra_hash - FROM mercury_transmit_requests - WHERE job_id = $1 AND server_url = $2 - ORDER BY epoch DESC, round DESC - `, jobID, serverURL) - if err != nil { - return nil, err - } - defer rows.Close() - - var transmissions []*Transmission - for rows.Next() { - transmission := &Transmission{Req: &pb.TransmitRequest{}} - var digest, extraHash common.Hash - - err := rows.Scan( - &transmission.Req.Payload, - &digest, - &transmission.ReportCtx.Epoch, - &transmission.ReportCtx.Round, - &extraHash, - ) - if err != nil { - return nil, err - } - transmission.ReportCtx.ConfigDigest = ocrtypes.ConfigDigest(digest) - transmission.ReportCtx.ExtraHash = extraHash - - transmissions = append(transmissions, transmission) - } - if err := rows.Err(); err != nil { - return nil, err - } - - return transmissions, nil -} - -// PruneTransmitRequests keeps at most maxSize rows for the given job ID, -// deleting the oldest transactions. -func (o *orm) PruneTransmitRequests(ctx context.Context, serverURL string, jobID int32, maxSize int) error { - // Prune the oldest requests by epoch and round. - _, err := o.ds.ExecContext(ctx, ` - DELETE FROM mercury_transmit_requests - WHERE job_id = $1 AND server_url = $2 AND - payload_hash NOT IN ( - SELECT payload_hash - FROM mercury_transmit_requests - WHERE job_id = $1 AND server_url = $2 - ORDER BY epoch DESC, round DESC - LIMIT $3 - ) - `, jobID, serverURL, maxSize) - return err -} - -func (o *orm) LatestReport(ctx context.Context, feedID [32]byte) (report []byte, err error) { - err = o.ds.GetContext(ctx, &report, `SELECT report FROM feed_latest_reports WHERE feed_id = $1`, feedID[:]) - if errors.Is(err, sql.ErrNoRows) { - return nil, nil - } - return report, err -} - -func hashPayload(payload []byte) []byte { - checksum := sha256.Sum256(payload) - return checksum[:] -} diff --git a/core/services/relay/evm/mercury/orm_test.go b/core/services/relay/evm/mercury/orm_test.go deleted file mode 100644 index 090b3f06eac..00000000000 --- a/core/services/relay/evm/mercury/orm_test.go +++ /dev/null @@ -1,378 +0,0 @@ -package mercury - -import ( - "testing" - - "github.com/cometbft/cometbft/libs/rand" - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/pgtest" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/wsrpc/pb" -) - -var ( - sURL = "wss://example.com/mercury" - sURL2 = "wss://mercuryserver.test" - sURL3 = "wss://mercuryserver.example/foo" -) - -func TestORM(t *testing.T) { - ctx := testutils.Context(t) - db := pgtest.NewSqlxDB(t) - - jobID := rand.Int32() // foreign key constraints disabled so value doesn't matter - pgtest.MustExec(t, db, `SET CONSTRAINTS mercury_transmit_requests_job_id_fkey DEFERRED`) - pgtest.MustExec(t, db, `SET CONSTRAINTS feed_latest_reports_job_id_fkey DEFERRED`) - orm := NewORM(db) - feedID := sampleFeedID - - reports := sampleReports - reportContexts := make([]ocrtypes.ReportContext, 4) - for i := range reportContexts { - reportContexts[i] = ocrtypes.ReportContext{ - ReportTimestamp: ocrtypes.ReportTimestamp{ - ConfigDigest: ocrtypes.ConfigDigest{'1'}, - Epoch: 10, - Round: uint8(i), - }, - ExtraHash: [32]byte{'2'}, - } - } - - l, err := orm.LatestReport(testutils.Context(t), feedID) - require.NoError(t, err) - assert.Nil(t, l) - - // Test insert and get requests. - // s1 - err = orm.InsertTransmitRequest(ctx, []string{sURL}, &pb.TransmitRequest{Payload: reports[0]}, jobID, reportContexts[0]) - require.NoError(t, err) - err = orm.InsertTransmitRequest(ctx, []string{sURL}, &pb.TransmitRequest{Payload: reports[1]}, jobID, reportContexts[1]) - require.NoError(t, err) - err = orm.InsertTransmitRequest(ctx, []string{sURL}, &pb.TransmitRequest{Payload: reports[2]}, jobID, reportContexts[2]) - require.NoError(t, err) - - // s2 - err = orm.InsertTransmitRequest(ctx, []string{sURL2}, &pb.TransmitRequest{Payload: reports[3]}, jobID, reportContexts[0]) - require.NoError(t, err) - - transmissions, err := orm.GetTransmitRequests(ctx, sURL, jobID) - require.NoError(t, err) - require.Equal(t, transmissions, []*Transmission{ - {Req: &pb.TransmitRequest{Payload: reports[2]}, ReportCtx: reportContexts[2]}, - {Req: &pb.TransmitRequest{Payload: reports[1]}, ReportCtx: reportContexts[1]}, - {Req: &pb.TransmitRequest{Payload: reports[0]}, ReportCtx: reportContexts[0]}, - }) - transmissions, err = orm.GetTransmitRequests(ctx, sURL2, jobID) - require.NoError(t, err) - require.Equal(t, transmissions, []*Transmission{ - {Req: &pb.TransmitRequest{Payload: reports[3]}, ReportCtx: reportContexts[0]}, - }) - - l, err = orm.LatestReport(testutils.Context(t), feedID) - require.NoError(t, err) - assert.NotEqual(t, reports[0], l) - assert.Equal(t, reports[2], l) - - // Test requests can be deleted. - err = orm.DeleteTransmitRequests(ctx, sURL, []*pb.TransmitRequest{{Payload: reports[1]}}) - require.NoError(t, err) - - transmissions, err = orm.GetTransmitRequests(ctx, sURL, jobID) - require.NoError(t, err) - require.Equal(t, transmissions, []*Transmission{ - {Req: &pb.TransmitRequest{Payload: reports[2]}, ReportCtx: reportContexts[2]}, - {Req: &pb.TransmitRequest{Payload: reports[0]}, ReportCtx: reportContexts[0]}, - }) - - l, err = orm.LatestReport(testutils.Context(t), feedID) - require.NoError(t, err) - assert.Equal(t, reports[2], l) - - // Test deleting non-existent requests does not error. - err = orm.DeleteTransmitRequests(ctx, sURL, []*pb.TransmitRequest{{Payload: []byte("does-not-exist")}}) - require.NoError(t, err) - - transmissions, err = orm.GetTransmitRequests(ctx, sURL, jobID) - require.NoError(t, err) - require.Equal(t, transmissions, []*Transmission{ - {Req: &pb.TransmitRequest{Payload: reports[2]}, ReportCtx: reportContexts[2]}, - {Req: &pb.TransmitRequest{Payload: reports[0]}, ReportCtx: reportContexts[0]}, - }) - - // Test deleting multiple requests. - err = orm.DeleteTransmitRequests(ctx, sURL, []*pb.TransmitRequest{ - {Payload: reports[0]}, - {Payload: reports[2]}, - }) - require.NoError(t, err) - - l, err = orm.LatestReport(testutils.Context(t), feedID) - require.NoError(t, err) - assert.Equal(t, reports[2], l) - - transmissions, err = orm.GetTransmitRequests(ctx, sURL, jobID) - require.NoError(t, err) - require.Empty(t, transmissions) - - // More inserts. - err = orm.InsertTransmitRequest(ctx, []string{sURL}, &pb.TransmitRequest{Payload: reports[3]}, jobID, reportContexts[3]) - require.NoError(t, err) - - transmissions, err = orm.GetTransmitRequests(ctx, sURL, jobID) - require.NoError(t, err) - require.Equal(t, transmissions, []*Transmission{ - {Req: &pb.TransmitRequest{Payload: reports[3]}, ReportCtx: reportContexts[3]}, - }) - - // Duplicate requests are ignored. - err = orm.InsertTransmitRequest(ctx, []string{sURL}, &pb.TransmitRequest{Payload: reports[3]}, jobID, reportContexts[3]) - require.NoError(t, err) - err = orm.InsertTransmitRequest(ctx, []string{sURL}, &pb.TransmitRequest{Payload: reports[3]}, jobID, reportContexts[3]) - require.NoError(t, err) - - transmissions, err = orm.GetTransmitRequests(ctx, sURL, jobID) - require.NoError(t, err) - require.Equal(t, transmissions, []*Transmission{ - {Req: &pb.TransmitRequest{Payload: reports[3]}, ReportCtx: reportContexts[3]}, - }) - - l, err = orm.LatestReport(testutils.Context(t), feedID) - require.NoError(t, err) - assert.Equal(t, reports[3], l) - - // s2 not affected by deletion - transmissions, err = orm.GetTransmitRequests(ctx, sURL2, jobID) - require.NoError(t, err) - require.Len(t, transmissions, 1) -} - -func TestORM_InsertTransmitRequest_MultipleServerURLs(t *testing.T) { - ctx := testutils.Context(t) - db := pgtest.NewSqlxDB(t) - - jobID := rand.Int32() // foreign key constraints disabled so value doesn't matter - pgtest.MustExec(t, db, `SET CONSTRAINTS mercury_transmit_requests_job_id_fkey DEFERRED`) - pgtest.MustExec(t, db, `SET CONSTRAINTS feed_latest_reports_job_id_fkey DEFERRED`) - orm := NewORM(db) - feedID := sampleFeedID - - reports := sampleReports - reportContexts := make([]ocrtypes.ReportContext, 4) - for i := range reportContexts { - reportContexts[i] = ocrtypes.ReportContext{ - ReportTimestamp: ocrtypes.ReportTimestamp{ - ConfigDigest: ocrtypes.ConfigDigest{'1'}, - Epoch: 10, - Round: uint8(i), - }, - ExtraHash: [32]byte{'2'}, - } - } - err := orm.InsertTransmitRequest(ctx, []string{sURL, sURL2, sURL3}, &pb.TransmitRequest{Payload: reports[0]}, jobID, reportContexts[0]) - require.NoError(t, err) - - transmissions, err := orm.GetTransmitRequests(ctx, sURL, jobID) - require.NoError(t, err) - require.Len(t, transmissions, 1) - assert.Equal(t, &Transmission{Req: &pb.TransmitRequest{Payload: reports[0]}, ReportCtx: reportContexts[0]}, transmissions[0]) - - transmissions, err = orm.GetTransmitRequests(ctx, sURL2, jobID) - require.NoError(t, err) - require.Len(t, transmissions, 1) - assert.Equal(t, &Transmission{Req: &pb.TransmitRequest{Payload: reports[0]}, ReportCtx: reportContexts[0]}, transmissions[0]) - - transmissions, err = orm.GetTransmitRequests(ctx, sURL3, jobID) - require.NoError(t, err) - require.Len(t, transmissions, 1) - assert.Equal(t, &Transmission{Req: &pb.TransmitRequest{Payload: reports[0]}, ReportCtx: reportContexts[0]}, transmissions[0]) - - l, err := orm.LatestReport(testutils.Context(t), feedID) - require.NoError(t, err) - assert.Equal(t, reports[0], l) -} - -func TestORM_PruneTransmitRequests(t *testing.T) { - ctx := testutils.Context(t) - db := pgtest.NewSqlxDB(t) - jobID := rand.Int32() // foreign key constraints disabled so value doesn't matter - pgtest.MustExec(t, db, `SET CONSTRAINTS mercury_transmit_requests_job_id_fkey DEFERRED`) - pgtest.MustExec(t, db, `SET CONSTRAINTS feed_latest_reports_job_id_fkey DEFERRED`) - - orm := NewORM(db) - - reports := sampleReports - - makeReportContext := func(epoch uint32, round uint8) ocrtypes.ReportContext { - return ocrtypes.ReportContext{ - ReportTimestamp: ocrtypes.ReportTimestamp{ - ConfigDigest: ocrtypes.ConfigDigest{'1'}, - Epoch: epoch, - Round: round, - }, - ExtraHash: [32]byte{'2'}, - } - } - - // s1 - err := orm.InsertTransmitRequest(ctx, []string{sURL}, &pb.TransmitRequest{Payload: reports[0]}, jobID, makeReportContext(1, 1)) - require.NoError(t, err) - err = orm.InsertTransmitRequest(ctx, []string{sURL}, &pb.TransmitRequest{Payload: reports[1]}, jobID, makeReportContext(1, 2)) - require.NoError(t, err) - // s2 - should not be touched - err = orm.InsertTransmitRequest(ctx, []string{sURL2}, &pb.TransmitRequest{Payload: reports[0]}, jobID, makeReportContext(1, 0)) - require.NoError(t, err) - err = orm.InsertTransmitRequest(ctx, []string{sURL2}, &pb.TransmitRequest{Payload: reports[0]}, jobID, makeReportContext(1, 1)) - require.NoError(t, err) - err = orm.InsertTransmitRequest(ctx, []string{sURL2}, &pb.TransmitRequest{Payload: reports[1]}, jobID, makeReportContext(1, 2)) - require.NoError(t, err) - err = orm.InsertTransmitRequest(ctx, []string{sURL2}, &pb.TransmitRequest{Payload: reports[2]}, jobID, makeReportContext(1, 3)) - require.NoError(t, err) - - // Max size greater than number of records, expect no-op - err = orm.PruneTransmitRequests(ctx, sURL, jobID, 5) - require.NoError(t, err) - - transmissions, err := orm.GetTransmitRequests(ctx, sURL, jobID) - require.NoError(t, err) - require.Equal(t, transmissions, []*Transmission{ - {Req: &pb.TransmitRequest{Payload: reports[1]}, ReportCtx: makeReportContext(1, 2)}, - {Req: &pb.TransmitRequest{Payload: reports[0]}, ReportCtx: makeReportContext(1, 1)}, - }) - - // Max size equal to number of records, expect no-op - err = orm.PruneTransmitRequests(ctx, sURL, jobID, 2) - require.NoError(t, err) - - transmissions, err = orm.GetTransmitRequests(ctx, sURL, jobID) - require.NoError(t, err) - require.Equal(t, transmissions, []*Transmission{ - {Req: &pb.TransmitRequest{Payload: reports[1]}, ReportCtx: makeReportContext(1, 2)}, - {Req: &pb.TransmitRequest{Payload: reports[0]}, ReportCtx: makeReportContext(1, 1)}, - }) - - // Max size is number of records + 1, but jobID differs, expect no-op - err = orm.PruneTransmitRequests(ctx, sURL, -1, 2) - require.NoError(t, err) - - transmissions, err = orm.GetTransmitRequests(ctx, sURL, jobID) - require.NoError(t, err) - require.Equal(t, []*Transmission{ - {Req: &pb.TransmitRequest{Payload: reports[1]}, ReportCtx: makeReportContext(1, 2)}, - {Req: &pb.TransmitRequest{Payload: reports[0]}, ReportCtx: makeReportContext(1, 1)}, - }, transmissions) - - err = orm.InsertTransmitRequest(ctx, []string{sURL}, &pb.TransmitRequest{Payload: reports[2]}, jobID, makeReportContext(2, 1)) - require.NoError(t, err) - err = orm.InsertTransmitRequest(ctx, []string{sURL}, &pb.TransmitRequest{Payload: reports[3]}, jobID, makeReportContext(2, 2)) - require.NoError(t, err) - - // Max size is table size - 1, expect the oldest row to be pruned. - err = orm.PruneTransmitRequests(ctx, sURL, jobID, 3) - require.NoError(t, err) - - transmissions, err = orm.GetTransmitRequests(ctx, sURL, jobID) - require.NoError(t, err) - require.Equal(t, []*Transmission{ - {Req: &pb.TransmitRequest{Payload: reports[3]}, ReportCtx: makeReportContext(2, 2)}, - {Req: &pb.TransmitRequest{Payload: reports[2]}, ReportCtx: makeReportContext(2, 1)}, - {Req: &pb.TransmitRequest{Payload: reports[1]}, ReportCtx: makeReportContext(1, 2)}, - }, transmissions) - - // s2 not touched - transmissions, err = orm.GetTransmitRequests(ctx, sURL2, jobID) - require.NoError(t, err) - assert.Len(t, transmissions, 3) -} - -func TestORM_InsertTransmitRequest_LatestReport(t *testing.T) { - ctx := testutils.Context(t) - db := pgtest.NewSqlxDB(t) - jobID := rand.Int32() // foreign key constraints disabled so value doesn't matter - pgtest.MustExec(t, db, `SET CONSTRAINTS mercury_transmit_requests_job_id_fkey DEFERRED`) - pgtest.MustExec(t, db, `SET CONSTRAINTS feed_latest_reports_job_id_fkey DEFERRED`) - - orm := NewORM(db) - feedID := sampleFeedID - - reports := sampleReports - - makeReportContext := func(epoch uint32, round uint8) ocrtypes.ReportContext { - return ocrtypes.ReportContext{ - ReportTimestamp: ocrtypes.ReportTimestamp{ - ConfigDigest: ocrtypes.ConfigDigest{'1'}, - Epoch: epoch, - Round: round, - }, - ExtraHash: [32]byte{'2'}, - } - } - - err := orm.InsertTransmitRequest(ctx, []string{sURL}, &pb.TransmitRequest{Payload: reports[0]}, jobID, makeReportContext( - 0, 0, - )) - require.NoError(t, err) - - // this should be ignored, because report context is the same - err = orm.InsertTransmitRequest(ctx, []string{sURL2}, &pb.TransmitRequest{Payload: reports[1]}, jobID, makeReportContext( - 0, 0, - )) - require.NoError(t, err) - - l, err := orm.LatestReport(testutils.Context(t), feedID) - require.NoError(t, err) - assert.Equal(t, reports[0], l) - - t.Run("replaces if epoch and round are larger", func(t *testing.T) { - err = orm.InsertTransmitRequest(ctx, []string{"foo"}, &pb.TransmitRequest{Payload: reports[1]}, jobID, makeReportContext(1, 1)) - require.NoError(t, err) - - l, err = orm.LatestReport(testutils.Context(t), feedID) - require.NoError(t, err) - assert.Equal(t, reports[1], l) - }) - t.Run("replaces if epoch is the same but round is greater", func(t *testing.T) { - err = orm.InsertTransmitRequest(ctx, []string{sURL}, &pb.TransmitRequest{Payload: reports[2]}, jobID, makeReportContext(1, 2)) - require.NoError(t, err) - - l, err = orm.LatestReport(testutils.Context(t), feedID) - require.NoError(t, err) - assert.Equal(t, reports[2], l) - }) - t.Run("replaces if epoch is larger but round is smaller", func(t *testing.T) { - err = orm.InsertTransmitRequest(ctx, []string{"bar"}, &pb.TransmitRequest{Payload: reports[3]}, jobID, makeReportContext(2, 1)) - require.NoError(t, err) - - l, err = orm.LatestReport(testutils.Context(t), feedID) - require.NoError(t, err) - assert.Equal(t, reports[3], l) - }) - t.Run("does not overwrite if epoch/round is the same", func(t *testing.T) { - err = orm.InsertTransmitRequest(ctx, []string{sURL}, &pb.TransmitRequest{Payload: reports[0]}, jobID, makeReportContext(2, 1)) - require.NoError(t, err) - - l, err = orm.LatestReport(testutils.Context(t), feedID) - require.NoError(t, err) - assert.Equal(t, reports[3], l) - }) -} - -func Test_ReportCodec_FeedIDFromReport(t *testing.T) { - t.Run("FeedIDFromReport extracts the current block number from a valid report", func(t *testing.T) { - report := buildSampleV1Report(42) - - f, err := FeedIDFromReport(report) - require.NoError(t, err) - - assert.Equal(t, sampleFeedID[:], f[:]) - }) - t.Run("FeedIDFromReport returns error if report is invalid", func(t *testing.T) { - report := []byte{1} - - _, err := FeedIDFromReport(report) - assert.EqualError(t, err, "invalid length for report: 1") - }) -} diff --git a/core/services/relay/evm/mercury/payload_types.go b/core/services/relay/evm/mercury/payload_types.go new file mode 100644 index 00000000000..ef4c48f55f5 --- /dev/null +++ b/core/services/relay/evm/mercury/payload_types.go @@ -0,0 +1,26 @@ +package mercury + +import ( + "fmt" + + "github.com/ethereum/go-ethereum/accounts/abi" +) + +var PayloadTypes = getPayloadTypes() + +func getPayloadTypes() abi.Arguments { + mustNewType := func(t string) abi.Type { + result, err := abi.NewType(t, "", []abi.ArgumentMarshaling{}) + if err != nil { + panic(fmt.Sprintf("Unexpected error during abi.NewType: %s", err)) + } + return result + } + return abi.Arguments([]abi.Argument{ + {Name: "reportContext", Type: mustNewType("bytes32[3]")}, + {Name: "report", Type: mustNewType("bytes")}, + {Name: "rawRs", Type: mustNewType("bytes32[]")}, + {Name: "rawSs", Type: mustNewType("bytes32[]")}, + {Name: "rawVs", Type: mustNewType("bytes32")}, + }) +} diff --git a/core/services/relay/evm/mercury/persistence_manager.go b/core/services/relay/evm/mercury/persistence_manager.go deleted file mode 100644 index 68137d04c14..00000000000 --- a/core/services/relay/evm/mercury/persistence_manager.go +++ /dev/null @@ -1,149 +0,0 @@ -package mercury - -import ( - "context" - "sync" - "time" - - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/services" - "github.com/smartcontractkit/chainlink-common/pkg/sqlutil" - - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/wsrpc/pb" -) - -var ( - flushDeletesFrequency = time.Second - pruneFrequency = time.Hour -) - -type PersistenceManager struct { - lggr logger.Logger - orm ORM - serverURL string - - once services.StateMachine - stopCh services.StopChan - wg sync.WaitGroup - - deleteMu sync.Mutex - deleteQueue []*pb.TransmitRequest - - jobID int32 - - maxTransmitQueueSize int - flushDeletesFrequency time.Duration - pruneFrequency time.Duration -} - -func NewPersistenceManager(lggr logger.Logger, serverURL string, orm ORM, jobID int32, maxTransmitQueueSize int, flushDeletesFrequency, pruneFrequency time.Duration) *PersistenceManager { - return &PersistenceManager{ - lggr: logger.Sugared(lggr).Named("MercuryPersistenceManager").With("serverURL", serverURL), - orm: orm, - serverURL: serverURL, - stopCh: make(services.StopChan), - jobID: jobID, - maxTransmitQueueSize: maxTransmitQueueSize, - flushDeletesFrequency: flushDeletesFrequency, - pruneFrequency: pruneFrequency, - } -} - -func (pm *PersistenceManager) Start(ctx context.Context) error { - return pm.once.StartOnce("MercuryPersistenceManager", func() error { - pm.wg.Add(2) - go pm.runFlushDeletesLoop() - go pm.runPruneLoop() - return nil - }) -} - -func (pm *PersistenceManager) Close() error { - return pm.once.StopOnce("MercuryPersistenceManager", func() error { - close(pm.stopCh) - pm.wg.Wait() - return nil - }) -} - -func (pm *PersistenceManager) Insert(ctx context.Context, req *pb.TransmitRequest, reportCtx ocrtypes.ReportContext) error { - return pm.orm.InsertTransmitRequest(ctx, []string{pm.serverURL}, req, pm.jobID, reportCtx) -} - -func (pm *PersistenceManager) Delete(ctx context.Context, req *pb.TransmitRequest) error { - return pm.orm.DeleteTransmitRequests(ctx, pm.serverURL, []*pb.TransmitRequest{req}) -} - -func (pm *PersistenceManager) AsyncDelete(req *pb.TransmitRequest) { - pm.addToDeleteQueue(req) -} - -func (pm *PersistenceManager) Load(ctx context.Context) ([]*Transmission, error) { - return pm.orm.GetTransmitRequests(ctx, pm.serverURL, pm.jobID) -} - -func (pm *PersistenceManager) runFlushDeletesLoop() { - defer pm.wg.Done() - - ctx, cancel := pm.stopCh.NewCtx() - defer cancel() - - ticker := services.NewTicker(pm.flushDeletesFrequency) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - queuedReqs := pm.resetDeleteQueue() - if err := pm.orm.DeleteTransmitRequests(ctx, pm.serverURL, queuedReqs); err != nil { - pm.lggr.Errorw("Failed to delete queued transmit requests", "err", err) - pm.addToDeleteQueue(queuedReqs...) - } else { - pm.lggr.Debugw("Deleted queued transmit requests") - } - } - } -} - -func (pm *PersistenceManager) runPruneLoop() { - defer pm.wg.Done() - - ctx, cancel := pm.stopCh.NewCtx() - defer cancel() - - ticker := services.NewTicker(pm.pruneFrequency) - defer ticker.Stop() - for { - select { - case <-ctx.Done(): - return - case <-ticker.C: - func(ctx context.Context) { - ctx, cancelPrune := context.WithTimeout(sqlutil.WithoutDefaultTimeout(ctx), time.Minute) - defer cancelPrune() - if err := pm.orm.PruneTransmitRequests(ctx, pm.serverURL, pm.jobID, pm.maxTransmitQueueSize); err != nil { - pm.lggr.Errorw("Failed to prune transmit requests table", "err", err) - } else { - pm.lggr.Debugw("Pruned transmit requests table") - } - }(ctx) - } - } -} - -func (pm *PersistenceManager) addToDeleteQueue(reqs ...*pb.TransmitRequest) { - pm.deleteMu.Lock() - defer pm.deleteMu.Unlock() - pm.deleteQueue = append(pm.deleteQueue, reqs...) -} - -func (pm *PersistenceManager) resetDeleteQueue() []*pb.TransmitRequest { - pm.deleteMu.Lock() - defer pm.deleteMu.Unlock() - queue := pm.deleteQueue - pm.deleteQueue = nil - return queue -} diff --git a/core/services/relay/evm/mercury/persistence_manager_test.go b/core/services/relay/evm/mercury/persistence_manager_test.go deleted file mode 100644 index f2f4c379a4c..00000000000 --- a/core/services/relay/evm/mercury/persistence_manager_test.go +++ /dev/null @@ -1,179 +0,0 @@ -package mercury - -import ( - "testing" - "time" - - "github.com/cometbft/cometbft/libs/rand" - "github.com/jmoiron/sqlx" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/zap/zapcore" - "go.uber.org/zap/zaptest/observer" - - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/pgtest" - "github.com/smartcontractkit/chainlink/v2/core/logger" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/wsrpc/pb" -) - -func bootstrapPersistenceManager(t *testing.T, jobID int32, db *sqlx.DB) (*PersistenceManager, *observer.ObservedLogs) { - t.Helper() - lggr, observedLogs := logger.TestLoggerObserved(t, zapcore.DebugLevel) - orm := NewORM(db) - return NewPersistenceManager(lggr, "mercuryserver.example", orm, jobID, 2, 5*time.Millisecond, 5*time.Millisecond), observedLogs -} - -func TestPersistenceManager(t *testing.T) { - jobID1 := rand.Int32() - jobID2 := jobID1 + 1 - - ctx := testutils.Context(t) - db := pgtest.NewSqlxDB(t) - pgtest.MustExec(t, db, `SET CONSTRAINTS mercury_transmit_requests_job_id_fkey DEFERRED`) - pgtest.MustExec(t, db, `SET CONSTRAINTS feed_latest_reports_job_id_fkey DEFERRED`) - pm, _ := bootstrapPersistenceManager(t, jobID1, db) - - reports := sampleReports - - err := pm.Insert(ctx, &pb.TransmitRequest{Payload: reports[0]}, ocrtypes.ReportContext{}) - require.NoError(t, err) - err = pm.Insert(ctx, &pb.TransmitRequest{Payload: reports[1]}, ocrtypes.ReportContext{}) - require.NoError(t, err) - - transmissions, err := pm.Load(ctx) - require.NoError(t, err) - require.Equal(t, []*Transmission{ - {Req: &pb.TransmitRequest{Payload: reports[0]}}, - {Req: &pb.TransmitRequest{Payload: reports[1]}}, - }, transmissions) - - err = pm.Delete(ctx, &pb.TransmitRequest{Payload: reports[0]}) - require.NoError(t, err) - - transmissions, err = pm.Load(ctx) - require.NoError(t, err) - require.Equal(t, []*Transmission{ - {Req: &pb.TransmitRequest{Payload: reports[1]}}, - }, transmissions) - - t.Run("scopes load to only transmissions with matching job ID", func(t *testing.T) { - pm2, _ := bootstrapPersistenceManager(t, jobID2, db) - transmissions, err = pm2.Load(ctx) - require.NoError(t, err) - - assert.Len(t, transmissions, 0) - }) -} - -func TestPersistenceManagerAsyncDelete(t *testing.T) { - ctx := testutils.Context(t) - jobID := rand.Int32() - db := pgtest.NewSqlxDB(t) - pgtest.MustExec(t, db, `SET CONSTRAINTS mercury_transmit_requests_job_id_fkey DEFERRED`) - pgtest.MustExec(t, db, `SET CONSTRAINTS feed_latest_reports_job_id_fkey DEFERRED`) - pm, observedLogs := bootstrapPersistenceManager(t, jobID, db) - - reports := sampleReports - - err := pm.Insert(ctx, &pb.TransmitRequest{Payload: reports[0]}, ocrtypes.ReportContext{}) - require.NoError(t, err) - err = pm.Insert(ctx, &pb.TransmitRequest{Payload: reports[1]}, ocrtypes.ReportContext{}) - require.NoError(t, err) - - err = pm.Start(ctx) - require.NoError(t, err) - - pm.AsyncDelete(&pb.TransmitRequest{Payload: reports[0]}) - - // Wait for next poll. - observedLogs.TakeAll() - testutils.WaitForLogMessage(t, observedLogs, "Deleted queued transmit requests") - - transmissions, err := pm.Load(ctx) - require.NoError(t, err) - require.Equal(t, []*Transmission{ - {Req: &pb.TransmitRequest{Payload: reports[1]}}, - }, transmissions) - - // Test AsyncDelete is a no-op after Close. - err = pm.Close() - require.NoError(t, err) - - pm.AsyncDelete(&pb.TransmitRequest{Payload: reports[1]}) - - time.Sleep(15 * time.Millisecond) - - transmissions, err = pm.Load(ctx) - require.NoError(t, err) - require.Equal(t, []*Transmission{ - {Req: &pb.TransmitRequest{Payload: reports[1]}}, - }, transmissions) -} - -func TestPersistenceManagerPrune(t *testing.T) { - jobID1 := rand.Int32() - jobID2 := jobID1 + 1 - db := pgtest.NewSqlxDB(t) - pgtest.MustExec(t, db, `SET CONSTRAINTS mercury_transmit_requests_job_id_fkey DEFERRED`) - pgtest.MustExec(t, db, `SET CONSTRAINTS feed_latest_reports_job_id_fkey DEFERRED`) - - ctx := testutils.Context(t) - - reports := make([][]byte, 25) - for i := 0; i < 25; i++ { - reports[i] = buildSampleV1Report(int64(i)) - } - - pm2, _ := bootstrapPersistenceManager(t, jobID2, db) - for i := 0; i < 20; i++ { - err := pm2.Insert(ctx, &pb.TransmitRequest{Payload: reports[i]}, ocrtypes.ReportContext{ReportTimestamp: ocrtypes.ReportTimestamp{Epoch: uint32(i)}}) //nolint:gosec // G115 - require.NoError(t, err) - } - - pm, observedLogs := bootstrapPersistenceManager(t, jobID1, db) - - err := pm.Insert(ctx, &pb.TransmitRequest{Payload: reports[21]}, ocrtypes.ReportContext{ReportTimestamp: ocrtypes.ReportTimestamp{Epoch: 21}}) - require.NoError(t, err) - err = pm.Insert(ctx, &pb.TransmitRequest{Payload: reports[22]}, ocrtypes.ReportContext{ReportTimestamp: ocrtypes.ReportTimestamp{Epoch: 22}}) - require.NoError(t, err) - err = pm.Insert(ctx, &pb.TransmitRequest{Payload: reports[23]}, ocrtypes.ReportContext{ReportTimestamp: ocrtypes.ReportTimestamp{Epoch: 23}}) - require.NoError(t, err) - - err = pm.Start(ctx) - require.NoError(t, err) - - // Wait for next poll. - observedLogs.TakeAll() - testutils.WaitForLogMessage(t, observedLogs, "Pruned transmit requests table") - - transmissions, err := pm.Load(ctx) - require.NoError(t, err) - require.Equal(t, []*Transmission{ - {Req: &pb.TransmitRequest{Payload: reports[23]}, ReportCtx: ocrtypes.ReportContext{ReportTimestamp: ocrtypes.ReportTimestamp{Epoch: 23}}}, - {Req: &pb.TransmitRequest{Payload: reports[22]}, ReportCtx: ocrtypes.ReportContext{ReportTimestamp: ocrtypes.ReportTimestamp{Epoch: 22}}}, - }, transmissions) - - // Test pruning stops after Close. - err = pm.Close() - require.NoError(t, err) - - err = pm.Insert(ctx, &pb.TransmitRequest{Payload: reports[24]}, ocrtypes.ReportContext{ReportTimestamp: ocrtypes.ReportTimestamp{Epoch: 24}}) - require.NoError(t, err) - - transmissions, err = pm.Load(ctx) - require.NoError(t, err) - require.Equal(t, []*Transmission{ - {Req: &pb.TransmitRequest{Payload: reports[24]}, ReportCtx: ocrtypes.ReportContext{ReportTimestamp: ocrtypes.ReportTimestamp{Epoch: 24}}}, - {Req: &pb.TransmitRequest{Payload: reports[23]}, ReportCtx: ocrtypes.ReportContext{ReportTimestamp: ocrtypes.ReportTimestamp{Epoch: 23}}}, - {Req: &pb.TransmitRequest{Payload: reports[22]}, ReportCtx: ocrtypes.ReportContext{ReportTimestamp: ocrtypes.ReportTimestamp{Epoch: 22}}}, - }, transmissions) - - t.Run("prune was scoped to job ID", func(t *testing.T) { - transmissions, err = pm2.Load(ctx) - require.NoError(t, err) - assert.Len(t, transmissions, 20) - }) -} diff --git a/core/services/relay/evm/mercury/queue.go b/core/services/relay/evm/mercury/queue.go deleted file mode 100644 index a450d21af6e..00000000000 --- a/core/services/relay/evm/mercury/queue.go +++ /dev/null @@ -1,259 +0,0 @@ -package mercury - -import ( - "context" - "errors" - "fmt" - "sync" - "time" - - heap "github.com/esote/minmaxheap" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/services" - - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/wsrpc/pb" -) - -type asyncDeleter interface { - AsyncDelete(req *pb.TransmitRequest) -} - -var _ services.Service = (*transmitQueue)(nil) - -var transmitQueueLoad = promauto.NewGaugeVec(prometheus.GaugeOpts{ - Name: "mercury_transmit_queue_load", - Help: "Current count of items in the transmit queue", -}, - []string{"feedID", "serverURL", "capacity"}, -) - -// Prometheus' default interval is 15s, set this to under 7.5s to avoid -// aliasing (see: https://en.wikipedia.org/wiki/Nyquist_frequency) -const promInterval = 6500 * time.Millisecond - -// TransmitQueue is the high-level package that everything outside of this file should be using -// It stores pending transmissions, yielding the latest (highest priority) first to the caller -type transmitQueue struct { - services.StateMachine - - cond sync.Cond - lggr logger.SugaredLogger - asyncDeleter asyncDeleter - mu *sync.RWMutex - - pq *priorityQueue - maxlen int - closed bool - - // monitor loop - stopMonitor func() - transmitQueueLoad prometheus.Gauge -} - -type Transmission struct { - Req *pb.TransmitRequest // the payload to transmit - ReportCtx ocrtypes.ReportContext // contains priority information (latest epoch/round wins) -} - -type TransmitQueue interface { - services.Service - - BlockingPop() (t *Transmission) - Push(req *pb.TransmitRequest, reportCtx ocrtypes.ReportContext) (ok bool) - Init(transmissions []*Transmission) - IsEmpty() bool -} - -// maxlen controls how many items will be stored in the queue -// 0 means unlimited - be careful, this can cause memory leaks -func NewTransmitQueue(lggr logger.Logger, serverURL, feedID string, maxlen int, asyncDeleter asyncDeleter) TransmitQueue { - mu := new(sync.RWMutex) - return &transmitQueue{ - services.StateMachine{}, - sync.Cond{L: mu}, - logger.Sugared(lggr).Named("TransmitQueue"), - asyncDeleter, - mu, - nil, // pq needs to be initialized by calling tq.Init before use - maxlen, - false, - nil, - transmitQueueLoad.WithLabelValues(feedID, serverURL, fmt.Sprintf("%d", maxlen)), - } -} - -func (tq *transmitQueue) Init(transmissions []*Transmission) { - pq := priorityQueue(transmissions) - heap.Init(&pq) // ensure the heap is ordered - tq.pq = &pq -} - -func (tq *transmitQueue) Push(req *pb.TransmitRequest, reportCtx ocrtypes.ReportContext) (ok bool) { - tq.cond.L.Lock() - defer tq.cond.L.Unlock() - - if tq.closed { - return false - } - - if tq.maxlen != 0 && tq.pq.Len() == tq.maxlen { - // evict oldest entry to make room - tq.lggr.Criticalf("Transmit queue is full; dropping oldest transmission (reached max length of %d)", tq.maxlen) - removed := heap.PopMax(tq.pq) - if transmission, ok := removed.(*Transmission); ok { - tq.asyncDeleter.AsyncDelete(transmission.Req) - } - } - - heap.Push(tq.pq, &Transmission{req, reportCtx}) - tq.cond.Signal() - - return true -} - -// BlockingPop will block until at least one item is in the heap, and then return it -// If the queue is closed, it will immediately return nil -func (tq *transmitQueue) BlockingPop() (t *Transmission) { - tq.cond.L.Lock() - defer tq.cond.L.Unlock() - if tq.closed { - return nil - } - for t = tq.pop(); t == nil; t = tq.pop() { - tq.cond.Wait() - if tq.closed { - return nil - } - } - return t -} - -func (tq *transmitQueue) IsEmpty() bool { - tq.mu.RLock() - defer tq.mu.RUnlock() - return tq.pq.Len() == 0 -} - -func (tq *transmitQueue) Start(context.Context) error { - return tq.StartOnce("TransmitQueue", func() error { - t := services.NewTicker(promInterval) - wg := new(sync.WaitGroup) - chStop := make(chan struct{}) - tq.stopMonitor = func() { - t.Stop() - close(chStop) - wg.Wait() - } - wg.Add(1) - go tq.monitorLoop(t.C, chStop, wg) - return nil - }) -} - -func (tq *transmitQueue) Close() error { - return tq.StopOnce("TransmitQueue", func() error { - tq.cond.L.Lock() - tq.closed = true - tq.cond.L.Unlock() - tq.cond.Broadcast() - tq.stopMonitor() - return nil - }) -} - -func (tq *transmitQueue) monitorLoop(c <-chan time.Time, chStop <-chan struct{}, wg *sync.WaitGroup) { - defer wg.Done() - - for { - select { - case <-c: - tq.report() - case <-chStop: - return - } - } -} - -func (tq *transmitQueue) report() { - tq.mu.RLock() - length := tq.pq.Len() - tq.mu.RUnlock() - tq.transmitQueueLoad.Set(float64(length)) -} - -func (tq *transmitQueue) Ready() error { - return nil -} -func (tq *transmitQueue) Name() string { return tq.lggr.Name() } -func (tq *transmitQueue) HealthReport() map[string]error { - report := map[string]error{tq.Name(): errors.Join( - tq.status(), - )} - return report -} - -func (tq *transmitQueue) status() (merr error) { - tq.mu.RLock() - length := tq.pq.Len() - closed := tq.closed - tq.mu.RUnlock() - if tq.maxlen != 0 && length > (tq.maxlen/2) { - merr = errors.Join(merr, fmt.Errorf("transmit priority queue is greater than 50%% full (%d/%d)", length, tq.maxlen)) - } - if closed { - merr = errors.New("transmit queue is closed") - } - return merr -} - -// pop latest Transmission from the heap -// Not thread-safe -func (tq *transmitQueue) pop() *Transmission { - if tq.pq.Len() == 0 { - return nil - } - return heap.Pop(tq.pq).(*Transmission) -} - -// HEAP -// Adapted from https://pkg.go.dev/container/heap#example-package-PriorityQueue - -// WARNING: None of these methods are thread-safe, caller must synchronize - -var _ heap.Interface = &priorityQueue{} - -type priorityQueue []*Transmission - -func (pq priorityQueue) Len() int { return len(pq) } - -func (pq priorityQueue) Less(i, j int) bool { - // We want Pop to give us the latest round, so we use greater than here - // i.e. a later epoch/round is "less" than an earlier one - return pq[i].ReportCtx.ReportTimestamp.Epoch > pq[j].ReportCtx.ReportTimestamp.Epoch && - pq[i].ReportCtx.ReportTimestamp.Round > pq[j].ReportCtx.ReportTimestamp.Round -} - -func (pq priorityQueue) Swap(i, j int) { - pq[i], pq[j] = pq[j], pq[i] -} - -func (pq *priorityQueue) Pop() any { - n := len(*pq) - if n == 0 { - return nil - } - old := *pq - item := old[n-1] - old[n-1] = nil // avoid memory leak - *pq = old[0 : n-1] - return item -} - -func (pq *priorityQueue) Push(x any) { - *pq = append(*pq, x.(*Transmission)) -} diff --git a/core/services/relay/evm/mercury/queue_test.go b/core/services/relay/evm/mercury/queue_test.go deleted file mode 100644 index 8e5a0caf614..00000000000 --- a/core/services/relay/evm/mercury/queue_test.go +++ /dev/null @@ -1,149 +0,0 @@ -package mercury - -import ( - "sync" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "go.uber.org/zap/zapcore" - - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" - "github.com/smartcontractkit/chainlink/v2/core/logger" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/mocks" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/wsrpc/pb" -) - -type TestTransmissionWithReport struct { - tr *pb.TransmitRequest - ctx ocrtypes.ReportContext -} - -func createTestTransmissions(t *testing.T) []TestTransmissionWithReport { - t.Helper() - return []TestTransmissionWithReport{ - { - tr: &pb.TransmitRequest{ - Payload: []byte("test1"), - }, - ctx: ocrtypes.ReportContext{ - ReportTimestamp: ocrtypes.ReportTimestamp{ - Epoch: 1, - Round: 1, - ConfigDigest: ocrtypes.ConfigDigest{}, - }, - }, - }, - { - tr: &pb.TransmitRequest{ - Payload: []byte("test2"), - }, - ctx: ocrtypes.ReportContext{ - ReportTimestamp: ocrtypes.ReportTimestamp{ - Epoch: 2, - Round: 2, - ConfigDigest: ocrtypes.ConfigDigest{}, - }, - }, - }, - { - tr: &pb.TransmitRequest{ - Payload: []byte("test3"), - }, - ctx: ocrtypes.ReportContext{ - ReportTimestamp: ocrtypes.ReportTimestamp{ - Epoch: 3, - Round: 3, - ConfigDigest: ocrtypes.ConfigDigest{}, - }, - }, - }, - } -} - -func Test_Queue(t *testing.T) { - t.Parallel() - lggr, observedLogs := logger.TestLoggerObserved(t, zapcore.ErrorLevel) - testTransmissions := createTestTransmissions(t) - deleter := mocks.NewAsyncDeleter(t) - transmitQueue := NewTransmitQueue(lggr, sURL, "foo feed ID", 7, deleter) - transmitQueue.Init([]*Transmission{}) - - t.Run("successfully add transmissions to transmit queue", func(t *testing.T) { - for _, tt := range testTransmissions { - ok := transmitQueue.Push(tt.tr, tt.ctx) - require.True(t, ok) - } - report := transmitQueue.HealthReport() - assert.Nil(t, report[transmitQueue.Name()]) - }) - - t.Run("transmit queue is more than 50% full", func(t *testing.T) { - transmitQueue.Push(testTransmissions[2].tr, testTransmissions[2].ctx) - report := transmitQueue.HealthReport() - assert.Equal(t, report[transmitQueue.Name()].Error(), "transmit priority queue is greater than 50% full (4/7)") - }) - - t.Run("transmit queue pops the highest priority transmission", func(t *testing.T) { - tr := transmitQueue.BlockingPop() - assert.Equal(t, testTransmissions[2].tr, tr.Req) - }) - - t.Run("transmit queue is full and evicts the oldest transmission", func(t *testing.T) { - deleter.On("AsyncDelete", testTransmissions[0].tr).Once() - - // add 5 more transmissions to overflow the queue by 1 - for i := 0; i < 5; i++ { - transmitQueue.Push(testTransmissions[1].tr, testTransmissions[1].ctx) - } - - // expecting testTransmissions[0] to get evicted and not present in the queue anymore - testutils.WaitForLogMessage(t, observedLogs, "Transmit queue is full; dropping oldest transmission (reached max length of 7)") - for i := 0; i < 7; i++ { - tr := transmitQueue.BlockingPop() - assert.NotEqual(t, tr.Req, testTransmissions[0].tr) - } - }) - - t.Run("transmit queue blocks when empty and resumes when tranmission available", func(t *testing.T) { - assert.True(t, transmitQueue.IsEmpty()) - - var wg sync.WaitGroup - wg.Add(2) - go func() { - defer wg.Done() - tr := transmitQueue.BlockingPop() - assert.Equal(t, tr.Req, testTransmissions[0].tr) - }() - go func() { - defer wg.Done() - transmitQueue.Push(testTransmissions[0].tr, testTransmissions[0].ctx) - }() - wg.Wait() - }) - - t.Run("initializes transmissions", func(t *testing.T) { - transmissions := []*Transmission{ - { - Req: &pb.TransmitRequest{ - Payload: []byte("new1"), - }, - ReportCtx: ocrtypes.ReportContext{ - ReportTimestamp: ocrtypes.ReportTimestamp{ - Epoch: 1, - Round: 1, - ConfigDigest: ocrtypes.ConfigDigest{}, - }, - }, - }, - } - transmitQueue := NewTransmitQueue(lggr, sURL, "foo feed ID", 7, deleter) - transmitQueue.Init(transmissions) - - transmission := transmitQueue.BlockingPop() - assert.Equal(t, transmission.Req.Payload, []byte("new1")) - assert.True(t, transmitQueue.IsEmpty()) - }) -} diff --git a/core/services/relay/evm/mercury/test_helpers.go b/core/services/relay/evm/mercury/test_helpers.go deleted file mode 100644 index 27093268b3e..00000000000 --- a/core/services/relay/evm/mercury/test_helpers.go +++ /dev/null @@ -1,39 +0,0 @@ -package mercury - -import ( - "github.com/ethereum/go-ethereum/common/hexutil" - - "github.com/smartcontractkit/libocr/offchainreporting2plus/chains/evmutil" - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" -) - -func BuildSamplePayload(report []byte, reportCtx ocrtypes.ReportContext, sigs []ocrtypes.AttributedOnchainSignature) []byte { - var rs [][32]byte - var ss [][32]byte - var vs [32]byte - for i, as := range sigs { - r, s, v, err := evmutil.SplitSignature(as.Signature) - if err != nil { - panic("eventTransmit(ev): error in SplitSignature") - } - rs = append(rs, r) - ss = append(ss, s) - vs[i] = v - } - rawReportCtx := evmutil.RawReportContext(reportCtx) - payload, err := PayloadTypes.Pack(rawReportCtx, report, rs, ss, vs) - if err != nil { - panic(err) - } - return payload -} - -func MustHexToConfigDigest(s string) (cd ocrtypes.ConfigDigest) { - b := hexutil.MustDecode(s) - var err error - cd, err = ocrtypes.BytesToConfigDigest(b) - if err != nil { - panic(err) - } - return -} diff --git a/core/services/relay/evm/mercury/transmitter.go b/core/services/relay/evm/mercury/transmitter.go deleted file mode 100644 index be500593bf3..00000000000 --- a/core/services/relay/evm/mercury/transmitter.go +++ /dev/null @@ -1,613 +0,0 @@ -package mercury - -import ( - "bytes" - "context" - "crypto/ed25519" - "errors" - "fmt" - "io" - "math/big" - "sort" - "sync" - "time" - - "github.com/ethereum/go-ethereum/accounts/abi" - "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/jpillora/backoff" - pkgerrors "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - "golang.org/x/exp/maps" - "golang.org/x/sync/errgroup" - - "github.com/smartcontractkit/libocr/offchainreporting2plus/chains/evmutil" - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - - capStreams "github.com/smartcontractkit/chainlink-common/pkg/capabilities/datastreams" - "github.com/smartcontractkit/chainlink-common/pkg/capabilities/triggers" - commonconfig "github.com/smartcontractkit/chainlink-common/pkg/config" - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/services" - "github.com/smartcontractkit/chainlink-common/pkg/types/mercury" - - mercuryutils "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/utils" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/wsrpc" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/wsrpc/pb" - "github.com/smartcontractkit/chainlink/v2/core/utils" -) - -const ( - // Mercury server error codes - DuplicateReport = 2 -) - -var ( - transmitSuccessCount = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "mercury_transmit_success_count", - Help: "Number of successful transmissions (duplicates are counted as success)", - }, - []string{"feedID", "serverURL"}, - ) - transmitDuplicateCount = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "mercury_transmit_duplicate_count", - Help: "Number of transmissions where the server told us it was a duplicate", - }, - []string{"feedID", "serverURL"}, - ) - transmitConnectionErrorCount = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "mercury_transmit_connection_error_count", - Help: "Number of errored transmissions that failed due to problem with the connection", - }, - []string{"feedID", "serverURL"}, - ) - transmitQueueDeleteErrorCount = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "mercury_transmit_queue_delete_error_count", - Help: "Running count of DB errors when trying to delete an item from the queue DB", - }, - []string{"feedID", "serverURL"}, - ) - transmitQueueInsertErrorCount = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "mercury_transmit_queue_insert_error_count", - Help: "Running count of DB errors when trying to insert an item into the queue DB", - }, - []string{"feedID", "serverURL"}, - ) - transmitQueuePushErrorCount = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "mercury_transmit_queue_push_error_count", - Help: "Running count of DB errors when trying to push an item onto the queue", - }, - []string{"feedID", "serverURL"}, - ) - transmitServerErrorCount = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "mercury_transmit_server_error_count", - Help: "Number of errored transmissions that failed due to an error returned by the mercury server", - }, - []string{"feedID", "serverURL", "code"}, - ) -) - -type Transmitter interface { - mercury.Transmitter - services.Service -} - -type ConfigTracker interface { - LatestConfigDetails(ctx context.Context) (changedInBlock uint64, configDigest ocrtypes.ConfigDigest, err error) -} - -type TransmitterReportDecoder interface { - BenchmarkPriceFromReport(ctx context.Context, report ocrtypes.Report) (*big.Int, error) - ObservationTimestampFromReport(ctx context.Context, report ocrtypes.Report) (uint32, error) -} - -type BenchmarkPriceDecoder func(ctx context.Context, feedID mercuryutils.FeedID, report ocrtypes.Report) (*big.Int, error) - -var _ Transmitter = (*mercuryTransmitter)(nil) - -type TransmitterConfig interface { - TransmitTimeout() commonconfig.Duration -} - -type mercuryTransmitter struct { - services.StateMachine - lggr logger.SugaredLogger - cfg TransmitterConfig - - orm ORM - servers map[string]*server - - codec TransmitterReportDecoder - benchmarkPriceDecoder BenchmarkPriceDecoder - triggerCapability *triggers.MercuryTriggerService - - feedID mercuryutils.FeedID - jobID int32 - fromAccount string - - stopCh services.StopChan - wg *sync.WaitGroup -} - -var PayloadTypes = getPayloadTypes() - -func getPayloadTypes() abi.Arguments { - mustNewType := func(t string) abi.Type { - result, err := abi.NewType(t, "", []abi.ArgumentMarshaling{}) - if err != nil { - panic(fmt.Sprintf("Unexpected error during abi.NewType: %s", err)) - } - return result - } - return abi.Arguments([]abi.Argument{ - {Name: "reportContext", Type: mustNewType("bytes32[3]")}, - {Name: "report", Type: mustNewType("bytes")}, - {Name: "rawRs", Type: mustNewType("bytes32[]")}, - {Name: "rawSs", Type: mustNewType("bytes32[]")}, - {Name: "rawVs", Type: mustNewType("bytes32")}, - }) -} - -type server struct { - lggr logger.SugaredLogger - - transmitTimeout time.Duration - - c wsrpc.Client - pm *PersistenceManager - q TransmitQueue - - deleteQueue chan *pb.TransmitRequest - - url string - - transmitSuccessCount prometheus.Counter - transmitDuplicateCount prometheus.Counter - transmitConnectionErrorCount prometheus.Counter - transmitQueueDeleteErrorCount prometheus.Counter - transmitQueueInsertErrorCount prometheus.Counter - transmitQueuePushErrorCount prometheus.Counter -} - -func (s *server) HealthReport() map[string]error { - report := map[string]error{} - services.CopyHealth(report, s.c.HealthReport()) - services.CopyHealth(report, s.q.HealthReport()) - return report -} - -func (s *server) runDeleteQueueLoop(stopCh services.StopChan, wg *sync.WaitGroup) { - defer wg.Done() - ctx, cancel := stopCh.NewCtx() - defer cancel() - - // Exponential backoff for very rarely occurring errors (DB disconnect etc) - b := backoff.Backoff{ - Min: 1 * time.Second, - Max: 120 * time.Second, - Factor: 2, - Jitter: true, - } - - for { - select { - case req := <-s.deleteQueue: - for { - if err := s.pm.Delete(ctx, req); err != nil { - s.lggr.Errorw("Failed to delete transmit request record", "err", err, "req.Payload", req.Payload) - s.transmitQueueDeleteErrorCount.Inc() - select { - case <-time.After(b.Duration()): - // Wait a backoff duration before trying to delete again - continue - case <-stopCh: - // abort and return immediately on stop even if items remain in queue - return - } - } - break - } - // success - b.Reset() - case <-stopCh: - // abort and return immediately on stop even if items remain in queue - return - } - } -} - -func (s *server) runQueueLoop(stopCh services.StopChan, wg *sync.WaitGroup, feedIDHex string) { - defer wg.Done() - // Exponential backoff with very short retry interval (since latency is a priority) - // 5ms, 10ms, 20ms, 40ms etc - b := backoff.Backoff{ - Min: 5 * time.Millisecond, - Max: 1 * time.Second, - Factor: 2, - Jitter: true, - } - ctx, cancel := stopCh.NewCtx() - defer cancel() - for { - t := s.q.BlockingPop() - if t == nil { - // queue was closed - return - } - res, err := func(ctx context.Context) (*pb.TransmitResponse, error) { - ctx, cancel := context.WithTimeout(ctx, utils.WithJitter(s.transmitTimeout)) - defer cancel() - return s.c.Transmit(ctx, t.Req) - }(ctx) - if ctx.Err() != nil { - // only canceled on transmitter close so we can exit - return - } else if err != nil { - s.transmitConnectionErrorCount.Inc() - s.lggr.Errorw("Transmit report failed", "err", err, "reportCtx", t.ReportCtx) - if ok := s.q.Push(t.Req, t.ReportCtx); !ok { - s.lggr.Error("Failed to push report to transmit queue; queue is closed") - return - } - // Wait a backoff duration before pulling the most recent transmission - // the heap - select { - case <-time.After(b.Duration()): - continue - case <-stopCh: - return - } - } - - b.Reset() - if res.Error == "" { - s.transmitSuccessCount.Inc() - s.lggr.Debugw("Transmit report success", "payload", hexutil.Encode(t.Req.Payload), "response", res, "repts", t.ReportCtx.ReportTimestamp) - } else { - // We don't need to retry here because the mercury server - // has confirmed it received the report. We only need to retry - // on networking/unknown errors - switch res.Code { - case DuplicateReport: - s.transmitSuccessCount.Inc() - s.transmitDuplicateCount.Inc() - s.lggr.Debugw("Transmit report success; duplicate report", "payload", hexutil.Encode(t.Req.Payload), "response", res, "repts", t.ReportCtx.ReportTimestamp) - default: - transmitServerErrorCount.WithLabelValues(feedIDHex, s.url, fmt.Sprintf("%d", res.Code)).Inc() - s.lggr.Errorw("Transmit report failed; mercury server returned error", "response", res, "reportCtx", t.ReportCtx, "err", res.Error, "code", res.Code) - } - } - - select { - case s.deleteQueue <- t.Req: - default: - s.lggr.Criticalw("Delete queue is full", "reportCtx", t.ReportCtx) - } - } -} - -const TransmitQueueMaxSize = 10_000 // hardcode this for legacy transmitter since we want the config var to apply only to LLO - -func newServer(lggr logger.Logger, cfg TransmitterConfig, client wsrpc.Client, pm *PersistenceManager, serverURL, feedIDHex string) *server { - return &server{ - logger.Sugared(lggr), - cfg.TransmitTimeout().Duration(), - client, - pm, - NewTransmitQueue(lggr, serverURL, feedIDHex, TransmitQueueMaxSize, pm), - make(chan *pb.TransmitRequest, TransmitQueueMaxSize), - serverURL, - transmitSuccessCount.WithLabelValues(feedIDHex, serverURL), - transmitDuplicateCount.WithLabelValues(feedIDHex, serverURL), - transmitConnectionErrorCount.WithLabelValues(feedIDHex, serverURL), - transmitQueueDeleteErrorCount.WithLabelValues(feedIDHex, serverURL), - transmitQueueInsertErrorCount.WithLabelValues(feedIDHex, serverURL), - transmitQueuePushErrorCount.WithLabelValues(feedIDHex, serverURL), - } -} - -func NewTransmitter(lggr logger.Logger, cfg TransmitterConfig, clients map[string]wsrpc.Client, fromAccount ed25519.PublicKey, jobID int32, feedID [32]byte, orm ORM, codec TransmitterReportDecoder, benchmarkPriceDecoder BenchmarkPriceDecoder, triggerCapability *triggers.MercuryTriggerService) *mercuryTransmitter { - sugared := logger.Sugared(lggr) - feedIDHex := fmt.Sprintf("0x%x", feedID[:]) - servers := make(map[string]*server, len(clients)) - for serverURL, client := range clients { - cLggr := sugared.Named(serverURL).With("serverURL", serverURL) - pm := NewPersistenceManager(cLggr, serverURL, orm, jobID, TransmitQueueMaxSize, flushDeletesFrequency, pruneFrequency) - servers[serverURL] = newServer(cLggr, cfg, client, pm, serverURL, feedIDHex) - } - return &mercuryTransmitter{ - services.StateMachine{}, - sugared.Named("MercuryTransmitter").With("feedID", feedIDHex), - cfg, - orm, - servers, - codec, - benchmarkPriceDecoder, - triggerCapability, - feedID, - jobID, - fmt.Sprintf("%x", fromAccount), - make(services.StopChan), - &sync.WaitGroup{}, - } -} - -func (mt *mercuryTransmitter) Start(ctx context.Context) (err error) { - return mt.StartOnce("MercuryTransmitter", func() error { - mt.lggr.Debugw("Loading transmit requests from database") - - { - var startClosers []services.StartClose - for _, s := range mt.servers { - transmissions, err := s.pm.Load(ctx) - if err != nil { - return err - } - s.q.Init(transmissions) - // starting pm after loading from it is fine because it simply spawns some garbage collection/prune goroutines - startClosers = append(startClosers, s.c, s.q, s.pm) - - mt.wg.Add(2) - go s.runDeleteQueueLoop(mt.stopCh, mt.wg) - go s.runQueueLoop(mt.stopCh, mt.wg, mt.feedID.Hex()) - } - if err := (&services.MultiStart{}).Start(ctx, startClosers...); err != nil { - return err - } - } - - return nil - }) -} - -func (mt *mercuryTransmitter) Close() error { - return mt.StopOnce("MercuryTransmitter", func() error { - // Drain all the queues first - var qs []io.Closer - for _, s := range mt.servers { - qs = append(qs, s.q) - } - if err := services.CloseAll(qs...); err != nil { - return err - } - - close(mt.stopCh) - mt.wg.Wait() - - // Close all the persistence managers - // Close all the clients - var closers []io.Closer - for _, s := range mt.servers { - closers = append(closers, s.pm) - closers = append(closers, s.c) - } - return services.CloseAll(closers...) - }) -} - -func (mt *mercuryTransmitter) Name() string { return mt.lggr.Name() } - -func (mt *mercuryTransmitter) HealthReport() map[string]error { - report := map[string]error{mt.Name(): mt.Healthy()} - for _, s := range mt.servers { - services.CopyHealth(report, s.HealthReport()) - } - return report -} - -func (mt *mercuryTransmitter) sendToTrigger(report ocrtypes.Report, rawReportCtx [3][32]byte, signatures []ocrtypes.AttributedOnchainSignature) error { - rawSignatures := [][]byte{} - for _, sig := range signatures { - rawSignatures = append(rawSignatures, sig.Signature) - } - - reportContextFlat := []byte{} - reportContextFlat = append(reportContextFlat, rawReportCtx[0][:]...) - reportContextFlat = append(reportContextFlat, rawReportCtx[1][:]...) - reportContextFlat = append(reportContextFlat, rawReportCtx[2][:]...) - - converted := capStreams.FeedReport{ - FeedID: mt.feedID.Hex(), - FullReport: report, - ReportContext: reportContextFlat, - Signatures: rawSignatures, - // NOTE: Skipping fields derived from FullReport, they will be filled out at a later stage - // after decoding and validating signatures. - } - return mt.triggerCapability.ProcessReport([]capStreams.FeedReport{converted}) -} - -// Transmit sends the report to the on-chain smart contract's Transmit method. -func (mt *mercuryTransmitter) Transmit(ctx context.Context, reportCtx ocrtypes.ReportContext, report ocrtypes.Report, signatures []ocrtypes.AttributedOnchainSignature) error { - rawReportCtx := evmutil.RawReportContext(reportCtx) - if mt.triggerCapability != nil { - // Acting as a Capability - send report to trigger service and exit. - return mt.sendToTrigger(report, rawReportCtx, signatures) - } - - var rs [][32]byte - var ss [][32]byte - var vs [32]byte - for i, as := range signatures { - r, s, v, err := evmutil.SplitSignature(as.Signature) - if err != nil { - panic("eventTransmit(ev): error in SplitSignature") - } - rs = append(rs, r) - ss = append(ss, s) - vs[i] = v - } - - payload, err := PayloadTypes.Pack(rawReportCtx, []byte(report), rs, ss, vs) - if err != nil { - return pkgerrors.Wrap(err, "abi.Pack failed") - } - - req := &pb.TransmitRequest{ - Payload: payload, - } - - ts, err := mt.codec.ObservationTimestampFromReport(ctx, report) - if err != nil { - mt.lggr.Warnw("Failed to get observation timestamp from report", "err", err) - } - mt.lggr.Debugw("Transmit enqueue", "req.Payload", hexutil.Encode(req.Payload), "report", report, "repts", reportCtx.ReportTimestamp, "signatures", signatures, "observationsTimestamp", ts) - - if err := mt.orm.InsertTransmitRequest(ctx, maps.Keys(mt.servers), req, mt.jobID, reportCtx); err != nil { - return err - } - - g := new(errgroup.Group) - for _, s := range mt.servers { - s := s // https://golang.org/doc/faq#closures_and_goroutines - g.Go(func() error { - if ok := s.q.Push(req, reportCtx); !ok { - s.transmitQueuePushErrorCount.Inc() - return errors.New("transmit queue is closed") - } - return nil - }) - } - - return g.Wait() -} - -// FromAccount returns the stringified (hex) CSA public key -func (mt *mercuryTransmitter) FromAccount(ctx context.Context) (ocrtypes.Account, error) { - return ocrtypes.Account(mt.fromAccount), nil -} - -// LatestConfigDigestAndEpoch retrieves the latest config digest and epoch from the OCR2 contract. -func (mt *mercuryTransmitter) LatestConfigDigestAndEpoch(ctx context.Context) (cd ocrtypes.ConfigDigest, epoch uint32, err error) { - panic("not needed for OCR3") -} - -func (mt *mercuryTransmitter) FetchInitialMaxFinalizedBlockNumber(ctx context.Context) (*int64, error) { - mt.lggr.Trace("FetchInitialMaxFinalizedBlockNumber") - - report, err := mt.latestReport(ctx, mt.feedID) - if err != nil { - return nil, err - } - - if report == nil { - mt.lggr.Debugw("FetchInitialMaxFinalizedBlockNumber success; got nil report") - return nil, nil - } - - mt.lggr.Debugw("FetchInitialMaxFinalizedBlockNumber success", "currentBlockNum", report.CurrentBlockNumber) - - return &report.CurrentBlockNumber, nil -} - -func (mt *mercuryTransmitter) LatestPrice(ctx context.Context, feedID [32]byte) (*big.Int, error) { - mt.lggr.Trace("LatestPrice") - - fullReport, err := mt.latestReport(ctx, feedID) - if err != nil { - return nil, err - } - if fullReport == nil { - return nil, nil - } - payload := fullReport.Payload - m := make(map[string]interface{}) - if err := PayloadTypes.UnpackIntoMap(m, payload); err != nil { - return nil, err - } - report, is := m["report"].([]byte) - if !is { - return nil, fmt.Errorf("expected report to be []byte, but it was %T", m["report"]) - } - return mt.benchmarkPriceDecoder(ctx, feedID, report) -} - -// LatestTimestamp will return -1, nil if the feed is missing -func (mt *mercuryTransmitter) LatestTimestamp(ctx context.Context) (int64, error) { - mt.lggr.Trace("LatestTimestamp") - - report, err := mt.latestReport(ctx, mt.feedID) - if err != nil { - return 0, err - } - - if report == nil { - mt.lggr.Debugw("LatestTimestamp success; got nil report") - return -1, nil - } - - mt.lggr.Debugw("LatestTimestamp success", "timestamp", report.ObservationsTimestamp) - - return report.ObservationsTimestamp, nil -} - -func (mt *mercuryTransmitter) latestReport(ctx context.Context, feedID [32]byte) (*pb.Report, error) { - mt.lggr.Trace("latestReport") - - req := &pb.LatestReportRequest{ - FeedId: feedID[:], - } - - var reports []*pb.Report - mu := sync.Mutex{} - var g errgroup.Group - for _, s := range mt.servers { - s := s - g.Go(func() error { - resp, err := s.c.LatestReport(ctx, req) - if err != nil { - s.lggr.Warnw("latestReport failed", "err", err) - return err - } - if resp == nil { - err = errors.New("latestReport expected non-nil response from server") - s.lggr.Warn(err.Error()) - return err - } - if resp.Error != "" { - err = errors.New(resp.Error) - s.lggr.Warnw("latestReport failed; mercury server returned error", "err", err) - return fmt.Errorf("latestReport failed; mercury server returned error: %s", resp.Error) - } - if resp.Report == nil { - s.lggr.Tracew("latestReport success: returned nil") - } else if !bytes.Equal(resp.Report.FeedId, feedID[:]) { - err = fmt.Errorf("latestReport failed; mismatched feed IDs, expected: 0x%x, got: 0x%x", mt.feedID[:], resp.Report.FeedId) - s.lggr.Errorw("latestReport failed", "err", err) - return err - } else { - s.lggr.Tracew("latestReport success", "observationsTimestamp", resp.Report.ObservationsTimestamp, "currentBlockNum", resp.Report.CurrentBlockNumber) - } - mu.Lock() - defer mu.Unlock() - reports = append(reports, resp.Report) - return nil - }) - } - err := g.Wait() - - if len(reports) == 0 { - return nil, fmt.Errorf("latestReport failed; all servers returned an error: %w", err) - } - - sortReportsLatestFirst(reports) - - return reports[0], nil -} - -func sortReportsLatestFirst(reports []*pb.Report) { - sort.Slice(reports, func(i, j int) bool { - // nils are "earliest" so they go to the end - if reports[i] == nil { - return false - } else if reports[j] == nil { - return true - } - // Handle block number case - if reports[i].ObservationsTimestamp == reports[j].ObservationsTimestamp { - return reports[i].CurrentBlockNumber > reports[j].CurrentBlockNumber - } - // Timestamp case - return reports[i].ObservationsTimestamp > reports[j].ObservationsTimestamp - }) -} diff --git a/core/services/relay/evm/mercury/transmitter_test.go b/core/services/relay/evm/mercury/transmitter_test.go deleted file mode 100644 index 00eb63b4e8c..00000000000 --- a/core/services/relay/evm/mercury/transmitter_test.go +++ /dev/null @@ -1,599 +0,0 @@ -package mercury - -import ( - "context" - "math/big" - "sync" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common/hexutil" - "github.com/pkg/errors" - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/smartcontractkit/chainlink-common/pkg/capabilities/triggers" - commonconfig "github.com/smartcontractkit/chainlink-common/pkg/config" - - "github.com/smartcontractkit/chainlink/v2/core/config" - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils/pgtest" - "github.com/smartcontractkit/chainlink/v2/core/logger" - mercurytypes "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/types" - mercuryutils "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/utils" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/wsrpc" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/wsrpc/mocks" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/wsrpc/pb" - "github.com/smartcontractkit/chainlink/v2/evm/utils" -) - -type mockCfg struct{} - -func (m mockCfg) Protocol() config.MercuryTransmitterProtocol { - return config.MercuryTransmitterProtocolGRPC -} - -func (m mockCfg) TransmitQueueMaxSize() uint32 { - return 100_000 -} - -func (m mockCfg) TransmitTimeout() commonconfig.Duration { - return *commonconfig.MustNewDuration(1 * time.Hour) -} - -func Test_MercuryTransmitter_Transmit(t *testing.T) { - lggr := logger.TestLogger(t) - db := pgtest.NewSqlxDB(t) - var jobID int32 - pgtest.MustExec(t, db, `SET CONSTRAINTS mercury_transmit_requests_job_id_fkey DEFERRED`) - pgtest.MustExec(t, db, `SET CONSTRAINTS feed_latest_reports_job_id_fkey DEFERRED`) - codec := new(mockCodec) - benchmarkPriceDecoder := func(ctx context.Context, feedID mercuryutils.FeedID, report ocrtypes.Report) (*big.Int, error) { - return codec.BenchmarkPriceFromReport(ctx, report) - } - orm := NewORM(db) - clients := map[string]wsrpc.Client{} - - t.Run("with one mercury server", func(t *testing.T) { - t.Run("v1 report transmission successfully enqueued", func(t *testing.T) { - report := sampleV1Report - c := &mocks.MockWSRPCClient{} - clients[sURL] = c - mt := NewTransmitter(lggr, mockCfg{}, clients, sampleClientPubKey, jobID, sampleFeedID, orm, codec, benchmarkPriceDecoder, nil) - // init the queue since we skipped starting transmitter - mt.servers[sURL].q.Init([]*Transmission{}) - err := mt.Transmit(testutils.Context(t), sampleReportContext, report, sampleSigs) - require.NoError(t, err) - - // ensure it was added to the queue - require.Equal(t, mt.servers[sURL].q.(*transmitQueue).pq.Len(), 1) - assert.Subset(t, mt.servers[sURL].q.(*transmitQueue).pq.Pop().(*Transmission).Req.Payload, report) - }) - t.Run("v2 report transmission successfully enqueued", func(t *testing.T) { - report := sampleV2Report - c := &mocks.MockWSRPCClient{} - clients[sURL] = c - mt := NewTransmitter(lggr, mockCfg{}, clients, sampleClientPubKey, jobID, sampleFeedID, orm, codec, benchmarkPriceDecoder, nil) - // init the queue since we skipped starting transmitter - mt.servers[sURL].q.Init([]*Transmission{}) - err := mt.Transmit(testutils.Context(t), sampleReportContext, report, sampleSigs) - require.NoError(t, err) - - // ensure it was added to the queue - require.Equal(t, mt.servers[sURL].q.(*transmitQueue).pq.Len(), 1) - assert.Subset(t, mt.servers[sURL].q.(*transmitQueue).pq.Pop().(*Transmission).Req.Payload, report) - }) - t.Run("v3 report transmission successfully enqueued", func(t *testing.T) { - report := sampleV3Report - c := &mocks.MockWSRPCClient{} - clients[sURL] = c - mt := NewTransmitter(lggr, mockCfg{}, clients, sampleClientPubKey, jobID, sampleFeedID, orm, codec, benchmarkPriceDecoder, nil) - // init the queue since we skipped starting transmitter - mt.servers[sURL].q.Init([]*Transmission{}) - err := mt.Transmit(testutils.Context(t), sampleReportContext, report, sampleSigs) - require.NoError(t, err) - - // ensure it was added to the queue - require.Equal(t, mt.servers[sURL].q.(*transmitQueue).pq.Len(), 1) - assert.Subset(t, mt.servers[sURL].q.(*transmitQueue).pq.Pop().(*Transmission).Req.Payload, report) - }) - t.Run("v3 report transmission sent only to trigger service", func(t *testing.T) { - report := sampleV3Report - c := &mocks.MockWSRPCClient{} - clients[sURL] = c - triggerService, err := triggers.NewMercuryTriggerService(0, "", "", lggr) - require.NoError(t, err) - mt := NewTransmitter(lggr, mockCfg{}, clients, sampleClientPubKey, jobID, sampleFeedID, orm, codec, benchmarkPriceDecoder, triggerService) - // init the queue since we skipped starting transmitter - mt.servers[sURL].q.Init([]*Transmission{}) - err = mt.Transmit(testutils.Context(t), sampleReportContext, report, sampleSigs) - require.NoError(t, err) - // queue is empty - require.Equal(t, mt.servers[sURL].q.(*transmitQueue).pq.Len(), 0) - }) - }) - - t.Run("with multiple mercury servers", func(t *testing.T) { - report := sampleV3Report - c := &mocks.MockWSRPCClient{} - clients[sURL] = c - clients[sURL2] = c - clients[sURL3] = c - - mt := NewTransmitter(lggr, mockCfg{}, clients, sampleClientPubKey, jobID, sampleFeedID, orm, codec, benchmarkPriceDecoder, nil) - // init the queue since we skipped starting transmitter - mt.servers[sURL].q.Init([]*Transmission{}) - mt.servers[sURL2].q.Init([]*Transmission{}) - mt.servers[sURL3].q.Init([]*Transmission{}) - - err := mt.Transmit(testutils.Context(t), sampleReportContext, report, sampleSigs) - require.NoError(t, err) - - // ensure it was added to the queue - require.Equal(t, mt.servers[sURL].q.(*transmitQueue).pq.Len(), 1) - assert.Subset(t, mt.servers[sURL].q.(*transmitQueue).pq.Pop().(*Transmission).Req.Payload, report) - require.Equal(t, mt.servers[sURL2].q.(*transmitQueue).pq.Len(), 1) - assert.Subset(t, mt.servers[sURL2].q.(*transmitQueue).pq.Pop().(*Transmission).Req.Payload, report) - require.Equal(t, mt.servers[sURL3].q.(*transmitQueue).pq.Len(), 1) - assert.Subset(t, mt.servers[sURL3].q.(*transmitQueue).pq.Pop().(*Transmission).Req.Payload, report) - }) -} - -func Test_MercuryTransmitter_LatestTimestamp(t *testing.T) { - t.Parallel() - lggr := logger.TestLogger(t) - db := pgtest.NewSqlxDB(t) - var jobID int32 - codec := new(mockCodec) - benchmarkPriceDecoder := func(ctx context.Context, feedID mercuryutils.FeedID, report ocrtypes.Report) (*big.Int, error) { - return codec.BenchmarkPriceFromReport(ctx, report) - } - - orm := NewORM(db) - clients := map[string]wsrpc.Client{} - - t.Run("successful query", func(t *testing.T) { - c := &mocks.MockWSRPCClient{ - LatestReportF: func(ctx context.Context, in *pb.LatestReportRequest) (out *pb.LatestReportResponse, err error) { - require.NotNil(t, in) - assert.Equal(t, hexutil.Encode(sampleFeedID[:]), hexutil.Encode(in.FeedId)) - out = new(pb.LatestReportResponse) - out.Report = new(pb.Report) - out.Report.FeedId = sampleFeedID[:] - out.Report.ObservationsTimestamp = 42 - return out, nil - }, - } - clients[sURL] = c - mt := NewTransmitter(lggr, mockCfg{}, clients, sampleClientPubKey, jobID, sampleFeedID, orm, codec, benchmarkPriceDecoder, nil) - ts, err := mt.LatestTimestamp(testutils.Context(t)) - require.NoError(t, err) - - assert.Equal(t, int64(42), ts) - }) - - t.Run("successful query returning nil report (new feed) gives latest timestamp = -1", func(t *testing.T) { - c := &mocks.MockWSRPCClient{ - LatestReportF: func(ctx context.Context, in *pb.LatestReportRequest) (out *pb.LatestReportResponse, err error) { - out = new(pb.LatestReportResponse) - out.Report = nil - return out, nil - }, - } - clients[sURL] = c - mt := NewTransmitter(lggr, mockCfg{}, clients, sampleClientPubKey, jobID, sampleFeedID, orm, codec, benchmarkPriceDecoder, nil) - ts, err := mt.LatestTimestamp(testutils.Context(t)) - require.NoError(t, err) - - assert.Equal(t, int64(-1), ts) - }) - - t.Run("failing query", func(t *testing.T) { - c := &mocks.MockWSRPCClient{ - LatestReportF: func(ctx context.Context, in *pb.LatestReportRequest) (out *pb.LatestReportResponse, err error) { - return nil, errors.New("something exploded") - }, - } - clients[sURL] = c - mt := NewTransmitter(lggr, mockCfg{}, clients, sampleClientPubKey, jobID, sampleFeedID, orm, codec, benchmarkPriceDecoder, nil) - _, err := mt.LatestTimestamp(testutils.Context(t)) - require.Error(t, err) - assert.Contains(t, err.Error(), "something exploded") - }) - - t.Run("with multiple servers, uses latest", func(t *testing.T) { - clients[sURL] = &mocks.MockWSRPCClient{ - LatestReportF: func(ctx context.Context, in *pb.LatestReportRequest) (out *pb.LatestReportResponse, err error) { - return nil, errors.New("something exploded") - }, - } - clients[sURL2] = &mocks.MockWSRPCClient{ - LatestReportF: func(ctx context.Context, in *pb.LatestReportRequest) (out *pb.LatestReportResponse, err error) { - out = new(pb.LatestReportResponse) - out.Report = new(pb.Report) - out.Report.FeedId = sampleFeedID[:] - out.Report.ObservationsTimestamp = 42 - return out, nil - }, - } - clients[sURL3] = &mocks.MockWSRPCClient{ - LatestReportF: func(ctx context.Context, in *pb.LatestReportRequest) (out *pb.LatestReportResponse, err error) { - out = new(pb.LatestReportResponse) - out.Report = new(pb.Report) - out.Report.FeedId = sampleFeedID[:] - out.Report.ObservationsTimestamp = 41 - return out, nil - }, - } - mt := NewTransmitter(lggr, mockCfg{}, clients, sampleClientPubKey, jobID, sampleFeedID, orm, codec, benchmarkPriceDecoder, nil) - ts, err := mt.LatestTimestamp(testutils.Context(t)) - require.NoError(t, err) - - assert.Equal(t, int64(42), ts) - }) -} - -type mockCodec struct { - val *big.Int - err error -} - -var _ mercurytypes.ReportCodec = &mockCodec{} - -func (m *mockCodec) BenchmarkPriceFromReport(ctx context.Context, _ ocrtypes.Report) (*big.Int, error) { - return m.val, m.err -} - -func (m *mockCodec) ObservationTimestampFromReport(ctx context.Context, report ocrtypes.Report) (uint32, error) { - return 0, nil -} - -func Test_MercuryTransmitter_LatestPrice(t *testing.T) { - t.Parallel() - lggr := logger.TestLogger(t) - db := pgtest.NewSqlxDB(t) - var jobID int32 - - codec := new(mockCodec) - benchmarkPriceDecoder := func(ctx context.Context, feedID mercuryutils.FeedID, report ocrtypes.Report) (*big.Int, error) { - return codec.BenchmarkPriceFromReport(ctx, report) - } - orm := NewORM(db) - clients := map[string]wsrpc.Client{} - - t.Run("successful query", func(t *testing.T) { - originalPrice := big.NewInt(123456789) - c := &mocks.MockWSRPCClient{ - LatestReportF: func(ctx context.Context, in *pb.LatestReportRequest) (out *pb.LatestReportResponse, err error) { - require.NotNil(t, in) - assert.Equal(t, hexutil.Encode(sampleFeedID[:]), hexutil.Encode(in.FeedId)) - out = new(pb.LatestReportResponse) - out.Report = new(pb.Report) - out.Report.FeedId = sampleFeedID[:] - out.Report.Payload = buildSamplePayload([]byte("doesn't matter")) - return out, nil - }, - } - clients[sURL] = c - mt := NewTransmitter(lggr, mockCfg{}, clients, sampleClientPubKey, jobID, sampleFeedID, orm, codec, benchmarkPriceDecoder, nil) - - t.Run("BenchmarkPriceFromReport succeeds", func(t *testing.T) { - codec.val = originalPrice - codec.err = nil - - price, err := mt.LatestPrice(testutils.Context(t), sampleFeedID) - require.NoError(t, err) - - assert.Equal(t, originalPrice, price) - }) - t.Run("BenchmarkPriceFromReport fails", func(t *testing.T) { - codec.val = nil - codec.err = errors.New("something exploded") - - _, err := mt.LatestPrice(testutils.Context(t), sampleFeedID) - require.Error(t, err) - - assert.EqualError(t, err, "something exploded") - }) - }) - - t.Run("successful query returning nil report (new feed)", func(t *testing.T) { - c := &mocks.MockWSRPCClient{ - LatestReportF: func(ctx context.Context, in *pb.LatestReportRequest) (out *pb.LatestReportResponse, err error) { - out = new(pb.LatestReportResponse) - out.Report = nil - return out, nil - }, - } - clients[sURL] = c - mt := NewTransmitter(lggr, mockCfg{}, clients, sampleClientPubKey, jobID, sampleFeedID, orm, codec, benchmarkPriceDecoder, nil) - price, err := mt.LatestPrice(testutils.Context(t), sampleFeedID) - require.NoError(t, err) - - assert.Nil(t, price) - }) - - t.Run("failing query", func(t *testing.T) { - c := &mocks.MockWSRPCClient{ - LatestReportF: func(ctx context.Context, in *pb.LatestReportRequest) (out *pb.LatestReportResponse, err error) { - return nil, errors.New("something exploded") - }, - } - clients[sURL] = c - mt := NewTransmitter(lggr, mockCfg{}, clients, sampleClientPubKey, jobID, sampleFeedID, orm, codec, benchmarkPriceDecoder, nil) - _, err := mt.LatestPrice(testutils.Context(t), sampleFeedID) - require.Error(t, err) - assert.Contains(t, err.Error(), "something exploded") - }) -} - -func Test_MercuryTransmitter_FetchInitialMaxFinalizedBlockNumber(t *testing.T) { - t.Parallel() - - lggr := logger.TestLogger(t) - db := pgtest.NewSqlxDB(t) - var jobID int32 - codec := new(mockCodec) - benchmarkPriceDecoder := func(ctx context.Context, feedID mercuryutils.FeedID, report ocrtypes.Report) (*big.Int, error) { - return codec.BenchmarkPriceFromReport(ctx, report) - } - orm := NewORM(db) - clients := map[string]wsrpc.Client{} - - t.Run("successful query", func(t *testing.T) { - c := &mocks.MockWSRPCClient{ - LatestReportF: func(ctx context.Context, in *pb.LatestReportRequest) (out *pb.LatestReportResponse, err error) { - require.NotNil(t, in) - assert.Equal(t, hexutil.Encode(sampleFeedID[:]), hexutil.Encode(in.FeedId)) - out = new(pb.LatestReportResponse) - out.Report = new(pb.Report) - out.Report.FeedId = sampleFeedID[:] - out.Report.CurrentBlockNumber = 42 - return out, nil - }, - } - clients[sURL] = c - mt := NewTransmitter(lggr, mockCfg{}, clients, sampleClientPubKey, jobID, sampleFeedID, orm, codec, benchmarkPriceDecoder, nil) - bn, err := mt.FetchInitialMaxFinalizedBlockNumber(testutils.Context(t)) - require.NoError(t, err) - - require.NotNil(t, bn) - assert.Equal(t, 42, int(*bn)) - }) - t.Run("successful query returning nil report (new feed)", func(t *testing.T) { - c := &mocks.MockWSRPCClient{ - LatestReportF: func(ctx context.Context, in *pb.LatestReportRequest) (out *pb.LatestReportResponse, err error) { - out = new(pb.LatestReportResponse) - out.Report = nil - return out, nil - }, - } - clients[sURL] = c - mt := NewTransmitter(lggr, mockCfg{}, clients, sampleClientPubKey, jobID, sampleFeedID, orm, codec, benchmarkPriceDecoder, nil) - bn, err := mt.FetchInitialMaxFinalizedBlockNumber(testutils.Context(t)) - require.NoError(t, err) - - assert.Nil(t, bn) - }) - t.Run("failing query", func(t *testing.T) { - c := &mocks.MockWSRPCClient{ - LatestReportF: func(ctx context.Context, in *pb.LatestReportRequest) (out *pb.LatestReportResponse, err error) { - return nil, errors.New("something exploded") - }, - } - clients[sURL] = c - mt := NewTransmitter(lggr, mockCfg{}, clients, sampleClientPubKey, jobID, sampleFeedID, orm, codec, benchmarkPriceDecoder, nil) - _, err := mt.FetchInitialMaxFinalizedBlockNumber(testutils.Context(t)) - require.Error(t, err) - assert.Contains(t, err.Error(), "something exploded") - }) - t.Run("return feed ID is wrong", func(t *testing.T) { - c := &mocks.MockWSRPCClient{ - LatestReportF: func(ctx context.Context, in *pb.LatestReportRequest) (out *pb.LatestReportResponse, err error) { - require.NotNil(t, in) - assert.Equal(t, hexutil.Encode(sampleFeedID[:]), hexutil.Encode(in.FeedId)) - out = new(pb.LatestReportResponse) - out.Report = new(pb.Report) - out.Report.CurrentBlockNumber = 42 - out.Report.FeedId = []byte{1, 2} - return out, nil - }, - } - clients[sURL] = c - mt := NewTransmitter(lggr, mockCfg{}, clients, sampleClientPubKey, jobID, sampleFeedID, orm, codec, benchmarkPriceDecoder, nil) - _, err := mt.FetchInitialMaxFinalizedBlockNumber(testutils.Context(t)) - require.Error(t, err) - assert.Contains(t, err.Error(), "latestReport failed; mismatched feed IDs, expected: 0x1c916b4aa7e57ca7b68ae1bf45653f56b656fd3aa335ef7fae696b663f1b8472, got: 0x") - }) -} - -func Test_sortReportsLatestFirst(t *testing.T) { - reports := []*pb.Report{ - nil, - {ObservationsTimestamp: 1}, - {ObservationsTimestamp: 1}, - {ObservationsTimestamp: 2}, - {CurrentBlockNumber: 1}, - nil, - {CurrentBlockNumber: 2}, - {}, - } - - sortReportsLatestFirst(reports) - - assert.Equal(t, int64(2), reports[0].ObservationsTimestamp) - assert.Equal(t, int64(1), reports[1].ObservationsTimestamp) - assert.Equal(t, int64(1), reports[2].ObservationsTimestamp) - assert.Equal(t, int64(0), reports[3].ObservationsTimestamp) - assert.Equal(t, int64(2), reports[3].CurrentBlockNumber) - assert.Equal(t, int64(0), reports[4].ObservationsTimestamp) - assert.Equal(t, int64(1), reports[4].CurrentBlockNumber) - assert.Equal(t, int64(0), reports[5].ObservationsTimestamp) - assert.Equal(t, int64(0), reports[5].CurrentBlockNumber) - assert.Nil(t, reports[6]) - assert.Nil(t, reports[7]) -} - -type mockQ struct { - ch chan *Transmission -} - -func newMockQ() *mockQ { - return &mockQ{make(chan *Transmission, 100)} -} - -func (m *mockQ) Start(context.Context) error { return nil } -func (m *mockQ) Close() error { - m.ch <- nil - return nil -} -func (m *mockQ) Ready() error { return nil } -func (m *mockQ) HealthReport() map[string]error { return nil } -func (m *mockQ) Name() string { return "" } -func (m *mockQ) BlockingPop() (t *Transmission) { - val := <-m.ch - return val -} -func (m *mockQ) Push(req *pb.TransmitRequest, reportCtx ocrtypes.ReportContext) (ok bool) { - m.ch <- &Transmission{Req: req, ReportCtx: reportCtx} - return true -} -func (m *mockQ) Init(transmissions []*Transmission) {} -func (m *mockQ) IsEmpty() bool { return false } - -func Test_MercuryTransmitter_runQueueLoop(t *testing.T) { - feedIDHex := utils.NewHash().Hex() - lggr := logger.TestLogger(t) - c := &mocks.MockWSRPCClient{} - db := pgtest.NewSqlxDB(t) - orm := NewORM(db) - pm := NewPersistenceManager(lggr, sURL, orm, 0, 0, 0, 0) - cfg := mockCfg{} - - s := newServer(lggr, cfg, c, pm, sURL, feedIDHex) - - req := &pb.TransmitRequest{ - Payload: []byte{1, 2, 3}, - ReportFormat: 32, - } - - t.Run("pulls from queue and transmits successfully", func(t *testing.T) { - transmit := make(chan *pb.TransmitRequest, 1) - c.TransmitF = func(ctx context.Context, in *pb.TransmitRequest) (*pb.TransmitResponse, error) { - transmit <- in - return &pb.TransmitResponse{Code: 0, Error: ""}, nil - } - q := newMockQ() - s.q = q - wg := &sync.WaitGroup{} - wg.Add(1) - - go s.runQueueLoop(nil, wg, feedIDHex) - - q.Push(req, sampleReportContext) - - select { - case tr := <-transmit: - assert.Equal(t, []byte{1, 2, 3}, tr.Payload) - assert.Equal(t, 32, int(tr.ReportFormat)) - // case <-time.After(testutils.WaitTimeout(t)): - case <-time.After(1 * time.Second): - t.Fatal("expected a transmit request to be sent") - } - - q.Close() - wg.Wait() - }) - - t.Run("on duplicate, success", func(t *testing.T) { - transmit := make(chan *pb.TransmitRequest, 1) - c.TransmitF = func(ctx context.Context, in *pb.TransmitRequest) (*pb.TransmitResponse, error) { - transmit <- in - return &pb.TransmitResponse{Code: DuplicateReport, Error: ""}, nil - } - q := newMockQ() - s.q = q - wg := &sync.WaitGroup{} - wg.Add(1) - - go s.runQueueLoop(nil, wg, feedIDHex) - - q.Push(req, sampleReportContext) - - select { - case tr := <-transmit: - assert.Equal(t, []byte{1, 2, 3}, tr.Payload) - assert.Equal(t, 32, int(tr.ReportFormat)) - // case <-time.After(testutils.WaitTimeout(t)): - case <-time.After(1 * time.Second): - t.Fatal("expected a transmit request to be sent") - } - - q.Close() - wg.Wait() - }) - t.Run("on server-side error, does not retry", func(t *testing.T) { - transmit := make(chan *pb.TransmitRequest, 1) - c.TransmitF = func(ctx context.Context, in *pb.TransmitRequest) (*pb.TransmitResponse, error) { - transmit <- in - return &pb.TransmitResponse{Code: DuplicateReport, Error: ""}, nil - } - q := newMockQ() - s.q = q - wg := &sync.WaitGroup{} - wg.Add(1) - - go s.runQueueLoop(nil, wg, feedIDHex) - - q.Push(req, sampleReportContext) - - select { - case tr := <-transmit: - assert.Equal(t, []byte{1, 2, 3}, tr.Payload) - assert.Equal(t, 32, int(tr.ReportFormat)) - // case <-time.After(testutils.WaitTimeout(t)): - case <-time.After(1 * time.Second): - t.Fatal("expected a transmit request to be sent") - } - - q.Close() - wg.Wait() - }) - t.Run("on transmit error, retries", func(t *testing.T) { - transmit := make(chan *pb.TransmitRequest, 1) - c.TransmitF = func(ctx context.Context, in *pb.TransmitRequest) (*pb.TransmitResponse, error) { - transmit <- in - return &pb.TransmitResponse{}, errors.New("transmission error") - } - q := newMockQ() - s.q = q - wg := &sync.WaitGroup{} - wg.Add(1) - stopCh := make(chan struct{}, 1) - - go s.runQueueLoop(stopCh, wg, feedIDHex) - - q.Push(req, sampleReportContext) - - cnt := 0 - Loop: - for { - select { - case tr := <-transmit: - assert.Equal(t, []byte{1, 2, 3}, tr.Payload) - assert.Equal(t, 32, int(tr.ReportFormat)) - if cnt > 2 { - break Loop - } - cnt++ - // case <-time.After(testutils.WaitTimeout(t)): - case <-time.After(1 * time.Second): - t.Fatal("expected 3 transmit requests to be sent") - } - } - - close(stopCh) - wg.Wait() - }) -} diff --git a/core/services/relay/evm/mercury/types/types.go b/core/services/relay/evm/mercury/types/types.go deleted file mode 100644 index 98910887111..00000000000 --- a/core/services/relay/evm/mercury/types/types.go +++ /dev/null @@ -1,33 +0,0 @@ -package types - -import ( - "context" - "math/big" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" -) - -type DataSourceORM interface { - LatestReport(ctx context.Context, feedID [32]byte) (report []byte, err error) -} - -type ReportCodec interface { - BenchmarkPriceFromReport(ctx context.Context, report ocrtypes.Report) (*big.Int, error) -} - -var ( - PriceFeedMissingCount = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "mercury_price_feed_missing", - Help: "Running count of times mercury tried to query a price feed for billing from mercury server, but it was missing", - }, - []string{"queriedFeedID"}, - ) - PriceFeedErrorCount = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "mercury_price_feed_errors", - Help: "Running count of times mercury tried to query a price feed for billing from mercury server, but got an error", - }, - []string{"queriedFeedID"}, - ) -) diff --git a/core/services/relay/evm/mercury/v1/data_source.go b/core/services/relay/evm/mercury/v1/data_source.go deleted file mode 100644 index 0b9b6727fcf..00000000000 --- a/core/services/relay/evm/mercury/v1/data_source.go +++ /dev/null @@ -1,327 +0,0 @@ -package v1 - -import ( - "context" - "errors" - "fmt" - "math/big" - "sync" - - pkgerrors "github.com/pkg/errors" - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" - - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - - "github.com/smartcontractkit/chainlink-common/pkg/types/mercury" - v1types "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v1" - v1 "github.com/smartcontractkit/chainlink-data-streams/mercury/v1" - - "github.com/smartcontractkit/chainlink/v2/core/logger" - "github.com/smartcontractkit/chainlink/v2/core/services/job" - "github.com/smartcontractkit/chainlink/v2/core/services/ocrcommon" - "github.com/smartcontractkit/chainlink/v2/core/services/pipeline" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/types" - mercuryutils "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/utils" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v1/reportcodec" - "github.com/smartcontractkit/chainlink/v2/core/utils" -) - -var ( - insufficientBlocksCount = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "mercury_insufficient_blocks_count", - Help: fmt.Sprintf("Count of times that there were not enough blocks in the chain during observation (need: %d)", nBlocksObservation), - }, - []string{"feedID"}, - ) - zeroBlocksCount = promauto.NewCounterVec(prometheus.CounterOpts{ - Name: "mercury_zero_blocks_count", - Help: "Count of times that there were zero blocks in the chain during observation", - }, - []string{"feedID"}, - ) -) - -const nBlocksObservation int = v1.MaxAllowedBlocks - -type Runner interface { - ExecuteRun(ctx context.Context, spec pipeline.Spec, vars pipeline.Vars) (run *pipeline.Run, trrs pipeline.TaskRunResults, err error) -} - -// Fetcher fetcher data from Mercury server -type Fetcher interface { - // FetchInitialMaxFinalizedBlockNumber should fetch the initial max finalized block number - FetchInitialMaxFinalizedBlockNumber(context.Context) (*int64, error) -} - -type datasource struct { - pipelineRunner Runner - jb job.Job - spec pipeline.Spec - lggr logger.Logger - saver ocrcommon.Saver - orm types.DataSourceORM - codec reportcodec.ReportCodec - feedID [32]byte - - mu sync.RWMutex - - chEnhancedTelem chan<- ocrcommon.EnhancedTelemetryMercuryData - mercuryChainReader mercury.ChainReader - fetcher Fetcher - initialBlockNumber *int64 - - insufficientBlocksCounter prometheus.Counter - zeroBlocksCounter prometheus.Counter -} - -var _ v1.DataSource = &datasource{} - -func NewDataSource(orm types.DataSourceORM, pr pipeline.Runner, jb job.Job, spec pipeline.Spec, lggr logger.Logger, s ocrcommon.Saver, enhancedTelemChan chan ocrcommon.EnhancedTelemetryMercuryData, mercuryChainReader mercury.ChainReader, fetcher Fetcher, initialBlockNumber *int64, feedID mercuryutils.FeedID) *datasource { - return &datasource{pr, jb, spec, lggr, s, orm, reportcodec.ReportCodec{}, feedID, sync.RWMutex{}, enhancedTelemChan, mercuryChainReader, fetcher, initialBlockNumber, insufficientBlocksCount.WithLabelValues(feedID.String()), zeroBlocksCount.WithLabelValues(feedID.String())} -} - -type ErrEmptyLatestReport struct { - Err error -} - -func (e ErrEmptyLatestReport) Unwrap() error { return e.Err } - -func (e ErrEmptyLatestReport) Error() string { - return fmt.Sprintf("FetchInitialMaxFinalizedBlockNumber returned empty LatestReport; this is a new feed. No initialBlockNumber was set, tried to use current block number to determine maxFinalizedBlockNumber but got error: %v", e.Err) -} - -func (ds *datasource) Observe(ctx context.Context, repts ocrtypes.ReportTimestamp, fetchMaxFinalizedBlockNum bool) (obs v1types.Observation, pipelineExecutionErr error) { - // setLatestBlocks must come chronologically before observations, along - // with observationTimestamp, to avoid front-running - - // Errors are not expected when reading from the underlying ChainReader - if err := ds.setLatestBlocks(ctx, &obs); err != nil { - return obs, err - } - - var wg sync.WaitGroup - if fetchMaxFinalizedBlockNum { - wg.Add(1) - go func() { - defer wg.Done() - latest, dbErr := ds.orm.LatestReport(ctx, ds.feedID) - if dbErr != nil { - obs.MaxFinalizedBlockNumber.Err = dbErr - return - } - if latest != nil { - obs.MaxFinalizedBlockNumber.Val, obs.MaxFinalizedBlockNumber.Err = ds.codec.CurrentBlockNumFromReport(ctx, latest) - return - } - val, fetchErr := ds.fetcher.FetchInitialMaxFinalizedBlockNumber(ctx) - if fetchErr != nil { - obs.MaxFinalizedBlockNumber.Err = fetchErr - return - } - if val != nil { - obs.MaxFinalizedBlockNumber.Val = *val - return - } - if ds.initialBlockNumber == nil { - if obs.CurrentBlockNum.Err != nil { - obs.MaxFinalizedBlockNumber.Err = ErrEmptyLatestReport{Err: obs.CurrentBlockNum.Err} - } else { - // Subract 1 here because we will later add 1 to the - // maxFinalizedBlockNumber to get the first validFromBlockNum, which - // ought to be the same as current block num. - obs.MaxFinalizedBlockNumber.Val = obs.CurrentBlockNum.Val - 1 - ds.lggr.Infof("FetchInitialMaxFinalizedBlockNumber returned empty LatestReport; this is a new feed so maxFinalizedBlockNumber=%d (initialBlockNumber unset, using currentBlockNum=%d-1)", obs.MaxFinalizedBlockNumber.Val, obs.CurrentBlockNum.Val) - } - } else { - // NOTE: It's important to subtract 1 if the server is missing any past - // report (brand new feed) since we will add 1 to the - // maxFinalizedBlockNumber to get the first validFromBlockNum, which - // ought to be zero. - // - // If "initialBlockNumber" is set to zero, this will give a starting block of zero. - obs.MaxFinalizedBlockNumber.Val = *ds.initialBlockNumber - 1 - ds.lggr.Infof("FetchInitialMaxFinalizedBlockNumber returned empty LatestReport; this is a new feed so maxFinalizedBlockNumber=%d (initialBlockNumber=%d)", obs.MaxFinalizedBlockNumber.Val, *ds.initialBlockNumber) - } - }() - } else { - obs.MaxFinalizedBlockNumber.Err = errors.New("fetchMaxFinalizedBlockNum=false") - } - var trrs pipeline.TaskRunResults - wg.Add(1) - go func() { - defer wg.Done() - var run *pipeline.Run - run, trrs, pipelineExecutionErr = ds.executeRun(ctx) - if pipelineExecutionErr != nil { - pipelineExecutionErr = fmt.Errorf("Observe failed while executing run: %w", pipelineExecutionErr) - return - } - - ds.saver.Save(run) - - // NOTE: trrs comes back as _all_ tasks, but we only want the terminal ones - // They are guaranteed to be sorted by index asc so should be in the correct order - var finaltrrs []pipeline.TaskRunResult - for _, trr := range trrs { - if trr.IsTerminal() { - finaltrrs = append(finaltrrs, trr) - } - } - - var parsed parseOutput - parsed, pipelineExecutionErr = ds.parse(finaltrrs) - if pipelineExecutionErr != nil { - pipelineExecutionErr = fmt.Errorf("Observe failed while parsing run results: %w", pipelineExecutionErr) - return - } - obs.BenchmarkPrice = parsed.benchmarkPrice - obs.Bid = parsed.bid - obs.Ask = parsed.ask - }() - - wg.Wait() - - if pipelineExecutionErr != nil { - return - } - - ocrcommon.MaybeEnqueueEnhancedTelem(ds.jb, ds.chEnhancedTelem, ocrcommon.EnhancedTelemetryMercuryData{ - V1Observation: &obs, - TaskRunResults: trrs, - RepTimestamp: repts, - FeedVersion: mercuryutils.REPORT_V1, - }) - - return obs, nil -} - -func toBigInt(val interface{}) (*big.Int, error) { - dec, err := utils.ToDecimal(val) - if err != nil { - return nil, err - } - return dec.BigInt(), nil -} - -type parseOutput struct { - benchmarkPrice mercury.ObsResult[*big.Int] - bid mercury.ObsResult[*big.Int] - ask mercury.ObsResult[*big.Int] -} - -// parse expects the output of observe to be three values, in the following order: -// 1. benchmark price -// 2. bid -// 3. ask -// -// returns error on parse errors: if something is the wrong type -func (ds *datasource) parse(trrs pipeline.TaskRunResults) (o parseOutput, merr error) { - var finaltrrs []pipeline.TaskRunResult - for _, trr := range trrs { - // only return terminal trrs from executeRun - if trr.IsTerminal() { - finaltrrs = append(finaltrrs, trr) - } - } - - // pipeline.TaskRunResults comes ordered asc by index, this is guaranteed - // by the pipeline executor - if len(finaltrrs) != 3 { - return o, fmt.Errorf("invalid number of results, expected: 3, got: %d", len(finaltrrs)) - } - merr = errors.Join( - setBenchmarkPrice(&o, finaltrrs[0].Result), - setBid(&o, finaltrrs[1].Result), - setAsk(&o, finaltrrs[2].Result), - ) - - return o, merr -} - -func setBenchmarkPrice(o *parseOutput, res pipeline.Result) error { - if res.Error != nil { - o.benchmarkPrice.Err = res.Error - } else if val, err := toBigInt(res.Value); err != nil { - return fmt.Errorf("failed to parse BenchmarkPrice: %w", err) - } else { - o.benchmarkPrice.Val = val - } - return nil -} - -func setBid(o *parseOutput, res pipeline.Result) error { - if res.Error != nil { - o.bid.Err = res.Error - } else if val, err := toBigInt(res.Value); err != nil { - return fmt.Errorf("failed to parse Bid: %w", err) - } else { - o.bid.Val = val - } - return nil -} - -func setAsk(o *parseOutput, res pipeline.Result) error { - if res.Error != nil { - o.ask.Err = res.Error - } else if val, err := toBigInt(res.Value); err != nil { - return fmt.Errorf("failed to parse Ask: %w", err) - } else { - o.ask.Val = val - } - return nil -} - -// The context passed in here has a timeout of (ObservationTimeout + ObservationGracePeriod). -// Upon context cancellation, its expected that we return any usable values within ObservationGracePeriod. -func (ds *datasource) executeRun(ctx context.Context) (*pipeline.Run, pipeline.TaskRunResults, error) { - vars := pipeline.NewVarsFrom(map[string]interface{}{ - "jb": map[string]interface{}{ - "databaseID": ds.jb.ID, - "externalJobID": ds.jb.ExternalJobID, - "name": ds.jb.Name.ValueOrZero(), - }, - }) - - run, trrs, err := ds.pipelineRunner.ExecuteRun(ctx, ds.spec, vars) - if err != nil { - return nil, nil, pkgerrors.Wrapf(err, "error executing run for spec ID %v", ds.spec.ID) - } - - return run, trrs, err -} - -func (ds *datasource) setLatestBlocks(ctx context.Context, obs *v1types.Observation) error { - latestBlocks, err := ds.mercuryChainReader.LatestHeads(ctx, nBlocksObservation) - - if err != nil { - ds.lggr.Errorw("failed to read latest blocks", "err", err) - return err - } - - if len(latestBlocks) < nBlocksObservation { - ds.insufficientBlocksCounter.Inc() - ds.lggr.Warnw("Insufficient blocks", "latestBlocks", latestBlocks, "lenLatestBlocks", len(latestBlocks), "nBlocksObservation", nBlocksObservation) - } - - // TODO: remove with https://smartcontract-it.atlassian.net/browse/BCF-2209 - if len(latestBlocks) == 0 { - obsErr := fmt.Errorf("no blocks available") - ds.zeroBlocksCounter.Inc() - obs.CurrentBlockNum.Err = obsErr - obs.CurrentBlockHash.Err = obsErr - obs.CurrentBlockTimestamp.Err = obsErr - } else { - obs.CurrentBlockNum.Val = int64(latestBlocks[0].Number) - obs.CurrentBlockHash.Val = latestBlocks[0].Hash - obs.CurrentBlockTimestamp.Val = latestBlocks[0].Timestamp - } - - for _, block := range latestBlocks { - obs.LatestBlocks = append( - obs.LatestBlocks, - v1types.NewBlock(int64(block.Number), block.Hash, block.Timestamp)) - } - - return nil -} diff --git a/core/services/relay/evm/mercury/v1/data_source_test.go b/core/services/relay/evm/mercury/v1/data_source_test.go deleted file mode 100644 index 5cf37a1c315..00000000000 --- a/core/services/relay/evm/mercury/v1/data_source_test.go +++ /dev/null @@ -1,468 +0,0 @@ -package v1 - -import ( - "context" - "fmt" - "io" - "math/big" - "math/rand" - "testing" - "time" - - "github.com/ethereum/go-ethereum/common" - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - - mercurytypes "github.com/smartcontractkit/chainlink-common/pkg/types/mercury" - v1 "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v1" - - htmocks "github.com/smartcontractkit/chainlink/v2/common/headtracker/mocks" - evmtypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/types" - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" - "github.com/smartcontractkit/chainlink/v2/core/logger" - "github.com/smartcontractkit/chainlink/v2/core/services/job" - "github.com/smartcontractkit/chainlink/v2/core/services/pipeline" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm" - mercurymocks "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/mocks" - mercuryutils "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/utils" - reportcodecv1 "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v1/reportcodec" - "github.com/smartcontractkit/chainlink/v2/evm/assets" - "github.com/smartcontractkit/chainlink/v2/evm/utils" -) - -var _ mercurytypes.ServerFetcher = &mockFetcher{} - -type mockFetcher struct { - num *int64 - err error -} - -func (m *mockFetcher) FetchInitialMaxFinalizedBlockNumber(context.Context) (*int64, error) { - return m.num, m.err -} - -func (m *mockFetcher) LatestPrice(ctx context.Context, feedID [32]byte) (*big.Int, error) { - return nil, nil -} - -func (m *mockFetcher) LatestTimestamp(context.Context) (int64, error) { - return 0, nil -} - -type mockSaver struct { - r *pipeline.Run -} - -func (ms *mockSaver) Save(r *pipeline.Run) { - ms.r = r -} - -type mockORM struct { - report []byte - err error -} - -func (m *mockORM) LatestReport(ctx context.Context, feedID [32]byte) (report []byte, err error) { - return m.report, m.err -} - -type mockChainReader struct { - err error - obs []mercurytypes.Head -} - -func (m *mockChainReader) LatestHeads(context.Context, int) ([]mercurytypes.Head, error) { - return m.obs, m.err -} - -func TestMercury_Observe(t *testing.T) { - orm := &mockORM{} - lggr := logger.TestLogger(t) - ds := NewDataSource(orm, nil, job.Job{}, pipeline.Spec{}, lggr, nil, nil, nil, nil, nil, mercuryutils.FeedID{}) - ctx := testutils.Context(t) - repts := ocrtypes.ReportTimestamp{} - - fetcher := &mockFetcher{} - ds.fetcher = fetcher - - saver := &mockSaver{} - ds.saver = saver - - trrs := []pipeline.TaskRunResult{ - { - // benchmark price - Result: pipeline.Result{Value: "122.345"}, - Task: &mercurymocks.MockTask{}, - }, - { - // bid - Result: pipeline.Result{Value: "121.993"}, - Task: &mercurymocks.MockTask{}, - }, - { - // ask - Result: pipeline.Result{Value: "123.111"}, - Task: &mercurymocks.MockTask{}, - }, - } - - runner := &mercurymocks.MockRunner{ - Trrs: trrs, - } - ds.pipelineRunner = runner - - spec := pipeline.Spec{} - ds.spec = spec - - h := htmocks.NewHeadTracker[*evmtypes.Head, common.Hash](t) - ds.mercuryChainReader = evm.NewMercuryChainReader(h) - - head := &evmtypes.Head{ - Number: int64(rand.Int31()), - Hash: utils.NewHash(), - Timestamp: time.Now(), - } - h.On("LatestChain").Return(head) - - t.Run("when fetchMaxFinalizedBlockNum=true", func(t *testing.T) { - t.Run("with latest report in database", func(t *testing.T) { - orm.report = buildSampleV1Report() - orm.err = nil - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.NoError(t, obs.MaxFinalizedBlockNumber.Err) - assert.Equal(t, int64(143), obs.MaxFinalizedBlockNumber.Val) - }) - t.Run("if querying latest report fails", func(t *testing.T) { - orm.report = nil - orm.err = errors.New("something exploded") - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.EqualError(t, obs.MaxFinalizedBlockNumber.Err, "something exploded") - assert.Zero(t, obs.MaxFinalizedBlockNumber.Val) - }) - t.Run("if decoding latest report fails", func(t *testing.T) { - orm.report = []byte{1, 2, 3} - orm.err = nil - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.EqualError(t, obs.MaxFinalizedBlockNumber.Err, "failed to decode report: abi: cannot marshal in to go type: length insufficient 3 require 32") - assert.Zero(t, obs.MaxFinalizedBlockNumber.Val) - }) - - orm.report = nil - orm.err = nil - - t.Run("without latest report in database", func(t *testing.T) { - t.Run("if FetchInitialMaxFinalizedBlockNumber returns error", func(t *testing.T) { - fetcher.err = errors.New("mock fetcher error") - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.EqualError(t, obs.MaxFinalizedBlockNumber.Err, "mock fetcher error") - assert.Zero(t, obs.MaxFinalizedBlockNumber.Val) - }) - t.Run("if FetchInitialMaxFinalizedBlockNumber succeeds", func(t *testing.T) { - fetcher.err = nil - var num int64 = 32 - fetcher.num = &num - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.NoError(t, obs.MaxFinalizedBlockNumber.Err) - assert.Equal(t, int64(32), obs.MaxFinalizedBlockNumber.Val) - }) - t.Run("if FetchInitialMaxFinalizedBlockNumber returns nil (new feed) and initialBlockNumber is set", func(t *testing.T) { - var initialBlockNumber int64 = 50 - ds.initialBlockNumber = &initialBlockNumber - fetcher.err = nil - fetcher.num = nil - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.NoError(t, obs.MaxFinalizedBlockNumber.Err) - assert.Equal(t, int64(49), obs.MaxFinalizedBlockNumber.Val) - }) - t.Run("if FetchInitialMaxFinalizedBlockNumber returns nil (new feed) and initialBlockNumber is not set", func(t *testing.T) { - ds.initialBlockNumber = nil - t.Run("if current block num is valid", func(t *testing.T) { - fetcher.err = nil - fetcher.num = nil - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.NoError(t, obs.MaxFinalizedBlockNumber.Err) - assert.Equal(t, head.Number-1, obs.MaxFinalizedBlockNumber.Val) - }) - t.Run("if no current block available", func(t *testing.T) { - h2 := htmocks.NewHeadTracker[*evmtypes.Head, common.Hash](t) - h2.On("LatestChain").Return((*evmtypes.Head)(nil)) - ds.mercuryChainReader = evm.NewMercuryChainReader(h2) - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.EqualError(t, obs.MaxFinalizedBlockNumber.Err, "FetchInitialMaxFinalizedBlockNumber returned empty LatestReport; this is a new feed. No initialBlockNumber was set, tried to use current block number to determine maxFinalizedBlockNumber but got error: no blocks available") - }) - }) - }) - }) - - ds.mercuryChainReader = evm.NewMercuryChainReader(h) - - t.Run("when fetchMaxFinalizedBlockNum=false", func(t *testing.T) { - t.Run("when run execution fails, returns error", func(t *testing.T) { - t.Cleanup(func() { - runner.Err = nil - }) - runner.Err = errors.New("run execution failed") - - _, err := ds.Observe(ctx, repts, false) - assert.EqualError(t, err, "Observe failed while executing run: error executing run for spec ID 0: run execution failed") - }) - t.Run("makes observation using pipeline, when all tasks succeed", func(t *testing.T) { - obs, err := ds.Observe(ctx, repts, false) - assert.NoError(t, err) - - assert.Equal(t, big.NewInt(122), obs.BenchmarkPrice.Val) - assert.NoError(t, obs.BenchmarkPrice.Err) - assert.Equal(t, big.NewInt(121), obs.Bid.Val) - assert.NoError(t, obs.Bid.Err) - assert.Equal(t, big.NewInt(123), obs.Ask.Val) - assert.NoError(t, obs.Ask.Err) - assert.Equal(t, head.Number, obs.CurrentBlockNum.Val) - assert.NoError(t, obs.CurrentBlockNum.Err) - assert.Equal(t, fmt.Sprintf("%x", head.Hash), fmt.Sprintf("%x", obs.CurrentBlockHash.Val)) - assert.NoError(t, obs.CurrentBlockHash.Err) - assert.Equal(t, uint64(head.Timestamp.Unix()), obs.CurrentBlockTimestamp.Val) - assert.NoError(t, obs.CurrentBlockTimestamp.Err) - - assert.Zero(t, obs.MaxFinalizedBlockNumber.Val) - assert.EqualError(t, obs.MaxFinalizedBlockNumber.Err, "fetchMaxFinalizedBlockNum=false") - }) - t.Run("makes observation using pipeline, with erroring tasks", func(t *testing.T) { - for i := range trrs { - trrs[i].Result.Error = fmt.Errorf("task error %d", i) - } - - obs, err := ds.Observe(ctx, repts, false) - assert.NoError(t, err) - - assert.Zero(t, obs.BenchmarkPrice.Val) - assert.EqualError(t, obs.BenchmarkPrice.Err, "task error 0") - assert.Zero(t, obs.Bid.Val) - assert.EqualError(t, obs.Bid.Err, "task error 1") - assert.Zero(t, obs.Ask.Val) - assert.EqualError(t, obs.Ask.Err, "task error 2") - assert.Equal(t, head.Number, obs.CurrentBlockNum.Val) - assert.NoError(t, obs.CurrentBlockNum.Err) - assert.Equal(t, fmt.Sprintf("%x", head.Hash), fmt.Sprintf("%x", obs.CurrentBlockHash.Val)) - assert.NoError(t, obs.CurrentBlockHash.Err) - assert.Equal(t, uint64(head.Timestamp.Unix()), obs.CurrentBlockTimestamp.Val) - assert.NoError(t, obs.CurrentBlockTimestamp.Err) - - assert.Zero(t, obs.MaxFinalizedBlockNumber.Val) - assert.EqualError(t, obs.MaxFinalizedBlockNumber.Err, "fetchMaxFinalizedBlockNum=false") - }) - t.Run("makes partial observation using pipeline, if only some results have errored", func(t *testing.T) { - trrs[0].Result.Error = fmt.Errorf("task failed") - trrs[1].Result.Value = "33" - trrs[1].Result.Error = nil - trrs[2].Result.Value = nil - trrs[2].Result.Error = fmt.Errorf("task failed") - - obs, err := ds.Observe(ctx, repts, false) - assert.NoError(t, err) - - assert.Zero(t, obs.BenchmarkPrice.Val) - assert.EqualError(t, obs.BenchmarkPrice.Err, "task failed") - assert.Equal(t, big.NewInt(33), obs.Bid.Val) - assert.NoError(t, obs.Bid.Err) - assert.Zero(t, obs.Ask.Val) - assert.EqualError(t, obs.Ask.Err, "task failed") - }) - t.Run("returns error if at least one result is unparseable", func(t *testing.T) { - trrs[0].Result.Error = fmt.Errorf("task failed") - trrs[1].Result.Value = "foo" - trrs[1].Result.Error = nil - trrs[2].Result.Value = "123456" - trrs[2].Result.Error = nil - - _, err := ds.Observe(ctx, repts, false) - assert.EqualError(t, err, "Observe failed while parsing run results: failed to parse Bid: can't convert foo to decimal") - }) - t.Run("saves run", func(t *testing.T) { - for i := range trrs { - trrs[i].Result.Value = "123" - trrs[i].Result.Error = nil - } - - _, err := ds.Observe(ctx, repts, false) - assert.NoError(t, err) - - assert.Equal(t, int64(42), saver.r.ID) - }) - }) - - t.Run("LatestBlocks is populated correctly", func(t *testing.T) { - t.Run("when chain length is zero", func(t *testing.T) { - ht2 := htmocks.NewHeadTracker[*evmtypes.Head, common.Hash](t) - ht2.On("LatestChain").Return((*evmtypes.Head)(nil)) - ds.mercuryChainReader = evm.NewMercuryChainReader(ht2) - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.Len(t, obs.LatestBlocks, 0) - - ht2.AssertExpectations(t) - }) - t.Run("when chain is too short", func(t *testing.T) { - h4 := &evmtypes.Head{ - Number: 4, - } - h5 := &evmtypes.Head{ - Number: 5, - } - h5.Parent.Store(h4) - h6 := &evmtypes.Head{ - Number: 6, - } - h6.Parent.Store(h5) - - ht2 := htmocks.NewHeadTracker[*evmtypes.Head, common.Hash](t) - ht2.On("LatestChain").Return(h6) - ds.mercuryChainReader = evm.NewMercuryChainReader(ht2) - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.Len(t, obs.LatestBlocks, 3) - assert.Equal(t, 6, int(obs.LatestBlocks[0].Num)) - assert.Equal(t, 5, int(obs.LatestBlocks[1].Num)) - assert.Equal(t, 4, int(obs.LatestBlocks[2].Num)) - - ht2.AssertExpectations(t) - }) - t.Run("when chain is long enough", func(t *testing.T) { - heads := make([]*evmtypes.Head, nBlocksObservation+5) - for i := range heads { - heads[i] = &evmtypes.Head{Number: int64(i)} - if i > 0 { - heads[i].Parent.Store(heads[i-1]) - } - } - - ht2 := htmocks.NewHeadTracker[*evmtypes.Head, common.Hash](t) - ht2.On("LatestChain").Return(heads[len(heads)-1]) - ds.mercuryChainReader = evm.NewMercuryChainReader(ht2) - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.Len(t, obs.LatestBlocks, nBlocksObservation) - highestBlockNum := heads[len(heads)-1].Number - for i := range obs.LatestBlocks { - assert.Equal(t, int(highestBlockNum)-i, int(obs.LatestBlocks[i].Num)) - } - - ht2.AssertExpectations(t) - }) - - t.Run("when chain reader returns an error", func(t *testing.T) { - ds.mercuryChainReader = &mockChainReader{ - err: io.EOF, - obs: nil, - } - - obs, err := ds.Observe(ctx, repts, true) - assert.Error(t, err) - assert.Equal(t, obs, v1.Observation{}) - }) - }) -} - -func TestMercury_SetLatestBlocks(t *testing.T) { - lggr := logger.TestLogger(t) - ds := NewDataSource(nil, nil, job.Job{}, pipeline.Spec{}, lggr, nil, nil, nil, nil, nil, mercuryutils.FeedID{}) - - h := evmtypes.Head{ - Number: testutils.NewRandomPositiveInt64(), - Hash: utils.NewHash(), - ParentHash: utils.NewHash(), - Timestamp: time.Now(), - BaseFeePerGas: assets.NewWeiI(testutils.NewRandomPositiveInt64()), - ReceiptsRoot: utils.NewHash(), - TransactionsRoot: utils.NewHash(), - StateRoot: utils.NewHash(), - } - - t.Run("returns head from headtracker if present", func(t *testing.T) { - headTracker := htmocks.NewHeadTracker[*evmtypes.Head, common.Hash](t) - headTracker.On("LatestChain").Return(&h, nil) - ds.mercuryChainReader = evm.NewMercuryChainReader(headTracker) - - obs := v1.Observation{} - err := ds.setLatestBlocks(testutils.Context(t), &obs) - - assert.NoError(t, err) - assert.Equal(t, h.Number, obs.CurrentBlockNum.Val) - assert.Equal(t, h.Hash.Bytes(), obs.CurrentBlockHash.Val) - assert.Equal(t, uint64(h.Timestamp.Unix()), obs.CurrentBlockTimestamp.Val) - - assert.Len(t, obs.LatestBlocks, 1) - headTracker.AssertExpectations(t) - }) - - t.Run("if headtracker returns nil head", func(t *testing.T) { - headTracker := htmocks.NewHeadTracker[*evmtypes.Head, common.Hash](t) - // This can happen in some cases e.g. RPC node is offline - headTracker.On("LatestChain").Return((*evmtypes.Head)(nil)) - ds.mercuryChainReader = evm.NewChainReader(headTracker) - obs := v1.Observation{} - err := ds.setLatestBlocks(testutils.Context(t), &obs) - - assert.NoError(t, err) - assert.Zero(t, obs.CurrentBlockNum.Val) - assert.Zero(t, obs.CurrentBlockHash.Val) - assert.Zero(t, obs.CurrentBlockTimestamp.Val) - assert.EqualError(t, obs.CurrentBlockNum.Err, "no blocks available") - assert.EqualError(t, obs.CurrentBlockHash.Err, "no blocks available") - assert.EqualError(t, obs.CurrentBlockTimestamp.Err, "no blocks available") - - assert.Len(t, obs.LatestBlocks, 0) - headTracker.AssertExpectations(t) - }) -} - -var sampleFeedID = [32]uint8{28, 145, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114} - -func buildSampleV1Report() []byte { - feedID := sampleFeedID - timestamp := uint32(42) - bp := big.NewInt(242) - bid := big.NewInt(243) - ask := big.NewInt(244) - currentBlockNumber := uint64(143) - currentBlockHash := utils.NewHash() - currentBlockTimestamp := uint64(123) - validFromBlockNum := uint64(142) - - b, err := reportcodecv1.ReportTypes.Pack(feedID, timestamp, bp, bid, ask, currentBlockNumber, currentBlockHash, currentBlockTimestamp, validFromBlockNum) - if err != nil { - panic(err) - } - return b -} diff --git a/core/services/relay/evm/mercury/v1/reportcodec/report_codec.go b/core/services/relay/evm/mercury/v1/reportcodec/report_codec.go deleted file mode 100644 index 52cdeff96cb..00000000000 --- a/core/services/relay/evm/mercury/v1/reportcodec/report_codec.go +++ /dev/null @@ -1,99 +0,0 @@ -package reportcodec - -import ( - "context" - "errors" - "fmt" - "math" - "math/big" - - "github.com/ethereum/go-ethereum/common" - pkgerrors "github.com/pkg/errors" - - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - - "github.com/smartcontractkit/chainlink-common/pkg/logger" - v1 "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v1" - - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/utils" - reporttypes "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v1/types" -) - -// NOTE: -// This report codec is based on the original median evmreportcodec -// here: -// https://github.com/smartcontractkit/offchain-reporting/blob/master/lib/offchainreporting2/reportingplugin/median/evmreportcodec/reportcodec.go -var ReportTypes = reporttypes.GetSchema() -var maxReportLength = 32 * len(ReportTypes) // each arg is 256 bit EVM word - -var _ v1.ReportCodec = &ReportCodec{} - -type ReportCodec struct { - logger logger.Logger - feedID utils.FeedID -} - -func NewReportCodec(feedID [32]byte, lggr logger.Logger) *ReportCodec { - return &ReportCodec{lggr, feedID} -} - -func (r *ReportCodec) BuildReport(ctx context.Context, rf v1.ReportFields) (ocrtypes.Report, error) { - var merr error - if rf.BenchmarkPrice == nil { - merr = errors.Join(merr, errors.New("benchmarkPrice may not be nil")) - } - if rf.Bid == nil { - merr = errors.Join(merr, errors.New("bid may not be nil")) - } - if rf.Ask == nil { - merr = errors.Join(merr, errors.New("ask may not be nil")) - } - if len(rf.CurrentBlockHash) != 32 { - merr = errors.Join(merr, fmt.Errorf("invalid length for currentBlockHash, expected: 32, got: %d", len(rf.CurrentBlockHash))) - } - if merr != nil { - return nil, merr - } - var currentBlockHash common.Hash - copy(currentBlockHash[:], rf.CurrentBlockHash) - - reportBytes, err := ReportTypes.Pack(r.feedID, rf.Timestamp, rf.BenchmarkPrice, rf.Bid, rf.Ask, uint64(rf.CurrentBlockNum), currentBlockHash, uint64(rf.ValidFromBlockNum), rf.CurrentBlockTimestamp) - return ocrtypes.Report(reportBytes), pkgerrors.Wrap(err, "failed to pack report blob") -} - -// Maximum length in bytes of Report returned by BuildReport. Used for -// defending against spam attacks. -func (r *ReportCodec) MaxReportLength(ctx context.Context, n int) (int, error) { - return maxReportLength, nil -} - -func (r *ReportCodec) CurrentBlockNumFromReport(ctx context.Context, report ocrtypes.Report) (int64, error) { - decoded, err := r.Decode(report) - if err != nil { - return 0, err - } - if decoded.CurrentBlockNum > math.MaxInt64 { - return 0, fmt.Errorf("CurrentBlockNum=%d overflows max int64", decoded.CurrentBlockNum) - } - return int64(decoded.CurrentBlockNum), nil -} - -func (r *ReportCodec) Decode(report ocrtypes.Report) (*reporttypes.Report, error) { - return reporttypes.Decode(report) -} - -func (r *ReportCodec) BenchmarkPriceFromReport(ctx context.Context, report ocrtypes.Report) (*big.Int, error) { - decoded, err := r.Decode(report) - if err != nil { - return nil, err - } - return decoded.BenchmarkPrice, nil -} - -func (r *ReportCodec) ObservationTimestampFromReport(ctx context.Context, report ocrtypes.Report) (uint32, error) { - decoded, err := r.Decode(report) - if err != nil { - return 0, err - } - return decoded.ObservationsTimestamp, nil -} diff --git a/core/services/relay/evm/mercury/v1/reportcodec/report_codec_test.go b/core/services/relay/evm/mercury/v1/reportcodec/report_codec_test.go deleted file mode 100644 index 44f098f8eae..00000000000 --- a/core/services/relay/evm/mercury/v1/reportcodec/report_codec_test.go +++ /dev/null @@ -1,204 +0,0 @@ -package reportcodec - -import ( - "fmt" - "math" - "math/big" - "testing" - - "github.com/ethereum/go-ethereum/common" - "github.com/ethereum/go-ethereum/common/hexutil" - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2/types" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - - v1 "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v1" - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" - "github.com/smartcontractkit/chainlink/v2/evm/utils" -) - -var hash = hexutil.MustDecode("0x552c2cea3ab43bae137d89ee6142a01db3ae2b5678bc3c9bd5f509f537bea57b") - -func newValidReportFields() v1.ReportFields { - return v1.ReportFields{ - Timestamp: 242, - BenchmarkPrice: big.NewInt(243), - Bid: big.NewInt(244), - Ask: big.NewInt(245), - CurrentBlockNum: 248, - CurrentBlockHash: hash, - ValidFromBlockNum: 46, - CurrentBlockTimestamp: 123, - } -} - -func Test_ReportCodec(t *testing.T) { - r := ReportCodec{} - - t.Run("BuildReport errors on zero fields", func(t *testing.T) { - ctx := testutils.Context(t) - _, err := r.BuildReport(ctx, v1.ReportFields{}) - require.Error(t, err) - assert.Contains(t, err.Error(), "benchmarkPrice may not be nil") - assert.Contains(t, err.Error(), "bid may not be nil") - assert.Contains(t, err.Error(), "ask may not be nil") - assert.Contains(t, err.Error(), "invalid length for currentBlockHash, expected: 32, got: 0") - }) - - t.Run("BuildReport constructs a report from observations", func(t *testing.T) { - ctx := testutils.Context(t) - rf := newValidReportFields() - // only need to test happy path since validations are done in relaymercury - - report, err := r.BuildReport(ctx, rf) - require.NoError(t, err) - - reportElems := make(map[string]interface{}) - err = ReportTypes.UnpackIntoMap(reportElems, report) - require.NoError(t, err) - - assert.Equal(t, int(reportElems["observationsTimestamp"].(uint32)), 242) - assert.Equal(t, reportElems["benchmarkPrice"].(*big.Int).Int64(), int64(243)) - assert.Equal(t, reportElems["bid"].(*big.Int).Int64(), int64(244)) - assert.Equal(t, reportElems["ask"].(*big.Int).Int64(), int64(245)) - assert.Equal(t, reportElems["currentBlockNum"].(uint64), uint64(248)) - assert.Equal(t, common.Hash(reportElems["currentBlockHash"].([32]byte)), common.BytesToHash(hash)) - assert.Equal(t, reportElems["currentBlockTimestamp"].(uint64), uint64(123)) - assert.Equal(t, reportElems["validFromBlockNum"].(uint64), uint64(46)) - - assert.Equal(t, types.Report{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf4, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf5, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf8, 0x55, 0x2c, 0x2c, 0xea, 0x3a, 0xb4, 0x3b, 0xae, 0x13, 0x7d, 0x89, 0xee, 0x61, 0x42, 0xa0, 0x1d, 0xb3, 0xae, 0x2b, 0x56, 0x78, 0xbc, 0x3c, 0x9b, 0xd5, 0xf5, 0x9, 0xf5, 0x37, 0xbe, 0xa5, 0x7b, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x2e, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7b}, report) - - max, err := r.MaxReportLength(ctx, 4) - require.NoError(t, err) - assert.LessOrEqual(t, len(report), max) - - t.Run("Decode decodes the report", func(t *testing.T) { - decoded, err := r.Decode(report) - require.NoError(t, err) - - require.NotNil(t, decoded) - - assert.Equal(t, uint32(242), decoded.ObservationsTimestamp) - assert.Equal(t, big.NewInt(243), decoded.BenchmarkPrice) - assert.Equal(t, big.NewInt(244), decoded.Bid) - assert.Equal(t, big.NewInt(245), decoded.Ask) - assert.Equal(t, uint64(248), decoded.CurrentBlockNum) - assert.Equal(t, [32]byte(common.BytesToHash(hash)), decoded.CurrentBlockHash) - assert.Equal(t, uint64(123), decoded.CurrentBlockTimestamp) - assert.Equal(t, uint64(46), decoded.ValidFromBlockNum) - }) - }) - - t.Run("Decode errors on invalid report", func(t *testing.T) { - _, err := r.Decode([]byte{1, 2, 3}) - assert.EqualError(t, err, "failed to decode report: abi: cannot marshal in to go type: length insufficient 3 require 32") - - longBad := make([]byte, 64) - for i := 0; i < len(longBad); i++ { - longBad[i] = byte(i) - } - _, err = r.Decode(longBad) - assert.EqualError(t, err, "failed to decode report: abi: improperly encoded uint32 value") - }) -} - -func buildSampleReport(bn, validFromBn int64, feedID [32]byte) []byte { - timestamp := uint32(42) - bp := big.NewInt(242) - bid := big.NewInt(243) - ask := big.NewInt(244) - currentBlockNumber := uint64(bn) - currentBlockHash := utils.NewHash() - currentBlockTimestamp := uint64(123) - validFromBlockNum := uint64(validFromBn) - - b, err := ReportTypes.Pack(feedID, timestamp, bp, bid, ask, currentBlockNumber, currentBlockHash, validFromBlockNum, currentBlockTimestamp) - if err != nil { - panic(err) - } - return b -} - -func Test_ReportCodec_CurrentBlockNumFromReport(t *testing.T) { - r := ReportCodec{} - feedID := utils.NewHash() - - var validBn int64 = 42 - var invalidBn int64 = -1 - - t.Run("CurrentBlockNumFromReport extracts the current block number from a valid report", func(t *testing.T) { - report := buildSampleReport(validBn, 143, feedID) - - ctx := testutils.Context(t) - bn, err := r.CurrentBlockNumFromReport(ctx, report) - require.NoError(t, err) - - assert.Equal(t, validBn, bn) - }) - t.Run("CurrentBlockNumFromReport returns error if block num is too large", func(t *testing.T) { - report := buildSampleReport(invalidBn, 143, feedID) - - ctx := testutils.Context(t) - _, err := r.CurrentBlockNumFromReport(ctx, report) - require.Error(t, err) - - assert.Contains(t, err.Error(), "CurrentBlockNum=18446744073709551615 overflows max int64") - }) -} - -func (r *ReportCodec) ValidFromBlockNumFromReport(report ocrtypes.Report) (int64, error) { - decoded, err := r.Decode(report) - if err != nil { - return 0, err - } - n := decoded.ValidFromBlockNum - if n > math.MaxInt64 { - return 0, fmt.Errorf("ValidFromBlockNum=%d overflows max int64", n) - } - return int64(n), nil -} - -func Test_ReportCodec_ValidFromBlockNumFromReport(t *testing.T) { - r := ReportCodec{} - feedID := utils.NewHash() - - t.Run("ValidFromBlockNumFromReport extracts the valid from block number from a valid report", func(t *testing.T) { - report := buildSampleReport(42, 999, feedID) - - bn, err := r.ValidFromBlockNumFromReport(report) - require.NoError(t, err) - - assert.Equal(t, int64(999), bn) - }) - t.Run("ValidFromBlockNumFromReport returns error if valid from block number is too large", func(t *testing.T) { - report := buildSampleReport(42, -1, feedID) - - _, err := r.ValidFromBlockNumFromReport(report) - require.Error(t, err) - - assert.Contains(t, err.Error(), "ValidFromBlockNum=18446744073709551615 overflows max int64") - }) -} - -func Test_ReportCodec_BenchmarkPriceFromReport(t *testing.T) { - r := ReportCodec{} - feedID := utils.NewHash() - - t.Run("BenchmarkPriceFromReport extracts the benchmark price from valid report", func(t *testing.T) { - ctx := testutils.Context(t) - report := buildSampleReport(42, 999, feedID) - - bp, err := r.BenchmarkPriceFromReport(ctx, report) - require.NoError(t, err) - - assert.Equal(t, big.NewInt(242), bp) - }) - t.Run("BenchmarkPriceFromReport errors on invalid report", func(t *testing.T) { - ctx := testutils.Context(t) - _, err := r.BenchmarkPriceFromReport(ctx, []byte{1, 2, 3}) - require.Error(t, err) - assert.EqualError(t, err, "failed to decode report: abi: cannot marshal in to go type: length insufficient 3 require 32") - }) -} diff --git a/core/services/relay/evm/mercury/v1/types/types.go b/core/services/relay/evm/mercury/v1/types/types.go deleted file mode 100644 index 709fd856a21..00000000000 --- a/core/services/relay/evm/mercury/v1/types/types.go +++ /dev/null @@ -1,56 +0,0 @@ -package reporttypes - -import ( - "fmt" - "math/big" - - "github.com/ethereum/go-ethereum/accounts/abi" -) - -var schema = GetSchema() - -func GetSchema() abi.Arguments { - mustNewType := func(t string) abi.Type { - result, err := abi.NewType(t, "", []abi.ArgumentMarshaling{}) - if err != nil { - panic(fmt.Sprintf("Unexpected error during abi.NewType: %s", err)) - } - return result - } - return abi.Arguments([]abi.Argument{ - {Name: "feedId", Type: mustNewType("bytes32")}, - {Name: "observationsTimestamp", Type: mustNewType("uint32")}, - {Name: "benchmarkPrice", Type: mustNewType("int192")}, - {Name: "bid", Type: mustNewType("int192")}, - {Name: "ask", Type: mustNewType("int192")}, - {Name: "currentBlockNum", Type: mustNewType("uint64")}, - {Name: "currentBlockHash", Type: mustNewType("bytes32")}, - {Name: "validFromBlockNum", Type: mustNewType("uint64")}, - {Name: "currentBlockTimestamp", Type: mustNewType("uint64")}, - }) -} - -type Report struct { - FeedId [32]byte - ObservationsTimestamp uint32 - BenchmarkPrice *big.Int - Bid *big.Int - Ask *big.Int - CurrentBlockNum uint64 - CurrentBlockHash [32]byte - ValidFromBlockNum uint64 - CurrentBlockTimestamp uint64 -} - -// Decode is made available to external users (i.e. mercury server) -func Decode(report []byte) (*Report, error) { - values, err := schema.Unpack(report) - if err != nil { - return nil, fmt.Errorf("failed to decode report: %w", err) - } - decoded := new(Report) - if err = schema.Copy(decoded, values); err != nil { - return nil, fmt.Errorf("failed to copy report values to struct: %w", err) - } - return decoded, nil -} diff --git a/core/services/relay/evm/mercury/v2/data_source.go b/core/services/relay/evm/mercury/v2/data_source.go deleted file mode 100644 index 30649916fc8..00000000000 --- a/core/services/relay/evm/mercury/v2/data_source.go +++ /dev/null @@ -1,241 +0,0 @@ -package v2 - -import ( - "context" - "fmt" - "math/big" - "sync" - - pkgerrors "github.com/pkg/errors" - - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - - "github.com/smartcontractkit/chainlink-common/pkg/types/mercury" - v2types "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v2" - v2 "github.com/smartcontractkit/chainlink-data-streams/mercury/v2" - - "github.com/smartcontractkit/chainlink/v2/core/logger" - "github.com/smartcontractkit/chainlink/v2/core/services/job" - "github.com/smartcontractkit/chainlink/v2/core/services/ocrcommon" - "github.com/smartcontractkit/chainlink/v2/core/services/pipeline" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/types" - mercurytypes "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/types" - mercuryutils "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/utils" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v2/reportcodec" - "github.com/smartcontractkit/chainlink/v2/core/utils" -) - -type Runner interface { - ExecuteRun(ctx context.Context, spec pipeline.Spec, vars pipeline.Vars) (run *pipeline.Run, trrs pipeline.TaskRunResults, err error) -} - -type LatestReportFetcher interface { - LatestPrice(ctx context.Context, feedID [32]byte) (*big.Int, error) - LatestTimestamp(context.Context) (int64, error) -} - -type datasource struct { - pipelineRunner Runner - jb job.Job - spec pipeline.Spec - feedID mercuryutils.FeedID - lggr logger.Logger - saver ocrcommon.Saver - orm types.DataSourceORM - codec reportcodec.ReportCodec - - fetcher LatestReportFetcher - linkFeedID mercuryutils.FeedID - nativeFeedID mercuryutils.FeedID - - mu sync.RWMutex - - chEnhancedTelem chan<- ocrcommon.EnhancedTelemetryMercuryData -} - -var _ v2.DataSource = &datasource{} - -func NewDataSource(orm types.DataSourceORM, pr pipeline.Runner, jb job.Job, spec pipeline.Spec, feedID mercuryutils.FeedID, lggr logger.Logger, s ocrcommon.Saver, enhancedTelemChan chan ocrcommon.EnhancedTelemetryMercuryData, fetcher LatestReportFetcher, linkFeedID, nativeFeedID mercuryutils.FeedID) *datasource { - return &datasource{pr, jb, spec, feedID, lggr, s, orm, reportcodec.ReportCodec{}, fetcher, linkFeedID, nativeFeedID, sync.RWMutex{}, enhancedTelemChan} -} - -func (ds *datasource) Observe(ctx context.Context, repts ocrtypes.ReportTimestamp, fetchMaxFinalizedTimestamp bool) (obs v2types.Observation, pipelineExecutionErr error) { - var wg sync.WaitGroup - ctx, cancel := context.WithCancel(ctx) - - if fetchMaxFinalizedTimestamp { - wg.Add(1) - go func() { - defer wg.Done() - latest, dbErr := ds.orm.LatestReport(ctx, ds.feedID) - if dbErr != nil { - obs.MaxFinalizedTimestamp.Err = dbErr - return - } - if latest != nil { - maxFinalizedBlockNumber, decodeErr := ds.codec.ObservationTimestampFromReport(ctx, latest) - obs.MaxFinalizedTimestamp.Val, obs.MaxFinalizedTimestamp.Err = int64(maxFinalizedBlockNumber), decodeErr - return - } - obs.MaxFinalizedTimestamp.Val, obs.MaxFinalizedTimestamp.Err = ds.fetcher.LatestTimestamp(ctx) - }() - } - - var trrs pipeline.TaskRunResults - wg.Add(1) - go func() { - defer wg.Done() - var run *pipeline.Run - run, trrs, pipelineExecutionErr = ds.executeRun(ctx) - if pipelineExecutionErr != nil { - cancel() - pipelineExecutionErr = fmt.Errorf("Observe failed while executing run: %w", pipelineExecutionErr) - return - } - - ds.saver.Save(run) - - var parsed parseOutput - parsed, pipelineExecutionErr = ds.parse(trrs) - if pipelineExecutionErr != nil { - cancel() - // This is not expected under normal circumstances - ds.lggr.Errorw("Observe failed while parsing run results", "err", pipelineExecutionErr) - pipelineExecutionErr = fmt.Errorf("Observe failed while parsing run results: %w", pipelineExecutionErr) - return - } - obs.BenchmarkPrice = parsed.benchmarkPrice - }() - - var isLink, isNative bool - if len(ds.jb.OCR2OracleSpec.PluginConfig) == 0 { - obs.LinkPrice.Val = v2.MissingPrice - } else if ds.feedID == ds.linkFeedID { - isLink = true - } else { - wg.Add(1) - go func() { - defer wg.Done() - obs.LinkPrice.Val, obs.LinkPrice.Err = ds.fetcher.LatestPrice(ctx, ds.linkFeedID) - if obs.LinkPrice.Val == nil && obs.LinkPrice.Err == nil { - mercurytypes.PriceFeedMissingCount.WithLabelValues(ds.linkFeedID.String()).Inc() - ds.lggr.Warnw(fmt.Sprintf("Mercury server was missing LINK feed, using sentinel value of %s", v2.MissingPrice), "linkFeedID", ds.linkFeedID) - obs.LinkPrice.Val = v2.MissingPrice - } else if obs.LinkPrice.Err != nil { - mercurytypes.PriceFeedErrorCount.WithLabelValues(ds.linkFeedID.String()).Inc() - ds.lggr.Errorw("Mercury server returned error querying LINK price feed", "err", obs.LinkPrice.Err, "linkFeedID", ds.linkFeedID) - } - }() - } - - if len(ds.jb.OCR2OracleSpec.PluginConfig) == 0 { - obs.NativePrice.Val = v2.MissingPrice - } else if ds.feedID == ds.nativeFeedID { - isNative = true - } else { - wg.Add(1) - go func() { - defer wg.Done() - obs.NativePrice.Val, obs.NativePrice.Err = ds.fetcher.LatestPrice(ctx, ds.nativeFeedID) - if obs.NativePrice.Val == nil && obs.NativePrice.Err == nil { - mercurytypes.PriceFeedMissingCount.WithLabelValues(ds.nativeFeedID.String()).Inc() - ds.lggr.Warnw(fmt.Sprintf("Mercury server was missing native feed, using sentinel value of %s", v2.MissingPrice), "nativeFeedID", ds.nativeFeedID) - obs.NativePrice.Val = v2.MissingPrice - } else if obs.NativePrice.Err != nil { - mercurytypes.PriceFeedErrorCount.WithLabelValues(ds.nativeFeedID.String()).Inc() - ds.lggr.Errorw("Mercury server returned error querying native price feed", "err", obs.NativePrice.Err, "nativeFeedID", ds.nativeFeedID) - } - }() - } - - wg.Wait() - cancel() - - if pipelineExecutionErr != nil { - return - } - - if isLink || isNative { - // run has now completed so it is safe to use benchmark price - if isLink { - // This IS the LINK feed, use our observed price - obs.LinkPrice.Val, obs.LinkPrice.Err = obs.BenchmarkPrice.Val, obs.BenchmarkPrice.Err - } - if isNative { - // This IS the native feed, use our observed price - obs.NativePrice.Val, obs.NativePrice.Err = obs.BenchmarkPrice.Val, obs.BenchmarkPrice.Err - } - } - - ocrcommon.MaybeEnqueueEnhancedTelem(ds.jb, ds.chEnhancedTelem, ocrcommon.EnhancedTelemetryMercuryData{ - V2Observation: &obs, - TaskRunResults: trrs, - RepTimestamp: repts, - FeedVersion: mercuryutils.REPORT_V2, - FetchMaxFinalizedTimestamp: fetchMaxFinalizedTimestamp, - IsLinkFeed: isLink, - IsNativeFeed: isNative, - }) - - return obs, nil -} - -func toBigInt(val interface{}) (*big.Int, error) { - dec, err := utils.ToDecimal(val) - if err != nil { - return nil, err - } - return dec.BigInt(), nil -} - -type parseOutput struct { - benchmarkPrice mercury.ObsResult[*big.Int] -} - -func (ds *datasource) parse(trrs pipeline.TaskRunResults) (o parseOutput, merr error) { - var finaltrrs []pipeline.TaskRunResult - for _, trr := range trrs { - // only return terminal trrs from executeRun - if trr.IsTerminal() { - finaltrrs = append(finaltrrs, trr) - } - } - - if len(finaltrrs) != 1 { - return o, fmt.Errorf("invalid number of results, expected: 1, got: %d", len(finaltrrs)) - } - - return o, setBenchmarkPrice(&o, finaltrrs[0].Result) -} - -func setBenchmarkPrice(o *parseOutput, res pipeline.Result) error { - if res.Error != nil { - o.benchmarkPrice.Err = res.Error - return res.Error - } - val, err := toBigInt(res.Value) - if err != nil { - return fmt.Errorf("failed to parse BenchmarkPrice: %w", err) - } - o.benchmarkPrice.Val = val - return nil -} - -// The context passed in here has a timeout of (ObservationTimeout + ObservationGracePeriod). -// Upon context cancellation, its expected that we return any usable values within ObservationGracePeriod. -func (ds *datasource) executeRun(ctx context.Context) (*pipeline.Run, pipeline.TaskRunResults, error) { - vars := pipeline.NewVarsFrom(map[string]interface{}{ - "jb": map[string]interface{}{ - "databaseID": ds.jb.ID, - "externalJobID": ds.jb.ExternalJobID, - "name": ds.jb.Name.ValueOrZero(), - }, - }) - - run, trrs, err := ds.pipelineRunner.ExecuteRun(ctx, ds.spec, vars) - if err != nil { - return nil, nil, pkgerrors.Wrapf(err, "error executing run for spec ID %v", ds.spec.ID) - } - - return run, trrs, err -} diff --git a/core/services/relay/evm/mercury/v2/data_source_test.go b/core/services/relay/evm/mercury/v2/data_source_test.go deleted file mode 100644 index 25716521d86..00000000000 --- a/core/services/relay/evm/mercury/v2/data_source_test.go +++ /dev/null @@ -1,335 +0,0 @@ -package v2 - -import ( - "context" - "math/big" - "testing" - - "github.com/pkg/errors" - "github.com/stretchr/testify/assert" - - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - - "github.com/smartcontractkit/chainlink-common/pkg/types/mercury" - v2 "github.com/smartcontractkit/chainlink-data-streams/mercury/v2" - - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" - "github.com/smartcontractkit/chainlink/v2/core/logger" - "github.com/smartcontractkit/chainlink/v2/core/services/job" - "github.com/smartcontractkit/chainlink/v2/core/services/pipeline" - mercurymocks "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/mocks" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/utils" - reportcodecv2 "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v2/reportcodec" -) - -var _ mercury.ServerFetcher = &mockFetcher{} - -type mockFetcher struct { - ts int64 - tsErr error - linkPrice *big.Int - linkPriceErr error - nativePrice *big.Int - nativePriceErr error -} - -var feedId utils.FeedID = [32]byte{1} -var linkFeedId utils.FeedID = [32]byte{2} -var nativeFeedId utils.FeedID = [32]byte{3} - -func (m *mockFetcher) FetchInitialMaxFinalizedBlockNumber(context.Context) (*int64, error) { - return nil, nil -} - -func (m *mockFetcher) LatestPrice(ctx context.Context, fId [32]byte) (*big.Int, error) { - if fId == linkFeedId { - return m.linkPrice, m.linkPriceErr - } else if fId == nativeFeedId { - return m.nativePrice, m.nativePriceErr - } - return nil, nil -} - -func (m *mockFetcher) LatestTimestamp(context.Context) (int64, error) { - return m.ts, m.tsErr -} - -type mockORM struct { - report []byte - err error -} - -func (m *mockORM) LatestReport(ctx context.Context, feedID [32]byte) (report []byte, err error) { - return m.report, m.err -} - -type mockSaver struct { - r *pipeline.Run -} - -func (ms *mockSaver) Save(r *pipeline.Run) { - ms.r = r -} - -func Test_Datasource(t *testing.T) { - orm := &mockORM{} - jb := job.Job{ - Type: job.Type(pipeline.OffchainReporting2JobType), - OCR2OracleSpec: &job.OCR2OracleSpec{ - CaptureEATelemetry: true, - PluginConfig: map[string]interface{}{ - "serverURL": "a", - }, - }, - } - ds := &datasource{orm: orm, lggr: logger.TestLogger(t), jb: jb} - ctx := testutils.Context(t) - repts := ocrtypes.ReportTimestamp{} - - fetcher := &mockFetcher{} - ds.fetcher = fetcher - - saver := &mockSaver{} - ds.saver = saver - - goodTrrs := []pipeline.TaskRunResult{ - { - // bp - Result: pipeline.Result{Value: "122.345"}, - Task: &mercurymocks.MockTask{}, - }, - } - - ds.pipelineRunner = &mercurymocks.MockRunner{ - Trrs: goodTrrs, - } - - spec := pipeline.Spec{} - ds.spec = spec - - t.Run("when fetchMaxFinalizedTimestamp=true", func(t *testing.T) { - t.Run("with latest report in database", func(t *testing.T) { - orm.report = buildSampleV2Report() - orm.err = nil - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.NoError(t, obs.MaxFinalizedTimestamp.Err) - assert.Equal(t, int64(124), obs.MaxFinalizedTimestamp.Val) - }) - t.Run("if querying latest report fails", func(t *testing.T) { - orm.report = nil - orm.err = errors.New("something exploded") - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.EqualError(t, obs.MaxFinalizedTimestamp.Err, "something exploded") - assert.Zero(t, obs.MaxFinalizedTimestamp.Val) - }) - t.Run("if codec fails to decode", func(t *testing.T) { - orm.report = []byte{1, 2, 3} - orm.err = nil - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.EqualError(t, obs.MaxFinalizedTimestamp.Err, "failed to decode report: abi: cannot marshal in to go type: length insufficient 3 require 32") - assert.Zero(t, obs.MaxFinalizedTimestamp.Val) - }) - - orm.report = nil - orm.err = nil - - t.Run("if LatestTimestamp returns error", func(t *testing.T) { - fetcher.tsErr = errors.New("some error") - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.EqualError(t, obs.MaxFinalizedTimestamp.Err, "some error") - assert.Zero(t, obs.MaxFinalizedTimestamp.Val) - }) - - t.Run("if LatestTimestamp succeeds", func(t *testing.T) { - fetcher.tsErr = nil - fetcher.ts = 123 - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.Equal(t, int64(123), obs.MaxFinalizedTimestamp.Val) - assert.NoError(t, obs.MaxFinalizedTimestamp.Err) - }) - - t.Run("if LatestTimestamp succeeds but ts=0 (new feed)", func(t *testing.T) { - fetcher.tsErr = nil - fetcher.ts = 0 - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.NoError(t, obs.MaxFinalizedTimestamp.Err) - assert.Zero(t, obs.MaxFinalizedTimestamp.Val) - }) - - t.Run("when run execution succeeded", func(t *testing.T) { - t.Run("when feedId=linkFeedID=nativeFeedId", func(t *testing.T) { - t.Cleanup(func() { - ds.feedID, ds.linkFeedID, ds.nativeFeedID = feedId, linkFeedId, nativeFeedId - }) - - ds.feedID, ds.linkFeedID, ds.nativeFeedID = feedId, feedId, feedId - - fetcher.ts = 123123 - fetcher.tsErr = nil - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.Equal(t, big.NewInt(122), obs.BenchmarkPrice.Val) - assert.NoError(t, obs.BenchmarkPrice.Err) - assert.Equal(t, int64(123123), obs.MaxFinalizedTimestamp.Val) - assert.NoError(t, obs.MaxFinalizedTimestamp.Err) - assert.Equal(t, big.NewInt(122), obs.LinkPrice.Val) - assert.NoError(t, obs.LinkPrice.Err) - assert.Equal(t, big.NewInt(122), obs.NativePrice.Val) - assert.NoError(t, obs.NativePrice.Err) - }) - }) - }) - - t.Run("when fetchMaxFinalizedTimestamp=false", func(t *testing.T) { - t.Run("when run execution fails, returns error", func(t *testing.T) { - t.Cleanup(func() { - ds.pipelineRunner = &mercurymocks.MockRunner{ - Trrs: goodTrrs, - Err: nil, - } - }) - - ds.pipelineRunner = &mercurymocks.MockRunner{ - Trrs: goodTrrs, - Err: errors.New("run execution failed"), - } - - _, err := ds.Observe(ctx, repts, false) - assert.EqualError(t, err, "Observe failed while executing run: error executing run for spec ID 0: run execution failed") - }) - - t.Run("when parsing run results fails, return error", func(t *testing.T) { - t.Cleanup(func() { - runner := &mercurymocks.MockRunner{ - Trrs: goodTrrs, - Err: nil, - } - ds.pipelineRunner = runner - }) - - badTrrs := []pipeline.TaskRunResult{ - { - // benchmark price - Result: pipeline.Result{Error: errors.New("some error with bp")}, - Task: &mercurymocks.MockTask{}, - }, - } - - ds.pipelineRunner = &mercurymocks.MockRunner{ - Trrs: badTrrs, - Err: nil, - } - - _, err := ds.Observe(ctx, repts, false) - assert.EqualError(t, err, "Observe failed while parsing run results: some error with bp") - }) - - t.Run("when run execution succeeded", func(t *testing.T) { - t.Run("when feedId=linkFeedID=nativeFeedId", func(t *testing.T) { - t.Cleanup(func() { - ds.feedID, ds.linkFeedID, ds.nativeFeedID = feedId, linkFeedId, nativeFeedId - }) - - var feedId utils.FeedID = [32]byte{1} - ds.feedID, ds.linkFeedID, ds.nativeFeedID = feedId, feedId, feedId - - obs, err := ds.Observe(ctx, repts, false) - assert.NoError(t, err) - - assert.Equal(t, big.NewInt(122), obs.BenchmarkPrice.Val) - assert.NoError(t, obs.BenchmarkPrice.Err) - assert.Equal(t, int64(0), obs.MaxFinalizedTimestamp.Val) - assert.NoError(t, obs.MaxFinalizedTimestamp.Err) - assert.Equal(t, big.NewInt(122), obs.LinkPrice.Val) - assert.NoError(t, obs.LinkPrice.Err) - assert.Equal(t, big.NewInt(122), obs.NativePrice.Val) - assert.NoError(t, obs.NativePrice.Err) - }) - - t.Run("when fails to fetch linkPrice or nativePrice", func(t *testing.T) { - t.Cleanup(func() { - fetcher.linkPriceErr = nil - fetcher.nativePriceErr = nil - }) - - fetcher.linkPriceErr = errors.New("some error fetching link price") - fetcher.nativePriceErr = errors.New("some error fetching native price") - - obs, err := ds.Observe(ctx, repts, false) - assert.NoError(t, err) - - assert.Nil(t, obs.LinkPrice.Val) - assert.EqualError(t, obs.LinkPrice.Err, "some error fetching link price") - assert.Nil(t, obs.NativePrice.Val) - assert.EqualError(t, obs.NativePrice.Err, "some error fetching native price") - }) - - t.Run("when PluginConfig is empty", func(t *testing.T) { - t.Cleanup(func() { - ds.jb = jb - }) - - fetcher.linkPriceErr = errors.New("some error fetching link price") - fetcher.nativePriceErr = errors.New("some error fetching native price") - - ds.jb.OCR2OracleSpec.PluginConfig = job.JSONConfig{} - - obs, err := ds.Observe(ctx, repts, false) - assert.NoError(t, err) - assert.Nil(t, obs.LinkPrice.Err) - assert.Equal(t, obs.LinkPrice.Val, v2.MissingPrice) - assert.Nil(t, obs.NativePrice.Err) - assert.Equal(t, obs.NativePrice.Val, v2.MissingPrice) - assert.Equal(t, big.NewInt(122), obs.BenchmarkPrice.Val) - }) - - t.Run("when succeeds to fetch linkPrice or nativePrice but got nil (new feed)", func(t *testing.T) { - obs, err := ds.Observe(ctx, repts, false) - assert.NoError(t, err) - - assert.Equal(t, obs.LinkPrice.Val, v2.MissingPrice) - assert.Nil(t, obs.LinkPrice.Err) - assert.Equal(t, obs.NativePrice.Val, v2.MissingPrice) - assert.Nil(t, obs.NativePrice.Err) - }) - }) - }) -} - -var sampleFeedID = [32]uint8{28, 145, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114} - -func buildSampleV2Report() []byte { - feedID := sampleFeedID - timestamp := uint32(124) - bp := big.NewInt(242) - validFromTimestamp := uint32(123) - expiresAt := uint32(456) - linkFee := big.NewInt(3334455) - nativeFee := big.NewInt(556677) - - b, err := reportcodecv2.ReportTypes.Pack(feedID, validFromTimestamp, timestamp, nativeFee, linkFee, expiresAt, bp) - if err != nil { - panic(err) - } - return b -} diff --git a/core/services/relay/evm/mercury/v2/reportcodec/report_codec.go b/core/services/relay/evm/mercury/v2/reportcodec/report_codec.go deleted file mode 100644 index d35621da01b..00000000000 --- a/core/services/relay/evm/mercury/v2/reportcodec/report_codec.go +++ /dev/null @@ -1,79 +0,0 @@ -package reportcodec - -import ( - "context" - "errors" - "fmt" - "math/big" - - pkgerrors "github.com/pkg/errors" - - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - - "github.com/smartcontractkit/chainlink-common/pkg/logger" - v2 "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v2" - - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/utils" - reporttypes "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v2/types" -) - -var ReportTypes = reporttypes.GetSchema() -var maxReportLength = 32 * len(ReportTypes) // each arg is 256 bit EVM word -var zero = big.NewInt(0) - -var _ v2.ReportCodec = &ReportCodec{} - -type ReportCodec struct { - logger logger.Logger - feedID utils.FeedID -} - -func NewReportCodec(feedID [32]byte, lggr logger.Logger) *ReportCodec { - return &ReportCodec{lggr, feedID} -} - -func (r *ReportCodec) BuildReport(ctx context.Context, rf v2.ReportFields) (ocrtypes.Report, error) { - var merr error - if rf.BenchmarkPrice == nil { - merr = errors.Join(merr, errors.New("benchmarkPrice may not be nil")) - } - if rf.LinkFee == nil { - merr = errors.Join(merr, errors.New("linkFee may not be nil")) - } else if rf.LinkFee.Cmp(zero) < 0 { - merr = errors.Join(merr, fmt.Errorf("linkFee may not be negative (got: %s)", rf.LinkFee)) - } - if rf.NativeFee == nil { - merr = errors.Join(merr, errors.New("nativeFee may not be nil")) - } else if rf.NativeFee.Cmp(zero) < 0 { - merr = errors.Join(merr, fmt.Errorf("nativeFee may not be negative (got: %s)", rf.NativeFee)) - } - if merr != nil { - return nil, merr - } - reportBytes, err := ReportTypes.Pack(r.feedID, rf.ValidFromTimestamp, rf.Timestamp, rf.NativeFee, rf.LinkFee, rf.ExpiresAt, rf.BenchmarkPrice) - return ocrtypes.Report(reportBytes), pkgerrors.Wrap(err, "failed to pack report blob") -} - -func (r *ReportCodec) MaxReportLength(ctx context.Context, n int) (int, error) { - return maxReportLength, nil -} - -func (r *ReportCodec) ObservationTimestampFromReport(ctx context.Context, report ocrtypes.Report) (uint32, error) { - decoded, err := r.Decode(ctx, report) - if err != nil { - return 0, err - } - return decoded.ObservationsTimestamp, nil -} - -func (r *ReportCodec) Decode(ctx context.Context, report ocrtypes.Report) (*reporttypes.Report, error) { - return reporttypes.Decode(report) -} - -func (r *ReportCodec) BenchmarkPriceFromReport(ctx context.Context, report ocrtypes.Report) (*big.Int, error) { - decoded, err := r.Decode(ctx, report) - if err != nil { - return nil, err - } - return decoded.BenchmarkPrice, nil -} diff --git a/core/services/relay/evm/mercury/v2/reportcodec/report_codec_test.go b/core/services/relay/evm/mercury/v2/reportcodec/report_codec_test.go deleted file mode 100644 index 809869282b7..00000000000 --- a/core/services/relay/evm/mercury/v2/reportcodec/report_codec_test.go +++ /dev/null @@ -1,162 +0,0 @@ -package reportcodec - -import ( - "math/big" - "testing" - - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - - v2 "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v2" - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" -) - -func newValidReportFields() v2.ReportFields { - return v2.ReportFields{ - Timestamp: 242, - BenchmarkPrice: big.NewInt(243), - ValidFromTimestamp: 123, - ExpiresAt: 20, - LinkFee: big.NewInt(456), - NativeFee: big.NewInt(457), - } -} - -func Test_ReportCodec_BuildReport(t *testing.T) { - r := ReportCodec{} - - t.Run("BuildReport errors on zero values", func(t *testing.T) { - ctx := testutils.Context(t) - _, err := r.BuildReport(ctx, v2.ReportFields{}) - require.Error(t, err) - assert.Contains(t, err.Error(), "benchmarkPrice may not be nil") - assert.Contains(t, err.Error(), "linkFee may not be nil") - assert.Contains(t, err.Error(), "nativeFee may not be nil") - }) - - t.Run("BuildReport constructs a report from observations", func(t *testing.T) { - ctx := testutils.Context(t) - rf := newValidReportFields() - // only need to test happy path since validations are done in relaymercury - - report, err := r.BuildReport(ctx, rf) - require.NoError(t, err) - - reportElems := make(map[string]interface{}) - err = ReportTypes.UnpackIntoMap(reportElems, report) - require.NoError(t, err) - - assert.Equal(t, int(reportElems["observationsTimestamp"].(uint32)), 242) - assert.Equal(t, reportElems["benchmarkPrice"].(*big.Int).Int64(), int64(243)) - assert.Equal(t, reportElems["validFromTimestamp"].(uint32), uint32(123)) - assert.Equal(t, reportElems["expiresAt"].(uint32), uint32(20)) - assert.Equal(t, reportElems["linkFee"].(*big.Int).Int64(), int64(456)) - assert.Equal(t, reportElems["nativeFee"].(*big.Int).Int64(), int64(457)) - - assert.Equal(t, types.Report{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7b, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0xc9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0xc8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x14, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf3}, report) - max, err := r.MaxReportLength(ctx, 4) - require.NoError(t, err) - assert.LessOrEqual(t, len(report), max) - - t.Run("Decode decodes the report", func(t *testing.T) { - ctx := testutils.Context(t) - decoded, err := r.Decode(ctx, report) - require.NoError(t, err) - - require.NotNil(t, decoded) - - assert.Equal(t, uint32(242), decoded.ObservationsTimestamp) - assert.Equal(t, big.NewInt(243), decoded.BenchmarkPrice) - assert.Equal(t, uint32(123), decoded.ValidFromTimestamp) - assert.Equal(t, uint32(20), decoded.ExpiresAt) - assert.Equal(t, big.NewInt(456), decoded.LinkFee) - assert.Equal(t, big.NewInt(457), decoded.NativeFee) - }) - }) - - t.Run("errors on negative fee", func(t *testing.T) { - rf := newValidReportFields() - rf.LinkFee = big.NewInt(-1) - rf.NativeFee = big.NewInt(-1) - ctx := testutils.Context(t) - _, err := r.BuildReport(ctx, rf) - require.Error(t, err) - - assert.Contains(t, err.Error(), "linkFee may not be negative (got: -1)") - assert.Contains(t, err.Error(), "nativeFee may not be negative (got: -1)") - }) - - t.Run("Decode errors on invalid report", func(t *testing.T) { - ctx := testutils.Context(t) - _, err := r.Decode(ctx, []byte{1, 2, 3}) - assert.EqualError(t, err, "failed to decode report: abi: cannot marshal in to go type: length insufficient 3 require 32") - - longBad := make([]byte, 64) - for i := 0; i < len(longBad); i++ { - longBad[i] = byte(i) - } - _, err = r.Decode(ctx, longBad) - assert.EqualError(t, err, "failed to decode report: abi: improperly encoded uint32 value") - }) -} - -func buildSampleReport(ts int64) []byte { - feedID := [32]byte{'f', 'o', 'o'} - timestamp := uint32(ts) - bp := big.NewInt(242) - validFromTimestamp := uint32(123) - expiresAt := uint32(456) - linkFee := big.NewInt(3334455) - nativeFee := big.NewInt(556677) - - b, err := ReportTypes.Pack(feedID, validFromTimestamp, timestamp, nativeFee, linkFee, expiresAt, bp) - if err != nil { - panic(err) - } - return b -} - -func Test_ReportCodec_ObservationTimestampFromReport(t *testing.T) { - r := ReportCodec{} - - t.Run("ObservationTimestampFromReport extracts observation timestamp from a valid report", func(t *testing.T) { - report := buildSampleReport(123) - - ctx := testutils.Context(t) - ts, err := r.ObservationTimestampFromReport(ctx, report) - require.NoError(t, err) - - assert.Equal(t, ts, uint32(123)) - }) - t.Run("ObservationTimestampFromReport returns error when report is invalid", func(t *testing.T) { - report := []byte{1, 2, 3} - - ctx := testutils.Context(t) - _, err := r.ObservationTimestampFromReport(ctx, report) - require.Error(t, err) - - assert.EqualError(t, err, "failed to decode report: abi: cannot marshal in to go type: length insufficient 3 require 32") - }) -} - -func Test_ReportCodec_BenchmarkPriceFromReport(t *testing.T) { - r := ReportCodec{} - - t.Run("BenchmarkPriceFromReport extracts the benchmark price from valid report", func(t *testing.T) { - ctx := testutils.Context(t) - report := buildSampleReport(123) - - bp, err := r.BenchmarkPriceFromReport(ctx, report) - require.NoError(t, err) - - assert.Equal(t, big.NewInt(242), bp) - }) - t.Run("BenchmarkPriceFromReport errors on invalid report", func(t *testing.T) { - ctx := testutils.Context(t) - _, err := r.BenchmarkPriceFromReport(ctx, []byte{1, 2, 3}) - require.Error(t, err) - assert.EqualError(t, err, "failed to decode report: abi: cannot marshal in to go type: length insufficient 3 require 32") - }) -} diff --git a/core/services/relay/evm/mercury/v2/types/types.go b/core/services/relay/evm/mercury/v2/types/types.go deleted file mode 100644 index 3c1df286d14..00000000000 --- a/core/services/relay/evm/mercury/v2/types/types.go +++ /dev/null @@ -1,52 +0,0 @@ -package reporttypes - -import ( - "fmt" - "math/big" - - "github.com/ethereum/go-ethereum/accounts/abi" -) - -var schema = GetSchema() - -func GetSchema() abi.Arguments { - mustNewType := func(t string) abi.Type { - result, err := abi.NewType(t, "", []abi.ArgumentMarshaling{}) - if err != nil { - panic(fmt.Sprintf("Unexpected error during abi.NewType: %s", err)) - } - return result - } - return abi.Arguments([]abi.Argument{ - {Name: "feedId", Type: mustNewType("bytes32")}, - {Name: "validFromTimestamp", Type: mustNewType("uint32")}, - {Name: "observationsTimestamp", Type: mustNewType("uint32")}, - {Name: "nativeFee", Type: mustNewType("uint192")}, - {Name: "linkFee", Type: mustNewType("uint192")}, - {Name: "expiresAt", Type: mustNewType("uint32")}, - {Name: "benchmarkPrice", Type: mustNewType("int192")}, - }) -} - -type Report struct { - FeedId [32]byte - ObservationsTimestamp uint32 - BenchmarkPrice *big.Int - ValidFromTimestamp uint32 - ExpiresAt uint32 - LinkFee *big.Int - NativeFee *big.Int -} - -// Decode is made available to external users (i.e. mercury server) -func Decode(report []byte) (*Report, error) { - values, err := schema.Unpack(report) - if err != nil { - return nil, fmt.Errorf("failed to decode report: %w", err) - } - decoded := new(Report) - if err = schema.Copy(decoded, values); err != nil { - return nil, fmt.Errorf("failed to copy report values to struct: %w", err) - } - return decoded, nil -} diff --git a/core/services/relay/evm/mercury/v3/data_source.go b/core/services/relay/evm/mercury/v3/data_source.go deleted file mode 100644 index 776e2fb2f47..00000000000 --- a/core/services/relay/evm/mercury/v3/data_source.go +++ /dev/null @@ -1,294 +0,0 @@ -package v3 - -import ( - "context" - "errors" - "fmt" - "math/big" - "sync" - - pkgerrors "github.com/pkg/errors" - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - - "github.com/smartcontractkit/chainlink-common/pkg/types/mercury" - v3types "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v3" - v3 "github.com/smartcontractkit/chainlink-data-streams/mercury/v3" - "github.com/smartcontractkit/chainlink/v2/core/logger" - "github.com/smartcontractkit/chainlink/v2/core/services/job" - "github.com/smartcontractkit/chainlink/v2/core/services/ocrcommon" - "github.com/smartcontractkit/chainlink/v2/core/services/pipeline" - "github.com/smartcontractkit/chainlink/v2/core/services/pipeline/eautils" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/types" - mercurytypes "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/types" - mercuryutils "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/utils" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v3/reportcodec" - "github.com/smartcontractkit/chainlink/v2/core/utils" -) - -const adapterLWBAErrorName = "AdapterLWBAError" - -type Runner interface { - ExecuteRun(ctx context.Context, spec pipeline.Spec, vars pipeline.Vars) (run *pipeline.Run, trrs pipeline.TaskRunResults, err error) -} - -type LatestReportFetcher interface { - LatestPrice(ctx context.Context, feedID [32]byte) (*big.Int, error) - LatestTimestamp(context.Context) (int64, error) -} - -type datasource struct { - pipelineRunner Runner - jb job.Job - spec pipeline.Spec - feedID mercuryutils.FeedID - lggr logger.Logger - saver ocrcommon.Saver - orm types.DataSourceORM - codec reportcodec.ReportCodec - - fetcher LatestReportFetcher - linkFeedID mercuryutils.FeedID - nativeFeedID mercuryutils.FeedID - - mu sync.RWMutex - - chEnhancedTelem chan<- ocrcommon.EnhancedTelemetryMercuryData -} - -var _ v3.DataSource = &datasource{} - -func NewDataSource(orm types.DataSourceORM, pr pipeline.Runner, jb job.Job, spec pipeline.Spec, feedID mercuryutils.FeedID, lggr logger.Logger, s ocrcommon.Saver, enhancedTelemChan chan ocrcommon.EnhancedTelemetryMercuryData, fetcher LatestReportFetcher, linkFeedID, nativeFeedID mercuryutils.FeedID) *datasource { - return &datasource{pr, jb, spec, feedID, lggr, s, orm, reportcodec.ReportCodec{}, fetcher, linkFeedID, nativeFeedID, sync.RWMutex{}, enhancedTelemChan} -} - -func (ds *datasource) Observe(ctx context.Context, repts ocrtypes.ReportTimestamp, fetchMaxFinalizedTimestamp bool) (obs v3types.Observation, pipelineExecutionErr error) { - var wg sync.WaitGroup - ctx, cancel := context.WithCancel(ctx) - - if fetchMaxFinalizedTimestamp { - wg.Add(1) - go func() { - defer wg.Done() - latest, dbErr := ds.orm.LatestReport(ctx, ds.feedID) - if dbErr != nil { - obs.MaxFinalizedTimestamp.Err = dbErr - return - } - if latest != nil { - maxFinalizedBlockNumber, decodeErr := ds.codec.ObservationTimestampFromReport(ctx, latest) - obs.MaxFinalizedTimestamp.Val, obs.MaxFinalizedTimestamp.Err = int64(maxFinalizedBlockNumber), decodeErr - return - } - obs.MaxFinalizedTimestamp.Val, obs.MaxFinalizedTimestamp.Err = ds.fetcher.LatestTimestamp(ctx) - }() - } - - var trrs pipeline.TaskRunResults - wg.Add(1) - go func() { - defer wg.Done() - var run *pipeline.Run - run, trrs, pipelineExecutionErr = ds.executeRun(ctx) - if pipelineExecutionErr != nil { - cancel() - pipelineExecutionErr = fmt.Errorf("Observe failed while executing run: %w", pipelineExecutionErr) - return - } - - ds.saver.Save(run) - - var parsed parseOutput - parsed, pipelineExecutionErr = ds.parse(trrs) - if pipelineExecutionErr != nil { - cancel() - // This is not expected under normal circumstances - ds.lggr.Errorw("Observe failed while parsing run results", "err", pipelineExecutionErr) - pipelineExecutionErr = fmt.Errorf("Observe failed while parsing run results: %w", pipelineExecutionErr) - return - } - obs.BenchmarkPrice = parsed.benchmarkPrice - obs.Bid = parsed.bid - obs.Ask = parsed.ask - }() - - var isLink, isNative bool - if len(ds.jb.OCR2OracleSpec.PluginConfig) == 0 { - obs.LinkPrice.Val = v3.MissingPrice - } else if ds.feedID == ds.linkFeedID { - isLink = true - } else { - wg.Add(1) - go func() { - defer wg.Done() - obs.LinkPrice.Val, obs.LinkPrice.Err = ds.fetcher.LatestPrice(ctx, ds.linkFeedID) - if obs.LinkPrice.Val == nil && obs.LinkPrice.Err == nil { - mercurytypes.PriceFeedMissingCount.WithLabelValues(ds.linkFeedID.String()).Inc() - ds.lggr.Warnw(fmt.Sprintf("Mercury server was missing LINK feed, using sentinel value of %s", v3.MissingPrice), "linkFeedID", ds.linkFeedID) - obs.LinkPrice.Val = v3.MissingPrice - } else if obs.LinkPrice.Err != nil { - mercurytypes.PriceFeedErrorCount.WithLabelValues(ds.linkFeedID.String()).Inc() - ds.lggr.Errorw("Mercury server returned error querying LINK price feed", "err", obs.LinkPrice.Err, "linkFeedID", ds.linkFeedID) - } - }() - } - - if len(ds.jb.OCR2OracleSpec.PluginConfig) == 0 { - obs.NativePrice.Val = v3.MissingPrice - } else if ds.feedID == ds.nativeFeedID { - isNative = true - } else { - wg.Add(1) - go func() { - defer wg.Done() - obs.NativePrice.Val, obs.NativePrice.Err = ds.fetcher.LatestPrice(ctx, ds.nativeFeedID) - if obs.NativePrice.Val == nil && obs.NativePrice.Err == nil { - mercurytypes.PriceFeedMissingCount.WithLabelValues(ds.nativeFeedID.String()).Inc() - ds.lggr.Warnw(fmt.Sprintf("Mercury server was missing native feed, using sentinel value of %s", v3.MissingPrice), "nativeFeedID", ds.nativeFeedID) - obs.NativePrice.Val = v3.MissingPrice - } else if obs.NativePrice.Err != nil { - mercurytypes.PriceFeedErrorCount.WithLabelValues(ds.nativeFeedID.String()).Inc() - ds.lggr.Errorw("Mercury server returned error querying native price feed", "err", obs.NativePrice.Err, "nativeFeedID", ds.nativeFeedID) - } - }() - } - - wg.Wait() - cancel() - - if pipelineExecutionErr != nil { - var adapterError *eautils.AdapterError - if errors.As(pipelineExecutionErr, &adapterError) && adapterError.Name == adapterLWBAErrorName { - ocrcommon.MaybeEnqueueEnhancedTelem(ds.jb, ds.chEnhancedTelem, ocrcommon.EnhancedTelemetryMercuryData{ - V3Observation: &obs, - TaskRunResults: trrs, - RepTimestamp: repts, - FeedVersion: mercuryutils.REPORT_V3, - FetchMaxFinalizedTimestamp: fetchMaxFinalizedTimestamp, - IsLinkFeed: isLink, - IsNativeFeed: isNative, - DpInvariantViolationDetected: true, - }) - } - return - } - - if isLink || isNative { - // run has now completed so it is safe to use benchmark price - if isLink { - // This IS the LINK feed, use our observed price - obs.LinkPrice.Val, obs.LinkPrice.Err = obs.BenchmarkPrice.Val, obs.BenchmarkPrice.Err - } - if isNative { - // This IS the native feed, use our observed price - obs.NativePrice.Val, obs.NativePrice.Err = obs.BenchmarkPrice.Val, obs.BenchmarkPrice.Err - } - } - - ocrcommon.MaybeEnqueueEnhancedTelem(ds.jb, ds.chEnhancedTelem, ocrcommon.EnhancedTelemetryMercuryData{ - V3Observation: &obs, - TaskRunResults: trrs, - RepTimestamp: repts, - FeedVersion: mercuryutils.REPORT_V3, - FetchMaxFinalizedTimestamp: fetchMaxFinalizedTimestamp, - IsLinkFeed: isLink, - IsNativeFeed: isNative, - }) - - return obs, nil -} - -func toBigInt(val interface{}) (*big.Int, error) { - dec, err := utils.ToDecimal(val) - if err != nil { - return nil, err - } - return dec.BigInt(), nil -} - -type parseOutput struct { - benchmarkPrice mercury.ObsResult[*big.Int] - bid mercury.ObsResult[*big.Int] - ask mercury.ObsResult[*big.Int] -} - -func (ds *datasource) parse(trrs pipeline.TaskRunResults) (o parseOutput, merr error) { - var finaltrrs []pipeline.TaskRunResult - for _, trr := range trrs { - // only return terminal trrs from executeRun - if trr.IsTerminal() { - finaltrrs = append(finaltrrs, trr) - } - } - - // pipeline.TaskRunResults comes ordered asc by index, this is guaranteed - // by the pipeline executor - if len(finaltrrs) != 3 { - return o, fmt.Errorf("invalid number of results, expected: 3, got: %d", len(finaltrrs)) - } - - merr = errors.Join( - setBenchmarkPrice(&o, finaltrrs[0].Result), - setBid(&o, finaltrrs[1].Result), - setAsk(&o, finaltrrs[2].Result), - ) - - return o, merr -} - -func setBenchmarkPrice(o *parseOutput, res pipeline.Result) error { - if res.Error != nil { - o.benchmarkPrice.Err = res.Error - return res.Error - } - val, err := toBigInt(res.Value) - if err != nil { - return fmt.Errorf("failed to parse BenchmarkPrice: %w", err) - } - o.benchmarkPrice.Val = val - return nil -} - -func setBid(o *parseOutput, res pipeline.Result) error { - if res.Error != nil { - o.bid.Err = res.Error - return res.Error - } - val, err := toBigInt(res.Value) - if err != nil { - return fmt.Errorf("failed to parse Bid: %w", err) - } - o.bid.Val = val - return nil -} - -func setAsk(o *parseOutput, res pipeline.Result) error { - if res.Error != nil { - o.ask.Err = res.Error - return res.Error - } - val, err := toBigInt(res.Value) - if err != nil { - return fmt.Errorf("failed to parse Ask: %w", err) - } - o.ask.Val = val - return nil -} - -// The context passed in here has a timeout of (ObservationTimeout + ObservationGracePeriod). -// Upon context cancellation, its expected that we return any usable values within ObservationGracePeriod. -func (ds *datasource) executeRun(ctx context.Context) (*pipeline.Run, pipeline.TaskRunResults, error) { - vars := pipeline.NewVarsFrom(map[string]interface{}{ - "jb": map[string]interface{}{ - "databaseID": ds.jb.ID, - "externalJobID": ds.jb.ExternalJobID, - "name": ds.jb.Name.ValueOrZero(), - }, - }) - - run, trrs, err := ds.pipelineRunner.ExecuteRun(ctx, ds.spec, vars) - if err != nil { - return nil, nil, pkgerrors.Wrapf(err, "error executing run for spec ID %v", ds.spec.ID) - } - - return run, trrs, err -} diff --git a/core/services/relay/evm/mercury/v3/data_source_test.go b/core/services/relay/evm/mercury/v3/data_source_test.go deleted file mode 100644 index 518fabb12c9..00000000000 --- a/core/services/relay/evm/mercury/v3/data_source_test.go +++ /dev/null @@ -1,417 +0,0 @@ -package v3 - -import ( - "context" - "math/big" - "testing" - - relaymercuryv3 "github.com/smartcontractkit/chainlink-data-streams/mercury/v3" - "github.com/smartcontractkit/chainlink/v2/core/services/ocrcommon" - "github.com/smartcontractkit/chainlink/v2/core/services/pipeline/eautils" - - "github.com/pkg/errors" - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - "github.com/stretchr/testify/assert" - - mercurytypes "github.com/smartcontractkit/chainlink-common/pkg/types/mercury" - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" - "github.com/smartcontractkit/chainlink/v2/core/logger" - "github.com/smartcontractkit/chainlink/v2/core/services/job" - "github.com/smartcontractkit/chainlink/v2/core/services/pipeline" - mercurymocks "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/mocks" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/utils" - reportcodecv3 "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v3/reportcodec" -) - -var _ mercurytypes.ServerFetcher = &mockFetcher{} - -type mockFetcher struct { - ts int64 - tsErr error - linkPrice *big.Int - linkPriceErr error - nativePrice *big.Int - nativePriceErr error -} - -var feedId utils.FeedID = [32]byte{1} -var linkFeedId utils.FeedID = [32]byte{2} -var nativeFeedId utils.FeedID = [32]byte{3} - -func (m *mockFetcher) FetchInitialMaxFinalizedBlockNumber(context.Context) (*int64, error) { - return nil, nil -} - -func (m *mockFetcher) LatestPrice(ctx context.Context, fId [32]byte) (*big.Int, error) { - if fId == linkFeedId { - return m.linkPrice, m.linkPriceErr - } else if fId == nativeFeedId { - return m.nativePrice, m.nativePriceErr - } - return nil, nil -} - -func (m *mockFetcher) LatestTimestamp(context.Context) (int64, error) { - return m.ts, m.tsErr -} - -type mockORM struct { - report []byte - err error -} - -func (m *mockORM) LatestReport(ctx context.Context, feedID [32]byte) (report []byte, err error) { - return m.report, m.err -} - -type mockSaver struct { - r *pipeline.Run -} - -func (ms *mockSaver) Save(r *pipeline.Run) { - ms.r = r -} - -func Test_Datasource(t *testing.T) { - orm := &mockORM{} - jb := job.Job{ - Type: job.Type(pipeline.OffchainReporting2JobType), - OCR2OracleSpec: &job.OCR2OracleSpec{ - CaptureEATelemetry: true, - PluginConfig: map[string]interface{}{ - "serverURL": "a", - }, - }, - } - ds := &datasource{orm: orm, lggr: logger.TestLogger(t), jb: jb} - ctx := testutils.Context(t) - repts := ocrtypes.ReportTimestamp{} - - fetcher := &mockFetcher{} - ds.fetcher = fetcher - - saver := &mockSaver{} - ds.saver = saver - - goodTrrs := []pipeline.TaskRunResult{ - { - // bp - Result: pipeline.Result{Value: "122.345"}, - Task: &mercurymocks.MockTask{}, - }, - { - // bid - Result: pipeline.Result{Value: "121.993"}, - Task: &mercurymocks.MockTask{}, - }, - { - // ask - Result: pipeline.Result{Value: "123.111"}, - Task: &mercurymocks.MockTask{}, - }, - } - - ds.pipelineRunner = &mercurymocks.MockRunner{ - Trrs: goodTrrs, - } - - spec := pipeline.Spec{} - ds.spec = spec - - t.Run("when fetchMaxFinalizedTimestamp=true", func(t *testing.T) { - t.Run("with latest report in database", func(t *testing.T) { - orm.report = buildSampleV3Report() - orm.err = nil - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.NoError(t, obs.MaxFinalizedTimestamp.Err) - assert.Equal(t, int64(124), obs.MaxFinalizedTimestamp.Val) - }) - t.Run("if querying latest report fails", func(t *testing.T) { - orm.report = nil - orm.err = errors.New("something exploded") - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.EqualError(t, obs.MaxFinalizedTimestamp.Err, "something exploded") - assert.Zero(t, obs.MaxFinalizedTimestamp.Val) - }) - t.Run("if codec fails to decode", func(t *testing.T) { - orm.report = []byte{1, 2, 3} - orm.err = nil - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.EqualError(t, obs.MaxFinalizedTimestamp.Err, "failed to decode report: abi: cannot marshal in to go type: length insufficient 3 require 32") - assert.Zero(t, obs.MaxFinalizedTimestamp.Val) - }) - - orm.report = nil - orm.err = nil - - t.Run("if LatestTimestamp returns error", func(t *testing.T) { - fetcher.tsErr = errors.New("some error") - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.EqualError(t, obs.MaxFinalizedTimestamp.Err, "some error") - assert.Zero(t, obs.MaxFinalizedTimestamp.Val) - }) - - t.Run("if LatestTimestamp succeeds", func(t *testing.T) { - fetcher.tsErr = nil - fetcher.ts = 123 - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.Equal(t, int64(123), obs.MaxFinalizedTimestamp.Val) - assert.NoError(t, obs.MaxFinalizedTimestamp.Err) - }) - - t.Run("if LatestTimestamp succeeds but ts=0 (new feed)", func(t *testing.T) { - fetcher.tsErr = nil - fetcher.ts = 0 - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.NoError(t, obs.MaxFinalizedTimestamp.Err) - assert.Zero(t, obs.MaxFinalizedTimestamp.Val) - }) - - t.Run("when run execution succeeded", func(t *testing.T) { - t.Run("when feedId=linkFeedID=nativeFeedId", func(t *testing.T) { - t.Cleanup(func() { - ds.feedID, ds.linkFeedID, ds.nativeFeedID = feedId, linkFeedId, nativeFeedId - }) - - ds.feedID, ds.linkFeedID, ds.nativeFeedID = feedId, feedId, feedId - - fetcher.ts = 123123 - fetcher.tsErr = nil - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.Equal(t, big.NewInt(122), obs.BenchmarkPrice.Val) - assert.NoError(t, obs.BenchmarkPrice.Err) - assert.Equal(t, big.NewInt(121), obs.Bid.Val) - assert.NoError(t, obs.Bid.Err) - assert.Equal(t, big.NewInt(123), obs.Ask.Val) - assert.NoError(t, obs.Ask.Err) - assert.Equal(t, int64(123123), obs.MaxFinalizedTimestamp.Val) - assert.NoError(t, obs.MaxFinalizedTimestamp.Err) - assert.Equal(t, big.NewInt(122), obs.LinkPrice.Val) - assert.NoError(t, obs.LinkPrice.Err) - assert.Equal(t, big.NewInt(122), obs.NativePrice.Val) - assert.NoError(t, obs.NativePrice.Err) - }) - }) - }) - - t.Run("when fetchMaxFinalizedTimestamp=false", func(t *testing.T) { - t.Run("when run execution fails, returns error", func(t *testing.T) { - t.Cleanup(func() { - ds.pipelineRunner = &mercurymocks.MockRunner{ - Trrs: goodTrrs, - Err: nil, - } - }) - - ds.pipelineRunner = &mercurymocks.MockRunner{ - Trrs: goodTrrs, - Err: errors.New("run execution failed"), - } - - _, err := ds.Observe(ctx, repts, false) - assert.EqualError(t, err, "Observe failed while executing run: error executing run for spec ID 0: run execution failed") - }) - - t.Run("when parsing run results fails, return error", func(t *testing.T) { - t.Cleanup(func() { - runner := &mercurymocks.MockRunner{ - Trrs: goodTrrs, - Err: nil, - } - ds.pipelineRunner = runner - }) - - badTrrs := []pipeline.TaskRunResult{ - { - // benchmark price - Result: pipeline.Result{Value: "122.345"}, - Task: &mercurymocks.MockTask{}, - }, - { - // bid - Result: pipeline.Result{Value: "121.993"}, - Task: &mercurymocks.MockTask{}, - }, - { - // ask - Result: pipeline.Result{Error: errors.New("some error with ask")}, - Task: &mercurymocks.MockTask{}, - }, - } - - ds.pipelineRunner = &mercurymocks.MockRunner{ - Trrs: badTrrs, - Err: nil, - } - - chEnhancedTelem := make(chan ocrcommon.EnhancedTelemetryMercuryData, 1) - ds.chEnhancedTelem = chEnhancedTelem - - _, err := ds.Observe(ctx, repts, false) - assert.EqualError(t, err, "Observe failed while parsing run results: some error with ask") - - select { - case <-chEnhancedTelem: - assert.Fail(t, "did not expect to receive telemetry") - default: - } - }) - - t.Run("when run results fails with a bid ask violation", func(t *testing.T) { - t.Cleanup(func() { - runner := &mercurymocks.MockRunner{ - Trrs: goodTrrs, - Err: nil, - } - ds.pipelineRunner = runner - }) - - badTrrs := []pipeline.TaskRunResult{ - { - // benchmark price - Result: pipeline.Result{Value: "122.345"}, - Task: &mercurymocks.MockTask{}, - }, - { - // bid - Result: pipeline.Result{Value: "121.993"}, - Task: &mercurymocks.MockTask{}, - }, - { - // ask - Result: pipeline.Result{Error: &eautils.AdapterError{Name: adapterLWBAErrorName, Message: "bid ask violation"}}, - Task: &mercurymocks.MockTask{}, - }, - } - - ds.pipelineRunner = &mercurymocks.MockRunner{ - Trrs: badTrrs, - Err: nil, - } - - chEnhancedTelem := make(chan ocrcommon.EnhancedTelemetryMercuryData, 1) - ds.chEnhancedTelem = chEnhancedTelem - - _, err := ds.Observe(ctx, repts, false) - assert.EqualError(t, err, "Observe failed while parsing run results: AdapterLWBAError: bid ask violation") - - telem := <-chEnhancedTelem - assert.True(t, telem.DpInvariantViolationDetected) - }) - - t.Run("when run execution succeeded", func(t *testing.T) { - t.Run("when feedId=linkFeedID=nativeFeedId", func(t *testing.T) { - t.Cleanup(func() { - ds.feedID, ds.linkFeedID, ds.nativeFeedID = feedId, linkFeedId, nativeFeedId - }) - - var feedId utils.FeedID = [32]byte{1} - ds.feedID, ds.linkFeedID, ds.nativeFeedID = feedId, feedId, feedId - - obs, err := ds.Observe(ctx, repts, false) - assert.NoError(t, err) - - assert.Equal(t, big.NewInt(122), obs.BenchmarkPrice.Val) - assert.NoError(t, obs.BenchmarkPrice.Err) - assert.Equal(t, big.NewInt(121), obs.Bid.Val) - assert.NoError(t, obs.Bid.Err) - assert.Equal(t, big.NewInt(123), obs.Ask.Val) - assert.NoError(t, obs.Ask.Err) - assert.Equal(t, int64(0), obs.MaxFinalizedTimestamp.Val) - assert.NoError(t, obs.MaxFinalizedTimestamp.Err) - assert.Equal(t, big.NewInt(122), obs.LinkPrice.Val) - assert.NoError(t, obs.LinkPrice.Err) - assert.Equal(t, big.NewInt(122), obs.NativePrice.Val) - assert.NoError(t, obs.NativePrice.Err) - }) - - t.Run("when fails to fetch linkPrice or nativePrice", func(t *testing.T) { - t.Cleanup(func() { - fetcher.linkPriceErr = nil - fetcher.nativePriceErr = nil - }) - - fetcher.linkPriceErr = errors.New("some error fetching link price") - fetcher.nativePriceErr = errors.New("some error fetching native price") - - obs, err := ds.Observe(ctx, repts, false) - assert.NoError(t, err) - - assert.Nil(t, obs.LinkPrice.Val) - assert.EqualError(t, obs.LinkPrice.Err, "some error fetching link price") - assert.Nil(t, obs.NativePrice.Val) - assert.EqualError(t, obs.NativePrice.Err, "some error fetching native price") - }) - - t.Run("when PluginConfig is empty", func(t *testing.T) { - t.Cleanup(func() { - ds.jb = jb - }) - - fetcher.linkPriceErr = errors.New("some error fetching link price") - fetcher.nativePriceErr = errors.New("some error fetching native price") - - ds.jb.OCR2OracleSpec.PluginConfig = job.JSONConfig{} - - obs, err := ds.Observe(ctx, repts, false) - assert.NoError(t, err) - assert.Nil(t, obs.LinkPrice.Err) - assert.Equal(t, obs.LinkPrice.Val, relaymercuryv3.MissingPrice) - assert.Nil(t, obs.NativePrice.Err) - assert.Equal(t, obs.NativePrice.Val, relaymercuryv3.MissingPrice) - assert.Equal(t, big.NewInt(122), obs.BenchmarkPrice.Val) - }) - - t.Run("when succeeds to fetch linkPrice or nativePrice but got nil (new feed)", func(t *testing.T) { - obs, err := ds.Observe(ctx, repts, false) - assert.NoError(t, err) - - assert.Equal(t, obs.LinkPrice.Val, relaymercuryv3.MissingPrice) - assert.Nil(t, obs.LinkPrice.Err) - assert.Equal(t, obs.NativePrice.Val, relaymercuryv3.MissingPrice) - assert.Nil(t, obs.NativePrice.Err) - }) - }) - }) -} - -var sampleFeedID = [32]uint8{28, 145, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114} - -func buildSampleV3Report() []byte { - feedID := sampleFeedID - timestamp := uint32(124) - bp := big.NewInt(242) - bid := big.NewInt(243) - ask := big.NewInt(244) - validFromTimestamp := uint32(123) - expiresAt := uint32(456) - linkFee := big.NewInt(3334455) - nativeFee := big.NewInt(556677) - - b, err := reportcodecv3.ReportTypes.Pack(feedID, validFromTimestamp, timestamp, nativeFee, linkFee, expiresAt, bp, bid, ask) - if err != nil { - panic(err) - } - return b -} diff --git a/core/services/relay/evm/mercury/v4/data_source.go b/core/services/relay/evm/mercury/v4/data_source.go deleted file mode 100644 index 46e34f0b9c5..00000000000 --- a/core/services/relay/evm/mercury/v4/data_source.go +++ /dev/null @@ -1,262 +0,0 @@ -package v4 - -import ( - "context" - "errors" - "fmt" - "math/big" - "sync" - - pkgerrors "github.com/pkg/errors" - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - - "github.com/smartcontractkit/chainlink-common/pkg/types/mercury" - v4types "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v4" - v4 "github.com/smartcontractkit/chainlink-data-streams/mercury/v4" - "github.com/smartcontractkit/chainlink/v2/core/logger" - "github.com/smartcontractkit/chainlink/v2/core/services/job" - "github.com/smartcontractkit/chainlink/v2/core/services/ocrcommon" - "github.com/smartcontractkit/chainlink/v2/core/services/pipeline" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/types" - mercurytypes "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/types" - mercuryutils "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/utils" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v4/reportcodec" - "github.com/smartcontractkit/chainlink/v2/core/utils" -) - -type Runner interface { - ExecuteRun(ctx context.Context, spec pipeline.Spec, vars pipeline.Vars) (run *pipeline.Run, trrs pipeline.TaskRunResults, err error) -} - -type LatestReportFetcher interface { - LatestPrice(ctx context.Context, feedID [32]byte) (*big.Int, error) - LatestTimestamp(context.Context) (int64, error) -} - -type datasource struct { - pipelineRunner Runner - jb job.Job - spec pipeline.Spec - feedID mercuryutils.FeedID - lggr logger.Logger - saver ocrcommon.Saver - orm types.DataSourceORM - codec reportcodec.ReportCodec - - fetcher LatestReportFetcher - linkFeedID mercuryutils.FeedID - nativeFeedID mercuryutils.FeedID - - mu sync.RWMutex - - chEnhancedTelem chan<- ocrcommon.EnhancedTelemetryMercuryData -} - -var _ v4.DataSource = &datasource{} - -func NewDataSource(orm types.DataSourceORM, pr pipeline.Runner, jb job.Job, spec pipeline.Spec, feedID mercuryutils.FeedID, lggr logger.Logger, s ocrcommon.Saver, enhancedTelemChan chan ocrcommon.EnhancedTelemetryMercuryData, fetcher LatestReportFetcher, linkFeedID, nativeFeedID mercuryutils.FeedID) *datasource { - return &datasource{pr, jb, spec, feedID, lggr, s, orm, reportcodec.ReportCodec{}, fetcher, linkFeedID, nativeFeedID, sync.RWMutex{}, enhancedTelemChan} -} - -func (ds *datasource) Observe(ctx context.Context, repts ocrtypes.ReportTimestamp, fetchMaxFinalizedTimestamp bool) (obs v4types.Observation, pipelineExecutionErr error) { - var wg sync.WaitGroup - ctx, cancel := context.WithCancel(ctx) - - if fetchMaxFinalizedTimestamp { - wg.Add(1) - go func() { - defer wg.Done() - latest, dbErr := ds.orm.LatestReport(ctx, ds.feedID) - if dbErr != nil { - obs.MaxFinalizedTimestamp.Err = dbErr - return - } - if latest != nil { - maxFinalizedBlockNumber, decodeErr := ds.codec.ObservationTimestampFromReport(ctx, latest) - obs.MaxFinalizedTimestamp.Val, obs.MaxFinalizedTimestamp.Err = int64(maxFinalizedBlockNumber), decodeErr - return - } - obs.MaxFinalizedTimestamp.Val, obs.MaxFinalizedTimestamp.Err = ds.fetcher.LatestTimestamp(ctx) - }() - } - - var trrs pipeline.TaskRunResults - wg.Add(1) - go func() { - defer wg.Done() - var run *pipeline.Run - run, trrs, pipelineExecutionErr = ds.executeRun(ctx) - if pipelineExecutionErr != nil { - cancel() - pipelineExecutionErr = fmt.Errorf("Observe failed while executing run: %w", pipelineExecutionErr) - return - } - - ds.saver.Save(run) - - var parsed parseOutput - parsed, pipelineExecutionErr = ds.parse(trrs) - if pipelineExecutionErr != nil { - cancel() - // This is not expected under normal circumstances - ds.lggr.Errorw("Observe failed while parsing run results", "err", pipelineExecutionErr) - pipelineExecutionErr = fmt.Errorf("Observe failed while parsing run results: %w", pipelineExecutionErr) - return - } - obs.BenchmarkPrice = parsed.benchmarkPrice - obs.MarketStatus = parsed.marketStatus - }() - - var isLink, isNative bool - if len(ds.jb.OCR2OracleSpec.PluginConfig) == 0 { - obs.LinkPrice.Val = v4.MissingPrice - } else if ds.feedID == ds.linkFeedID { - isLink = true - } else { - wg.Add(1) - go func() { - defer wg.Done() - obs.LinkPrice.Val, obs.LinkPrice.Err = ds.fetcher.LatestPrice(ctx, ds.linkFeedID) - if obs.LinkPrice.Val == nil && obs.LinkPrice.Err == nil { - mercurytypes.PriceFeedMissingCount.WithLabelValues(ds.linkFeedID.String()).Inc() - ds.lggr.Warnw(fmt.Sprintf("Mercury server was missing LINK feed, using sentinel value of %s", v4.MissingPrice), "linkFeedID", ds.linkFeedID) - obs.LinkPrice.Val = v4.MissingPrice - } else if obs.LinkPrice.Err != nil { - mercurytypes.PriceFeedErrorCount.WithLabelValues(ds.linkFeedID.String()).Inc() - ds.lggr.Errorw("Mercury server returned error querying LINK price feed", "err", obs.LinkPrice.Err, "linkFeedID", ds.linkFeedID) - } - }() - } - - if len(ds.jb.OCR2OracleSpec.PluginConfig) == 0 { - obs.NativePrice.Val = v4.MissingPrice - } else if ds.feedID == ds.nativeFeedID { - isNative = true - } else { - wg.Add(1) - go func() { - defer wg.Done() - obs.NativePrice.Val, obs.NativePrice.Err = ds.fetcher.LatestPrice(ctx, ds.nativeFeedID) - if obs.NativePrice.Val == nil && obs.NativePrice.Err == nil { - mercurytypes.PriceFeedMissingCount.WithLabelValues(ds.nativeFeedID.String()).Inc() - ds.lggr.Warnw(fmt.Sprintf("Mercury server was missing native feed, using sentinel value of %s", v4.MissingPrice), "nativeFeedID", ds.nativeFeedID) - obs.NativePrice.Val = v4.MissingPrice - } else if obs.NativePrice.Err != nil { - mercurytypes.PriceFeedErrorCount.WithLabelValues(ds.nativeFeedID.String()).Inc() - ds.lggr.Errorw("Mercury server returned error querying native price feed", "err", obs.NativePrice.Err, "nativeFeedID", ds.nativeFeedID) - } - }() - } - - wg.Wait() - cancel() - - if pipelineExecutionErr != nil { - return - } - - if isLink || isNative { - // run has now completed so it is safe to use benchmark price - if isLink { - // This IS the LINK feed, use our observed price - obs.LinkPrice.Val, obs.LinkPrice.Err = obs.BenchmarkPrice.Val, obs.BenchmarkPrice.Err - } - if isNative { - // This IS the native feed, use our observed price - obs.NativePrice.Val, obs.NativePrice.Err = obs.BenchmarkPrice.Val, obs.BenchmarkPrice.Err - } - } - - ocrcommon.MaybeEnqueueEnhancedTelem(ds.jb, ds.chEnhancedTelem, ocrcommon.EnhancedTelemetryMercuryData{ - V4Observation: &obs, - TaskRunResults: trrs, - RepTimestamp: repts, - FeedVersion: mercuryutils.REPORT_V4, - FetchMaxFinalizedTimestamp: fetchMaxFinalizedTimestamp, - IsLinkFeed: isLink, - IsNativeFeed: isNative, - }) - - return obs, nil -} - -func toBigInt(val interface{}) (*big.Int, error) { - dec, err := utils.ToDecimal(val) - if err != nil { - return nil, err - } - return dec.BigInt(), nil -} - -type parseOutput struct { - benchmarkPrice mercury.ObsResult[*big.Int] - marketStatus mercury.ObsResult[uint32] -} - -func (ds *datasource) parse(trrs pipeline.TaskRunResults) (o parseOutput, merr error) { - var finaltrrs []pipeline.TaskRunResult - for _, trr := range trrs { - // only return terminal trrs from executeRun - if trr.IsTerminal() { - finaltrrs = append(finaltrrs, trr) - } - } - - // pipeline.TaskRunResults comes ordered asc by index, this is guaranteed - // by the pipeline executor - if len(finaltrrs) != 2 { - return o, fmt.Errorf("invalid number of results, expected: 2, got: %d", len(finaltrrs)) - } - - merr = errors.Join( - setBenchmarkPrice(&o, finaltrrs[0].Result), - setMarketStatus(&o, finaltrrs[1].Result), - ) - - return o, merr -} - -func setBenchmarkPrice(o *parseOutput, res pipeline.Result) error { - if res.Error != nil { - o.benchmarkPrice.Err = res.Error - return res.Error - } - val, err := toBigInt(res.Value) - if err != nil { - return fmt.Errorf("failed to parse BenchmarkPrice: %w", err) - } - o.benchmarkPrice.Val = val - return nil -} - -func setMarketStatus(o *parseOutput, res pipeline.Result) error { - if res.Error != nil { - o.marketStatus.Err = res.Error - return res.Error - } - val, err := toBigInt(res.Value) - if err != nil { - return fmt.Errorf("failed to parse MarketStatus: %w", err) - } - o.marketStatus.Val = uint32(val.Int64()) - return nil -} - -// The context passed in here has a timeout of (ObservationTimeout + ObservationGracePeriod). -// Upon context cancellation, its expected that we return any usable values within ObservationGracePeriod. -func (ds *datasource) executeRun(ctx context.Context) (*pipeline.Run, pipeline.TaskRunResults, error) { - vars := pipeline.NewVarsFrom(map[string]interface{}{ - "jb": map[string]interface{}{ - "databaseID": ds.jb.ID, - "externalJobID": ds.jb.ExternalJobID, - "name": ds.jb.Name.ValueOrZero(), - }, - }) - - run, trrs, err := ds.pipelineRunner.ExecuteRun(ctx, ds.spec, vars) - if err != nil { - return nil, nil, pkgerrors.Wrapf(err, "error executing run for spec ID %v", ds.spec.ID) - } - - return run, trrs, err -} diff --git a/core/services/relay/evm/mercury/v4/data_source_test.go b/core/services/relay/evm/mercury/v4/data_source_test.go deleted file mode 100644 index 48aec5989a6..00000000000 --- a/core/services/relay/evm/mercury/v4/data_source_test.go +++ /dev/null @@ -1,348 +0,0 @@ -package v4 - -import ( - "context" - "math/big" - "testing" - - "github.com/pkg/errors" - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - "github.com/stretchr/testify/assert" - - mercurytypes "github.com/smartcontractkit/chainlink-common/pkg/types/mercury" - relaymercuryv4 "github.com/smartcontractkit/chainlink-data-streams/mercury/v4" - "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" - "github.com/smartcontractkit/chainlink/v2/core/logger" - "github.com/smartcontractkit/chainlink/v2/core/services/job" - "github.com/smartcontractkit/chainlink/v2/core/services/pipeline" - mercurymocks "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/mocks" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/utils" - reportcodecv4 "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v4/reportcodec" -) - -var _ mercurytypes.ServerFetcher = &mockFetcher{} - -type mockFetcher struct { - ts int64 - tsErr error - linkPrice *big.Int - linkPriceErr error - nativePrice *big.Int - nativePriceErr error -} - -var feedId utils.FeedID = [32]byte{1} -var linkFeedId utils.FeedID = [32]byte{2} -var nativeFeedId utils.FeedID = [32]byte{3} - -func (m *mockFetcher) FetchInitialMaxFinalizedBlockNumber(context.Context) (*int64, error) { - return nil, nil -} - -func (m *mockFetcher) LatestPrice(ctx context.Context, fId [32]byte) (*big.Int, error) { - if fId == linkFeedId { - return m.linkPrice, m.linkPriceErr - } else if fId == nativeFeedId { - return m.nativePrice, m.nativePriceErr - } - return nil, nil -} - -func (m *mockFetcher) LatestTimestamp(context.Context) (int64, error) { - return m.ts, m.tsErr -} - -type mockORM struct { - report []byte - err error -} - -func (m *mockORM) LatestReport(ctx context.Context, feedID [32]byte) (report []byte, err error) { - return m.report, m.err -} - -type mockSaver struct { - r *pipeline.Run -} - -func (ms *mockSaver) Save(r *pipeline.Run) { - ms.r = r -} - -func Test_Datasource(t *testing.T) { - orm := &mockORM{} - jb := job.Job{ - Type: job.Type(pipeline.OffchainReporting2JobType), - OCR2OracleSpec: &job.OCR2OracleSpec{ - CaptureEATelemetry: true, - PluginConfig: map[string]interface{}{ - "serverURL": "a", - }, - }, - } - ds := &datasource{orm: orm, lggr: logger.TestLogger(t), jb: jb} - ctx := testutils.Context(t) - repts := ocrtypes.ReportTimestamp{} - - fetcher := &mockFetcher{} - ds.fetcher = fetcher - - saver := &mockSaver{} - ds.saver = saver - - goodTrrs := []pipeline.TaskRunResult{ - { - // bp - Result: pipeline.Result{Value: "122.345"}, - Task: &mercurymocks.MockTask{}, - }, - { - // marketStatus - Result: pipeline.Result{Value: "1"}, - Task: &mercurymocks.MockTask{}, - }, - } - - ds.pipelineRunner = &mercurymocks.MockRunner{ - Trrs: goodTrrs, - } - - spec := pipeline.Spec{} - ds.spec = spec - - t.Run("when fetchMaxFinalizedTimestamp=true", func(t *testing.T) { - t.Run("with latest report in database", func(t *testing.T) { - orm.report = buildSamplev4Report() - orm.err = nil - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.NoError(t, obs.MaxFinalizedTimestamp.Err) - assert.Equal(t, int64(124), obs.MaxFinalizedTimestamp.Val) - }) - t.Run("if querying latest report fails", func(t *testing.T) { - orm.report = nil - orm.err = errors.New("something exploded") - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.EqualError(t, obs.MaxFinalizedTimestamp.Err, "something exploded") - assert.Zero(t, obs.MaxFinalizedTimestamp.Val) - }) - t.Run("if codec fails to decode", func(t *testing.T) { - orm.report = []byte{1, 2, 3} - orm.err = nil - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.EqualError(t, obs.MaxFinalizedTimestamp.Err, "failed to decode report: abi: cannot marshal in to go type: length insufficient 3 require 32") - assert.Zero(t, obs.MaxFinalizedTimestamp.Val) - }) - - orm.report = nil - orm.err = nil - - t.Run("if LatestTimestamp returns error", func(t *testing.T) { - fetcher.tsErr = errors.New("some error") - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.EqualError(t, obs.MaxFinalizedTimestamp.Err, "some error") - assert.Zero(t, obs.MaxFinalizedTimestamp.Val) - }) - - t.Run("if LatestTimestamp succeeds", func(t *testing.T) { - fetcher.tsErr = nil - fetcher.ts = 123 - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.Equal(t, int64(123), obs.MaxFinalizedTimestamp.Val) - assert.NoError(t, obs.MaxFinalizedTimestamp.Err) - }) - - t.Run("if LatestTimestamp succeeds but ts=0 (new feed)", func(t *testing.T) { - fetcher.tsErr = nil - fetcher.ts = 0 - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.NoError(t, obs.MaxFinalizedTimestamp.Err) - assert.Zero(t, obs.MaxFinalizedTimestamp.Val) - }) - - t.Run("when run execution succeeded", func(t *testing.T) { - t.Run("when feedId=linkFeedID=nativeFeedId", func(t *testing.T) { - t.Cleanup(func() { - ds.feedID, ds.linkFeedID, ds.nativeFeedID = feedId, linkFeedId, nativeFeedId - }) - - ds.feedID, ds.linkFeedID, ds.nativeFeedID = feedId, feedId, feedId - - fetcher.ts = 123123 - fetcher.tsErr = nil - - obs, err := ds.Observe(ctx, repts, true) - assert.NoError(t, err) - - assert.Equal(t, big.NewInt(122), obs.BenchmarkPrice.Val) - assert.NoError(t, obs.BenchmarkPrice.Err) - assert.Equal(t, int64(123123), obs.MaxFinalizedTimestamp.Val) - assert.NoError(t, obs.MaxFinalizedTimestamp.Err) - assert.Equal(t, big.NewInt(122), obs.LinkPrice.Val) - assert.NoError(t, obs.LinkPrice.Err) - assert.Equal(t, big.NewInt(122), obs.NativePrice.Val) - assert.NoError(t, obs.NativePrice.Err) - assert.Equal(t, uint32(1), obs.MarketStatus.Val) - assert.NoError(t, obs.MarketStatus.Err) - }) - }) - }) - - t.Run("when fetchMaxFinalizedTimestamp=false", func(t *testing.T) { - t.Run("when run execution fails, returns error", func(t *testing.T) { - t.Cleanup(func() { - ds.pipelineRunner = &mercurymocks.MockRunner{ - Trrs: goodTrrs, - Err: nil, - } - }) - - ds.pipelineRunner = &mercurymocks.MockRunner{ - Trrs: goodTrrs, - Err: errors.New("run execution failed"), - } - - _, err := ds.Observe(ctx, repts, false) - assert.EqualError(t, err, "Observe failed while executing run: error executing run for spec ID 0: run execution failed") - }) - - t.Run("when parsing run results fails, return error", func(t *testing.T) { - t.Cleanup(func() { - runner := &mercurymocks.MockRunner{ - Trrs: goodTrrs, - Err: nil, - } - ds.pipelineRunner = runner - }) - - badTrrs := []pipeline.TaskRunResult{ - { - // benchmark price - Result: pipeline.Result{Error: errors.New("some error with bp")}, - Task: &mercurymocks.MockTask{}, - }, - { - // marketStatus - Result: pipeline.Result{Value: "1"}, - Task: &mercurymocks.MockTask{}, - }, - } - - ds.pipelineRunner = &mercurymocks.MockRunner{ - Trrs: badTrrs, - Err: nil, - } - - _, err := ds.Observe(ctx, repts, false) - assert.EqualError(t, err, "Observe failed while parsing run results: some error with bp") - }) - - t.Run("when run execution succeeded", func(t *testing.T) { - t.Run("when feedId=linkFeedID=nativeFeedId", func(t *testing.T) { - t.Cleanup(func() { - ds.feedID, ds.linkFeedID, ds.nativeFeedID = feedId, linkFeedId, nativeFeedId - }) - - var feedId utils.FeedID = [32]byte{1} - ds.feedID, ds.linkFeedID, ds.nativeFeedID = feedId, feedId, feedId - - obs, err := ds.Observe(ctx, repts, false) - assert.NoError(t, err) - - assert.Equal(t, big.NewInt(122), obs.BenchmarkPrice.Val) - assert.NoError(t, obs.BenchmarkPrice.Err) - assert.Equal(t, int64(0), obs.MaxFinalizedTimestamp.Val) - assert.NoError(t, obs.MaxFinalizedTimestamp.Err) - assert.Equal(t, big.NewInt(122), obs.LinkPrice.Val) - assert.NoError(t, obs.LinkPrice.Err) - assert.Equal(t, big.NewInt(122), obs.NativePrice.Val) - assert.NoError(t, obs.NativePrice.Err) - assert.Equal(t, uint32(1), obs.MarketStatus.Val) - assert.NoError(t, obs.MarketStatus.Err) - }) - - t.Run("when fails to fetch linkPrice or nativePrice", func(t *testing.T) { - t.Cleanup(func() { - fetcher.linkPriceErr = nil - fetcher.nativePriceErr = nil - }) - - fetcher.linkPriceErr = errors.New("some error fetching link price") - fetcher.nativePriceErr = errors.New("some error fetching native price") - - obs, err := ds.Observe(ctx, repts, false) - assert.NoError(t, err) - - assert.Nil(t, obs.LinkPrice.Val) - assert.EqualError(t, obs.LinkPrice.Err, "some error fetching link price") - assert.Nil(t, obs.NativePrice.Val) - assert.EqualError(t, obs.NativePrice.Err, "some error fetching native price") - }) - - t.Run("when PluginConfig is empty", func(t *testing.T) { - t.Cleanup(func() { - ds.jb = jb - }) - - fetcher.linkPriceErr = errors.New("some error fetching link price") - fetcher.nativePriceErr = errors.New("some error fetching native price") - - ds.jb.OCR2OracleSpec.PluginConfig = job.JSONConfig{} - - obs, err := ds.Observe(ctx, repts, false) - assert.NoError(t, err) - assert.Nil(t, obs.LinkPrice.Err) - assert.Equal(t, obs.LinkPrice.Val, relaymercuryv4.MissingPrice) - assert.Nil(t, obs.NativePrice.Err) - assert.Equal(t, obs.NativePrice.Val, relaymercuryv4.MissingPrice) - assert.Equal(t, big.NewInt(122), obs.BenchmarkPrice.Val) - }) - - t.Run("when succeeds to fetch linkPrice or nativePrice but got nil (new feed)", func(t *testing.T) { - obs, err := ds.Observe(ctx, repts, false) - assert.NoError(t, err) - - assert.Equal(t, obs.LinkPrice.Val, relaymercuryv4.MissingPrice) - assert.Nil(t, obs.LinkPrice.Err) - assert.Equal(t, obs.NativePrice.Val, relaymercuryv4.MissingPrice) - assert.Nil(t, obs.NativePrice.Err) - }) - }) - }) -} - -var sampleFeedID = [32]uint8{28, 145, 107, 74, 167, 229, 124, 167, 182, 138, 225, 191, 69, 101, 63, 86, 182, 86, 253, 58, 163, 53, 239, 127, 174, 105, 107, 102, 63, 27, 132, 114} - -func buildSamplev4Report() []byte { - feedID := sampleFeedID - timestamp := uint32(124) - bp := big.NewInt(242) - validFromTimestamp := uint32(123) - expiresAt := uint32(456) - linkFee := big.NewInt(3334455) - nativeFee := big.NewInt(556677) - marketStatus := uint32(1) - - b, err := reportcodecv4.ReportTypes.Pack(feedID, validFromTimestamp, timestamp, nativeFee, linkFee, expiresAt, bp, marketStatus) - if err != nil { - panic(err) - } - return b -} diff --git a/core/services/relay/evm/mercury/v4/reportcodec/report_codec.go b/core/services/relay/evm/mercury/v4/reportcodec/report_codec.go deleted file mode 100644 index c5d32c02ed4..00000000000 --- a/core/services/relay/evm/mercury/v4/reportcodec/report_codec.go +++ /dev/null @@ -1,78 +0,0 @@ -package reportcodec - -import ( - "context" - "errors" - "fmt" - "math/big" - - pkgerrors "github.com/pkg/errors" - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - - "github.com/smartcontractkit/chainlink-common/pkg/logger" - v4 "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v4" - - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/utils" - reporttypes "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury/v4/types" -) - -var ReportTypes = reporttypes.GetSchema() -var maxReportLength = 32 * len(ReportTypes) // each arg is 256 bit EVM word -var zero = big.NewInt(0) - -var _ v4.ReportCodec = &ReportCodec{} - -type ReportCodec struct { - logger logger.Logger - feedID utils.FeedID -} - -func NewReportCodec(feedID [32]byte, lggr logger.Logger) *ReportCodec { - return &ReportCodec{lggr, feedID} -} - -func (r *ReportCodec) BuildReport(ctx context.Context, rf v4.ReportFields) (ocrtypes.Report, error) { - var merr error - if rf.BenchmarkPrice == nil { - merr = errors.Join(merr, errors.New("benchmarkPrice may not be nil")) - } - if rf.LinkFee == nil { - merr = errors.Join(merr, errors.New("linkFee may not be nil")) - } else if rf.LinkFee.Cmp(zero) < 0 { - merr = errors.Join(merr, fmt.Errorf("linkFee may not be negative (got: %s)", rf.LinkFee)) - } - if rf.NativeFee == nil { - merr = errors.Join(merr, errors.New("nativeFee may not be nil")) - } else if rf.NativeFee.Cmp(zero) < 0 { - merr = errors.Join(merr, fmt.Errorf("nativeFee may not be negative (got: %s)", rf.NativeFee)) - } - if merr != nil { - return nil, merr - } - reportBytes, err := ReportTypes.Pack(r.feedID, rf.ValidFromTimestamp, rf.Timestamp, rf.NativeFee, rf.LinkFee, rf.ExpiresAt, rf.BenchmarkPrice, rf.MarketStatus) - return ocrtypes.Report(reportBytes), pkgerrors.Wrap(err, "failed to pack report blob") -} - -func (r *ReportCodec) MaxReportLength(ctx context.Context, n int) (int, error) { - return maxReportLength, nil -} - -func (r *ReportCodec) ObservationTimestampFromReport(ctx context.Context, report ocrtypes.Report) (uint32, error) { - decoded, err := r.Decode(ctx, report) - if err != nil { - return 0, err - } - return decoded.ObservationsTimestamp, nil -} - -func (r *ReportCodec) Decode(ctx context.Context, report ocrtypes.Report) (*reporttypes.Report, error) { - return reporttypes.Decode(report) -} - -func (r *ReportCodec) BenchmarkPriceFromReport(ctx context.Context, report ocrtypes.Report) (*big.Int, error) { - decoded, err := r.Decode(ctx, report) - if err != nil { - return nil, err - } - return decoded.BenchmarkPrice, nil -} diff --git a/core/services/relay/evm/mercury/v4/reportcodec/report_codec_test.go b/core/services/relay/evm/mercury/v4/reportcodec/report_codec_test.go deleted file mode 100644 index 9813d422cc1..00000000000 --- a/core/services/relay/evm/mercury/v4/reportcodec/report_codec_test.go +++ /dev/null @@ -1,165 +0,0 @@ -package reportcodec - -import ( - "math/big" - "testing" - - "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - v4 "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v4" - "github.com/smartcontractkit/chainlink-common/pkg/utils/tests" -) - -func newValidReportFields() v4.ReportFields { - return v4.ReportFields{ - Timestamp: 242, - BenchmarkPrice: big.NewInt(243), - ValidFromTimestamp: 123, - ExpiresAt: 20, - LinkFee: big.NewInt(456), - NativeFee: big.NewInt(457), - MarketStatus: 1, - } -} - -func Test_ReportCodec_BuildReport(t *testing.T) { - r := ReportCodec{} - - t.Run("BuildReport errors on zero values", func(t *testing.T) { - ctx := tests.Context(t) - _, err := r.BuildReport(ctx, v4.ReportFields{}) - require.Error(t, err) - assert.Contains(t, err.Error(), "benchmarkPrice may not be nil") - assert.Contains(t, err.Error(), "linkFee may not be nil") - assert.Contains(t, err.Error(), "nativeFee may not be nil") - }) - - t.Run("BuildReport constructs a report from observations", func(t *testing.T) { - ctx := tests.Context(t) - rf := newValidReportFields() - // only need to test happy path since validations are done in relaymercury - - report, err := r.BuildReport(ctx, rf) - require.NoError(t, err) - - reportElems := make(map[string]interface{}) - err = ReportTypes.UnpackIntoMap(reportElems, report) - require.NoError(t, err) - - assert.Equal(t, int(reportElems["observationsTimestamp"].(uint32)), 242) - assert.Equal(t, reportElems["benchmarkPrice"].(*big.Int).Int64(), int64(243)) - assert.Equal(t, reportElems["validFromTimestamp"].(uint32), uint32(123)) - assert.Equal(t, reportElems["expiresAt"].(uint32), uint32(20)) - assert.Equal(t, reportElems["linkFee"].(*big.Int).Int64(), int64(456)) - assert.Equal(t, reportElems["nativeFee"].(*big.Int).Int64(), int64(457)) - assert.Equal(t, reportElems["marketStatus"].(uint32), uint32(1)) - - assert.Equal(t, types.Report{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x7b, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf2, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0xc9, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, 0xc8, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x14, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xf3, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1}, report) - max, err := r.MaxReportLength(ctx, 4) - require.NoError(t, err) - assert.LessOrEqual(t, len(report), max) - - t.Run("Decode decodes the report", func(t *testing.T) { - ctx := tests.Context(t) - decoded, err := r.Decode(ctx, report) - require.NoError(t, err) - - require.NotNil(t, decoded) - - assert.Equal(t, uint32(242), decoded.ObservationsTimestamp) - assert.Equal(t, big.NewInt(243), decoded.BenchmarkPrice) - assert.Equal(t, uint32(123), decoded.ValidFromTimestamp) - assert.Equal(t, uint32(20), decoded.ExpiresAt) - assert.Equal(t, big.NewInt(456), decoded.LinkFee) - assert.Equal(t, big.NewInt(457), decoded.NativeFee) - assert.Equal(t, uint32(1), decoded.MarketStatus) - }) - }) - - t.Run("errors on negative fee", func(t *testing.T) { - ctx := tests.Context(t) - rf := newValidReportFields() - rf.LinkFee = big.NewInt(-1) - rf.NativeFee = big.NewInt(-1) - _, err := r.BuildReport(ctx, rf) - require.Error(t, err) - - assert.Contains(t, err.Error(), "linkFee may not be negative (got: -1)") - assert.Contains(t, err.Error(), "nativeFee may not be negative (got: -1)") - }) - - t.Run("Decode errors on invalid report", func(t *testing.T) { - ctx := tests.Context(t) - _, err := r.Decode(ctx, []byte{1, 2, 3}) - assert.EqualError(t, err, "failed to decode report: abi: cannot marshal in to go type: length insufficient 3 require 32") - - longBad := make([]byte, 64) - for i := 0; i < len(longBad); i++ { - longBad[i] = byte(i) - } - _, err = r.Decode(ctx, longBad) - assert.EqualError(t, err, "failed to decode report: abi: improperly encoded uint32 value") - }) -} - -func buildSampleReport(ts int64) []byte { - feedID := [32]byte{'f', 'o', 'o'} - timestamp := uint32(ts) - bp := big.NewInt(242) - validFromTimestamp := uint32(123) - expiresAt := uint32(456) - linkFee := big.NewInt(3334455) - nativeFee := big.NewInt(556677) - marketStatus := uint32(1) - - b, err := ReportTypes.Pack(feedID, validFromTimestamp, timestamp, nativeFee, linkFee, expiresAt, bp, marketStatus) - if err != nil { - panic(err) - } - return b -} - -func Test_ReportCodec_ObservationTimestampFromReport(t *testing.T) { - r := ReportCodec{} - - t.Run("ObservationTimestampFromReport extracts observation timestamp from a valid report", func(t *testing.T) { - ctx := tests.Context(t) - report := buildSampleReport(123) - - ts, err := r.ObservationTimestampFromReport(ctx, report) - require.NoError(t, err) - - assert.Equal(t, ts, uint32(123)) - }) - t.Run("ObservationTimestampFromReport returns error when report is invalid", func(t *testing.T) { - ctx := tests.Context(t) - report := []byte{1, 2, 3} - - _, err := r.ObservationTimestampFromReport(ctx, report) - require.Error(t, err) - - assert.EqualError(t, err, "failed to decode report: abi: cannot marshal in to go type: length insufficient 3 require 32") - }) -} - -func Test_ReportCodec_BenchmarkPriceFromReport(t *testing.T) { - r := ReportCodec{} - - t.Run("BenchmarkPriceFromReport extracts the benchmark price from valid report", func(t *testing.T) { - ctx := tests.Context(t) - report := buildSampleReport(123) - - bp, err := r.BenchmarkPriceFromReport(ctx, report) - require.NoError(t, err) - - assert.Equal(t, big.NewInt(242), bp) - }) - t.Run("BenchmarkPriceFromReport errors on invalid report", func(t *testing.T) { - ctx := tests.Context(t) - _, err := r.BenchmarkPriceFromReport(ctx, []byte{1, 2, 3}) - require.Error(t, err) - assert.EqualError(t, err, "failed to decode report: abi: cannot marshal in to go type: length insufficient 3 require 32") - }) -} diff --git a/core/services/relay/evm/mercury/v4/types/types.go b/core/services/relay/evm/mercury/v4/types/types.go deleted file mode 100644 index 584836c1e9b..00000000000 --- a/core/services/relay/evm/mercury/v4/types/types.go +++ /dev/null @@ -1,54 +0,0 @@ -package reporttypes - -import ( - "fmt" - "math/big" - - "github.com/ethereum/go-ethereum/accounts/abi" -) - -var schema = GetSchema() - -func GetSchema() abi.Arguments { - mustNewType := func(t string) abi.Type { - result, err := abi.NewType(t, "", []abi.ArgumentMarshaling{}) - if err != nil { - panic(fmt.Sprintf("Unexpected error during abi.NewType: %s", err)) - } - return result - } - return abi.Arguments([]abi.Argument{ - {Name: "feedId", Type: mustNewType("bytes32")}, - {Name: "validFromTimestamp", Type: mustNewType("uint32")}, - {Name: "observationsTimestamp", Type: mustNewType("uint32")}, - {Name: "nativeFee", Type: mustNewType("uint192")}, - {Name: "linkFee", Type: mustNewType("uint192")}, - {Name: "expiresAt", Type: mustNewType("uint32")}, - {Name: "benchmarkPrice", Type: mustNewType("int192")}, - {Name: "marketStatus", Type: mustNewType("uint32")}, - }) -} - -type Report struct { - FeedId [32]byte - ObservationsTimestamp uint32 - BenchmarkPrice *big.Int - ValidFromTimestamp uint32 - ExpiresAt uint32 - LinkFee *big.Int - NativeFee *big.Int - MarketStatus uint32 -} - -// Decode is made available to external users (i.e. mercury server) -func Decode(report []byte) (*Report, error) { - values, err := schema.Unpack(report) - if err != nil { - return nil, fmt.Errorf("failed to decode report: %w", err) - } - decoded := new(Report) - if err = schema.Copy(decoded, values); err != nil { - return nil, fmt.Errorf("failed to copy report values to struct: %w", err) - } - return decoded, nil -} diff --git a/core/services/relay/evm/mercury_config_provider.go b/core/services/relay/evm/mercury_config_provider.go deleted file mode 100644 index 53bf8e22d24..00000000000 --- a/core/services/relay/evm/mercury_config_provider.go +++ /dev/null @@ -1,48 +0,0 @@ -package evm - -import ( - "context" - "errors" - "fmt" - - "github.com/ethereum/go-ethereum/common" - - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - - "github.com/smartcontractkit/chainlink-common/pkg/logger" - commontypes "github.com/smartcontractkit/chainlink-common/pkg/types" - - "github.com/smartcontractkit/chainlink/v2/core/chains/legacyevm" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury" - "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/types" -) - -func newMercuryConfigProvider(ctx context.Context, lggr logger.Logger, chain legacyevm.Chain, opts *types.RelayOpts) (commontypes.ConfigProvider, error) { - if !common.IsHexAddress(opts.ContractID) { - return nil, errors.New("invalid contractID, expected hex address") - } - - aggregatorAddress := common.HexToAddress(opts.ContractID) - - relayConfig, err := opts.RelayConfig() - if err != nil { - return nil, fmt.Errorf("failed to get relay config: %w", err) - } - if relayConfig.FeedID == nil { - return nil, errors.New("feed ID is required for tracking config on mercury contracts") - } - cp, err := mercury.NewConfigPoller( - ctx, - logger.Named(lggr, relayConfig.FeedID.String()), - chain.LogPoller(), - aggregatorAddress, - *relayConfig.FeedID, - // TODO: Does mercury need to support config contract? DF-19182 - ) - if err != nil { - return nil, err - } - - offchainConfigDigester := mercury.NewOffchainConfigDigester(*relayConfig.FeedID, chain.Config().EVM().ChainID(), aggregatorAddress, ocrtypes.ConfigDigestPrefixMercuryV02) - return newConfigWatcher(lggr, aggregatorAddress, offchainConfigDigester, cp, chain, relayConfig.FromBlock, opts.New), nil -} diff --git a/core/services/relay/evm/mercury_provider.go b/core/services/relay/evm/mercury_provider.go deleted file mode 100644 index 85f633e063a..00000000000 --- a/core/services/relay/evm/mercury_provider.go +++ /dev/null @@ -1,167 +0,0 @@ -package evm - -import ( - "context" - "errors" - - ocrtypes "github.com/smartcontractkit/libocr/offchainreporting2plus/types" - - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/services" - commontypes "github.com/smartcontractkit/chainlink-common/pkg/types" - mercurytypes "github.com/smartcontractkit/chainlink-common/pkg/types/mercury" - v1 "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v1" - v2 "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v2" - v3 "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v3" - v4 "github.com/smartcontractkit/chainlink-common/pkg/types/mercury/v4" - - "github.com/smartcontractkit/chainlink-data-streams/mercury" - - httypes "github.com/smartcontractkit/chainlink/v2/core/chains/evm/headtracker/types" - evmmercury "github.com/smartcontractkit/chainlink/v2/core/services/relay/evm/mercury" -) - -var _ commontypes.MercuryProvider = (*mercuryProvider)(nil) - -type mercuryProvider struct { - cp commontypes.ConfigProvider - codec commontypes.Codec - transmitter evmmercury.Transmitter - reportCodecV1 v1.ReportCodec - reportCodecV2 v2.ReportCodec - reportCodecV3 v3.ReportCodec - reportCodecV4 v4.ReportCodec - mercuryChainReader mercurytypes.ChainReader - logger logger.Logger - ms services.MultiStart -} - -func NewMercuryProvider( - cp commontypes.ConfigProvider, - codec commontypes.Codec, - mercuryChainReader mercurytypes.ChainReader, - transmitter evmmercury.Transmitter, - reportCodecV1 v1.ReportCodec, - reportCodecV2 v2.ReportCodec, - reportCodecV3 v3.ReportCodec, - reportCodecV4 v4.ReportCodec, - lggr logger.Logger, -) *mercuryProvider { - return &mercuryProvider{ - cp, - codec, - transmitter, - reportCodecV1, - reportCodecV2, - reportCodecV3, - reportCodecV4, - mercuryChainReader, - lggr, - services.MultiStart{}, - } -} - -func (p *mercuryProvider) Start(ctx context.Context) error { - return p.ms.Start(ctx, p.cp, p.transmitter) -} - -func (p *mercuryProvider) Close() error { - return p.ms.Close() -} - -func (p *mercuryProvider) Ready() error { - return errors.Join(p.cp.Ready(), p.transmitter.Ready()) -} - -func (p *mercuryProvider) Name() string { - return p.logger.Name() -} - -func (p *mercuryProvider) HealthReport() map[string]error { - report := map[string]error{} - services.CopyHealth(report, p.cp.HealthReport()) - services.CopyHealth(report, p.transmitter.HealthReport()) - return report -} - -func (p *mercuryProvider) MercuryChainReader() mercurytypes.ChainReader { - return p.mercuryChainReader -} - -func (p *mercuryProvider) Codec() commontypes.Codec { - return p.codec -} - -func (p *mercuryProvider) ContractConfigTracker() ocrtypes.ContractConfigTracker { - return p.cp.ContractConfigTracker() -} - -func (p *mercuryProvider) OffchainConfigDigester() ocrtypes.OffchainConfigDigester { - return p.cp.OffchainConfigDigester() -} - -func (p *mercuryProvider) OnchainConfigCodec() mercurytypes.OnchainConfigCodec { - return mercury.StandardOnchainConfigCodec{} -} - -func (p *mercuryProvider) ReportCodecV1() v1.ReportCodec { - return p.reportCodecV1 -} - -func (p *mercuryProvider) ReportCodecV2() v2.ReportCodec { - return p.reportCodecV2 -} - -func (p *mercuryProvider) ReportCodecV3() v3.ReportCodec { - return p.reportCodecV3 -} - -func (p *mercuryProvider) ReportCodecV4() v4.ReportCodec { - return p.reportCodecV4 -} - -func (p *mercuryProvider) ContractTransmitter() ocrtypes.ContractTransmitter { - return p.transmitter -} - -func (p *mercuryProvider) MercuryServerFetcher() mercurytypes.ServerFetcher { - return p.transmitter -} - -func (p *mercuryProvider) ContractReader() commontypes.ContractReader { - return nil -} - -var _ mercurytypes.ChainReader = (*mercuryChainReader)(nil) - -type mercuryChainReader struct { - tracker httypes.HeadTracker -} - -func NewChainReader(h httypes.HeadTracker) mercurytypes.ChainReader { - return &mercuryChainReader{h} -} - -func NewMercuryChainReader(h httypes.HeadTracker) mercurytypes.ChainReader { - return &mercuryChainReader{ - tracker: h, - } -} - -func (r *mercuryChainReader) LatestHeads(ctx context.Context, k int) ([]mercurytypes.Head, error) { - evmBlocks := r.tracker.LatestChain().AsSlice(k) - if len(evmBlocks) == 0 { - return nil, nil - } - - blocks := make([]mercurytypes.Head, len(evmBlocks)) - for x := 0; x < len(evmBlocks); x++ { - blocks[x] = mercurytypes.Head{ - Number: uint64(evmBlocks[x].BlockNumber()), - Hash: evmBlocks[x].Hash.Bytes(), - Timestamp: uint64(evmBlocks[x].Timestamp.Unix()), - } - } - - return blocks, nil -} diff --git a/core/services/relay/evm/types/types.go b/core/services/relay/evm/types/types.go index d6756d04662..f6586322e5b 100644 --- a/core/services/relay/evm/types/types.go +++ b/core/services/relay/evm/types/types.go @@ -184,7 +184,6 @@ func (r *ReadType) UnmarshalText(text []byte) error { type LLOConfigMode string const ( - LLOConfigModeMercury LLOConfigMode = "mercury" LLOConfigModeBlueGreen LLOConfigMode = "bluegreen" ) @@ -192,6 +191,16 @@ func (c LLOConfigMode) String() string { return string(c) } +func (c *LLOConfigMode) UnmarshalText(text []byte) error { + switch string(text) { + case "", "bluegreen": + *c = LLOConfigModeBlueGreen + default: + return fmt.Errorf("unrecognized LLOConfigMode: %s", string(text)) + } + return nil +} + type DualTransmissionConfig struct { ContractAddress common.Address `json:"contractAddress" toml:"contractAddress"` TransmitterAddress common.Address `json:"transmitterAddress" toml:"transmitterAddress"` @@ -212,12 +221,6 @@ type RelayConfig struct { // Contract-specific SendingKeys pq.StringArray `json:"sendingKeys"` - // Mercury-specific - FeedID *common.Hash `json:"feedID"` - EnableTriggerCapability bool `json:"enableTriggerCapability"` - TriggerCapabilityName string `json:"triggerCapabilityName"` - TriggerCapabilityVersion string `json:"triggerCapabilityVersion"` - // LLO-specific LLODONID uint32 `json:"lloDonID" toml:"lloDonID"` LLOConfigMode LLOConfigMode `json:"lloConfigMode" toml:"lloConfigMode"` diff --git a/core/services/relay/evm/types/types_test.go b/core/services/relay/evm/types/types_test.go index 0adfd0c355b..09b7f9b5c9c 100644 --- a/core/services/relay/evm/types/types_test.go +++ b/core/services/relay/evm/types/types_test.go @@ -24,18 +24,13 @@ import ( // // Contract-specific // EffectiveTransmitterAddress null.String `json:"effectiveTransmitterAddress"` // SendingKeys pq.StringArray `json:"sendingKeys"` - -// // Mercury-specific -// FeedID *common.Hash `json:"feedID"` func Test_RelayConfig(t *testing.T) { cid := testutils.NewRandomEVMChainID() fromBlock := uint64(2222) - feedID := utils.NewHash() rawToml := fmt.Sprintf(` ChainID = "%s" FromBlock = %d -FeedID = "0x%x" -`, cid, fromBlock, feedID[:]) +`, cid, fromBlock) var rc RelayConfig err := toml.Unmarshal([]byte(rawToml), &rc) @@ -43,7 +38,6 @@ FeedID = "0x%x" assert.Equal(t, cid.String(), rc.ChainID.String()) assert.Equal(t, fromBlock, rc.FromBlock) - assert.Equal(t, feedID.Hex(), rc.FeedID.Hex()) } func Test_ChainReaderConfig(t *testing.T) {