From efab5b315b131efdd57989fa8c6a92dca5ab6b00 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 14 Jun 2024 13:54:22 -0400 Subject: [PATCH 01/68] Add LedgerRangeReader interface --- cmd/soroban-rpc/internal/db/db.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/cmd/soroban-rpc/internal/db/db.go b/cmd/soroban-rpc/internal/db/db.go index 227e2115..d4b3021d 100644 --- a/cmd/soroban-rpc/internal/db/db.go +++ b/cmd/soroban-rpc/internal/db/db.go @@ -17,7 +17,9 @@ import ( "github.com/stellar/go/support/errors" "github.com/stellar/go/support/log" "github.com/stellar/go/xdr" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" ) //go:embed migrations/*.sql @@ -30,6 +32,10 @@ const ( latestLedgerSequenceMetaKey = "LatestLedgerSequence" ) +type LedgerRangeReader interface { + GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) +} + type ReadWriter interface { NewTx(ctx context.Context) (WriteTx, error) GetLatestLedgerSequence(ctx context.Context) (uint32, error) From 9966f57a15ca1a51e48cb060ea610553f8caf10d Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 14 Jun 2024 13:54:37 -0400 Subject: [PATCH 02/68] Add GetLedgerRange implementation for ledgers table --- cmd/soroban-rpc/internal/db/ledger.go | 54 +++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/cmd/soroban-rpc/internal/db/ledger.go b/cmd/soroban-rpc/internal/db/ledger.go index 97887281..ebda7ae9 100644 --- a/cmd/soroban-rpc/internal/db/ledger.go +++ b/cmd/soroban-rpc/internal/db/ledger.go @@ -5,8 +5,11 @@ import ( "fmt" sq "github.com/Masterminds/squirrel" + "github.com/stellar/go/support/errors" "github.com/stellar/go/xdr" + + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" ) const ( @@ -69,6 +72,57 @@ func (r ledgerReader) GetLedger(ctx context.Context, sequence uint32) (xdr.Ledge } } +// GetLedgerRange pulls the min/max ledger sequence numbers from the database. +func (r ledgerReader) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { + var ledgerRange ledgerbucketwindow.LedgerRange + // + // We use subqueries alongside a UNION ALL stitch in order to select the min + // and max from the ledger table in a single query and get around sqlite's + // limitations with parentheses (see https://stackoverflow.com/a/22609948). + // + newestQ := sq. + Select("m1.meta"). + FromSelect( + sq. + Select("meta"). + From(ledgerCloseMetaTableName). + OrderBy("sequence ASC"). + Limit(1), + "m1", + ) + sql, args, err := sq. + Select("m2.meta"). + FromSelect( + sq. + Select("meta"). + From(ledgerCloseMetaTableName). + OrderBy("sequence DESC"). + Limit(1), + "m2", + ).ToSql() + if err != nil { + return ledgerRange, errors.Wrap(err, "couldn't build ledger range query") + } + + var lcms []xdr.LedgerCloseMeta + if err = r.db.Select(ctx, &lcms, newestQ.Suffix("UNION ALL "+sql, args...)); err != nil { + return ledgerRange, errors.Wrap(err, "couldn't query ledger range") + } else if len(lcms) < 2 { + // There is almost certainly a row, but we want to avoid a race condition + // with ingestion as well as support test cases from an empty DB, so we need + // to sanity check that there is in fact a result. Note that no ledgers in + // the database isn't an error, it's just an empty range. + return ledgerRange, nil + } + + lcm1, lcm2 := lcms[0], lcms[1] + ledgerRange.FirstLedger.Sequence = lcm1.LedgerSequence() + ledgerRange.FirstLedger.CloseTime = lcm1.LedgerCloseTime() + ledgerRange.LastLedger.Sequence = lcm2.LedgerSequence() + ledgerRange.LastLedger.CloseTime = lcm2.LedgerCloseTime() + return ledgerRange, nil +} + type ledgerWriter struct { stmtCache *sq.StmtCache } From aad48ab30a0032cc7971cbff407a9306a795fd57 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 14 Jun 2024 15:04:36 -0400 Subject: [PATCH 03/68] Use GetLedgerRange from the ledgers table for getHealth and getFeeStats --- cmd/soroban-rpc/internal/db/ledger.go | 3 ++- cmd/soroban-rpc/internal/db/mocks.go | 4 ++++ cmd/soroban-rpc/internal/db/transaction.go | 2 +- cmd/soroban-rpc/internal/jsonrpc.go | 4 ++-- cmd/soroban-rpc/internal/methods/get_fee_stats.go | 2 +- cmd/soroban-rpc/internal/methods/health.go | 3 ++- 6 files changed, 12 insertions(+), 6 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/ledger.go b/cmd/soroban-rpc/internal/db/ledger.go index ebda7ae9..31e960a4 100644 --- a/cmd/soroban-rpc/internal/db/ledger.go +++ b/cmd/soroban-rpc/internal/db/ledger.go @@ -21,6 +21,7 @@ type StreamLedgerFn func(xdr.LedgerCloseMeta) error type LedgerReader interface { GetLedger(ctx context.Context, sequence uint32) (xdr.LedgerCloseMeta, bool, error) StreamAllLedgers(ctx context.Context, f StreamLedgerFn) error + LedgerRangeReader } type LedgerWriter interface { @@ -72,7 +73,7 @@ func (r ledgerReader) GetLedger(ctx context.Context, sequence uint32) (xdr.Ledge } } -// GetLedgerRange pulls the min/max ledger sequence numbers from the database. +// GetLedgerRange pulls the min/max ledger sequence numbers from the ledgers table. func (r ledgerReader) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { var ledgerRange ledgerbucketwindow.LedgerRange // diff --git a/cmd/soroban-rpc/internal/db/mocks.go b/cmd/soroban-rpc/internal/db/mocks.go index d2dfba4f..c72b7393 100644 --- a/cmd/soroban-rpc/internal/db/mocks.go +++ b/cmd/soroban-rpc/internal/db/mocks.go @@ -105,6 +105,10 @@ func (m *mockLedgerReader) StreamAllLedgers(ctx context.Context, f StreamLedgerF return nil } +func (m *mockLedgerReader) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { + return ledgerbucketwindow.LedgerRange{}, nil +} + var _ TransactionReader = &mockTransactionHandler{} var _ TransactionWriter = &mockTransactionHandler{} var _ LedgerReader = &mockLedgerReader{} diff --git a/cmd/soroban-rpc/internal/db/transaction.go b/cmd/soroban-rpc/internal/db/transaction.go index c703c39e..68ca0d75 100644 --- a/cmd/soroban-rpc/internal/db/transaction.go +++ b/cmd/soroban-rpc/internal/db/transaction.go @@ -42,7 +42,7 @@ type TransactionWriter interface { // TransactionReader provides all of the public ways to read from the DB. type TransactionReader interface { GetTransaction(ctx context.Context, hash xdr.Hash) (Transaction, ledgerbucketwindow.LedgerRange, error) - GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) + LedgerRangeReader } type transactionHandler struct { diff --git a/cmd/soroban-rpc/internal/jsonrpc.go b/cmd/soroban-rpc/internal/jsonrpc.go index 014a75a1..5d5189ab 100644 --- a/cmd/soroban-rpc/internal/jsonrpc.go +++ b/cmd/soroban-rpc/internal/jsonrpc.go @@ -151,7 +151,7 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler { { methodName: "getHealth", underlyingHandler: methods.NewHealthCheck( - retentionWindow, params.TransactionReader, cfg.MaxHealthyLedgerLatency), + retentionWindow, params.LedgerReader, cfg.MaxHealthyLedgerLatency), longName: "get_health", queueLimit: cfg.RequestBacklogGetHealthQueueLimit, requestDurationLimit: cfg.MaxGetHealthExecutionDuration, @@ -232,7 +232,7 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler { }, { methodName: "getFeeStats", - underlyingHandler: methods.NewGetFeeStatsHandler(params.FeeStatWindows, params.TransactionReader, params.Logger), + underlyingHandler: methods.NewGetFeeStatsHandler(params.FeeStatWindows, params.LedgerReader, params.Logger), longName: "get_fee_stats", queueLimit: cfg.RequestBacklogGetFeeStatsTransactionQueueLimit, requestDurationLimit: cfg.MaxGetFeeStatsExecutionDuration, diff --git a/cmd/soroban-rpc/internal/methods/get_fee_stats.go b/cmd/soroban-rpc/internal/methods/get_fee_stats.go index 0e02896a..f76cbe10 100644 --- a/cmd/soroban-rpc/internal/methods/get_fee_stats.go +++ b/cmd/soroban-rpc/internal/methods/get_fee_stats.go @@ -58,7 +58,7 @@ type GetFeeStatsResult struct { } // NewGetFeeStatsHandler returns a handler obtaining fee statistics -func NewGetFeeStatsHandler(windows *feewindow.FeeWindows, reader db.TransactionReader, logger *log.Entry) jrpc2.Handler { +func NewGetFeeStatsHandler(windows *feewindow.FeeWindows, reader db.LedgerReader, logger *log.Entry) jrpc2.Handler { return NewHandler(func(ctx context.Context) (GetFeeStatsResult, error) { ledgerInfo, err := reader.GetLedgerRange(ctx) if err != nil { // still not fatal diff --git a/cmd/soroban-rpc/internal/methods/health.go b/cmd/soroban-rpc/internal/methods/health.go index 267b7206..d013d53f 100644 --- a/cmd/soroban-rpc/internal/methods/health.go +++ b/cmd/soroban-rpc/internal/methods/health.go @@ -6,6 +6,7 @@ import ( "time" "github.com/creachadair/jrpc2" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" ) @@ -19,7 +20,7 @@ type HealthCheckResult struct { // NewHealthCheck returns a health check json rpc handler func NewHealthCheck( retentionWindow uint32, - reader db.TransactionReader, + reader db.LedgerReader, maxHealthyLedgerLatency time.Duration, ) jrpc2.Handler { return NewHandler(func(ctx context.Context) (HealthCheckResult, error) { From 9e82846efa701a00d66af6cdb8805186e924d19b Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 14 Jun 2024 15:11:37 -0400 Subject: [PATCH 04/68] Change varible name --- cmd/soroban-rpc/internal/methods/get_fee_stats.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/soroban-rpc/internal/methods/get_fee_stats.go b/cmd/soroban-rpc/internal/methods/get_fee_stats.go index f76cbe10..efccdf71 100644 --- a/cmd/soroban-rpc/internal/methods/get_fee_stats.go +++ b/cmd/soroban-rpc/internal/methods/get_fee_stats.go @@ -60,7 +60,7 @@ type GetFeeStatsResult struct { // NewGetFeeStatsHandler returns a handler obtaining fee statistics func NewGetFeeStatsHandler(windows *feewindow.FeeWindows, reader db.LedgerReader, logger *log.Entry) jrpc2.Handler { return NewHandler(func(ctx context.Context) (GetFeeStatsResult, error) { - ledgerInfo, err := reader.GetLedgerRange(ctx) + ledgerRange, err := reader.GetLedgerRange(ctx) if err != nil { // still not fatal logger.WithError(err). Error("could not fetch ledger range") @@ -69,7 +69,7 @@ func NewGetFeeStatsHandler(windows *feewindow.FeeWindows, reader db.LedgerReader result := GetFeeStatsResult{ SorobanInclusionFee: convertFeeDistribution(windows.SorobanInclusionFeeWindow.GetFeeDistribution()), InclusionFee: convertFeeDistribution(windows.ClassicFeeWindow.GetFeeDistribution()), - LatestLedger: ledgerInfo.LastLedger.Sequence, + LatestLedger: ledgerRange.LastLedger.Sequence, } return result, nil }) From d198238022e465a5601580ba1911ec2cab44ffd1 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 14 Jun 2024 16:31:45 -0400 Subject: [PATCH 05/68] Add GetLedgerRange method for transactions table --- cmd/soroban-rpc/internal/db/transaction.go | 39 ++++++++++++++-------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/transaction.go b/cmd/soroban-rpc/internal/db/transaction.go index 68ca0d75..f54b4e62 100644 --- a/cmd/soroban-rpc/internal/db/transaction.go +++ b/cmd/soroban-rpc/internal/db/transaction.go @@ -134,7 +134,7 @@ func (txn *transactionHandler) trimTransactions(latestLedgerSeq uint32, retentio return err } -// GetLedgerRange pulls the min/max ledger sequence numbers from the database. +// GetLedgerRange pulls the min/max ledger sequence numbers from the transactions table. func (txn *transactionHandler) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { var ledgerRange ledgerbucketwindow.LedgerRange @@ -143,32 +143,45 @@ func (txn *transactionHandler) GetLedgerRange(ctx context.Context) (ledgerbucket // and max from the ledger table in a single query and get around sqlite's // limitations with parentheses (see https://stackoverflow.com/a/22609948). // - newestQ := sq. - Select("m1.meta"). + // Queries to get the minimum and maximum ledger sequence from the transactions table + minLedgerSeqQ := sq. + Select("m1.ledger_sequence"). FromSelect( sq. - Select("meta"). - From(ledgerCloseMetaTableName). - OrderBy("sequence ASC"). + Select("ledger_sequence"). + From(transactionTableName). + OrderBy("ledger_sequence ASC"). Limit(1), "m1", ) - sql, args, err := sq. - Select("m2.meta"). + maxLedgerSeqQ, args, err := sq. + Select("m2.ledger_sequence"). FromSelect( sq. - Select("meta"). - From(ledgerCloseMetaTableName). - OrderBy("sequence DESC"). + Select("ledger_sequence"). + From(transactionTableName). + OrderBy("ledger_sequence DESC"). Limit(1), "m2", ).ToSql() if err != nil { - return ledgerRange, errors.Wrap(err, "couldn't build ledger range query") + return ledgerRange, errors.Wrap(err, "couldn't build max ledger sequence query") } + // Combine the min and max ledger sequence queries using UNION ALL + txnMinMaxLedgersQ, _, err := minLedgerSeqQ.Suffix("UNION ALL "+maxLedgerSeqQ, args...).ToSql() + if err != nil { + return ledgerRange, errors.Wrap(err, "couldn't build max ledger sequence query") + } + + // Final query to join ledger_close_meta table and the sequence numbers we got from txnMinMaxLedgersQ + finalSql := sq. + Select("lcm.meta"). + From(fmt.Sprintf("%s as lcm", ledgerCloseMetaTableName)). + JoinClause(fmt.Sprintf("JOIN (%s) as seqs ON lcm.sequence == seqs.ledger_sequence", txnMinMaxLedgersQ)) + var lcms []xdr.LedgerCloseMeta - if err = txn.db.Select(ctx, &lcms, newestQ.Suffix("UNION ALL "+sql, args...)); err != nil { + if err = txn.db.Select(ctx, &lcms, finalSql); err != nil { return ledgerRange, errors.Wrap(err, "couldn't query ledger range") } else if len(lcms) < 2 { // There is almost certainly a row, but we want to avoid a race condition From f12d930e891dd3e74114a51c3a5f8ba0fae9a9a1 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 14 Jun 2024 16:51:02 -0400 Subject: [PATCH 06/68] Add test for GetLedgerRange for ledgers table --- cmd/soroban-rpc/internal/db/ledger_test.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/cmd/soroban-rpc/internal/db/ledger_test.go b/cmd/soroban-rpc/internal/db/ledger_test.go index f6ebd70b..cdd58451 100644 --- a/cmd/soroban-rpc/internal/db/ledger_test.go +++ b/cmd/soroban-rpc/internal/db/ledger_test.go @@ -10,6 +10,7 @@ import ( "github.com/stellar/go/network" "github.com/stellar/go/support/log" "github.com/stellar/go/xdr" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces" ) @@ -24,6 +25,9 @@ func createLedger(ledgerSequence uint32) xdr.LedgerCloseMeta { Hash: xdr.Hash{}, Header: xdr.LedgerHeader{ LedgerSeq: xdr.Uint32(ledgerSequence), + ScpValue: xdr.StellarValue{ + CloseTime: xdr.TimePoint(ledgerSequence + 10), + }, }, }, TxSet: xdr.GeneralizedTransactionSet{ @@ -62,6 +66,12 @@ func assertLedgerRange(t *testing.T, reader LedgerReader, start, end uint32) { allLedgers = allLedgers[1:] } assert.Empty(t, allLedgers) + + ledgerRange, err := reader.GetLedgerRange(context.Background()) + assert.Equal(t, start, ledgerRange.FirstLedger.Sequence) + assert.Equal(t, int64(start+10), ledgerRange.FirstLedger.CloseTime) + assert.Equal(t, end, ledgerRange.LastLedger.Sequence) + assert.Equal(t, int64(end+10), ledgerRange.LastLedger.CloseTime) } func TestLedgers(t *testing.T) { From 3c9c7a6741baea8558dfd2ee82a4c69fad1860a9 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 14 Jun 2024 17:15:37 -0400 Subject: [PATCH 07/68] Add GetLedgerRange in ConstantLedgerReader --- cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go b/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go index 30e05afa..05e378ec 100644 --- a/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go +++ b/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" ) const ( @@ -58,6 +59,10 @@ func (ledgerReader *ConstantLedgerReader) StreamAllLedgers(ctx context.Context, return nil } +func (ledgerReader *ConstantLedgerReader) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { + return ledgerbucketwindow.LedgerRange{}, nil +} + func createLedger(ledgerSequence uint32, protocolVersion uint32, hash byte) xdr.LedgerCloseMeta { return xdr.LedgerCloseMeta{ V: 1, @@ -85,3 +90,5 @@ func TestGetLatestLedger(t *testing.T) { assert.Equal(t, expectedLatestLedgerProtocolVersion, latestLedgerResp.ProtocolVersion) assert.Equal(t, expectedLatestLedgerSequence, latestLedgerResp.Sequence) } + +var _ db.LedgerReader = &ConstantLedgerReader{} From fc56898d581a25379643b7112589eb8813861c09 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 14 Jun 2024 17:49:28 -0400 Subject: [PATCH 08/68] Add test for GetLedgerRange in transactions table --- .../internal/db/transaction_test.go | 22 +++++++++++++++++-- 1 file changed, 20 insertions(+), 2 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/transaction_test.go b/cmd/soroban-rpc/internal/db/transaction_test.go index 7257c1e1..24a36bd6 100644 --- a/cmd/soroban-rpc/internal/db/transaction_test.go +++ b/cmd/soroban-rpc/internal/db/transaction_test.go @@ -19,11 +19,21 @@ import ( func TestTransactionNotFound(t *testing.T) { db := NewTestDB(t) + ctx := context.TODO() log := log.DefaultLogger log.SetLevel(logrus.TraceLevel) reader := NewTransactionReader(log, db, passphrase) - _, _, err := reader.GetTransaction(context.TODO(), xdr.Hash{}) + + // Assert the ledger range + ledgerRange, err := reader.GetLedgerRange(ctx) + assert.NoError(t, err) + assert.Equal(t, uint32(0), ledgerRange.FirstLedger.Sequence) + assert.Equal(t, int64(0), ledgerRange.FirstLedger.CloseTime) + assert.Equal(t, uint32(0), ledgerRange.LastLedger.Sequence) + assert.Equal(t, int64(0), ledgerRange.LastLedger.CloseTime) + + _, _, err = reader.GetTransaction(context.TODO(), xdr.Hash{}) require.Error(t, err, ErrNoTransaction) } @@ -51,8 +61,16 @@ func TestTransactionFound(t *testing.T) { } require.NoError(t, write.Commit(lcms[len(lcms)-1].LedgerSequence())) - // check 404 case + // Assert the ledger range reader := NewTransactionReader(log, db, passphrase) + ledgerRange, err := reader.GetLedgerRange(ctx) + assert.NoError(t, err) + assert.Equal(t, uint32(1334), ledgerRange.FirstLedger.Sequence) + assert.Equal(t, ledgerCloseTime(1334), ledgerRange.FirstLedger.CloseTime) + assert.Equal(t, uint32(1337), ledgerRange.LastLedger.Sequence) + assert.Equal(t, ledgerCloseTime(1337), ledgerRange.LastLedger.CloseTime) + + // check 404 case _, _, err = reader.GetTransaction(ctx, xdr.Hash{}) require.Error(t, err, ErrNoTransaction) From a27a176ca5b08f0bd318712b4ef526d825ad7534 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 14 Jun 2024 17:51:41 -0400 Subject: [PATCH 09/68] Fix nil pointer bugs in getTransactions --- cmd/soroban-rpc/internal/methods/get_transactions.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/soroban-rpc/internal/methods/get_transactions.go b/cmd/soroban-rpc/internal/methods/get_transactions.go index baea5a72..04ce1703 100644 --- a/cmd/soroban-rpc/internal/methods/get_transactions.go +++ b/cmd/soroban-rpc/internal/methods/get_transactions.go @@ -134,7 +134,7 @@ func (h transactionsRPCHandler) getTransactionsByLedgerSequence(ctx context.Cont // Iterate through each ledger and its transactions until limit or end range is reached. // The latest ledger acts as the end ledger range for the request. var txns []TransactionInfo - var cursor *toid.ID + cursor := toid.New(0, 0, 0) LedgerLoop: for ledgerSeq := start.LedgerSequence; ledgerSeq <= int32(ledgerRange.LastLedger.Sequence); ledgerSeq++ { // Get ledger close meta from db @@ -167,7 +167,7 @@ LedgerLoop: if ierr := reader.Seek(startTxIdx - 1); ierr != nil && ierr != io.EOF { return GetTransactionsResponse{}, &jrpc2.Error{ Code: jrpc2.InternalError, - Message: err.Error(), + Message: ierr.Error(), } } } From ef35cb893e62fe42c7eb6b29fcfd8936b7f5ecdf Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 17 Jun 2024 11:16:16 -0400 Subject: [PATCH 10/68] Remove latest ledger assertion --- cmd/soroban-rpc/internal/test/transaction_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/soroban-rpc/internal/test/transaction_test.go b/cmd/soroban-rpc/internal/test/transaction_test.go index 4a8461f6..6c3fb83f 100644 --- a/cmd/soroban-rpc/internal/test/transaction_test.go +++ b/cmd/soroban-rpc/internal/test/transaction_test.go @@ -291,8 +291,6 @@ func sendSuccessfulTransaction(t *testing.T, client *jrpc2.Client, kp *keypair.F assert.NoError(t, err) t.Logf("error: %#v\n", txResult) } - assert.NotZero(t, result.LatestLedger) - assert.NotZero(t, result.LatestLedgerCloseTime) response := getTransaction(t, client, expectedHashHex) if !assert.Equal(t, methods.TransactionStatusSuccess, response.Status) { From 1d9140c2199eb24ed881c2935074599895699ec9 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 17 Jun 2024 11:39:18 -0400 Subject: [PATCH 11/68] Comment out the latest ledger assertion --- cmd/soroban-rpc/internal/test/transaction_test.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/cmd/soroban-rpc/internal/test/transaction_test.go b/cmd/soroban-rpc/internal/test/transaction_test.go index 6c3fb83f..f8644cd2 100644 --- a/cmd/soroban-rpc/internal/test/transaction_test.go +++ b/cmd/soroban-rpc/internal/test/transaction_test.go @@ -136,8 +136,8 @@ func TestSendTransactionBadSequence(t *testing.T) { err = client.CallResult(context.Background(), "sendTransaction", request, &result) assert.NoError(t, err) - assert.NotZero(t, result.LatestLedger) - assert.NotZero(t, result.LatestLedgerCloseTime) + //assert.NotZero(t, result.LatestLedger) + //assert.NotZero(t, result.LatestLedgerCloseTime) expectedHashHex, err := tx.HashHex(StandaloneNetworkPassphrase) assert.NoError(t, err) assert.Equal(t, expectedHashHex, result.Hash) @@ -245,8 +245,8 @@ func TestSendTransactionFailedInLedger(t *testing.T) { assert.NoError(t, err) fmt.Printf("error: %#v\n", txResult) } - assert.NotZero(t, result.LatestLedger) - assert.NotZero(t, result.LatestLedgerCloseTime) + //assert.NotZero(t, result.LatestLedger) + //assert.NotZero(t, result.LatestLedgerCloseTime) response := getTransaction(t, client, expectedHashHex) assert.Equal(t, methods.TransactionStatusFailed, response.Status) @@ -291,6 +291,8 @@ func sendSuccessfulTransaction(t *testing.T, client *jrpc2.Client, kp *keypair.F assert.NoError(t, err) t.Logf("error: %#v\n", txResult) } + //assert.NotZero(t, result.LatestLedger) + //assert.NotZero(t, result.LatestLedgerCloseTime) response := getTransaction(t, client, expectedHashHex) if !assert.Equal(t, methods.TransactionStatusSuccess, response.Status) { From 1b35c07a562d8117e38ffcf60a94fc26651ebbd8 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 17 Jun 2024 12:41:34 -0400 Subject: [PATCH 12/68] Remove GetLedgerRange from meta table and use ledgerRangeGetter for getHealth and getFeeStats --- cmd/soroban-rpc/internal/db/ledger.go | 56 ------------------- cmd/soroban-rpc/internal/db/ledger_test.go | 6 -- cmd/soroban-rpc/internal/db/mocks.go | 4 -- cmd/soroban-rpc/internal/events/events.go | 3 +- cmd/soroban-rpc/internal/jsonrpc.go | 15 +++-- .../internal/methods/get_fee_stats.go | 2 +- .../methods/get_latest_ledger_test.go | 5 -- cmd/soroban-rpc/internal/methods/health.go | 2 +- 8 files changed, 13 insertions(+), 80 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/ledger.go b/cmd/soroban-rpc/internal/db/ledger.go index 31e960a4..39ded8f8 100644 --- a/cmd/soroban-rpc/internal/db/ledger.go +++ b/cmd/soroban-rpc/internal/db/ledger.go @@ -5,11 +5,7 @@ import ( "fmt" sq "github.com/Masterminds/squirrel" - "github.com/stellar/go/support/errors" - "github.com/stellar/go/xdr" - - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" ) const ( @@ -21,7 +17,6 @@ type StreamLedgerFn func(xdr.LedgerCloseMeta) error type LedgerReader interface { GetLedger(ctx context.Context, sequence uint32) (xdr.LedgerCloseMeta, bool, error) StreamAllLedgers(ctx context.Context, f StreamLedgerFn) error - LedgerRangeReader } type LedgerWriter interface { @@ -73,57 +68,6 @@ func (r ledgerReader) GetLedger(ctx context.Context, sequence uint32) (xdr.Ledge } } -// GetLedgerRange pulls the min/max ledger sequence numbers from the ledgers table. -func (r ledgerReader) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { - var ledgerRange ledgerbucketwindow.LedgerRange - // - // We use subqueries alongside a UNION ALL stitch in order to select the min - // and max from the ledger table in a single query and get around sqlite's - // limitations with parentheses (see https://stackoverflow.com/a/22609948). - // - newestQ := sq. - Select("m1.meta"). - FromSelect( - sq. - Select("meta"). - From(ledgerCloseMetaTableName). - OrderBy("sequence ASC"). - Limit(1), - "m1", - ) - sql, args, err := sq. - Select("m2.meta"). - FromSelect( - sq. - Select("meta"). - From(ledgerCloseMetaTableName). - OrderBy("sequence DESC"). - Limit(1), - "m2", - ).ToSql() - if err != nil { - return ledgerRange, errors.Wrap(err, "couldn't build ledger range query") - } - - var lcms []xdr.LedgerCloseMeta - if err = r.db.Select(ctx, &lcms, newestQ.Suffix("UNION ALL "+sql, args...)); err != nil { - return ledgerRange, errors.Wrap(err, "couldn't query ledger range") - } else if len(lcms) < 2 { - // There is almost certainly a row, but we want to avoid a race condition - // with ingestion as well as support test cases from an empty DB, so we need - // to sanity check that there is in fact a result. Note that no ledgers in - // the database isn't an error, it's just an empty range. - return ledgerRange, nil - } - - lcm1, lcm2 := lcms[0], lcms[1] - ledgerRange.FirstLedger.Sequence = lcm1.LedgerSequence() - ledgerRange.FirstLedger.CloseTime = lcm1.LedgerCloseTime() - ledgerRange.LastLedger.Sequence = lcm2.LedgerSequence() - ledgerRange.LastLedger.CloseTime = lcm2.LedgerCloseTime() - return ledgerRange, nil -} - type ledgerWriter struct { stmtCache *sq.StmtCache } diff --git a/cmd/soroban-rpc/internal/db/ledger_test.go b/cmd/soroban-rpc/internal/db/ledger_test.go index 52ca44d2..562afc0f 100644 --- a/cmd/soroban-rpc/internal/db/ledger_test.go +++ b/cmd/soroban-rpc/internal/db/ledger_test.go @@ -67,12 +67,6 @@ func assertLedgerRange(t *testing.T, reader LedgerReader, start, end uint32) { allLedgers = allLedgers[1:] } assert.Empty(t, allLedgers) - - ledgerRange, err := reader.GetLedgerRange(context.Background()) - assert.Equal(t, start, ledgerRange.FirstLedger.Sequence) - assert.Equal(t, int64(start+10), ledgerRange.FirstLedger.CloseTime) - assert.Equal(t, end, ledgerRange.LastLedger.Sequence) - assert.Equal(t, int64(end+10), ledgerRange.LastLedger.CloseTime) } func TestLedgers(t *testing.T) { diff --git a/cmd/soroban-rpc/internal/db/mocks.go b/cmd/soroban-rpc/internal/db/mocks.go index c72b7393..d2dfba4f 100644 --- a/cmd/soroban-rpc/internal/db/mocks.go +++ b/cmd/soroban-rpc/internal/db/mocks.go @@ -105,10 +105,6 @@ func (m *mockLedgerReader) StreamAllLedgers(ctx context.Context, f StreamLedgerF return nil } -func (m *mockLedgerReader) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { - return ledgerbucketwindow.LedgerRange{}, nil -} - var _ TransactionReader = &mockTransactionHandler{} var _ TransactionWriter = &mockTransactionHandler{} var _ LedgerReader = &mockLedgerReader{} diff --git a/cmd/soroban-rpc/internal/events/events.go b/cmd/soroban-rpc/internal/events/events.go index b724fbf9..f428c0e7 100644 --- a/cmd/soroban-rpc/internal/events/events.go +++ b/cmd/soroban-rpc/internal/events/events.go @@ -1,6 +1,7 @@ package events import ( + "context" "errors" "io" "sort" @@ -267,7 +268,7 @@ func readEvents(networkPassphrase string, ledgerCloseMeta xdr.LedgerCloseMeta) ( } // GetLedgerRange returns the first and latest ledger available in the store. -func (m *MemoryStore) GetLedgerRange() (ledgerbucketwindow.LedgerRange, error) { +func (m *MemoryStore) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { m.lock.RLock() defer m.lock.RUnlock() return m.eventsByLedger.GetLedgerRange(), nil diff --git a/cmd/soroban-rpc/internal/jsonrpc.go b/cmd/soroban-rpc/internal/jsonrpc.go index 5d5189ab..7bc948c5 100644 --- a/cmd/soroban-rpc/internal/jsonrpc.go +++ b/cmd/soroban-rpc/internal/jsonrpc.go @@ -136,10 +136,13 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler { }, } - // While we transition from in-memory to database-oriented history storage, - // the on-disk (transaction) retention window will always be larger than the - // in-memory (events) one. - var retentionWindow = cfg.TransactionLedgerRetentionWindow + // Get the largest history window + var ledgerRangeGetter db.LedgerRangeReader = params.EventStore + var retentionWindow = cfg.EventLedgerRetentionWindow + if cfg.TransactionLedgerRetentionWindow > cfg.EventLedgerRetentionWindow { + retentionWindow = cfg.TransactionLedgerRetentionWindow + ledgerRangeGetter = params.TransactionReader + } handlers := []struct { methodName string @@ -151,7 +154,7 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler { { methodName: "getHealth", underlyingHandler: methods.NewHealthCheck( - retentionWindow, params.LedgerReader, cfg.MaxHealthyLedgerLatency), + retentionWindow, ledgerRangeGetter, cfg.MaxHealthyLedgerLatency), longName: "get_health", queueLimit: cfg.RequestBacklogGetHealthQueueLimit, requestDurationLimit: cfg.MaxGetHealthExecutionDuration, @@ -232,7 +235,7 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler { }, { methodName: "getFeeStats", - underlyingHandler: methods.NewGetFeeStatsHandler(params.FeeStatWindows, params.LedgerReader, params.Logger), + underlyingHandler: methods.NewGetFeeStatsHandler(params.FeeStatWindows, ledgerRangeGetter, params.Logger), longName: "get_fee_stats", queueLimit: cfg.RequestBacklogGetFeeStatsTransactionQueueLimit, requestDurationLimit: cfg.MaxGetFeeStatsExecutionDuration, diff --git a/cmd/soroban-rpc/internal/methods/get_fee_stats.go b/cmd/soroban-rpc/internal/methods/get_fee_stats.go index efccdf71..61592176 100644 --- a/cmd/soroban-rpc/internal/methods/get_fee_stats.go +++ b/cmd/soroban-rpc/internal/methods/get_fee_stats.go @@ -58,7 +58,7 @@ type GetFeeStatsResult struct { } // NewGetFeeStatsHandler returns a handler obtaining fee statistics -func NewGetFeeStatsHandler(windows *feewindow.FeeWindows, reader db.LedgerReader, logger *log.Entry) jrpc2.Handler { +func NewGetFeeStatsHandler(windows *feewindow.FeeWindows, reader db.LedgerRangeReader, logger *log.Entry) jrpc2.Handler { return NewHandler(func(ctx context.Context) (GetFeeStatsResult, error) { ledgerRange, err := reader.GetLedgerRange(ctx) if err != nil { // still not fatal diff --git a/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go b/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go index 05e378ec..e961510c 100644 --- a/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go +++ b/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go @@ -9,7 +9,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" ) const ( @@ -59,10 +58,6 @@ func (ledgerReader *ConstantLedgerReader) StreamAllLedgers(ctx context.Context, return nil } -func (ledgerReader *ConstantLedgerReader) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { - return ledgerbucketwindow.LedgerRange{}, nil -} - func createLedger(ledgerSequence uint32, protocolVersion uint32, hash byte) xdr.LedgerCloseMeta { return xdr.LedgerCloseMeta{ V: 1, diff --git a/cmd/soroban-rpc/internal/methods/health.go b/cmd/soroban-rpc/internal/methods/health.go index d013d53f..bef5e4b6 100644 --- a/cmd/soroban-rpc/internal/methods/health.go +++ b/cmd/soroban-rpc/internal/methods/health.go @@ -20,7 +20,7 @@ type HealthCheckResult struct { // NewHealthCheck returns a health check json rpc handler func NewHealthCheck( retentionWindow uint32, - reader db.LedgerReader, + reader db.LedgerRangeReader, maxHealthyLedgerLatency time.Duration, ) jrpc2.Handler { return NewHandler(func(ctx context.Context) (HealthCheckResult, error) { From 420adc05b7bb8b484566e60cfeff2e09c811d722 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 17 Jun 2024 12:44:14 -0400 Subject: [PATCH 13/68] Remove ledgerCloseTime --- cmd/soroban-rpc/internal/db/ledger_test.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/ledger_test.go b/cmd/soroban-rpc/internal/db/ledger_test.go index 562afc0f..25369fac 100644 --- a/cmd/soroban-rpc/internal/db/ledger_test.go +++ b/cmd/soroban-rpc/internal/db/ledger_test.go @@ -26,9 +26,6 @@ func createLedger(ledgerSequence uint32) xdr.LedgerCloseMeta { Hash: xdr.Hash{}, Header: xdr.LedgerHeader{ LedgerSeq: xdr.Uint32(ledgerSequence), - ScpValue: xdr.StellarValue{ - CloseTime: xdr.TimePoint(ledgerSequence + 10), - }, }, }, TxSet: xdr.GeneralizedTransactionSet{ From 2343207800fcea745f691b57cbe8a045b1ca43cc Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 17 Jun 2024 12:44:38 -0400 Subject: [PATCH 14/68] Revert newline change --- cmd/soroban-rpc/internal/db/ledger.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/soroban-rpc/internal/db/ledger.go b/cmd/soroban-rpc/internal/db/ledger.go index 39ded8f8..97887281 100644 --- a/cmd/soroban-rpc/internal/db/ledger.go +++ b/cmd/soroban-rpc/internal/db/ledger.go @@ -5,6 +5,7 @@ import ( "fmt" sq "github.com/Masterminds/squirrel" + "github.com/stellar/go/xdr" ) From 1dc4020996a58bff7a38ee068cebd51e0c221ac8 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 17 Jun 2024 14:48:53 -0400 Subject: [PATCH 15/68] Revert --- cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go b/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go index e961510c..30e05afa 100644 --- a/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go +++ b/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go @@ -85,5 +85,3 @@ func TestGetLatestLedger(t *testing.T) { assert.Equal(t, expectedLatestLedgerProtocolVersion, latestLedgerResp.ProtocolVersion) assert.Equal(t, expectedLatestLedgerSequence, latestLedgerResp.Sequence) } - -var _ db.LedgerReader = &ConstantLedgerReader{} From 81bdf1ee8c7778d480c5ed8e1521dcbb00ec8c66 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 17 Jun 2024 16:03:18 -0400 Subject: [PATCH 16/68] Remove assertions --- cmd/soroban-rpc/internal/test/transaction_test.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/cmd/soroban-rpc/internal/test/transaction_test.go b/cmd/soroban-rpc/internal/test/transaction_test.go index f8644cd2..e16bcc5e 100644 --- a/cmd/soroban-rpc/internal/test/transaction_test.go +++ b/cmd/soroban-rpc/internal/test/transaction_test.go @@ -21,7 +21,6 @@ import ( func TestSendTransactionSucceedsWithoutResults(t *testing.T) { test := NewTest(t, nil) - client := test.GetRPCLient() kp := keypair.Root(StandaloneNetworkPassphrase) @@ -136,8 +135,6 @@ func TestSendTransactionBadSequence(t *testing.T) { err = client.CallResult(context.Background(), "sendTransaction", request, &result) assert.NoError(t, err) - //assert.NotZero(t, result.LatestLedger) - //assert.NotZero(t, result.LatestLedgerCloseTime) expectedHashHex, err := tx.HashHex(StandaloneNetworkPassphrase) assert.NoError(t, err) assert.Equal(t, expectedHashHex, result.Hash) @@ -245,8 +242,6 @@ func TestSendTransactionFailedInLedger(t *testing.T) { assert.NoError(t, err) fmt.Printf("error: %#v\n", txResult) } - //assert.NotZero(t, result.LatestLedger) - //assert.NotZero(t, result.LatestLedgerCloseTime) response := getTransaction(t, client, expectedHashHex) assert.Equal(t, methods.TransactionStatusFailed, response.Status) @@ -291,8 +286,6 @@ func sendSuccessfulTransaction(t *testing.T, client *jrpc2.Client, kp *keypair.F assert.NoError(t, err) t.Logf("error: %#v\n", txResult) } - //assert.NotZero(t, result.LatestLedger) - //assert.NotZero(t, result.LatestLedgerCloseTime) response := getTransaction(t, client, expectedHashHex) if !assert.Equal(t, methods.TransactionStatusSuccess, response.Status) { From f7d1b226290d18d3d6ded5f9c4a48c5e58898259 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 17 Jun 2024 16:04:15 -0400 Subject: [PATCH 17/68] revert --- cmd/soroban-rpc/internal/test/transaction_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/cmd/soroban-rpc/internal/test/transaction_test.go b/cmd/soroban-rpc/internal/test/transaction_test.go index e16bcc5e..fe7f8569 100644 --- a/cmd/soroban-rpc/internal/test/transaction_test.go +++ b/cmd/soroban-rpc/internal/test/transaction_test.go @@ -21,6 +21,7 @@ import ( func TestSendTransactionSucceedsWithoutResults(t *testing.T) { test := NewTest(t, nil) + client := test.GetRPCLient() kp := keypair.Root(StandaloneNetworkPassphrase) From c8d0a83d03a12b2053327cfcd3261393ceac7c47 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Tue, 18 Jun 2024 09:08:33 -0400 Subject: [PATCH 18/68] Change interface name --- cmd/soroban-rpc/internal/db/db.go | 2 +- cmd/soroban-rpc/internal/db/transaction.go | 2 +- cmd/soroban-rpc/internal/jsonrpc.go | 2 +- cmd/soroban-rpc/internal/methods/get_fee_stats.go | 2 +- cmd/soroban-rpc/internal/methods/health.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/db.go b/cmd/soroban-rpc/internal/db/db.go index 0ec45889..57ad7e6c 100644 --- a/cmd/soroban-rpc/internal/db/db.go +++ b/cmd/soroban-rpc/internal/db/db.go @@ -32,7 +32,7 @@ const ( latestLedgerSequenceMetaKey = "LatestLedgerSequence" ) -type LedgerRangeReader interface { +type LedgerRangeGetter interface { GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) } diff --git a/cmd/soroban-rpc/internal/db/transaction.go b/cmd/soroban-rpc/internal/db/transaction.go index c75f6673..54dd0c7c 100644 --- a/cmd/soroban-rpc/internal/db/transaction.go +++ b/cmd/soroban-rpc/internal/db/transaction.go @@ -42,7 +42,7 @@ type TransactionWriter interface { // TransactionReader provides all the public ways to read from the DB. type TransactionReader interface { GetTransaction(ctx context.Context, hash xdr.Hash) (Transaction, ledgerbucketwindow.LedgerRange, error) - LedgerRangeReader + LedgerRangeGetter } type transactionHandler struct { diff --git a/cmd/soroban-rpc/internal/jsonrpc.go b/cmd/soroban-rpc/internal/jsonrpc.go index 7bc948c5..1ea542d0 100644 --- a/cmd/soroban-rpc/internal/jsonrpc.go +++ b/cmd/soroban-rpc/internal/jsonrpc.go @@ -137,7 +137,7 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler { } // Get the largest history window - var ledgerRangeGetter db.LedgerRangeReader = params.EventStore + var ledgerRangeGetter db.LedgerRangeGetter = params.EventStore var retentionWindow = cfg.EventLedgerRetentionWindow if cfg.TransactionLedgerRetentionWindow > cfg.EventLedgerRetentionWindow { retentionWindow = cfg.TransactionLedgerRetentionWindow diff --git a/cmd/soroban-rpc/internal/methods/get_fee_stats.go b/cmd/soroban-rpc/internal/methods/get_fee_stats.go index 61592176..c60dbbdc 100644 --- a/cmd/soroban-rpc/internal/methods/get_fee_stats.go +++ b/cmd/soroban-rpc/internal/methods/get_fee_stats.go @@ -58,7 +58,7 @@ type GetFeeStatsResult struct { } // NewGetFeeStatsHandler returns a handler obtaining fee statistics -func NewGetFeeStatsHandler(windows *feewindow.FeeWindows, reader db.LedgerRangeReader, logger *log.Entry) jrpc2.Handler { +func NewGetFeeStatsHandler(windows *feewindow.FeeWindows, reader db.LedgerRangeGetter, logger *log.Entry) jrpc2.Handler { return NewHandler(func(ctx context.Context) (GetFeeStatsResult, error) { ledgerRange, err := reader.GetLedgerRange(ctx) if err != nil { // still not fatal diff --git a/cmd/soroban-rpc/internal/methods/health.go b/cmd/soroban-rpc/internal/methods/health.go index bef5e4b6..b30492c7 100644 --- a/cmd/soroban-rpc/internal/methods/health.go +++ b/cmd/soroban-rpc/internal/methods/health.go @@ -20,7 +20,7 @@ type HealthCheckResult struct { // NewHealthCheck returns a health check json rpc handler func NewHealthCheck( retentionWindow uint32, - reader db.LedgerRangeReader, + reader db.LedgerRangeGetter, maxHealthyLedgerLatency time.Duration, ) jrpc2.Handler { return NewHandler(func(ctx context.Context) (HealthCheckResult, error) { From d9df979a89b554f73ed5fbdfa17fe7b9108c5d74 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Thu, 20 Jun 2024 09:54:44 -0400 Subject: [PATCH 19/68] insert txns during integration test setup - 1 --- cmd/soroban-rpc/internal/db/util.go | 145 ++++++++++++++++++ cmd/soroban-rpc/internal/test/integration.go | 37 ++++- .../internal/test/transaction_test.go | 6 + 3 files changed, 185 insertions(+), 3 deletions(-) create mode 100644 cmd/soroban-rpc/internal/db/util.go diff --git a/cmd/soroban-rpc/internal/db/util.go b/cmd/soroban-rpc/internal/db/util.go new file mode 100644 index 00000000..434716bf --- /dev/null +++ b/cmd/soroban-rpc/internal/db/util.go @@ -0,0 +1,145 @@ +package db + +import ( + "encoding/hex" + + "github.com/stellar/go/network" + "github.com/stellar/go/xdr" +) + +func ledgerCloseTime(ledgerSequence uint32) int64 { + return int64(ledgerSequence)*25 + 100 +} + +func TxHash(acctSeq uint32) xdr.Hash { + envelope := TxEnvelope(acctSeq) + hash, err := network.HashTransactionInEnvelope(envelope, network.FutureNetworkPassphrase) + if err != nil { + panic(err) + } + + return hash +} + +func TxEnvelope(acctSeq uint32) xdr.TransactionEnvelope { + envelope, err := xdr.NewTransactionEnvelope(xdr.EnvelopeTypeEnvelopeTypeTx, xdr.TransactionV1Envelope{ + Tx: xdr.Transaction{ + Fee: 1, + SeqNum: xdr.SequenceNumber(acctSeq), + SourceAccount: xdr.MustMuxedAddress("MA7QYNF7SOWQ3GLR2BGMZEHXAVIRZA4KVWLTJJFC7MGXUA74P7UJVAAAAAAAAAAAAAJLK"), + }, + }) + if err != nil { + panic(err) + } + return envelope +} + +func TransactionResult(successful bool) xdr.TransactionResult { + code := xdr.TransactionResultCodeTxBadSeq + if successful { + code = xdr.TransactionResultCodeTxSuccess + } + opResults := []xdr.OperationResult{} + return xdr.TransactionResult{ + FeeCharged: 100, + Result: xdr.TransactionResultResult{ + Code: code, + Results: &opResults, + }, + } +} + +func CreateTxMeta(acctSeq uint32, successful bool) xdr.LedgerCloseMeta { + envelope := TxEnvelope(acctSeq) + + txProcessing := []xdr.TransactionResultMeta{ + { + TxApplyProcessing: xdr.TransactionMeta{ + V: 3, + Operations: &[]xdr.OperationMeta{}, + V3: &xdr.TransactionMetaV3{}, + }, + Result: xdr.TransactionResultPair{ + TransactionHash: TxHash(acctSeq), + Result: TransactionResult(successful), + }, + }, + } + + components := []xdr.TxSetComponent{ + { + Type: xdr.TxSetComponentTypeTxsetCompTxsMaybeDiscountedFee, + TxsMaybeDiscountedFee: &xdr.TxSetComponentTxsMaybeDiscountedFee{ + BaseFee: nil, + Txs: []xdr.TransactionEnvelope{ + envelope, + }, + }, + }, + } + return xdr.LedgerCloseMeta{ + V: 1, + V1: &xdr.LedgerCloseMetaV1{ + LedgerHeader: xdr.LedgerHeaderHistoryEntry{ + Header: xdr.LedgerHeader{ + ScpValue: xdr.StellarValue{ + CloseTime: xdr.TimePoint(ledgerCloseTime(acctSeq + 100)), + }, + LedgerSeq: xdr.Uint32(acctSeq), + }, + }, + TxProcessing: txProcessing, + TxSet: xdr.GeneralizedTransactionSet{ + V: 1, + V1TxSet: &xdr.TransactionSetV1{ + PreviousLedgerHash: xdr.Hash{1}, + Phases: []xdr.TransactionPhase{ + { + V: 0, + V0Components: &components, + }, + }, + }, + }, + }, + } +} + +func TxMetaWithEvents(acctSeq uint32, successful bool) xdr.LedgerCloseMeta { + + meta := CreateTxMeta(acctSeq, successful) + + contractIDBytes, _ := hex.DecodeString("df06d62447fd25da07c0135eed7557e5a5497ee7d15b7fe345bd47e191d8f577") + var contractID xdr.Hash + copy(contractID[:], contractIDBytes) + counter := xdr.ScSymbol("COUNTER") + + meta.V1.TxProcessing[0].TxApplyProcessing.V3 = &xdr.TransactionMetaV3{ + SorobanMeta: &xdr.SorobanTransactionMeta{ + Events: []xdr.ContractEvent{{ + ContractId: &contractID, + Type: xdr.ContractEventTypeContract, + Body: xdr.ContractEventBody{ + V: 0, + V0: &xdr.ContractEventV0{ + Topics: []xdr.ScVal{{ + Type: xdr.ScValTypeScvSymbol, + Sym: &counter, + }}, + Data: xdr.ScVal{ + Type: xdr.ScValTypeScvSymbol, + Sym: &counter, + }, + }, + }, + }}, + ReturnValue: xdr.ScVal{ + Type: xdr.ScValTypeScvSymbol, + Sym: &counter, + }, + }, + } + + return meta +} diff --git a/cmd/soroban-rpc/internal/test/integration.go b/cmd/soroban-rpc/internal/test/integration.go index 531a162f..42ba6188 100644 --- a/cmd/soroban-rpc/internal/test/integration.go +++ b/cmd/soroban-rpc/internal/test/integration.go @@ -20,9 +20,15 @@ import ( "github.com/creachadair/jrpc2/jhttp" "github.com/stellar/go/clients/stellarcore" "github.com/stellar/go/keypair" + "github.com/stellar/go/network" + "github.com/stellar/go/support/log" "github.com/stellar/go/txnbuild" + "github.com/stellar/go/xdr" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/methods" @@ -62,6 +68,7 @@ type Test struct { rpcContainerConfigMountDir string rpcContainerSQLiteMountDir string rpcContainerLogsCommand *exec.Cmd + sqlitePath string rpcClient *jrpc2.Client coreClient *stellarcore.Client @@ -118,6 +125,10 @@ func NewTest(t *testing.T, cfg *TestConfig) *Test { } i.waitForRPC() + // We populate the transactions table with some rows so that GetLedgerRange called in transaction related + // integration tests returns a valid latest ledger. + i.insertTransactions() + return i } @@ -144,6 +155,26 @@ func (i *Test) adminURL() string { return fmt.Sprintf("http://localhost:%d", adminPort) } +func (i *Test) insertTransactions() { + testDb, err := db.OpenSQLiteDB(i.sqlitePath) + assert.NoError(i.t, err) + + writer := db.NewReadWriter(log.DefaultLogger, testDb, interfaces.MakeNoOpDeamon(), 100, 1_000_000, network.FutureNetworkPassphrase) + write, err := writer.NewTx(context.Background()) + assert.NoError(i.t, err) + + lcms := make([]xdr.LedgerCloseMeta, 0, 3) + for i := uint32(0); i < uint32(cap(lcms)); i++ { + lcms = append(lcms, db.CreateTxMeta(9+i, i%2 == 0)) + } + + _, txW := write.LedgerWriter(), write.TransactionWriter() + for _, lcm := range lcms { + assert.NoError(i.t, txW.InsertTransactions(lcm)) + } + assert.NoError(i.t, write.Commit(lcms[len(lcms)-1].LedgerSequence())) +} + func (i *Test) waitForCheckpoint() { i.t.Log("Waiting for core to be up...") for t := 30 * time.Second; t >= 0; t -= time.Second { @@ -167,7 +198,7 @@ func (i *Test) waitForCheckpoint() { func (i *Test) getRPConfig(sqlitePath string) map[string]string { if sqlitePath == "" { - sqlitePath = path.Join(i.t.TempDir(), "soroban_rpc.sqlite") + i.sqlitePath = path.Join(i.t.TempDir(), "soroban_rpc.sqlite") } // Container's default path to captive core @@ -200,7 +231,7 @@ func (i *Test) getRPConfig(sqlitePath string) map[string]string { bindHost = "0.0.0.0" // The container needs to use the sqlite mount point i.rpcContainerSQLiteMountDir = filepath.Dir(sqlitePath) - sqlitePath = "/db/" + filepath.Base(sqlitePath) + i.sqlitePath = "/db/" + filepath.Base(sqlitePath) stellarCoreURL = fmt.Sprintf("http://core:%d", stellarCorePort) } @@ -223,7 +254,7 @@ func (i *Test) getRPConfig(sqlitePath string) map[string]string { "NETWORK_PASSPHRASE": StandaloneNetworkPassphrase, "HISTORY_ARCHIVE_URLS": archiveURL, "LOG_LEVEL": "debug", - "DB_PATH": sqlitePath, + "DB_PATH": i.sqlitePath, "INGESTION_TIMEOUT": "10m", "EVENT_LEDGER_RETENTION_WINDOW": strconv.Itoa(ledgerbucketwindow.OneDayOfLedgers), "TRANSACTION_RETENTION_WINDOW": strconv.Itoa(ledgerbucketwindow.OneDayOfLedgers), diff --git a/cmd/soroban-rpc/internal/test/transaction_test.go b/cmd/soroban-rpc/internal/test/transaction_test.go index fe7f8569..4a8461f6 100644 --- a/cmd/soroban-rpc/internal/test/transaction_test.go +++ b/cmd/soroban-rpc/internal/test/transaction_test.go @@ -136,6 +136,8 @@ func TestSendTransactionBadSequence(t *testing.T) { err = client.CallResult(context.Background(), "sendTransaction", request, &result) assert.NoError(t, err) + assert.NotZero(t, result.LatestLedger) + assert.NotZero(t, result.LatestLedgerCloseTime) expectedHashHex, err := tx.HashHex(StandaloneNetworkPassphrase) assert.NoError(t, err) assert.Equal(t, expectedHashHex, result.Hash) @@ -243,6 +245,8 @@ func TestSendTransactionFailedInLedger(t *testing.T) { assert.NoError(t, err) fmt.Printf("error: %#v\n", txResult) } + assert.NotZero(t, result.LatestLedger) + assert.NotZero(t, result.LatestLedgerCloseTime) response := getTransaction(t, client, expectedHashHex) assert.Equal(t, methods.TransactionStatusFailed, response.Status) @@ -287,6 +291,8 @@ func sendSuccessfulTransaction(t *testing.T, client *jrpc2.Client, kp *keypair.F assert.NoError(t, err) t.Logf("error: %#v\n", txResult) } + assert.NotZero(t, result.LatestLedger) + assert.NotZero(t, result.LatestLedgerCloseTime) response := getTransaction(t, client, expectedHashHex) if !assert.Equal(t, methods.TransactionStatusSuccess, response.Status) { From 6db04d577cff79a40ea34a57f20127ac1b0e7939 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Thu, 20 Jun 2024 10:21:41 -0400 Subject: [PATCH 20/68] insert txns during integration test setup - 2 --- cmd/soroban-rpc/internal/db/util.go | 76 ++++---------------- cmd/soroban-rpc/internal/test/integration.go | 3 +- 2 files changed, 17 insertions(+), 62 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/util.go b/cmd/soroban-rpc/internal/db/util.go index 434716bf..4f221798 100644 --- a/cmd/soroban-rpc/internal/db/util.go +++ b/cmd/soroban-rpc/internal/db/util.go @@ -1,27 +1,20 @@ package db import ( - "encoding/hex" - "github.com/stellar/go/network" "github.com/stellar/go/xdr" ) -func ledgerCloseTime(ledgerSequence uint32) int64 { - return int64(ledgerSequence)*25 + 100 -} - -func TxHash(acctSeq uint32) xdr.Hash { - envelope := TxEnvelope(acctSeq) +func txHash(acctSeq uint32) xdr.Hash { + envelope := txEnvelope(acctSeq) hash, err := network.HashTransactionInEnvelope(envelope, network.FutureNetworkPassphrase) if err != nil { panic(err) } - return hash } -func TxEnvelope(acctSeq uint32) xdr.TransactionEnvelope { +func txEnvelope(acctSeq uint32) xdr.TransactionEnvelope { envelope, err := xdr.NewTransactionEnvelope(xdr.EnvelopeTypeEnvelopeTypeTx, xdr.TransactionV1Envelope{ Tx: xdr.Transaction{ Fee: 1, @@ -35,7 +28,7 @@ func TxEnvelope(acctSeq uint32) xdr.TransactionEnvelope { return envelope } -func TransactionResult(successful bool) xdr.TransactionResult { +func transactionResult(successful bool) xdr.TransactionResult { code := xdr.TransactionResultCodeTxBadSeq if successful { code = xdr.TransactionResultCodeTxSuccess @@ -51,8 +44,7 @@ func TransactionResult(successful bool) xdr.TransactionResult { } func CreateTxMeta(acctSeq uint32, successful bool) xdr.LedgerCloseMeta { - envelope := TxEnvelope(acctSeq) - + envelope := txEnvelope(acctSeq) txProcessing := []xdr.TransactionResultMeta{ { TxApplyProcessing: xdr.TransactionMeta{ @@ -61,23 +53,21 @@ func CreateTxMeta(acctSeq uint32, successful bool) xdr.LedgerCloseMeta { V3: &xdr.TransactionMetaV3{}, }, Result: xdr.TransactionResultPair{ - TransactionHash: TxHash(acctSeq), - Result: TransactionResult(successful), + TransactionHash: txHash(acctSeq), + Result: transactionResult(successful), }, }, } - components := []xdr.TxSetComponent{ { Type: xdr.TxSetComponentTypeTxsetCompTxsMaybeDiscountedFee, TxsMaybeDiscountedFee: &xdr.TxSetComponentTxsMaybeDiscountedFee{ BaseFee: nil, - Txs: []xdr.TransactionEnvelope{ - envelope, - }, + Txs: []xdr.TransactionEnvelope{envelope}, }, }, } + return xdr.LedgerCloseMeta{ V: 1, V1: &xdr.LedgerCloseMetaV1{ @@ -94,52 +84,16 @@ func CreateTxMeta(acctSeq uint32, successful bool) xdr.LedgerCloseMeta { V: 1, V1TxSet: &xdr.TransactionSetV1{ PreviousLedgerHash: xdr.Hash{1}, - Phases: []xdr.TransactionPhase{ - { - V: 0, - V0Components: &components, - }, - }, + Phases: []xdr.TransactionPhase{{ + V: 0, + V0Components: &components, + }}, }, }, }, } } -func TxMetaWithEvents(acctSeq uint32, successful bool) xdr.LedgerCloseMeta { - - meta := CreateTxMeta(acctSeq, successful) - - contractIDBytes, _ := hex.DecodeString("df06d62447fd25da07c0135eed7557e5a5497ee7d15b7fe345bd47e191d8f577") - var contractID xdr.Hash - copy(contractID[:], contractIDBytes) - counter := xdr.ScSymbol("COUNTER") - - meta.V1.TxProcessing[0].TxApplyProcessing.V3 = &xdr.TransactionMetaV3{ - SorobanMeta: &xdr.SorobanTransactionMeta{ - Events: []xdr.ContractEvent{{ - ContractId: &contractID, - Type: xdr.ContractEventTypeContract, - Body: xdr.ContractEventBody{ - V: 0, - V0: &xdr.ContractEventV0{ - Topics: []xdr.ScVal{{ - Type: xdr.ScValTypeScvSymbol, - Sym: &counter, - }}, - Data: xdr.ScVal{ - Type: xdr.ScValTypeScvSymbol, - Sym: &counter, - }, - }, - }, - }}, - ReturnValue: xdr.ScVal{ - Type: xdr.ScValTypeScvSymbol, - Sym: &counter, - }, - }, - } - - return meta +func ledgerCloseTime(ledgerSequence uint32) int64 { + return int64(ledgerSequence)*25 + 100 } diff --git a/cmd/soroban-rpc/internal/test/integration.go b/cmd/soroban-rpc/internal/test/integration.go index 42ba6188..2e4b226a 100644 --- a/cmd/soroban-rpc/internal/test/integration.go +++ b/cmd/soroban-rpc/internal/test/integration.go @@ -156,6 +156,7 @@ func (i *Test) adminURL() string { } func (i *Test) insertTransactions() { + i.t.Logf("inserting transactions") testDb, err := db.OpenSQLiteDB(i.sqlitePath) assert.NoError(i.t, err) @@ -165,7 +166,7 @@ func (i *Test) insertTransactions() { lcms := make([]xdr.LedgerCloseMeta, 0, 3) for i := uint32(0); i < uint32(cap(lcms)); i++ { - lcms = append(lcms, db.CreateTxMeta(9+i, i%2 == 0)) + lcms = append(lcms, db.CreateTxMeta(9+i, true)) } _, txW := write.LedgerWriter(), write.TransactionWriter() From 00d16d50f915c46da0db863aad59ffff6c7cfa8a Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Thu, 20 Jun 2024 11:17:16 -0400 Subject: [PATCH 21/68] insert txns during integration test setup - 3 --- .../internal/db/transaction_test.go | 124 ++---------------- cmd/soroban-rpc/internal/db/util.go | 80 +++++------ .../integrationtest/infrastructure/test.go | 28 ++++ 3 files changed, 82 insertions(+), 150 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/transaction_test.go b/cmd/soroban-rpc/internal/db/transaction_test.go index 24a36bd6..7af1dfae 100644 --- a/cmd/soroban-rpc/internal/db/transaction_test.go +++ b/cmd/soroban-rpc/internal/db/transaction_test.go @@ -7,14 +7,13 @@ import ( "testing" "github.com/sirupsen/logrus" - "github.com/stellar/go/network" "github.com/stellar/go/support/log" "github.com/stellar/go/xdr" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces" ) func TestTransactionNotFound(t *testing.T) { @@ -48,10 +47,10 @@ func TestTransactionFound(t *testing.T) { require.NoError(t, err) lcms := []xdr.LedgerCloseMeta{ - txMeta(1234, true), - txMeta(1235, true), - txMeta(1236, true), - txMeta(1237, true), + CreateTxMeta(1234, true), + CreateTxMeta(1235, true), + CreateTxMeta(1236, true), + CreateTxMeta(1237, true), } ledgerW, txW := write.LedgerWriter(), write.TransactionWriter() @@ -65,10 +64,10 @@ func TestTransactionFound(t *testing.T) { reader := NewTransactionReader(log, db, passphrase) ledgerRange, err := reader.GetLedgerRange(ctx) assert.NoError(t, err) - assert.Equal(t, uint32(1334), ledgerRange.FirstLedger.Sequence) - assert.Equal(t, ledgerCloseTime(1334), ledgerRange.FirstLedger.CloseTime) - assert.Equal(t, uint32(1337), ledgerRange.LastLedger.Sequence) - assert.Equal(t, ledgerCloseTime(1337), ledgerRange.LastLedger.CloseTime) + assert.Equal(t, uint32(1234), ledgerRange.FirstLedger.Sequence) + assert.Equal(t, LedgerCloseTime(1334), ledgerRange.FirstLedger.CloseTime) + assert.Equal(t, uint32(1237), ledgerRange.LastLedger.Sequence) + assert.Equal(t, LedgerCloseTime(1337), ledgerRange.LastLedger.CloseTime) // check 404 case _, _, err = reader.GetTransaction(ctx, xdr.Hash{}) @@ -79,8 +78,8 @@ func TestTransactionFound(t *testing.T) { h := lcm.TransactionHash(0) tx, lRange, err := reader.GetTransaction(ctx, h) require.NoError(t, err, "failed to find txhash %s in db", hex.EncodeToString(h[:])) - assert.EqualValues(t, 1234+100, lRange.FirstLedger.Sequence) - assert.EqualValues(t, 1237+100, lRange.LastLedger.Sequence) + assert.EqualValues(t, 1234, lRange.FirstLedger.Sequence) + assert.EqualValues(t, 1237, lRange.LastLedger.Sequence) assert.EqualValues(t, 1, tx.ApplicationOrder) expectedEnvelope, err := lcm.TransactionEnvelopes()[0].MarshalBinary() @@ -101,7 +100,7 @@ func BenchmarkTransactionFetch(b *testing.B) { // ingest 100k tx rows lcms := make([]xdr.LedgerCloseMeta, 0, 100_000) for i := uint32(0); i < uint32(cap(lcms)); i++ { - lcms = append(lcms, txMeta(1234+i, i%2 == 0)) + lcms = append(lcms, CreateTxMeta(1234+i, i%2 == 0)) } ledgerW, txW := write.LedgerWriter(), write.TransactionWriter() @@ -125,100 +124,3 @@ func BenchmarkTransactionFetch(b *testing.B) { assert.Equal(b, r%2 == 0, tx.Successful) } } - -// -// Structure creation methods below. -// - -func txHash(acctSeq uint32) xdr.Hash { - envelope := txEnvelope(acctSeq) - hash, err := network.HashTransactionInEnvelope(envelope, passphrase) - if err != nil { - panic(err) - } - return hash -} - -func txEnvelope(acctSeq uint32) xdr.TransactionEnvelope { - envelope, err := xdr.NewTransactionEnvelope(xdr.EnvelopeTypeEnvelopeTypeTx, xdr.TransactionV1Envelope{ - Tx: xdr.Transaction{ - Fee: 1, - SeqNum: xdr.SequenceNumber(acctSeq), - SourceAccount: xdr.MustMuxedAddress("MA7QYNF7SOWQ3GLR2BGMZEHXAVIRZA4KVWLTJJFC7MGXUA74P7UJVAAAAAAAAAAAAAJLK"), - }, - }) - if err != nil { - panic(err) - } - return envelope -} - -func transactionResult(successful bool) xdr.TransactionResult { - code := xdr.TransactionResultCodeTxBadSeq - if successful { - code = xdr.TransactionResultCodeTxSuccess - } - opResults := []xdr.OperationResult{} - return xdr.TransactionResult{ - FeeCharged: 100, - Result: xdr.TransactionResultResult{ - Code: code, - Results: &opResults, - }, - } -} - -func txMeta(acctSeq uint32, successful bool) xdr.LedgerCloseMeta { - envelope := txEnvelope(acctSeq) - txProcessing := []xdr.TransactionResultMeta{ - { - TxApplyProcessing: xdr.TransactionMeta{ - V: 3, - Operations: &[]xdr.OperationMeta{}, - V3: &xdr.TransactionMetaV3{}, - }, - Result: xdr.TransactionResultPair{ - TransactionHash: txHash(acctSeq), - Result: transactionResult(successful), - }, - }, - } - components := []xdr.TxSetComponent{ - { - Type: xdr.TxSetComponentTypeTxsetCompTxsMaybeDiscountedFee, - TxsMaybeDiscountedFee: &xdr.TxSetComponentTxsMaybeDiscountedFee{ - BaseFee: nil, - Txs: []xdr.TransactionEnvelope{envelope}, - }, - }, - } - - return xdr.LedgerCloseMeta{ - V: 1, - V1: &xdr.LedgerCloseMetaV1{ - LedgerHeader: xdr.LedgerHeaderHistoryEntry{ - Header: xdr.LedgerHeader{ - ScpValue: xdr.StellarValue{ - CloseTime: xdr.TimePoint(ledgerCloseTime(acctSeq + 100)), - }, - LedgerSeq: xdr.Uint32(acctSeq + 100), - }, - }, - TxProcessing: txProcessing, - TxSet: xdr.GeneralizedTransactionSet{ - V: 1, - V1TxSet: &xdr.TransactionSetV1{ - PreviousLedgerHash: xdr.Hash{1}, - Phases: []xdr.TransactionPhase{{ - V: 0, - V0Components: &components, - }}, - }, - }, - }, - } -} - -func ledgerCloseTime(ledgerSequence uint32) int64 { - return int64(ledgerSequence)*25 + 100 -} diff --git a/cmd/soroban-rpc/internal/db/util.go b/cmd/soroban-rpc/internal/db/util.go index 4f221798..6628ce12 100644 --- a/cmd/soroban-rpc/internal/db/util.go +++ b/cmd/soroban-rpc/internal/db/util.go @@ -5,43 +5,7 @@ import ( "github.com/stellar/go/xdr" ) -func txHash(acctSeq uint32) xdr.Hash { - envelope := txEnvelope(acctSeq) - hash, err := network.HashTransactionInEnvelope(envelope, network.FutureNetworkPassphrase) - if err != nil { - panic(err) - } - return hash -} - -func txEnvelope(acctSeq uint32) xdr.TransactionEnvelope { - envelope, err := xdr.NewTransactionEnvelope(xdr.EnvelopeTypeEnvelopeTypeTx, xdr.TransactionV1Envelope{ - Tx: xdr.Transaction{ - Fee: 1, - SeqNum: xdr.SequenceNumber(acctSeq), - SourceAccount: xdr.MustMuxedAddress("MA7QYNF7SOWQ3GLR2BGMZEHXAVIRZA4KVWLTJJFC7MGXUA74P7UJVAAAAAAAAAAAAAJLK"), - }, - }) - if err != nil { - panic(err) - } - return envelope -} - -func transactionResult(successful bool) xdr.TransactionResult { - code := xdr.TransactionResultCodeTxBadSeq - if successful { - code = xdr.TransactionResultCodeTxSuccess - } - opResults := []xdr.OperationResult{} - return xdr.TransactionResult{ - FeeCharged: 100, - Result: xdr.TransactionResultResult{ - Code: code, - Results: &opResults, - }, - } -} +// Methods for creating test txMeta func CreateTxMeta(acctSeq uint32, successful bool) xdr.LedgerCloseMeta { envelope := txEnvelope(acctSeq) @@ -74,7 +38,7 @@ func CreateTxMeta(acctSeq uint32, successful bool) xdr.LedgerCloseMeta { LedgerHeader: xdr.LedgerHeaderHistoryEntry{ Header: xdr.LedgerHeader{ ScpValue: xdr.StellarValue{ - CloseTime: xdr.TimePoint(ledgerCloseTime(acctSeq + 100)), + CloseTime: xdr.TimePoint(LedgerCloseTime(acctSeq + 100)), }, LedgerSeq: xdr.Uint32(acctSeq), }, @@ -94,6 +58,44 @@ func CreateTxMeta(acctSeq uint32, successful bool) xdr.LedgerCloseMeta { } } -func ledgerCloseTime(ledgerSequence uint32) int64 { +func txHash(acctSeq uint32) xdr.Hash { + envelope := txEnvelope(acctSeq) + hash, err := network.HashTransactionInEnvelope(envelope, network.FutureNetworkPassphrase) + if err != nil { + panic(err) + } + return hash +} + +func txEnvelope(acctSeq uint32) xdr.TransactionEnvelope { + envelope, err := xdr.NewTransactionEnvelope(xdr.EnvelopeTypeEnvelopeTypeTx, xdr.TransactionV1Envelope{ + Tx: xdr.Transaction{ + Fee: 1, + SeqNum: xdr.SequenceNumber(acctSeq), + SourceAccount: xdr.MustMuxedAddress("MA7QYNF7SOWQ3GLR2BGMZEHXAVIRZA4KVWLTJJFC7MGXUA74P7UJVAAAAAAAAAAAAAJLK"), + }, + }) + if err != nil { + panic(err) + } + return envelope +} + +func transactionResult(successful bool) xdr.TransactionResult { + code := xdr.TransactionResultCodeTxBadSeq + if successful { + code = xdr.TransactionResultCodeTxSuccess + } + opResults := []xdr.OperationResult{} + return xdr.TransactionResult{ + FeeCharged: 100, + Result: xdr.TransactionResultResult{ + Code: code, + Results: &opResults, + }, + } +} + +func LedgerCloseTime(ledgerSequence uint32) int64 { return int64(ledgerSequence)*25 + 100 } diff --git a/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go b/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go index 9279ae77..f1bd51ea 100644 --- a/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go +++ b/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go @@ -18,6 +18,7 @@ import ( "testing" "time" + "github.com/stellar/go/network" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -30,6 +31,8 @@ import ( "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/config" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/methods" ) @@ -170,6 +173,10 @@ func NewTest(t *testing.T, cfg *TestConfig) *Test { i.waitForRPC() } + // We populate the transactions table with some rows so that GetLedgerRange called in transaction related + // integration tests returns a valid latest ledger. + i.insertTransactions() + return i } @@ -202,6 +209,27 @@ func (i *Test) spawnContainers() { i.fillContainerPorts() } +func (i *Test) insertTransactions() { + i.t.Logf("inserting transactions") + testDb, err := db.OpenSQLiteDB(i.sqlitePath) + assert.NoError(i.t, err) + + writer := db.NewReadWriter(supportlog.New(), testDb, interfaces.MakeNoOpDeamon(), 100, 1_000_000, network.FutureNetworkPassphrase) + write, err := writer.NewTx(context.Background()) + assert.NoError(i.t, err) + + lcms := make([]xdr.LedgerCloseMeta, 0, 3) + for i := uint32(0); i < uint32(cap(lcms)); i++ { + lcms = append(lcms, db.CreateTxMeta(9+i, true)) + } + + _, txW := write.LedgerWriter(), write.TransactionWriter() + for _, lcm := range lcms { + assert.NoError(i.t, txW.InsertTransactions(lcm)) + } + assert.NoError(i.t, write.Commit(lcms[len(lcms)-1].LedgerSequence())) +} + func (i *Test) stopContainers() { // There were containerized workloads we should bring down downCmd := []string{"down"} From ff242370002b467c8293f663ac42fee5a7bdf7c7 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Thu, 20 Jun 2024 13:49:16 -0400 Subject: [PATCH 22/68] Fix linting errors --- cmd/soroban-rpc/internal/db/transaction.go | 4 ++-- cmd/soroban-rpc/internal/db/transaction_test.go | 4 +--- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/transaction.go b/cmd/soroban-rpc/internal/db/transaction.go index 6cb0f092..99fafce2 100644 --- a/cmd/soroban-rpc/internal/db/transaction.go +++ b/cmd/soroban-rpc/internal/db/transaction.go @@ -172,7 +172,7 @@ func (txn *transactionHandler) GetLedgerRange(ctx context.Context) (ledgerbucket // Combine the min and max ledger sequence queries using UNION ALL txnMinMaxLedgersQ, _, err := minLedgerSeqQ.Suffix("UNION ALL "+maxLedgerSeqQ, args...).ToSql() if err != nil { - return ledgerRange, errors.Wrap(err, "couldn't build max ledger sequence query") + return ledgerRange, fmt.Errorf("couldn't build ledger range query: %w", err) } // Final query to join ledger_close_meta table and the sequence numbers we got from txnMinMaxLedgersQ @@ -183,7 +183,7 @@ func (txn *transactionHandler) GetLedgerRange(ctx context.Context) (ledgerbucket var lcms []xdr.LedgerCloseMeta if err = txn.db.Select(ctx, &lcms, finalSql); err != nil { - return ledgerRange, errors.Wrap(err, "couldn't query ledger range") + return ledgerRange, fmt.Errorf("couldn't build ledger range query: %w", err) } else if len(lcms) < 2 { // There is almost certainly a row, but we want to avoid a race condition // with ingestion as well as support test cases from an empty DB, so we need diff --git a/cmd/soroban-rpc/internal/db/transaction_test.go b/cmd/soroban-rpc/internal/db/transaction_test.go index 40a24233..6073c038 100644 --- a/cmd/soroban-rpc/internal/db/transaction_test.go +++ b/cmd/soroban-rpc/internal/db/transaction_test.go @@ -10,7 +10,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/stellar/go/network" "github.com/stellar/go/support/log" "github.com/stellar/go/xdr" @@ -20,10 +19,9 @@ import ( func TestTransactionNotFound(t *testing.T) { db := NewTestDB(t) ctx := context.TODO() - log := log.DefaultLogger log.SetLevel(logrus.TraceLevel) - reader := NewTransactionReader(log, db, passphrase) + reader := NewTransactionReader(log.DefaultLogger, db, passphrase) // Assert the ledger range ledgerRange, err := reader.GetLedgerRange(ctx) From 29d588e883ae0c352f085b8bdf03637449438631 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Thu, 20 Jun 2024 13:53:26 -0400 Subject: [PATCH 23/68] Fix linting errors - 2 --- cmd/soroban-rpc/internal/db/transaction_test.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/transaction_test.go b/cmd/soroban-rpc/internal/db/transaction_test.go index 6073c038..97a40c98 100644 --- a/cmd/soroban-rpc/internal/db/transaction_test.go +++ b/cmd/soroban-rpc/internal/db/transaction_test.go @@ -25,14 +25,14 @@ func TestTransactionNotFound(t *testing.T) { // Assert the ledger range ledgerRange, err := reader.GetLedgerRange(ctx) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, uint32(0), ledgerRange.FirstLedger.Sequence) assert.Equal(t, int64(0), ledgerRange.FirstLedger.CloseTime) assert.Equal(t, uint32(0), ledgerRange.LastLedger.Sequence) assert.Equal(t, int64(0), ledgerRange.LastLedger.CloseTime) _, _, err = reader.GetTransaction(context.TODO(), xdr.Hash{}) - require.Error(t, err, ErrNoTransaction) + require.ErrorIs(t, err, ErrNoTransaction) } func TestTransactionFound(t *testing.T) { @@ -62,7 +62,7 @@ func TestTransactionFound(t *testing.T) { // Assert the ledger range reader := NewTransactionReader(log, db, passphrase) ledgerRange, err := reader.GetLedgerRange(ctx) - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, uint32(1234), ledgerRange.FirstLedger.Sequence) assert.Equal(t, LedgerCloseTime(1334), ledgerRange.FirstLedger.CloseTime) assert.Equal(t, uint32(1237), ledgerRange.LastLedger.Sequence) @@ -70,7 +70,7 @@ func TestTransactionFound(t *testing.T) { // check 404 case _, _, err = reader.GetTransaction(ctx, xdr.Hash{}) - require.Error(t, err, ErrNoTransaction) + require.ErrorIs(t, err, ErrNoTransaction) // check all 200 cases for _, lcm := range lcms { From e377bf54336b14f50cb6387ad3d7ada9aa3e4926 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Thu, 20 Jun 2024 14:29:20 -0400 Subject: [PATCH 24/68] Fix linting errors - 3 --- .../integrationtest/infrastructure/test.go | 42 ++++++++++++------- 1 file changed, 28 insertions(+), 14 deletions(-) diff --git a/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go b/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go index 097c9ba5..de21c34f 100644 --- a/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go +++ b/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go @@ -19,7 +19,6 @@ import ( "time" "github.com/stellar/go/network" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/stellar/go/clients/stellarcore" @@ -211,12 +210,12 @@ func (i *Test) spawnContainers() { func (i *Test) insertTransactions() { i.t.Logf("inserting transactions") - testDb, err := db.OpenSQLiteDB(i.sqlitePath) - assert.NoError(i.t, err) + testDB, err := db.OpenSQLiteDB(i.sqlitePath) + require.NoError(i.t, err) - writer := db.NewReadWriter(supportlog.New(), testDb, interfaces.MakeNoOpDeamon(), 100, 1_000_000, network.FutureNetworkPassphrase) + writer := db.NewReadWriter(supportlog.New(), testDB, interfaces.MakeNoOpDeamon(), 100, 1_000_000, network.FutureNetworkPassphrase) write, err := writer.NewTx(context.Background()) - assert.NoError(i.t, err) + require.NoError(i.t, err) lcms := make([]xdr.LedgerCloseMeta, 0, 3) for i := uint32(0); i < uint32(cap(lcms)); i++ { @@ -225,9 +224,9 @@ func (i *Test) insertTransactions() { _, txW := write.LedgerWriter(), write.TransactionWriter() for _, lcm := range lcms { - assert.NoError(i.t, txW.InsertTransactions(lcm)) + require.NoError(i.t, txW.InsertTransactions(lcm)) } - assert.NoError(i.t, write.Commit(lcms[len(lcms)-1].LedgerSequence())) + require.NoError(i.t, write.Commit(lcms[len(lcms)-1].LedgerSequence())) } func (i *Test) stopContainers() { @@ -443,7 +442,7 @@ type testLogWriter struct { testDone bool } -func (tw *testLogWriter) Write(p []byte) (n int, err error) { +func (tw *testLogWriter) Write(p []byte) (int, error) { tw.testDoneMx.RLock() if tw.testDone { // Workaround for https://github.com/stellar/go/issues/5342 @@ -564,17 +563,28 @@ func (i *Test) prepareShutdownHandlers() { i.shutdown = func() { close(done) if i.daemon != nil { - i.daemon.Close() + err := i.daemon.Close() + if err != nil { + fmt.Errorf("could not close RPC daemon: %v", err) + return + } i.daemon = nil } if i.rpcClient != nil { - i.rpcClient.Close() + err := i.rpcClient.Close() + if err != nil { + fmt.Errorf("could not close RPC client: %v", err) + return + } } if i.areThereContainers() { i.stopContainers() } if i.rpcContainerLogsCommand != nil { - i.rpcContainerLogsCommand.Wait() + err := i.rpcContainerLogsCommand.Wait() + if err != nil { + return + } } } @@ -647,7 +657,11 @@ func (i *Test) UpgradeProtocol(version uint32) { func (i *Test) StopRPC() { if i.daemon != nil { - i.daemon.Close() + err := i.daemon.Close() + if err != nil { + fmt.Errorf("could not close RPC daemon: %v", err) + return + } i.daemon = nil } if i.runRPCInContainer() { @@ -666,7 +680,7 @@ func (i *Test) GetDaemon() *daemon.Daemon { func (i *Test) SendMasterOperation(op txnbuild.Operation) methods.GetTransactionResponse { params := CreateTransactionParams(i.MasterAccount(), op) tx, err := txnbuild.NewTransaction(params) - assert.NoError(i.t, err) + require.NoError(i.t, err) return i.SendMasterTransaction(tx) } @@ -686,7 +700,7 @@ func (i *Test) PreflightAndSendMasterOperation(op txnbuild.Operation) methods.Ge ) params = PreflightTransactionParams(i.t, i.rpcClient, params) tx, err := txnbuild.NewTransaction(params) - assert.NoError(i.t, err) + require.NoError(i.t, err) return i.SendMasterTransaction(tx) } From fd66565bb6fe954cb0b1998345a2be7158e47971 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Thu, 20 Jun 2024 14:58:07 -0400 Subject: [PATCH 25/68] Fix linting errors - 4 --- cmd/soroban-rpc/internal/db/transaction.go | 22 ++++++++++++------- .../internal/db/transaction_test.go | 4 ++-- cmd/soroban-rpc/internal/db/util.go | 14 ++++++++---- cmd/soroban-rpc/internal/events/events.go | 5 +++-- .../integrationtest/infrastructure/test.go | 11 ++++++---- 5 files changed, 36 insertions(+), 20 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/transaction.go b/cmd/soroban-rpc/internal/db/transaction.go index 99fafce2..8dc42b0b 100644 --- a/cmd/soroban-rpc/internal/db/transaction.go +++ b/cmd/soroban-rpc/internal/db/transaction.go @@ -1,3 +1,4 @@ +//nolint:intrange package db import ( @@ -18,7 +19,10 @@ import ( "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" ) -const transactionTableName = "transactions" +const ( + transactionTableName = "transactions" + FirstLedgerToMigrate = 2 +) var ErrNoTransaction = errors.New("no transaction with this hash exists") @@ -176,13 +180,13 @@ func (txn *transactionHandler) GetLedgerRange(ctx context.Context) (ledgerbucket } // Final query to join ledger_close_meta table and the sequence numbers we got from txnMinMaxLedgersQ - finalSql := sq. + finalSQL := sq. Select("lcm.meta"). - From(fmt.Sprintf("%s as lcm", ledgerCloseMetaTableName)). + From(ledgerCloseMetaTableName + " as lcm"). JoinClause(fmt.Sprintf("JOIN (%s) as seqs ON lcm.sequence == seqs.ledger_sequence", txnMinMaxLedgersQ)) var lcms []xdr.LedgerCloseMeta - if err = txn.db.Select(ctx, &lcms, finalSql); err != nil { + if err = txn.db.Select(ctx, &lcms, finalSQL); err != nil { return ledgerRange, fmt.Errorf("couldn't build ledger range query: %w", err) } else if len(lcms) < 2 { // There is almost certainly a row, but we want to avoid a race condition @@ -216,7 +220,7 @@ func (txn *transactionHandler) GetTransaction(ctx context.Context, hash xdr.Hash tx := Transaction{} ledgerRange, err := txn.GetLedgerRange(ctx) - if err != nil && err != ErrEmptyDB { + if err != nil && !errors.Is(err, ErrEmptyDB) { return tx, ledgerRange, err } @@ -334,13 +338,15 @@ func (t *transactionTableMigration) ApplicableRange() *LedgerSeqRange { } } -func (t *transactionTableMigration) Apply(ctx context.Context, meta xdr.LedgerCloseMeta) error { +func (t *transactionTableMigration) Apply(_ context.Context, meta xdr.LedgerCloseMeta) error { return t.writer.InsertTransactions(meta) } -func newTransactionTableMigration(ctx context.Context, logger *log.Entry, retentionWindow uint32, passphrase string) migrationApplierFactory { +func newTransactionTableMigration(ctx context.Context, logger *log.Entry, + retentionWindow uint32, passphrase string, +) migrationApplierFactory { return migrationApplierFactoryF(func(db *DB, latestLedger uint32) (MigrationApplier, error) { - firstLedgerToMigrate := uint32(2) + firstLedgerToMigrate := uint32(FirstLedgerToMigrate) writer := &transactionHandler{ log: logger, db: db, diff --git a/cmd/soroban-rpc/internal/db/transaction_test.go b/cmd/soroban-rpc/internal/db/transaction_test.go index 97a40c98..7095878e 100644 --- a/cmd/soroban-rpc/internal/db/transaction_test.go +++ b/cmd/soroban-rpc/internal/db/transaction_test.go @@ -64,9 +64,9 @@ func TestTransactionFound(t *testing.T) { ledgerRange, err := reader.GetLedgerRange(ctx) require.NoError(t, err) assert.Equal(t, uint32(1234), ledgerRange.FirstLedger.Sequence) - assert.Equal(t, LedgerCloseTime(1334), ledgerRange.FirstLedger.CloseTime) + assert.Equal(t, LedgerCloseTime(1234), ledgerRange.FirstLedger.CloseTime) assert.Equal(t, uint32(1237), ledgerRange.LastLedger.Sequence) - assert.Equal(t, LedgerCloseTime(1337), ledgerRange.LastLedger.CloseTime) + assert.Equal(t, LedgerCloseTime(1237), ledgerRange.LastLedger.CloseTime) // check 404 case _, _, err = reader.GetTransaction(ctx, xdr.Hash{}) diff --git a/cmd/soroban-rpc/internal/db/util.go b/cmd/soroban-rpc/internal/db/util.go index 6628ce12..60c32019 100644 --- a/cmd/soroban-rpc/internal/db/util.go +++ b/cmd/soroban-rpc/internal/db/util.go @@ -7,12 +7,18 @@ import ( // Methods for creating test txMeta +const ( + TxMetaV = 3 + LedgerCloseTimeConstant = 100 + FeeCharged = 100 +) + func CreateTxMeta(acctSeq uint32, successful bool) xdr.LedgerCloseMeta { envelope := txEnvelope(acctSeq) txProcessing := []xdr.TransactionResultMeta{ { TxApplyProcessing: xdr.TransactionMeta{ - V: 3, + V: TxMetaV, Operations: &[]xdr.OperationMeta{}, V3: &xdr.TransactionMetaV3{}, }, @@ -38,7 +44,7 @@ func CreateTxMeta(acctSeq uint32, successful bool) xdr.LedgerCloseMeta { LedgerHeader: xdr.LedgerHeaderHistoryEntry{ Header: xdr.LedgerHeader{ ScpValue: xdr.StellarValue{ - CloseTime: xdr.TimePoint(LedgerCloseTime(acctSeq + 100)), + CloseTime: xdr.TimePoint(LedgerCloseTime(acctSeq)), }, LedgerSeq: xdr.Uint32(acctSeq), }, @@ -88,7 +94,7 @@ func transactionResult(successful bool) xdr.TransactionResult { } opResults := []xdr.OperationResult{} return xdr.TransactionResult{ - FeeCharged: 100, + FeeCharged: FeeCharged, Result: xdr.TransactionResultResult{ Code: code, Results: &opResults, @@ -97,5 +103,5 @@ func transactionResult(successful bool) xdr.TransactionResult { } func LedgerCloseTime(ledgerSequence uint32) int64 { - return int64(ledgerSequence)*25 + 100 + return int64(ledgerSequence)*25 + LedgerCloseTimeConstant } diff --git a/cmd/soroban-rpc/internal/events/events.go b/cmd/soroban-rpc/internal/events/events.go index 40205064..8be9121d 100644 --- a/cmd/soroban-rpc/internal/events/events.go +++ b/cmd/soroban-rpc/internal/events/events.go @@ -1,3 +1,4 @@ +//nolint:mnd package events import ( @@ -237,7 +238,7 @@ func readEvents(networkPassphrase string, ledgerCloseMeta xdr.LedgerCloseMeta) ( for { var tx ingest.LedgerTransaction tx, err = txReader.Read() - if err == io.EOF { + if errors.Is(err, io.EOF) { err = nil break } @@ -271,7 +272,7 @@ func readEvents(networkPassphrase string, ledgerCloseMeta xdr.LedgerCloseMeta) ( } // GetLedgerRange returns the first and latest ledger available in the store. -func (m *MemoryStore) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { +func (m *MemoryStore) GetLedgerRange(_ context.Context) (ledgerbucketwindow.LedgerRange, error) { m.lock.RLock() defer m.lock.RUnlock() return m.eventsByLedger.GetLedgerRange(), nil diff --git a/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go b/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go index de21c34f..a2e93a09 100644 --- a/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go +++ b/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go @@ -1,8 +1,10 @@ +//nolint:intrange package infrastructure import ( "context" "crypto/sha256" + "errors" "fmt" "net" "os" @@ -550,7 +552,8 @@ func (i *Test) runSuccessfulComposeCommand(args ...string) []byte { if err != nil { i.t.Log("Compose command failed, args:", args) } - if exitErr, ok := err.(*exec.ExitError); ok { + var exitErr *exec.ExitError + if errors.As(err, &exitErr) { i.t.Log("stdout:\n", string(out)) i.t.Log("stderr:\n", string(exitErr.Stderr)) } @@ -565,7 +568,7 @@ func (i *Test) prepareShutdownHandlers() { if i.daemon != nil { err := i.daemon.Close() if err != nil { - fmt.Errorf("could not close RPC daemon: %v", err) + i.t.Logf("could not close RPC daemon: %v", err) return } i.daemon = nil @@ -573,7 +576,7 @@ func (i *Test) prepareShutdownHandlers() { if i.rpcClient != nil { err := i.rpcClient.Close() if err != nil { - fmt.Errorf("could not close RPC client: %v", err) + i.t.Logf("could not close RPC client: %v", err) return } } @@ -659,7 +662,7 @@ func (i *Test) StopRPC() { if i.daemon != nil { err := i.daemon.Close() if err != nil { - fmt.Errorf("could not close RPC daemon: %v", err) + i.t.Logf("could not close RPC daemon: %v", err) return } i.daemon = nil From 056f712daf501ac8783eae8bce0bf219028d6f44 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Thu, 20 Jun 2024 18:27:09 -0400 Subject: [PATCH 26/68] Fix linting errors - 5 --- .../integrationtest/infrastructure/test.go | 3 ++- .../internal/methods/get_fee_stats.go | 4 +++- .../internal/methods/get_transaction_test.go | 20 +++++++++---------- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go b/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go index a2e93a09..c904dba1 100644 --- a/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go +++ b/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go @@ -215,7 +215,8 @@ func (i *Test) insertTransactions() { testDB, err := db.OpenSQLiteDB(i.sqlitePath) require.NoError(i.t, err) - writer := db.NewReadWriter(supportlog.New(), testDB, interfaces.MakeNoOpDeamon(), 100, 1_000_000, network.FutureNetworkPassphrase) + writer := db.NewReadWriter(supportlog.New(), testDB, interfaces.MakeNoOpDeamon(), + 100, 1_000_000, network.FutureNetworkPassphrase) write, err := writer.NewTx(context.Background()) require.NoError(i.t, err) diff --git a/cmd/soroban-rpc/internal/methods/get_fee_stats.go b/cmd/soroban-rpc/internal/methods/get_fee_stats.go index 66e2abb4..89e29a16 100644 --- a/cmd/soroban-rpc/internal/methods/get_fee_stats.go +++ b/cmd/soroban-rpc/internal/methods/get_fee_stats.go @@ -58,7 +58,9 @@ type GetFeeStatsResult struct { } // NewGetFeeStatsHandler returns a handler obtaining fee statistics -func NewGetFeeStatsHandler(windows *feewindow.FeeWindows, reader db.LedgerRangeGetter, logger *log.Entry) jrpc2.Handler { +func NewGetFeeStatsHandler(windows *feewindow.FeeWindows, reader db.LedgerRangeGetter, + logger *log.Entry, +) jrpc2.Handler { return NewHandler(func(ctx context.Context) (GetFeeStatsResult, error) { ledgerRange, err := reader.GetLedgerRange(ctx) if err != nil { // still not fatal diff --git a/cmd/soroban-rpc/internal/methods/get_transaction_test.go b/cmd/soroban-rpc/internal/methods/get_transaction_test.go index 65ceea29..b2a0cb9a 100644 --- a/cmd/soroban-rpc/internal/methods/get_transaction_test.go +++ b/cmd/soroban-rpc/internal/methods/get_transaction_test.go @@ -17,19 +17,19 @@ import ( func TestGetTransaction(t *testing.T) { var ( - ctx = context.TODO() - log = log.DefaultLogger - store = db.NewMockTransactionStore("passphrase") + ctx = context.TODO() + logger = log.DefaultLogger + store = db.NewMockTransactionStore("passphrase") ) log.SetLevel(logrus.DebugLevel) - _, err := GetTransaction(ctx, log, store, GetTransactionRequest{"ab"}) + _, err := GetTransaction(ctx, logger, store, GetTransactionRequest{"ab"}) require.EqualError(t, err, "[-32602] unexpected hash length (2)") - _, err = GetTransaction(ctx, log, store, GetTransactionRequest{"foo "}) + _, err = GetTransaction(ctx, logger, store, GetTransactionRequest{"foo "}) require.EqualError(t, err, "[-32602] incorrect hash: encoding/hex: invalid byte: U+006F 'o'") hash := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - tx, err := GetTransaction(ctx, log, store, GetTransactionRequest{hash}) + tx, err := GetTransaction(ctx, logger, store, GetTransactionRequest{hash}) require.NoError(t, err) require.Equal(t, GetTransactionResponse{Status: TransactionStatusNotFound}, tx) @@ -38,7 +38,7 @@ func TestGetTransaction(t *testing.T) { xdrHash := txHash(1) hash = hex.EncodeToString(xdrHash[:]) - tx, err = GetTransaction(ctx, log, store, GetTransactionRequest{hash}) + tx, err = GetTransaction(ctx, logger, store, GetTransactionRequest{hash}) require.NoError(t, err) expectedTxResult, err := xdr.MarshalBase64(meta.V1.TxProcessing[0].Result.Result) @@ -68,7 +68,7 @@ func TestGetTransaction(t *testing.T) { require.NoError(t, store.InsertTransactions(meta)) // the first transaction should still be there - tx, err = GetTransaction(ctx, log, store, GetTransactionRequest{hash}) + tx, err = GetTransaction(ctx, logger, store, GetTransactionRequest{hash}) require.NoError(t, err) require.Equal(t, GetTransactionResponse{ Status: TransactionStatusSuccess, @@ -97,7 +97,7 @@ func TestGetTransaction(t *testing.T) { expectedTxMeta, err = xdr.MarshalBase64(meta.V1.TxProcessing[0].TxApplyProcessing) require.NoError(t, err) - tx, err = GetTransaction(ctx, log, store, GetTransactionRequest{hash}) + tx, err = GetTransaction(ctx, logger, store, GetTransactionRequest{hash}) require.NoError(t, err) require.Equal(t, GetTransactionResponse{ Status: TransactionStatusFailed, @@ -134,7 +134,7 @@ func TestGetTransaction(t *testing.T) { expectedEventsMeta, err := xdr.MarshalBase64(diagnosticEvents[0]) require.NoError(t, err) - tx, err = GetTransaction(ctx, log, store, GetTransactionRequest{hash}) + tx, err = GetTransaction(ctx, logger, store, GetTransactionRequest{hash}) require.NoError(t, err) require.Equal(t, GetTransactionResponse{ Status: TransactionStatusSuccess, From 31dda7d0e8925b99f908b31ca351da13f553a39d Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 21 Jun 2024 10:28:41 -0400 Subject: [PATCH 27/68] Revert --- .../internal/methods/get_transaction_test.go | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/cmd/soroban-rpc/internal/methods/get_transaction_test.go b/cmd/soroban-rpc/internal/methods/get_transaction_test.go index b2a0cb9a..e49231f2 100644 --- a/cmd/soroban-rpc/internal/methods/get_transaction_test.go +++ b/cmd/soroban-rpc/internal/methods/get_transaction_test.go @@ -17,19 +17,19 @@ import ( func TestGetTransaction(t *testing.T) { var ( - ctx = context.TODO() - logger = log.DefaultLogger - store = db.NewMockTransactionStore("passphrase") + ctx = context.TODO() + log = log.DefaultLogger + store = db.NewMockTransactionStore("passphrase") ) log.SetLevel(logrus.DebugLevel) - _, err := GetTransaction(ctx, logger, store, GetTransactionRequest{"ab"}) + _, err := GetTransaction(ctx, log, store, GetTransactionRequest{"ab"}) require.EqualError(t, err, "[-32602] unexpected hash length (2)") - _, err = GetTransaction(ctx, logger, store, GetTransactionRequest{"foo "}) + _, err = GetTransaction(ctx, log, store, GetTransactionRequest{"foo"}) require.EqualError(t, err, "[-32602] incorrect hash: encoding/hex: invalid byte: U+006F 'o'") hash := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - tx, err := GetTransaction(ctx, logger, store, GetTransactionRequest{hash}) + tx, err := GetTransaction(ctx, log, store, GetTransactionRequest{hash}) require.NoError(t, err) require.Equal(t, GetTransactionResponse{Status: TransactionStatusNotFound}, tx) @@ -38,7 +38,7 @@ func TestGetTransaction(t *testing.T) { xdrHash := txHash(1) hash = hex.EncodeToString(xdrHash[:]) - tx, err = GetTransaction(ctx, logger, store, GetTransactionRequest{hash}) + tx, err = GetTransaction(ctx, log, store, GetTransactionRequest{hash}) require.NoError(t, err) expectedTxResult, err := xdr.MarshalBase64(meta.V1.TxProcessing[0].Result.Result) @@ -68,7 +68,7 @@ func TestGetTransaction(t *testing.T) { require.NoError(t, store.InsertTransactions(meta)) // the first transaction should still be there - tx, err = GetTransaction(ctx, logger, store, GetTransactionRequest{hash}) + tx, err = GetTransaction(ctx, log, store, GetTransactionRequest{hash}) require.NoError(t, err) require.Equal(t, GetTransactionResponse{ Status: TransactionStatusSuccess, @@ -97,7 +97,7 @@ func TestGetTransaction(t *testing.T) { expectedTxMeta, err = xdr.MarshalBase64(meta.V1.TxProcessing[0].TxApplyProcessing) require.NoError(t, err) - tx, err = GetTransaction(ctx, logger, store, GetTransactionRequest{hash}) + tx, err = GetTransaction(ctx, log, store, GetTransactionRequest{hash}) require.NoError(t, err) require.Equal(t, GetTransactionResponse{ Status: TransactionStatusFailed, @@ -134,7 +134,7 @@ func TestGetTransaction(t *testing.T) { expectedEventsMeta, err := xdr.MarshalBase64(diagnosticEvents[0]) require.NoError(t, err) - tx, err = GetTransaction(ctx, logger, store, GetTransactionRequest{hash}) + tx, err = GetTransaction(ctx, log, store, GetTransactionRequest{hash}) require.NoError(t, err) require.Equal(t, GetTransactionResponse{ Status: TransactionStatusSuccess, From 5a815b63bac22e8eff54333e8b88857cf8bf6f4f Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 21 Jun 2024 10:30:57 -0400 Subject: [PATCH 28/68] Revert-2 --- cmd/soroban-rpc/internal/methods/get_transaction_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/soroban-rpc/internal/methods/get_transaction_test.go b/cmd/soroban-rpc/internal/methods/get_transaction_test.go index e49231f2..65ceea29 100644 --- a/cmd/soroban-rpc/internal/methods/get_transaction_test.go +++ b/cmd/soroban-rpc/internal/methods/get_transaction_test.go @@ -25,7 +25,7 @@ func TestGetTransaction(t *testing.T) { _, err := GetTransaction(ctx, log, store, GetTransactionRequest{"ab"}) require.EqualError(t, err, "[-32602] unexpected hash length (2)") - _, err = GetTransaction(ctx, log, store, GetTransactionRequest{"foo"}) + _, err = GetTransaction(ctx, log, store, GetTransactionRequest{"foo "}) require.EqualError(t, err, "[-32602] incorrect hash: encoding/hex: invalid byte: U+006F 'o'") hash := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" From bf6ec288008d1652dcb00c7de05374bd96ddca5f Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 21 Jun 2024 10:36:31 -0400 Subject: [PATCH 29/68] change camel-case naming --- cmd/soroban-rpc/internal/db/transaction.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/transaction.go b/cmd/soroban-rpc/internal/db/transaction.go index 8dc42b0b..78f3f123 100644 --- a/cmd/soroban-rpc/internal/db/transaction.go +++ b/cmd/soroban-rpc/internal/db/transaction.go @@ -21,7 +21,7 @@ import ( const ( transactionTableName = "transactions" - FirstLedgerToMigrate = 2 + firstLedgerToMigrate = 2 ) var ErrNoTransaction = errors.New("no transaction with this hash exists") @@ -346,7 +346,7 @@ func newTransactionTableMigration(ctx context.Context, logger *log.Entry, retentionWindow uint32, passphrase string, ) migrationApplierFactory { return migrationApplierFactoryF(func(db *DB, latestLedger uint32) (MigrationApplier, error) { - firstLedgerToMigrate := uint32(FirstLedgerToMigrate) + firstLedgerToMigrate := uint32(firstLedgerToMigrate) writer := &transactionHandler{ log: logger, db: db, From ab4f82e1452a113c3fb57e426c71bfeb8362efad Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 21 Jun 2024 10:37:40 -0400 Subject: [PATCH 30/68] change camel-case naming - 2 --- cmd/soroban-rpc/internal/db/util.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/util.go b/cmd/soroban-rpc/internal/db/util.go index 60c32019..97a4164b 100644 --- a/cmd/soroban-rpc/internal/db/util.go +++ b/cmd/soroban-rpc/internal/db/util.go @@ -8,9 +8,9 @@ import ( // Methods for creating test txMeta const ( - TxMetaV = 3 - LedgerCloseTimeConstant = 100 - FeeCharged = 100 + txMetaV = 3 + ledgerCloseTimeConstant = 100 + feeCharged = 100 ) func CreateTxMeta(acctSeq uint32, successful bool) xdr.LedgerCloseMeta { @@ -18,7 +18,7 @@ func CreateTxMeta(acctSeq uint32, successful bool) xdr.LedgerCloseMeta { txProcessing := []xdr.TransactionResultMeta{ { TxApplyProcessing: xdr.TransactionMeta{ - V: TxMetaV, + V: txMetaV, Operations: &[]xdr.OperationMeta{}, V3: &xdr.TransactionMetaV3{}, }, @@ -94,7 +94,7 @@ func transactionResult(successful bool) xdr.TransactionResult { } opResults := []xdr.OperationResult{} return xdr.TransactionResult{ - FeeCharged: FeeCharged, + FeeCharged: feeCharged, Result: xdr.TransactionResultResult{ Code: code, Results: &opResults, @@ -103,5 +103,5 @@ func transactionResult(successful bool) xdr.TransactionResult { } func LedgerCloseTime(ledgerSequence uint32) int64 { - return int64(ledgerSequence)*25 + LedgerCloseTimeConstant + return int64(ledgerSequence)*25 + ledgerCloseTimeConstant } From 569ca783f378d47f9349fd51a0ea977f1ac04215 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 21 Jun 2024 14:15:51 -0400 Subject: [PATCH 31/68] Fix linting errrors - 6 --- cmd/soroban-rpc/internal/db/db.go | 16 ++++--- .../integrationtest/infrastructure/test.go | 4 +- cmd/soroban-rpc/internal/jsonrpc.go | 43 ++++++++++++------- 3 files changed, 40 insertions(+), 23 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/db.go b/cmd/soroban-rpc/internal/db/db.go index 38305d2c..f905704a 100644 --- a/cmd/soroban-rpc/internal/db/db.go +++ b/cmd/soroban-rpc/internal/db/db.go @@ -66,7 +66,8 @@ func openSQLiteDB(dbFilePath string) (*db.Session, error) { // 2. Disable WAL auto-checkpointing (we will do the checkpointing ourselves with wal_checkpoint pragmas // after every write transaction). // 3. Use synchronous=NORMAL, which is faster and still safe in WAL mode. - session, err := db.Open("sqlite3", fmt.Sprintf("file:%s?_journal_mode=WAL&_wal_autocheckpoint=0&_synchronous=NORMAL", dbFilePath)) + session, err := db.Open("sqlite3", + fmt.Sprintf("file:%s?_journal_mode=WAL&_wal_autocheckpoint=0&_synchronous=NORMAL", dbFilePath)) if err != nil { return nil, fmt.Errorf("open failed: %w", err) } @@ -78,7 +79,9 @@ func openSQLiteDB(dbFilePath string) (*db.Session, error) { return session, nil } -func OpenSQLiteDBWithPrometheusMetrics(dbFilePath string, namespace string, sub db.Subservice, registry *prometheus.Registry) (*DB, error) { +func OpenSQLiteDBWithPrometheusMetrics(dbFilePath string, namespace string, sub db.Subservice, + registry *prometheus.Registry, +) (*DB, error) { session, err := openSQLiteDB(dbFilePath) if err != nil { return nil, err @@ -133,7 +136,8 @@ func getMetaValue(ctx context.Context, q db.SessionInterface, key string) (strin case 1: // expected length on an initialized DB default: - return "", fmt.Errorf("multiple entries (%d) for key %q in table %q", len(results), latestLedgerSequenceMetaKey, metaTableName) + return "", fmt.Errorf("multiple entries (%d) for key %q in table %q", len(results), + latestLedgerSequenceMetaKey, metaTableName) } return results[0], nil } @@ -330,11 +334,11 @@ func (w writeTx) Rollback() error { // errors.New("not in transaction") is returned when rolling back a transaction which has // already been committed or rolled back. We can ignore those errors // because we allow rolling back after commits in defer statements. - if err := w.tx.Rollback(); err == nil || err.Error() == "not in transaction" { + err := w.tx.Rollback() + if err == nil || err.Error() == "not in transaction" { return nil - } else { - return err } + return err } func runSQLMigrations(db *sql.DB, dialect string) error { diff --git a/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go b/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go index c904dba1..9d309be1 100644 --- a/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go +++ b/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go @@ -20,11 +20,11 @@ import ( "testing" "time" - "github.com/stellar/go/network" "github.com/stretchr/testify/require" "github.com/stellar/go/clients/stellarcore" "github.com/stellar/go/keypair" + "github.com/stellar/go/network" proto "github.com/stellar/go/protocols/stellarcore" supportlog "github.com/stellar/go/support/log" "github.com/stellar/go/txnbuild" @@ -385,7 +385,7 @@ func (i *Test) generateCaptiveCoreCfgForContainer() { // Try the directory before the integration test refactoring // TODO: remove this hack after protocol 22 is released out, err = getOldVersionCaptiveCoreConfigVersion("../../test", captiveCoreConfigFilename) - outStr := strings.Replace(string(out), `ADDRESS="localhost"`, `ADDRESS="${CORE_HOST_PORT}"`, -1) + outStr := strings.ReplaceAll(string(out), `ADDRESS="localhost"`, `ADDRESS="${CORE_HOST_PORT}"`) out = []byte(outStr) } require.NoError(i.t, err) diff --git a/cmd/soroban-rpc/internal/jsonrpc.go b/cmd/soroban-rpc/internal/jsonrpc.go index b9234f16..54913fff 100644 --- a/cmd/soroban-rpc/internal/jsonrpc.go +++ b/cmd/soroban-rpc/internal/jsonrpc.go @@ -3,6 +3,7 @@ package internal import ( "context" "encoding/json" + "errors" "net/http" "strconv" "strings" @@ -29,7 +30,10 @@ import ( // maxHTTPRequestSize defines the largest request size that the http handler // would be willing to accept before dropping the request. The implementation // uses the default MaxBytesHandler to limit the request size. -const maxHTTPRequestSize = 512 * 1024 // half a megabyte +const ( + maxHTTPRequestSize = 512 * 1024 // half a megabyte + warningThresholdDenominator = 3 +) // Handler is the HTTP handler which serves the Soroban JSON RPC responses type Handler struct { @@ -67,7 +71,7 @@ func decorateHandlers(daemon interfaces.Daemon, logger *log.Entry, m handler.Map }, []string{"endpoint", "status"}) decorated := handler.Map{} for endpoint, h := range m { - // create copy of h so it can be used in closure bleow + // create copy of h, so it can be used in closure below h := h decorated[endpoint] = handler.New(func(ctx context.Context, r *jrpc2.Request) (interface{}, error) { reqID := strconv.FormatUint(middleware.NextRequestID(), 10) @@ -80,7 +84,8 @@ func decorateHandlers(daemon interfaces.Daemon, logger *log.Entry, m handler.Map if ok && simulateTransactionResponse.Error != "" { label["status"] = "error" } else if err != nil { - if jsonRPCErr, ok := err.(*jrpc2.Error); ok { + var jsonRPCErr *jrpc2.Error + if errors.As(err, &jsonRPCErr) { prometheusLabelReplacer := strings.NewReplacer(" ", "_", "-", "_", "(", "", ")", "") status := prometheusLabelReplacer.Replace(jsonRPCErr.Code.String()) label["status"] = status @@ -139,7 +144,7 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler { // Get the largest history window var ledgerRangeGetter db.LedgerRangeGetter = params.EventStore - var retentionWindow = cfg.EventLedgerRetentionWindow + retentionWindow := cfg.EventLedgerRetentionWindow if cfg.TransactionLedgerRetentionWindow > cfg.EventLedgerRetentionWindow { retentionWindow = cfg.TransactionLedgerRetentionWindow ledgerRangeGetter = params.TransactionReader @@ -176,8 +181,9 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler { requestDurationLimit: cfg.MaxGetNetworkExecutionDuration, }, { - methodName: "getVersionInfo", - underlyingHandler: methods.NewGetVersionInfoHandler(params.Logger, params.LedgerEntryReader, params.LedgerReader, params.Daemon), + methodName: "getVersionInfo", + underlyingHandler: methods.NewGetVersionInfoHandler(params.Logger, params.LedgerEntryReader, + params.LedgerReader, params.Daemon), longName: "get_version_info", queueLimit: cfg.RequestBacklogGetVersionInfoQueueLimit, requestDurationLimit: cfg.MaxGetVersionInfoExecutionDuration, @@ -211,8 +217,9 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler { requestDurationLimit: cfg.MaxGetTransactionExecutionDuration, }, { - methodName: "getTransactions", - underlyingHandler: methods.NewGetTransactionsHandler(params.Logger, params.LedgerReader, params.TransactionReader, cfg.MaxTransactionsLimit, cfg.DefaultTransactionsLimit, cfg.NetworkPassphrase), + methodName: "getTransactions", + underlyingHandler: methods.NewGetTransactionsHandler(params.Logger, params.LedgerReader, + params.TransactionReader, cfg.MaxTransactionsLimit, cfg.DefaultTransactionsLimit, cfg.NetworkPassphrase), longName: "get_transactions", queueLimit: cfg.RequestBacklogGetTransactionsQueueLimit, requestDurationLimit: cfg.MaxGetTransactionsExecutionDuration, @@ -260,8 +267,10 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler { durationWarnCounterName := handler.longName + "_execution_threshold_warning" durationLimitCounterName := handler.longName + "_execution_threshold_limit" - durationWarnCounterHelp := "The metric measures the count of " + handler.methodName + " requests that surpassed the warning threshold for execution time" - durationLimitCounterHelp := "The metric measures the count of " + handler.methodName + " requests that surpassed the limit threshold for execution time" + durationWarnCounterHelp := "The metric measures the count of " + handler.methodName + + " requests that surpassed the warning threshold for execution time" + durationLimitCounterHelp := "The metric measures the count of " + handler.methodName + + " requests that surpassed the limit threshold for execution time" requestDurationWarnCounter := prometheus.NewCounter(prometheus.CounterOpts{ Namespace: params.Daemon.MetricsNamespace(), Subsystem: "network", @@ -274,7 +283,7 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler { Help: durationLimitCounterHelp, }) // set the warning threshold to be one third of the limit. - requestDurationWarn := handler.requestDurationLimit / 3 + requestDurationWarn := handler.requestDurationLimit / warningThresholdDenominator durationLimiter := network.MakeJrpcRequestDurationLimiter( queueLimiter.Handle, requestDurationWarn, @@ -303,12 +312,16 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler { params.Logger) globalQueueRequestExecutionDurationWarningCounter := prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: params.Daemon.MetricsNamespace(), Subsystem: "network", Name: "global_request_execution_duration_threshold_warning", - Help: "The metric measures the count of requests that surpassed the warning threshold for execution time", + Namespace: params.Daemon.MetricsNamespace(), + Subsystem: "network", + Name: "global_request_execution_duration_threshold_warning", + Help: "The metric measures the count of requests that surpassed the warning threshold for execution time", }) globalQueueRequestExecutionDurationLimitCounter := prometheus.NewCounter(prometheus.CounterOpts{ - Namespace: params.Daemon.MetricsNamespace(), Subsystem: "network", Name: "global_request_execution_duration_threshold_limit", - Help: "The metric measures the count of requests that surpassed the limit threshold for execution time", + Namespace: params.Daemon.MetricsNamespace(), + Subsystem: "network", + Name: "global_request_execution_duration_threshold_limit", + Help: "The metric measures the count of requests that surpassed the limit threshold for execution time", }) handler := network.MakeHTTPRequestDurationLimiter( queueLimitedBridge, From 894f5c214e77d26154d4175bd8c31a9982cbbd96 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 21 Jun 2024 15:36:22 -0400 Subject: [PATCH 32/68] Simplify ledger range query --- cmd/soroban-rpc/internal/db/transaction.go | 119 ++++++++++++++------- 1 file changed, 81 insertions(+), 38 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/transaction.go b/cmd/soroban-rpc/internal/db/transaction.go index 78f3f123..a3404d8a 100644 --- a/cmd/soroban-rpc/internal/db/transaction.go +++ b/cmd/soroban-rpc/internal/db/transaction.go @@ -143,51 +143,27 @@ func (txn *transactionHandler) trimTransactions(latestLedgerSeq uint32, retentio func (txn *transactionHandler) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { var ledgerRange ledgerbucketwindow.LedgerRange - // - // We use subqueries alongside a UNION ALL stitch in order to select the min - // and max from the ledger table in a single query and get around sqlite's - // limitations with parentheses (see https://stackoverflow.com/a/22609948). - // - // Queries to get the minimum and maximum ledger sequence from the transactions table - minLedgerSeqQ := sq. - Select("m1.ledger_sequence"). - FromSelect( - sq. - Select("ledger_sequence"). - From(transactionTableName). - OrderBy("ledger_sequence ASC"). - Limit(1), - "m1", - ) - maxLedgerSeqQ, args, err := sq. - Select("m2.ledger_sequence"). - FromSelect( - sq. - Select("ledger_sequence"). - From(transactionTableName). - OrderBy("ledger_sequence DESC"). - Limit(1), - "m2", - ).ToSql() - if err != nil { - return ledgerRange, fmt.Errorf("couldn't build ledger range query: %w", err) + // Query to get the minimum and maximum ledger sequence from the transactions table + var ledgerSeqs struct { + MinLedgerSequence uint32 `db:"min_ledger_sequence"` + MaxLedgerSequence uint32 `db:"max_ledger_sequence"` } - - // Combine the min and max ledger sequence queries using UNION ALL - txnMinMaxLedgersQ, _, err := minLedgerSeqQ.Suffix("UNION ALL "+maxLedgerSeqQ, args...).ToSql() - if err != nil { - return ledgerRange, fmt.Errorf("couldn't build ledger range query: %w", err) + minMaxLedgerSequenceSql := sq. + Select("MIN(ledger_sequence) AS min_ledger_sequence, MAX(ledger_sequence) AS max_ledger_sequence"). + From(transactionTableName) + if err := txn.db.Get(ctx, &ledgerSeqs, minMaxLedgerSequenceSql); err != nil { + return ledgerRange, fmt.Errorf("couldn't query ledger range: %w", err) } - // Final query to join ledger_close_meta table and the sequence numbers we got from txnMinMaxLedgersQ - finalSQL := sq. + // Use the min and max ledger sequences to query the ledger_close_meta table + ledgerMetaSql := sq. Select("lcm.meta"). From(ledgerCloseMetaTableName + " as lcm"). - JoinClause(fmt.Sprintf("JOIN (%s) as seqs ON lcm.sequence == seqs.ledger_sequence", txnMinMaxLedgersQ)) + Where(sq.Eq{"lcm.sequence": []uint32{ledgerSeqs.MinLedgerSequence, ledgerSeqs.MaxLedgerSequence}}) var lcms []xdr.LedgerCloseMeta - if err = txn.db.Select(ctx, &lcms, finalSQL); err != nil { - return ledgerRange, fmt.Errorf("couldn't build ledger range query: %w", err) + if err := txn.db.Select(ctx, &lcms, ledgerMetaSql); err != nil { + return ledgerRange, fmt.Errorf("couldn't query ledger range: %w", err) } else if len(lcms) < 2 { // There is almost certainly a row, but we want to avoid a race condition // with ingestion as well as support test cases from an empty DB, so we need @@ -207,6 +183,73 @@ func (txn *transactionHandler) GetLedgerRange(ctx context.Context) (ledgerbucket return ledgerRange, nil } +//func (txn *transactionHandler) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { +// var ledgerRange ledgerbucketwindow.LedgerRange +// +// // +// // We use subqueries alongside a UNION ALL stitch in order to select the min +// // and max from the ledger table in a single query and get around sqlite's +// // limitations with parentheses (see https://stackoverflow.com/a/22609948). +// // +// // Queries to get the minimum and maximum ledger sequence from the transactions table +// minLedgerSeqQ := sq. +// Select("m1.ledger_sequence"). +// FromSelect( +// sq. +// Select("ledger_sequence"). +// From(transactionTableName). +// OrderBy("ledger_sequence ASC"). +// Limit(1), +// "m1", +// ) +// maxLedgerSeqQ, args, err := sq. +// Select("m2.ledger_sequence"). +// FromSelect( +// sq. +// Select("ledger_sequence"). +// From(transactionTableName). +// OrderBy("ledger_sequence DESC"). +// Limit(1), +// "m2", +// ).ToSql() +// if err != nil { +// return ledgerRange, fmt.Errorf("couldn't build ledger range query: %w", err) +// } +// +// // Combine the min and max ledger sequence queries using UNION ALL +// txnMinMaxLedgersQ, _, err := minLedgerSeqQ.Suffix("UNION ALL "+maxLedgerSeqQ, args...).ToSql() +// if err != nil { +// return ledgerRange, fmt.Errorf("couldn't build ledger range query: %w", err) +// } +// +// // Final query to join ledger_close_meta table and the sequence numbers we got from txnMinMaxLedgersQ +// finalSQL := sq. +// Select("lcm.meta"). +// From(ledgerCloseMetaTableName + " as lcm"). +// JoinClause(fmt.Sprintf("JOIN (%s) as seqs ON lcm.sequence == seqs.ledger_sequence", txnMinMaxLedgersQ)) +// +// var lcms []xdr.LedgerCloseMeta +// if err = txn.db.Select(ctx, &lcms, finalSQL); err != nil { +// return ledgerRange, fmt.Errorf("couldn't build ledger range query: %w", err) +// } else if len(lcms) < 2 { +// // There is almost certainly a row, but we want to avoid a race condition +// // with ingestion as well as support test cases from an empty DB, so we need +// // to sanity check that there is in fact a result. Note that no ledgers in +// // the database isn't an error, it's just an empty range. +// return ledgerRange, nil +// } +// +// lcm1, lcm2 := lcms[0], lcms[1] +// ledgerRange.FirstLedger.Sequence = lcm1.LedgerSequence() +// ledgerRange.FirstLedger.CloseTime = lcm1.LedgerCloseTime() +// ledgerRange.LastLedger.Sequence = lcm2.LedgerSequence() +// ledgerRange.LastLedger.CloseTime = lcm2.LedgerCloseTime() +// +// txn.log.Debugf("Database ledger range: [%d, %d]", +// ledgerRange.FirstLedger.Sequence, ledgerRange.LastLedger.Sequence) +// return ledgerRange, nil +//} + // GetTransaction conforms to the interface in // methods/get_transaction.go#NewGetTransactionHandler so that it can be used // directly against the RPC handler. From eb4ccac8d3528db5c419d1e0e46a344b31ab3ee9 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 21 Jun 2024 16:33:25 -0400 Subject: [PATCH 33/68] Simplify ledger range query - 2 --- cmd/soroban-rpc/internal/db/transaction.go | 75 ++-------------------- 1 file changed, 4 insertions(+), 71 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/transaction.go b/cmd/soroban-rpc/internal/db/transaction.go index a3404d8a..e0f38db6 100644 --- a/cmd/soroban-rpc/internal/db/transaction.go +++ b/cmd/soroban-rpc/internal/db/transaction.go @@ -148,21 +148,21 @@ func (txn *transactionHandler) GetLedgerRange(ctx context.Context) (ledgerbucket MinLedgerSequence uint32 `db:"min_ledger_sequence"` MaxLedgerSequence uint32 `db:"max_ledger_sequence"` } - minMaxLedgerSequenceSql := sq. + minMaxLedgerSequenceSQL := sq. Select("MIN(ledger_sequence) AS min_ledger_sequence, MAX(ledger_sequence) AS max_ledger_sequence"). From(transactionTableName) - if err := txn.db.Get(ctx, &ledgerSeqs, minMaxLedgerSequenceSql); err != nil { + if err := txn.db.Get(ctx, &ledgerSeqs, minMaxLedgerSequenceSQL); err != nil { return ledgerRange, fmt.Errorf("couldn't query ledger range: %w", err) } // Use the min and max ledger sequences to query the ledger_close_meta table - ledgerMetaSql := sq. + ledgerMetaSQL := sq. Select("lcm.meta"). From(ledgerCloseMetaTableName + " as lcm"). Where(sq.Eq{"lcm.sequence": []uint32{ledgerSeqs.MinLedgerSequence, ledgerSeqs.MaxLedgerSequence}}) var lcms []xdr.LedgerCloseMeta - if err := txn.db.Select(ctx, &lcms, ledgerMetaSql); err != nil { + if err := txn.db.Select(ctx, &lcms, ledgerMetaSQL); err != nil { return ledgerRange, fmt.Errorf("couldn't query ledger range: %w", err) } else if len(lcms) < 2 { // There is almost certainly a row, but we want to avoid a race condition @@ -183,73 +183,6 @@ func (txn *transactionHandler) GetLedgerRange(ctx context.Context) (ledgerbucket return ledgerRange, nil } -//func (txn *transactionHandler) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { -// var ledgerRange ledgerbucketwindow.LedgerRange -// -// // -// // We use subqueries alongside a UNION ALL stitch in order to select the min -// // and max from the ledger table in a single query and get around sqlite's -// // limitations with parentheses (see https://stackoverflow.com/a/22609948). -// // -// // Queries to get the minimum and maximum ledger sequence from the transactions table -// minLedgerSeqQ := sq. -// Select("m1.ledger_sequence"). -// FromSelect( -// sq. -// Select("ledger_sequence"). -// From(transactionTableName). -// OrderBy("ledger_sequence ASC"). -// Limit(1), -// "m1", -// ) -// maxLedgerSeqQ, args, err := sq. -// Select("m2.ledger_sequence"). -// FromSelect( -// sq. -// Select("ledger_sequence"). -// From(transactionTableName). -// OrderBy("ledger_sequence DESC"). -// Limit(1), -// "m2", -// ).ToSql() -// if err != nil { -// return ledgerRange, fmt.Errorf("couldn't build ledger range query: %w", err) -// } -// -// // Combine the min and max ledger sequence queries using UNION ALL -// txnMinMaxLedgersQ, _, err := minLedgerSeqQ.Suffix("UNION ALL "+maxLedgerSeqQ, args...).ToSql() -// if err != nil { -// return ledgerRange, fmt.Errorf("couldn't build ledger range query: %w", err) -// } -// -// // Final query to join ledger_close_meta table and the sequence numbers we got from txnMinMaxLedgersQ -// finalSQL := sq. -// Select("lcm.meta"). -// From(ledgerCloseMetaTableName + " as lcm"). -// JoinClause(fmt.Sprintf("JOIN (%s) as seqs ON lcm.sequence == seqs.ledger_sequence", txnMinMaxLedgersQ)) -// -// var lcms []xdr.LedgerCloseMeta -// if err = txn.db.Select(ctx, &lcms, finalSQL); err != nil { -// return ledgerRange, fmt.Errorf("couldn't build ledger range query: %w", err) -// } else if len(lcms) < 2 { -// // There is almost certainly a row, but we want to avoid a race condition -// // with ingestion as well as support test cases from an empty DB, so we need -// // to sanity check that there is in fact a result. Note that no ledgers in -// // the database isn't an error, it's just an empty range. -// return ledgerRange, nil -// } -// -// lcm1, lcm2 := lcms[0], lcms[1] -// ledgerRange.FirstLedger.Sequence = lcm1.LedgerSequence() -// ledgerRange.FirstLedger.CloseTime = lcm1.LedgerCloseTime() -// ledgerRange.LastLedger.Sequence = lcm2.LedgerSequence() -// ledgerRange.LastLedger.CloseTime = lcm2.LedgerCloseTime() -// -// txn.log.Debugf("Database ledger range: [%d, %d]", -// ledgerRange.FirstLedger.Sequence, ledgerRange.LastLedger.Sequence) -// return ledgerRange, nil -//} - // GetTransaction conforms to the interface in // methods/get_transaction.go#NewGetTransactionHandler so that it can be used // directly against the RPC handler. From 66df55fc2799f2d00eca57ac4fe282ec88a8e917 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 24 Jun 2024 09:56:37 -0400 Subject: [PATCH 34/68] Simplify ledger range query - 3 --- cmd/soroban-rpc/internal/db/transaction.go | 18 ++++++---- .../internal/db/transaction_test.go | 36 +++++++++++++++++++ 2 files changed, 48 insertions(+), 6 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/transaction.go b/cmd/soroban-rpc/internal/db/transaction.go index e0f38db6..63ad6478 100644 --- a/cmd/soroban-rpc/internal/db/transaction.go +++ b/cmd/soroban-rpc/internal/db/transaction.go @@ -20,8 +20,9 @@ import ( ) const ( - transactionTableName = "transactions" - firstLedgerToMigrate = 2 + transactionTableName = "transactions" + firstLedgerToMigrate = 2 + getLedgerRangeQueryCondition = 2 ) var ErrNoTransaction = errors.New("no transaction with this hash exists") @@ -145,8 +146,8 @@ func (txn *transactionHandler) GetLedgerRange(ctx context.Context) (ledgerbucket // Query to get the minimum and maximum ledger sequence from the transactions table var ledgerSeqs struct { - MinLedgerSequence uint32 `db:"min_ledger_sequence"` - MaxLedgerSequence uint32 `db:"max_ledger_sequence"` + MinLedgerSequence *uint32 `db:"min_ledger_sequence"` + MaxLedgerSequence *uint32 `db:"max_ledger_sequence"` } minMaxLedgerSequenceSQL := sq. Select("MIN(ledger_sequence) AS min_ledger_sequence, MAX(ledger_sequence) AS max_ledger_sequence"). @@ -155,16 +156,21 @@ func (txn *transactionHandler) GetLedgerRange(ctx context.Context) (ledgerbucket return ledgerRange, fmt.Errorf("couldn't query ledger range: %w", err) } + // Empty DB + if ledgerSeqs.MinLedgerSequence == nil || ledgerSeqs.MaxLedgerSequence == nil { + return ledgerRange, nil + } + // Use the min and max ledger sequences to query the ledger_close_meta table ledgerMetaSQL := sq. Select("lcm.meta"). From(ledgerCloseMetaTableName + " as lcm"). - Where(sq.Eq{"lcm.sequence": []uint32{ledgerSeqs.MinLedgerSequence, ledgerSeqs.MaxLedgerSequence}}) + Where(sq.Eq{"lcm.sequence": []uint32{*ledgerSeqs.MinLedgerSequence, *ledgerSeqs.MaxLedgerSequence}}) var lcms []xdr.LedgerCloseMeta if err := txn.db.Select(ctx, &lcms, ledgerMetaSQL); err != nil { return ledgerRange, fmt.Errorf("couldn't query ledger range: %w", err) - } else if len(lcms) < 2 { + } else if len(lcms) < getLedgerRangeQueryCondition { // There is almost certainly a row, but we want to avoid a race condition // with ingestion as well as support test cases from an empty DB, so we need // to sanity check that there is in fact a result. Note that no ledgers in diff --git a/cmd/soroban-rpc/internal/db/transaction_test.go b/cmd/soroban-rpc/internal/db/transaction_test.go index 7095878e..9c4a237d 100644 --- a/cmd/soroban-rpc/internal/db/transaction_test.go +++ b/cmd/soroban-rpc/internal/db/transaction_test.go @@ -123,3 +123,39 @@ func BenchmarkTransactionFetch(b *testing.B) { assert.Equal(b, r%2 == 0, tx.Successful) } } + +//func BenchmarkGetLedgerRange(b *testing.B) { +// logger := log.DefaultLogger +// +// // create 100k tx rows +// lcms := make([]xdr.LedgerCloseMeta, 0, 100_000) +// for i := uint32(0); i < uint32(cap(lcms)); i++ { +// lcms = append(lcms, CreateTxMeta(1234+i, i%2 == 0)) +// } +// +// randoms := make([]int, b.N) +// for i := 0; i < b.N; i++ { +// randoms[i] = rand.Intn(len(lcms)) +// } +// +// for i := 2; i < b.N; i++ { +// db := NewTestDB(b) +// writer := NewReadWriter(logger, db, interfaces.MakeNoOpDeamon(), 100, 1_000_000, passphrase) +// write, err := writer.NewTx(context.TODO()) +// require.NoError(b, err) +// +// ledgerW, txW := write.LedgerWriter(), write.TransactionWriter() +// for j := 0; j < i; j++ { +// require.NoError(b, ledgerW.InsertLedger(lcms[j])) +// require.NoError(b, txW.InsertTransactions(lcms[j])) +// } +// require.NoError(b, write.Commit(lcms[len(lcms)-1].LedgerSequence())) +// reader := NewTransactionReader(logger, db, passphrase) +// +// b.ResetTimer() +// ledgerRange, err := reader.GetLedgerRange(context.TODO()) +// require.NoError(b, err) +// assert.Equal(b, ledgerRange.FirstLedger.Sequence, lcms[0].LedgerSequence()) +// assert.Equal(b, ledgerRange.LastLedger.Sequence, lcms[i-1].LedgerSequence()) +// } +//} From bb9b338fcad6c8e0113e0014510181850c6af55d Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 24 Jun 2024 12:05:51 -0400 Subject: [PATCH 35/68] Add benchmarking for GetLedgerRange --- .../internal/db/transaction_test.go | 64 +++++++++---------- 1 file changed, 29 insertions(+), 35 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/transaction_test.go b/cmd/soroban-rpc/internal/db/transaction_test.go index 9c4a237d..65ca90bb 100644 --- a/cmd/soroban-rpc/internal/db/transaction_test.go +++ b/cmd/soroban-rpc/internal/db/transaction_test.go @@ -124,38 +124,32 @@ func BenchmarkTransactionFetch(b *testing.B) { } } -//func BenchmarkGetLedgerRange(b *testing.B) { -// logger := log.DefaultLogger -// -// // create 100k tx rows -// lcms := make([]xdr.LedgerCloseMeta, 0, 100_000) -// for i := uint32(0); i < uint32(cap(lcms)); i++ { -// lcms = append(lcms, CreateTxMeta(1234+i, i%2 == 0)) -// } -// -// randoms := make([]int, b.N) -// for i := 0; i < b.N; i++ { -// randoms[i] = rand.Intn(len(lcms)) -// } -// -// for i := 2; i < b.N; i++ { -// db := NewTestDB(b) -// writer := NewReadWriter(logger, db, interfaces.MakeNoOpDeamon(), 100, 1_000_000, passphrase) -// write, err := writer.NewTx(context.TODO()) -// require.NoError(b, err) -// -// ledgerW, txW := write.LedgerWriter(), write.TransactionWriter() -// for j := 0; j < i; j++ { -// require.NoError(b, ledgerW.InsertLedger(lcms[j])) -// require.NoError(b, txW.InsertTransactions(lcms[j])) -// } -// require.NoError(b, write.Commit(lcms[len(lcms)-1].LedgerSequence())) -// reader := NewTransactionReader(logger, db, passphrase) -// -// b.ResetTimer() -// ledgerRange, err := reader.GetLedgerRange(context.TODO()) -// require.NoError(b, err) -// assert.Equal(b, ledgerRange.FirstLedger.Sequence, lcms[0].LedgerSequence()) -// assert.Equal(b, ledgerRange.LastLedger.Sequence, lcms[i-1].LedgerSequence()) -// } -//} +func BenchmarkGetLedgerRange(b *testing.B) { + db := NewTestDB(b) + logger := log.DefaultLogger + writer := NewReadWriter(logger, db, interfaces.MakeNoOpDeamon(), 100, 1_000_000, passphrase) + write, err := writer.NewTx(context.TODO()) + require.NoError(b, err) + + // create 100k tx rows + lcms := make([]xdr.LedgerCloseMeta, 0, 100_000) + for i := uint32(0); i < uint32(cap(lcms)); i++ { + lcms = append(lcms, CreateTxMeta(1234+i, i%2 == 0)) + } + + ledgerW, txW := write.LedgerWriter(), write.TransactionWriter() + for _, lcm := range lcms { + require.NoError(b, ledgerW.InsertLedger(lcm)) + require.NoError(b, txW.InsertTransactions(lcm)) + } + require.NoError(b, write.Commit(lcms[len(lcms)-1].LedgerSequence())) + reader := NewTransactionReader(logger, db, passphrase) + + b.ResetTimer() + for i := 0; i < b.N; i++ { + ledgerRange, err := reader.GetLedgerRange(context.TODO()) + require.NoError(b, err) + assert.Equal(b, ledgerRange.FirstLedger.Sequence, lcms[0].LedgerSequence()) + assert.Equal(b, ledgerRange.LastLedger.Sequence, lcms[len(lcms)-1].LedgerSequence()) + } +} From 7e7ab78e603187e3a546082d5dc3b37dbd44d2d1 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 24 Jun 2024 13:21:48 -0400 Subject: [PATCH 36/68] Fix linter issues - 6 --- cmd/soroban-rpc/internal/db/transaction_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/transaction_test.go b/cmd/soroban-rpc/internal/db/transaction_test.go index 65ca90bb..a0cd9650 100644 --- a/cmd/soroban-rpc/internal/db/transaction_test.go +++ b/cmd/soroban-rpc/internal/db/transaction_test.go @@ -98,8 +98,8 @@ func BenchmarkTransactionFetch(b *testing.B) { // ingest 100k tx rows lcms := make([]xdr.LedgerCloseMeta, 0, 100_000) - for i := uint32(0); i < uint32(cap(lcms)); i++ { - lcms = append(lcms, CreateTxMeta(1234+i, i%2 == 0)) + for i := range cap(lcms) { + lcms = append(lcms, CreateTxMeta(uint32(1234+i), i%2 == 0)) } ledgerW, txW := write.LedgerWriter(), write.TransactionWriter() @@ -111,12 +111,12 @@ func BenchmarkTransactionFetch(b *testing.B) { reader := NewTransactionReader(log, db, passphrase) randoms := make([]int, b.N) - for i := 0; i < b.N; i++ { + for i := range b.N { randoms[i] = rand.Intn(len(lcms)) } b.ResetTimer() - for i := 0; i < b.N; i++ { + for i := range b.N { r := randoms[i] tx, _, err := reader.GetTransaction(ctx, lcms[r].TransactionHash(0)) require.NoError(b, err) @@ -133,8 +133,8 @@ func BenchmarkGetLedgerRange(b *testing.B) { // create 100k tx rows lcms := make([]xdr.LedgerCloseMeta, 0, 100_000) - for i := uint32(0); i < uint32(cap(lcms)); i++ { - lcms = append(lcms, CreateTxMeta(1234+i, i%2 == 0)) + for i := range cap(lcms) { + lcms = append(lcms, CreateTxMeta(uint32(1234+i), i%2 == 0)) } ledgerW, txW := write.LedgerWriter(), write.TransactionWriter() @@ -146,7 +146,7 @@ func BenchmarkGetLedgerRange(b *testing.B) { reader := NewTransactionReader(logger, db, passphrase) b.ResetTimer() - for i := 0; i < b.N; i++ { + for range b.N { ledgerRange, err := reader.GetLedgerRange(context.TODO()) require.NoError(b, err) assert.Equal(b, ledgerRange.FirstLedger.Sequence, lcms[0].LedgerSequence()) From 961b84ff8628c87c7d950572b2a0855e13fc05e0 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 24 Jun 2024 15:41:34 -0400 Subject: [PATCH 37/68] Remove else condition --- cmd/soroban-rpc/internal/db/transaction.go | 13 +++---------- 1 file changed, 3 insertions(+), 10 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/transaction.go b/cmd/soroban-rpc/internal/db/transaction.go index 63ad6478..032bf902 100644 --- a/cmd/soroban-rpc/internal/db/transaction.go +++ b/cmd/soroban-rpc/internal/db/transaction.go @@ -20,9 +20,8 @@ import ( ) const ( - transactionTableName = "transactions" - firstLedgerToMigrate = 2 - getLedgerRangeQueryCondition = 2 + transactionTableName = "transactions" + firstLedgerToMigrate = 2 ) var ErrNoTransaction = errors.New("no transaction with this hash exists") @@ -156,7 +155,7 @@ func (txn *transactionHandler) GetLedgerRange(ctx context.Context) (ledgerbucket return ledgerRange, fmt.Errorf("couldn't query ledger range: %w", err) } - // Empty DB + // Empty DB or just a single row in the DB. if ledgerSeqs.MinLedgerSequence == nil || ledgerSeqs.MaxLedgerSequence == nil { return ledgerRange, nil } @@ -170,12 +169,6 @@ func (txn *transactionHandler) GetLedgerRange(ctx context.Context) (ledgerbucket var lcms []xdr.LedgerCloseMeta if err := txn.db.Select(ctx, &lcms, ledgerMetaSQL); err != nil { return ledgerRange, fmt.Errorf("couldn't query ledger range: %w", err) - } else if len(lcms) < getLedgerRangeQueryCondition { - // There is almost certainly a row, but we want to avoid a race condition - // with ingestion as well as support test cases from an empty DB, so we need - // to sanity check that there is in fact a result. Note that no ledgers in - // the database isn't an error, it's just an empty range. - return ledgerRange, nil } lcm1, lcm2 := lcms[0], lcms[1] From 93906e4cd02af5204f1af624f98995ca6b88696a Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Tue, 25 Jun 2024 14:43:19 -0400 Subject: [PATCH 38/68] Optimise the GetLedgerRange query --- cmd/soroban-rpc/internal/db/transaction.go | 46 +++++------ .../internal/db/transaction_test.go | 80 +++++++++++++------ 2 files changed, 74 insertions(+), 52 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/transaction.go b/cmd/soroban-rpc/internal/db/transaction.go index 032bf902..cd15c073 100644 --- a/cmd/soroban-rpc/internal/db/transaction.go +++ b/cmd/soroban-rpc/internal/db/transaction.go @@ -20,8 +20,9 @@ import ( ) const ( - transactionTableName = "transactions" - firstLedgerToMigrate = 2 + transactionTableName = "transactions" + firstLedgerToMigrate = 2 + getLedgerRangeMetaArrayCheckValue = 2 ) var ErrNoTransaction = errors.New("no transaction with this hash exists") @@ -143,32 +144,24 @@ func (txn *transactionHandler) trimTransactions(latestLedgerSeq uint32, retentio func (txn *transactionHandler) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { var ledgerRange ledgerbucketwindow.LedgerRange - // Query to get the minimum and maximum ledger sequence from the transactions table - var ledgerSeqs struct { - MinLedgerSequence *uint32 `db:"min_ledger_sequence"` - MaxLedgerSequence *uint32 `db:"max_ledger_sequence"` - } - minMaxLedgerSequenceSQL := sq. - Select("MIN(ledger_sequence) AS min_ledger_sequence, MAX(ledger_sequence) AS max_ledger_sequence"). - From(transactionTableName) - if err := txn.db.Get(ctx, &ledgerSeqs, minMaxLedgerSequenceSQL); err != nil { - return ledgerRange, fmt.Errorf("couldn't query ledger range: %w", err) - } - - // Empty DB or just a single row in the DB. - if ledgerSeqs.MinLedgerSequence == nil || ledgerSeqs.MaxLedgerSequence == nil { - return ledgerRange, nil - } - - // Use the min and max ledger sequences to query the ledger_close_meta table - ledgerMetaSQL := sq. - Select("lcm.meta"). - From(ledgerCloseMetaTableName + " as lcm"). - Where(sq.Eq{"lcm.sequence": []uint32{*ledgerSeqs.MinLedgerSequence, *ledgerSeqs.MaxLedgerSequence}}) + query := sq.Select("lcm.meta"). + FromSelect( + sq.Select("meta"). + From(ledgerCloseMetaTableName+" as lcm"). + Where(sq.Expr("lcm.sequence IN (SELECT MIN(ledger_sequence) FROM "+transactionTableName+ + " UNION ALL SELECT MAX(ledger_sequence) FROM "+transactionTableName+")")), + "lcm", + ) var lcms []xdr.LedgerCloseMeta - if err := txn.db.Select(ctx, &lcms, ledgerMetaSQL); err != nil { + if err := txn.db.Select(ctx, &lcms, query); err != nil { return ledgerRange, fmt.Errorf("couldn't query ledger range: %w", err) + } else if len(lcms) < getLedgerRangeMetaArrayCheckValue { + // There is almost certainly a row, but we want to avoid a race condition + // with ingestion as well as support test cases from an empty DB, so we need + // to sanity check that there is in fact a result. Note that no ledgers in + // the database isn't an error, it's just an empty range. + return ledgerRange, nil } lcm1, lcm2 := lcms[0], lcms[1] @@ -177,8 +170,7 @@ func (txn *transactionHandler) GetLedgerRange(ctx context.Context) (ledgerbucket ledgerRange.LastLedger.Sequence = lcm2.LedgerSequence() ledgerRange.LastLedger.CloseTime = lcm2.LedgerCloseTime() - txn.log.Debugf("Database ledger range: [%d, %d]", - ledgerRange.FirstLedger.Sequence, ledgerRange.LastLedger.Sequence) + txn.log.Debugf("Database ledger range: [%d, %d]", ledgerRange.FirstLedger.Sequence, ledgerRange.LastLedger.Sequence) return ledgerRange, nil } diff --git a/cmd/soroban-rpc/internal/db/transaction_test.go b/cmd/soroban-rpc/internal/db/transaction_test.go index a0cd9650..f08ac09d 100644 --- a/cmd/soroban-rpc/internal/db/transaction_test.go +++ b/cmd/soroban-rpc/internal/db/transaction_test.go @@ -18,30 +18,20 @@ import ( func TestTransactionNotFound(t *testing.T) { db := NewTestDB(t) - ctx := context.TODO() log.SetLevel(logrus.TraceLevel) reader := NewTransactionReader(log.DefaultLogger, db, passphrase) - - // Assert the ledger range - ledgerRange, err := reader.GetLedgerRange(ctx) - require.NoError(t, err) - assert.Equal(t, uint32(0), ledgerRange.FirstLedger.Sequence) - assert.Equal(t, int64(0), ledgerRange.FirstLedger.CloseTime) - assert.Equal(t, uint32(0), ledgerRange.LastLedger.Sequence) - assert.Equal(t, int64(0), ledgerRange.LastLedger.CloseTime) - - _, _, err = reader.GetTransaction(context.TODO(), xdr.Hash{}) + _, _, err := reader.GetTransaction(context.TODO(), xdr.Hash{}) require.ErrorIs(t, err, ErrNoTransaction) } func TestTransactionFound(t *testing.T) { db := NewTestDB(t) ctx := context.TODO() - log := log.DefaultLogger + logger := log.DefaultLogger log.SetLevel(logrus.TraceLevel) - writer := NewReadWriter(log, db, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) + writer := NewReadWriter(logger, db, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) write, err := writer.NewTx(ctx) require.NoError(t, err) @@ -59,16 +49,8 @@ func TestTransactionFound(t *testing.T) { } require.NoError(t, write.Commit(lcms[len(lcms)-1].LedgerSequence())) - // Assert the ledger range - reader := NewTransactionReader(log, db, passphrase) - ledgerRange, err := reader.GetLedgerRange(ctx) - require.NoError(t, err) - assert.Equal(t, uint32(1234), ledgerRange.FirstLedger.Sequence) - assert.Equal(t, LedgerCloseTime(1234), ledgerRange.FirstLedger.CloseTime) - assert.Equal(t, uint32(1237), ledgerRange.LastLedger.Sequence) - assert.Equal(t, LedgerCloseTime(1237), ledgerRange.LastLedger.CloseTime) - // check 404 case + reader := NewTransactionReader(logger, db, passphrase) _, _, err = reader.GetTransaction(ctx, xdr.Hash{}) require.ErrorIs(t, err, ErrNoTransaction) @@ -87,12 +69,60 @@ func TestTransactionFound(t *testing.T) { } } +func TestGetLedgerRange_NonEmptyDB(t *testing.T) { + db := NewTestDB(t) + ctx := context.TODO() + logger := log.DefaultLogger + log.SetLevel(logrus.TraceLevel) + + writer := NewReadWriter(logger, db, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) + write, err := writer.NewTx(ctx) + require.NoError(t, err) + + lcms := []xdr.LedgerCloseMeta{ + CreateTxMeta(1234, true), + CreateTxMeta(1235, true), + CreateTxMeta(1236, true), + CreateTxMeta(1237, true), + } + + ledgerW, txW := write.LedgerWriter(), write.TransactionWriter() + for _, lcm := range lcms { + require.NoError(t, ledgerW.InsertLedger(lcm), "ingestion failed for ledger %+v", lcm.V1) + require.NoError(t, txW.InsertTransactions(lcm), "ingestion failed for ledger %+v", lcm.V1) + } + require.NoError(t, write.Commit(lcms[len(lcms)-1].LedgerSequence())) + + reader := NewTransactionReader(logger, db, passphrase) + ledgerRange, err := reader.GetLedgerRange(ctx) + require.NoError(t, err) + assert.Equal(t, uint32(1234), ledgerRange.FirstLedger.Sequence) + assert.Equal(t, LedgerCloseTime(1234), ledgerRange.FirstLedger.CloseTime) + assert.Equal(t, uint32(1237), ledgerRange.LastLedger.Sequence) + assert.Equal(t, LedgerCloseTime(1237), ledgerRange.LastLedger.CloseTime) +} + +func TestGetLedgerRange_EmptyDB(t *testing.T) { + db := NewTestDB(t) + ctx := context.TODO() + logger := log.DefaultLogger + log.SetLevel(logrus.TraceLevel) + + reader := NewTransactionReader(logger, db, passphrase) + ledgerRange, err := reader.GetLedgerRange(ctx) + require.NoError(t, err) + assert.Equal(t, uint32(0), ledgerRange.FirstLedger.Sequence) + assert.Equal(t, int64(0), ledgerRange.FirstLedger.CloseTime) + assert.Equal(t, uint32(0), ledgerRange.LastLedger.Sequence) + assert.Equal(t, int64(0), ledgerRange.LastLedger.CloseTime) +} + func BenchmarkTransactionFetch(b *testing.B) { db := NewTestDB(b) ctx := context.TODO() - log := log.DefaultLogger + logger := log.DefaultLogger - writer := NewReadWriter(log, db, interfaces.MakeNoOpDeamon(), 100, 1_000_000, passphrase) + writer := NewReadWriter(logger, db, interfaces.MakeNoOpDeamon(), 100, 1_000_000, passphrase) write, err := writer.NewTx(ctx) require.NoError(b, err) @@ -108,7 +138,7 @@ func BenchmarkTransactionFetch(b *testing.B) { require.NoError(b, txW.InsertTransactions(lcm)) } require.NoError(b, write.Commit(lcms[len(lcms)-1].LedgerSequence())) - reader := NewTransactionReader(log, db, passphrase) + reader := NewTransactionReader(logger, db, passphrase) randoms := make([]int, b.N) for i := range b.N { From ee6a8931c4192e649850c442cbf89b0404b18199 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Tue, 25 Jun 2024 14:57:05 -0400 Subject: [PATCH 39/68] Fix intrange linter --- cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go b/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go index 9d309be1..3e4a97d0 100644 --- a/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go +++ b/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go @@ -1,4 +1,3 @@ -//nolint:intrange package infrastructure import ( From 52fb803f65335d64ef74c4fa113da89ecd1b46d9 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Tue, 25 Jun 2024 14:58:12 -0400 Subject: [PATCH 40/68] Use require.NoError --- .../integrationtest/infrastructure/test.go | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) diff --git a/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go b/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go index 3e4a97d0..a7f4c95c 100644 --- a/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go +++ b/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go @@ -567,18 +567,12 @@ func (i *Test) prepareShutdownHandlers() { close(done) if i.daemon != nil { err := i.daemon.Close() - if err != nil { - i.t.Logf("could not close RPC daemon: %v", err) - return - } + require.NoError(i.t, err) i.daemon = nil } if i.rpcClient != nil { err := i.rpcClient.Close() - if err != nil { - i.t.Logf("could not close RPC client: %v", err) - return - } + require.NoError(i.t, err) } if i.areThereContainers() { i.stopContainers() @@ -661,10 +655,7 @@ func (i *Test) UpgradeProtocol(version uint32) { func (i *Test) StopRPC() { if i.daemon != nil { err := i.daemon.Close() - if err != nil { - i.t.Logf("could not close RPC daemon: %v", err) - return - } + require.NoError(i.t, err) i.daemon = nil } if i.runRPCInContainer() { From 545bd690dfc3000a1b325760a1387b5533254efe Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Tue, 25 Jun 2024 14:58:48 -0400 Subject: [PATCH 41/68] Move comment to definition --- cmd/soroban-rpc/internal/jsonrpc.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/soroban-rpc/internal/jsonrpc.go b/cmd/soroban-rpc/internal/jsonrpc.go index 54913fff..3757c4fb 100644 --- a/cmd/soroban-rpc/internal/jsonrpc.go +++ b/cmd/soroban-rpc/internal/jsonrpc.go @@ -27,10 +27,10 @@ import ( "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/network" ) -// maxHTTPRequestSize defines the largest request size that the http handler -// would be willing to accept before dropping the request. The implementation -// uses the default MaxBytesHandler to limit the request size. const ( + // maxHTTPRequestSize defines the largest request size that the http handler + // would be willing to accept before dropping the request. The implementation + // uses the default MaxBytesHandler to limit the request size. maxHTTPRequestSize = 512 * 1024 // half a megabyte warningThresholdDenominator = 3 ) From 5b7b8f1ad714d9d9f172ffb645ed9ce9bb067a7f Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Tue, 25 Jun 2024 15:32:32 -0400 Subject: [PATCH 42/68] Fix intrange linter - 2 --- .../internal/integrationtest/infrastructure/test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go b/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go index a7f4c95c..faaa659f 100644 --- a/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go +++ b/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go @@ -220,8 +220,8 @@ func (i *Test) insertTransactions() { require.NoError(i.t, err) lcms := make([]xdr.LedgerCloseMeta, 0, 3) - for i := uint32(0); i < uint32(cap(lcms)); i++ { - lcms = append(lcms, db.CreateTxMeta(9+i, true)) + for i := range cap(lcms) { + lcms = append(lcms, db.CreateTxMeta(uint32(9+i), true)) } _, txW := write.LedgerWriter(), write.TransactionWriter() From 9aa85846b7d723890c3e82591345711a21657340 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Tue, 25 Jun 2024 16:19:16 -0400 Subject: [PATCH 43/68] Fix nomnd linter --- cmd/soroban-rpc/internal/events/events.go | 26 ++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/cmd/soroban-rpc/internal/events/events.go b/cmd/soroban-rpc/internal/events/events.go index 8be9121d..2ac071e5 100644 --- a/cmd/soroban-rpc/internal/events/events.go +++ b/cmd/soroban-rpc/internal/events/events.go @@ -1,4 +1,3 @@ -//nolint:mnd package events import ( @@ -18,6 +17,15 @@ import ( "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" ) +const ( + QuantileKey1 = 0.5 + QuantileValue1 = 0.05 + QuantileKey2 = 0.9 + QuantileValue2 = 0.01 + QuantileKey3 = 0.99 + QuantileValue3 = 0.001 +) + type event struct { diagnosticEventXDR []byte txIndex uint32 @@ -62,16 +70,24 @@ func NewMemoryStore(daemon interfaces.Daemon, networkPassphrase string, retentio // eventsDurationMetric is a metric for measuring latency of event store operations eventsDurationMetric := prometheus.NewSummaryVec(prometheus.SummaryOpts{ Namespace: daemon.MetricsNamespace(), Subsystem: "events", Name: "operation_duration_seconds", - Help: "event store operation durations, sliding window = 10m", - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + Help: "event store operation durations, sliding window = 10m", + Objectives: map[float64]float64{ + QuantileKey1: QuantileValue1, + QuantileKey2: QuantileValue2, + QuantileKey3: QuantileValue3, + }, }, []string{"operation"}, ) eventCountMetric := prometheus.NewSummary(prometheus.SummaryOpts{ Namespace: daemon.MetricsNamespace(), Subsystem: "events", Name: "count", - Help: "count of events ingested, sliding window = 10m", - Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, + Help: "count of events ingested, sliding window = 10m", + Objectives: map[float64]float64{ + QuantileKey1: QuantileValue1, + QuantileKey2: QuantileValue2, + QuantileKey3: QuantileValue3, + }, }) daemon.MetricsRegistry().MustRegister(eventCountMetric, eventsDurationMetric) return &MemoryStore{ From 5eb006d61aab1b1783774d654406764a433543c9 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 26 Jun 2024 09:56:40 -0400 Subject: [PATCH 44/68] Fix intrange linter --- cmd/soroban-rpc/internal/db/transaction.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/transaction.go b/cmd/soroban-rpc/internal/db/transaction.go index cd15c073..41cdf1a8 100644 --- a/cmd/soroban-rpc/internal/db/transaction.go +++ b/cmd/soroban-rpc/internal/db/transaction.go @@ -1,4 +1,3 @@ -//nolint:intrange package db import ( @@ -93,7 +92,7 @@ func (txn *transactionHandler) InsertTransactions(lcm xdr.LedgerCloseMeta) error } transactions := make(map[xdr.Hash]ingest.LedgerTransaction, txCount) - for i := 0; i < txCount; i++ { + for i := range txCount { tx, err := reader.Read() if err != nil { return fmt.Errorf("failed reading tx %d: %w", i, err) From f8466da7d2aaf0be8c4cf7b5005e07eac25108cd Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 26 Jun 2024 10:06:29 -0400 Subject: [PATCH 45/68] Remove db/util.go and add txMeta methods to infrastructure --- .../internal/db/transaction_test.go | 128 ++++++++++++++++-- cmd/soroban-rpc/internal/db/util.go | 107 --------------- .../integrationtest/infrastructure/test.go | 2 +- .../integrationtest/infrastructure/util.go | 102 ++++++++++++++ 4 files changed, 219 insertions(+), 120 deletions(-) delete mode 100644 cmd/soroban-rpc/internal/db/util.go diff --git a/cmd/soroban-rpc/internal/db/transaction_test.go b/cmd/soroban-rpc/internal/db/transaction_test.go index f08ac09d..b319fcc8 100644 --- a/cmd/soroban-rpc/internal/db/transaction_test.go +++ b/cmd/soroban-rpc/internal/db/transaction_test.go @@ -7,6 +7,7 @@ import ( "testing" "github.com/sirupsen/logrus" + "github.com/stellar/go/network" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -16,6 +17,12 @@ import ( "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces" ) +const ( + txMetaV = 3 + ledgerCloseTimeConstant = 100 + feeCharged = 100 +) + func TestTransactionNotFound(t *testing.T) { db := NewTestDB(t) log.SetLevel(logrus.TraceLevel) @@ -36,10 +43,10 @@ func TestTransactionFound(t *testing.T) { require.NoError(t, err) lcms := []xdr.LedgerCloseMeta{ - CreateTxMeta(1234, true), - CreateTxMeta(1235, true), - CreateTxMeta(1236, true), - CreateTxMeta(1237, true), + txMeta(1234, true), + txMeta(1235, true), + txMeta(1236, true), + txMeta(1237, true), } ledgerW, txW := write.LedgerWriter(), write.TransactionWriter() @@ -80,10 +87,10 @@ func TestGetLedgerRange_NonEmptyDB(t *testing.T) { require.NoError(t, err) lcms := []xdr.LedgerCloseMeta{ - CreateTxMeta(1234, true), - CreateTxMeta(1235, true), - CreateTxMeta(1236, true), - CreateTxMeta(1237, true), + txMeta(1234, true), + txMeta(1235, true), + txMeta(1236, true), + txMeta(1237, true), } ledgerW, txW := write.LedgerWriter(), write.TransactionWriter() @@ -97,9 +104,9 @@ func TestGetLedgerRange_NonEmptyDB(t *testing.T) { ledgerRange, err := reader.GetLedgerRange(ctx) require.NoError(t, err) assert.Equal(t, uint32(1234), ledgerRange.FirstLedger.Sequence) - assert.Equal(t, LedgerCloseTime(1234), ledgerRange.FirstLedger.CloseTime) + assert.Equal(t, ledgerCloseTime(1234), ledgerRange.FirstLedger.CloseTime) assert.Equal(t, uint32(1237), ledgerRange.LastLedger.Sequence) - assert.Equal(t, LedgerCloseTime(1237), ledgerRange.LastLedger.CloseTime) + assert.Equal(t, ledgerCloseTime(1237), ledgerRange.LastLedger.CloseTime) } func TestGetLedgerRange_EmptyDB(t *testing.T) { @@ -129,7 +136,7 @@ func BenchmarkTransactionFetch(b *testing.B) { // ingest 100k tx rows lcms := make([]xdr.LedgerCloseMeta, 0, 100_000) for i := range cap(lcms) { - lcms = append(lcms, CreateTxMeta(uint32(1234+i), i%2 == 0)) + lcms = append(lcms, txMeta(uint32(1234+i), i%2 == 0)) } ledgerW, txW := write.LedgerWriter(), write.TransactionWriter() @@ -164,7 +171,7 @@ func BenchmarkGetLedgerRange(b *testing.B) { // create 100k tx rows lcms := make([]xdr.LedgerCloseMeta, 0, 100_000) for i := range cap(lcms) { - lcms = append(lcms, CreateTxMeta(uint32(1234+i), i%2 == 0)) + lcms = append(lcms, txMeta(uint32(1234+i), i%2 == 0)) } ledgerW, txW := write.LedgerWriter(), write.TransactionWriter() @@ -183,3 +190,100 @@ func BenchmarkGetLedgerRange(b *testing.B) { assert.Equal(b, ledgerRange.LastLedger.Sequence, lcms[len(lcms)-1].LedgerSequence()) } } + +// +// Structure creation methods below. +// + +func txHash(acctSeq uint32) xdr.Hash { + envelope := txEnvelope(acctSeq) + hash, err := network.HashTransactionInEnvelope(envelope, passphrase) + if err != nil { + panic(err) + } + return hash +} + +func txEnvelope(acctSeq uint32) xdr.TransactionEnvelope { + envelope, err := xdr.NewTransactionEnvelope(xdr.EnvelopeTypeEnvelopeTypeTx, xdr.TransactionV1Envelope{ + Tx: xdr.Transaction{ + Fee: feeCharged, + SeqNum: xdr.SequenceNumber(acctSeq), + SourceAccount: xdr.MustMuxedAddress("MA7QYNF7SOWQ3GLR2BGMZEHXAVIRZA4KVWLTJJFC7MGXUA74P7UJVAAAAAAAAAAAAAJLK"), + }, + }) + if err != nil { + panic(err) + } + return envelope +} + +func transactionResult(successful bool) xdr.TransactionResult { + code := xdr.TransactionResultCodeTxBadSeq + if successful { + code = xdr.TransactionResultCodeTxSuccess + } + opResults := []xdr.OperationResult{} + return xdr.TransactionResult{ + FeeCharged: feeCharged, + Result: xdr.TransactionResultResult{ + Code: code, + Results: &opResults, + }, + } +} + +func txMeta(acctSeq uint32, successful bool) xdr.LedgerCloseMeta { + envelope := txEnvelope(acctSeq) + txProcessing := []xdr.TransactionResultMeta{ + { + TxApplyProcessing: xdr.TransactionMeta{ + V: txMetaV, + Operations: &[]xdr.OperationMeta{}, + V3: &xdr.TransactionMetaV3{}, + }, + Result: xdr.TransactionResultPair{ + TransactionHash: txHash(acctSeq), + Result: transactionResult(successful), + }, + }, + } + components := []xdr.TxSetComponent{ + { + Type: xdr.TxSetComponentTypeTxsetCompTxsMaybeDiscountedFee, + TxsMaybeDiscountedFee: &xdr.TxSetComponentTxsMaybeDiscountedFee{ + BaseFee: nil, + Txs: []xdr.TransactionEnvelope{envelope}, + }, + }, + } + + return xdr.LedgerCloseMeta{ + V: 1, + V1: &xdr.LedgerCloseMetaV1{ + LedgerHeader: xdr.LedgerHeaderHistoryEntry{ + Header: xdr.LedgerHeader{ + ScpValue: xdr.StellarValue{ + CloseTime: xdr.TimePoint(ledgerCloseTime(acctSeq)), + }, + LedgerSeq: xdr.Uint32(acctSeq), + }, + }, + TxProcessing: txProcessing, + TxSet: xdr.GeneralizedTransactionSet{ + V: 1, + V1TxSet: &xdr.TransactionSetV1{ + PreviousLedgerHash: xdr.Hash{1}, + Phases: []xdr.TransactionPhase{{ + V: 0, + V0Components: &components, + }}, + }, + }, + }, + } +} + +func ledgerCloseTime(ledgerSequence uint32) int64 { + return int64(ledgerSequence)*25 + ledgerCloseTimeConstant +} diff --git a/cmd/soroban-rpc/internal/db/util.go b/cmd/soroban-rpc/internal/db/util.go deleted file mode 100644 index 97a4164b..00000000 --- a/cmd/soroban-rpc/internal/db/util.go +++ /dev/null @@ -1,107 +0,0 @@ -package db - -import ( - "github.com/stellar/go/network" - "github.com/stellar/go/xdr" -) - -// Methods for creating test txMeta - -const ( - txMetaV = 3 - ledgerCloseTimeConstant = 100 - feeCharged = 100 -) - -func CreateTxMeta(acctSeq uint32, successful bool) xdr.LedgerCloseMeta { - envelope := txEnvelope(acctSeq) - txProcessing := []xdr.TransactionResultMeta{ - { - TxApplyProcessing: xdr.TransactionMeta{ - V: txMetaV, - Operations: &[]xdr.OperationMeta{}, - V3: &xdr.TransactionMetaV3{}, - }, - Result: xdr.TransactionResultPair{ - TransactionHash: txHash(acctSeq), - Result: transactionResult(successful), - }, - }, - } - components := []xdr.TxSetComponent{ - { - Type: xdr.TxSetComponentTypeTxsetCompTxsMaybeDiscountedFee, - TxsMaybeDiscountedFee: &xdr.TxSetComponentTxsMaybeDiscountedFee{ - BaseFee: nil, - Txs: []xdr.TransactionEnvelope{envelope}, - }, - }, - } - - return xdr.LedgerCloseMeta{ - V: 1, - V1: &xdr.LedgerCloseMetaV1{ - LedgerHeader: xdr.LedgerHeaderHistoryEntry{ - Header: xdr.LedgerHeader{ - ScpValue: xdr.StellarValue{ - CloseTime: xdr.TimePoint(LedgerCloseTime(acctSeq)), - }, - LedgerSeq: xdr.Uint32(acctSeq), - }, - }, - TxProcessing: txProcessing, - TxSet: xdr.GeneralizedTransactionSet{ - V: 1, - V1TxSet: &xdr.TransactionSetV1{ - PreviousLedgerHash: xdr.Hash{1}, - Phases: []xdr.TransactionPhase{{ - V: 0, - V0Components: &components, - }}, - }, - }, - }, - } -} - -func txHash(acctSeq uint32) xdr.Hash { - envelope := txEnvelope(acctSeq) - hash, err := network.HashTransactionInEnvelope(envelope, network.FutureNetworkPassphrase) - if err != nil { - panic(err) - } - return hash -} - -func txEnvelope(acctSeq uint32) xdr.TransactionEnvelope { - envelope, err := xdr.NewTransactionEnvelope(xdr.EnvelopeTypeEnvelopeTypeTx, xdr.TransactionV1Envelope{ - Tx: xdr.Transaction{ - Fee: 1, - SeqNum: xdr.SequenceNumber(acctSeq), - SourceAccount: xdr.MustMuxedAddress("MA7QYNF7SOWQ3GLR2BGMZEHXAVIRZA4KVWLTJJFC7MGXUA74P7UJVAAAAAAAAAAAAAJLK"), - }, - }) - if err != nil { - panic(err) - } - return envelope -} - -func transactionResult(successful bool) xdr.TransactionResult { - code := xdr.TransactionResultCodeTxBadSeq - if successful { - code = xdr.TransactionResultCodeTxSuccess - } - opResults := []xdr.OperationResult{} - return xdr.TransactionResult{ - FeeCharged: feeCharged, - Result: xdr.TransactionResultResult{ - Code: code, - Results: &opResults, - }, - } -} - -func LedgerCloseTime(ledgerSequence uint32) int64 { - return int64(ledgerSequence)*25 + ledgerCloseTimeConstant -} diff --git a/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go b/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go index faaa659f..400664d8 100644 --- a/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go +++ b/cmd/soroban-rpc/internal/integrationtest/infrastructure/test.go @@ -221,7 +221,7 @@ func (i *Test) insertTransactions() { lcms := make([]xdr.LedgerCloseMeta, 0, 3) for i := range cap(lcms) { - lcms = append(lcms, db.CreateTxMeta(uint32(9+i), true)) + lcms = append(lcms, txMeta(uint32(9+i), true)) } _, txW := write.LedgerWriter(), write.TransactionWriter() diff --git a/cmd/soroban-rpc/internal/integrationtest/infrastructure/util.go b/cmd/soroban-rpc/internal/integrationtest/infrastructure/util.go index 1a34325a..163717f7 100644 --- a/cmd/soroban-rpc/internal/integrationtest/infrastructure/util.go +++ b/cmd/soroban-rpc/internal/integrationtest/infrastructure/util.go @@ -5,11 +5,20 @@ import ( "path/filepath" "runtime" + "github.com/stellar/go/network" + "github.com/stellar/go/xdr" "github.com/stretchr/testify/require" "github.com/stellar/go/txnbuild" ) +const ( + txMetaV = 3 + ledgerCloseTimeConstant = 100 + feeCharged = 100 + passphrase = network.FutureNetworkPassphrase +) + //go:noinline func GetCurrentDirectory() string { _, currentFilename, _, _ := runtime.Caller(1) @@ -27,6 +36,99 @@ func getFreeTCPPort(t require.TestingT) uint16 { return uint16(l.Addr().(*net.TCPAddr).Port) } +func txHash(acctSeq uint32) xdr.Hash { + envelope := txEnvelope(acctSeq) + hash, err := network.HashTransactionInEnvelope(envelope, passphrase) + if err != nil { + panic(err) + } + return hash +} + +func txEnvelope(acctSeq uint32) xdr.TransactionEnvelope { + envelope, err := xdr.NewTransactionEnvelope(xdr.EnvelopeTypeEnvelopeTypeTx, xdr.TransactionV1Envelope{ + Tx: xdr.Transaction{ + Fee: feeCharged, + SeqNum: xdr.SequenceNumber(acctSeq), + SourceAccount: xdr.MustMuxedAddress("MA7QYNF7SOWQ3GLR2BGMZEHXAVIRZA4KVWLTJJFC7MGXUA74P7UJVAAAAAAAAAAAAAJLK"), + }, + }) + if err != nil { + panic(err) + } + return envelope +} + +func transactionResult(successful bool) xdr.TransactionResult { + code := xdr.TransactionResultCodeTxBadSeq + if successful { + code = xdr.TransactionResultCodeTxSuccess + } + opResults := []xdr.OperationResult{} + return xdr.TransactionResult{ + FeeCharged: feeCharged, + Result: xdr.TransactionResultResult{ + Code: code, + Results: &opResults, + }, + } +} + +func txMeta(acctSeq uint32, successful bool) xdr.LedgerCloseMeta { + envelope := txEnvelope(acctSeq) + txProcessing := []xdr.TransactionResultMeta{ + { + TxApplyProcessing: xdr.TransactionMeta{ + V: txMetaV, + Operations: &[]xdr.OperationMeta{}, + V3: &xdr.TransactionMetaV3{}, + }, + Result: xdr.TransactionResultPair{ + TransactionHash: txHash(acctSeq), + Result: transactionResult(successful), + }, + }, + } + components := []xdr.TxSetComponent{ + { + Type: xdr.TxSetComponentTypeTxsetCompTxsMaybeDiscountedFee, + TxsMaybeDiscountedFee: &xdr.TxSetComponentTxsMaybeDiscountedFee{ + BaseFee: nil, + Txs: []xdr.TransactionEnvelope{envelope}, + }, + }, + } + + return xdr.LedgerCloseMeta{ + V: 1, + V1: &xdr.LedgerCloseMetaV1{ + LedgerHeader: xdr.LedgerHeaderHistoryEntry{ + Header: xdr.LedgerHeader{ + ScpValue: xdr.StellarValue{ + CloseTime: xdr.TimePoint(ledgerCloseTime(acctSeq)), + }, + LedgerSeq: xdr.Uint32(acctSeq), + }, + }, + TxProcessing: txProcessing, + TxSet: xdr.GeneralizedTransactionSet{ + V: 1, + V1TxSet: &xdr.TransactionSetV1{ + PreviousLedgerHash: xdr.Hash{1}, + Phases: []xdr.TransactionPhase{{ + V: 0, + V0Components: &components, + }}, + }, + }, + }, + } +} + +func ledgerCloseTime(ledgerSequence uint32) int64 { + return int64(ledgerSequence)*25 + ledgerCloseTimeConstant +} + func CreateTransactionParams(account txnbuild.Account, op txnbuild.Operation) txnbuild.TransactionParams { return txnbuild.TransactionParams{ SourceAccount: account, From 50cd4c7e620304fc73758837f4ad66427c4cb72c Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 26 Jun 2024 10:10:35 -0400 Subject: [PATCH 46/68] forgot to gci files again :/ --- cmd/soroban-rpc/internal/db/transaction_test.go | 2 +- .../internal/integrationtest/infrastructure/util.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/transaction_test.go b/cmd/soroban-rpc/internal/db/transaction_test.go index b319fcc8..7c6aafe3 100644 --- a/cmd/soroban-rpc/internal/db/transaction_test.go +++ b/cmd/soroban-rpc/internal/db/transaction_test.go @@ -7,10 +7,10 @@ import ( "testing" "github.com/sirupsen/logrus" - "github.com/stellar/go/network" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/stellar/go/network" "github.com/stellar/go/support/log" "github.com/stellar/go/xdr" diff --git a/cmd/soroban-rpc/internal/integrationtest/infrastructure/util.go b/cmd/soroban-rpc/internal/integrationtest/infrastructure/util.go index 163717f7..db88fa96 100644 --- a/cmd/soroban-rpc/internal/integrationtest/infrastructure/util.go +++ b/cmd/soroban-rpc/internal/integrationtest/infrastructure/util.go @@ -5,11 +5,11 @@ import ( "path/filepath" "runtime" - "github.com/stellar/go/network" - "github.com/stellar/go/xdr" "github.com/stretchr/testify/require" + "github.com/stellar/go/network" "github.com/stellar/go/txnbuild" + "github.com/stellar/go/xdr" ) const ( From 4643089b6248d55068f70b735a7fc4b5bf07c68a Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 26 Jun 2024 12:24:25 -0400 Subject: [PATCH 47/68] Remove migration FIXME --- cmd/soroban-rpc/internal/db/transaction.go | 3 --- 1 file changed, 3 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/transaction.go b/cmd/soroban-rpc/internal/db/transaction.go index 41cdf1a8..35a96854 100644 --- a/cmd/soroban-rpc/internal/db/transaction.go +++ b/cmd/soroban-rpc/internal/db/transaction.go @@ -324,9 +324,6 @@ func newTransactionTableMigration(ctx context.Context, logger *log.Entry, } // Truncate the table, since it may contain data, causing insert conflicts later on. // (the migration was shipped after the actual transactions table change) - // FIXME: this can be simply replaced by an upper limit in the ledgers to migrate - // but ... it can't be done until https://github.com/stellar/soroban-rpc/issues/208 - // is addressed _, err := db.Exec(ctx, sq.Delete(transactionTableName)) if err != nil { return nil, fmt.Errorf("couldn't delete table %q: %w", transactionTableName, err) From 2691eab05dc34eb24b29ae09d6093d77b74340ef Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 26 Jun 2024 12:42:24 -0400 Subject: [PATCH 48/68] Fix linter checks --- cmd/soroban-rpc/internal/db/migration.go | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/migration.go b/cmd/soroban-rpc/internal/db/migration.go index 74a88f87..bd362567 100644 --- a/cmd/soroban-rpc/internal/db/migration.go +++ b/cmd/soroban-rpc/internal/db/migration.go @@ -117,7 +117,9 @@ type guardedMigration struct { alreadyMigrated bool } -func newGuardedDataMigration(ctx context.Context, uniqueMigrationName string, factory migrationApplierFactory, db *DB) (Migration, error) { +func newGuardedDataMigration(ctx context.Context, uniqueMigrationName string, + factory migrationApplierFactory, db *DB, +) (Migration, error) { migrationDB := &DB{ cache: db.cache, SessionInterface: db.SessionInterface.Clone(), @@ -132,7 +134,7 @@ func newGuardedDataMigration(ctx context.Context, uniqueMigrationName string, fa return nil, err } latestLedger, err := NewLedgerEntryReader(db).GetLatestLedgerSequence(ctx) - if err != nil && err != ErrEmptyDB { + if err != nil && !errors.Is(err, ErrEmptyDB) { err = errors.Join(err, migrationDB.Rollback()) return nil, fmt.Errorf("failed to get latest ledger sequence: %w", err) } @@ -177,13 +179,14 @@ func (g *guardedMigration) Commit(ctx context.Context) error { return g.db.Commit() } -func (g *guardedMigration) Rollback(ctx context.Context) error { +func (g *guardedMigration) Rollback(_ context.Context) error { return g.db.Rollback() } func BuildMigrations(ctx context.Context, logger *log.Entry, db *DB, cfg *config.Config) (Migration, error) { migrationName := "TransactionsTable" - factory := newTransactionTableMigration(ctx, logger.WithField("migration", migrationName), cfg.TransactionLedgerRetentionWindow, cfg.NetworkPassphrase) + factory := newTransactionTableMigration(ctx, logger.WithField("migration", migrationName), + cfg.TransactionLedgerRetentionWindow, cfg.NetworkPassphrase) m, err := newGuardedDataMigration(ctx, migrationName, factory, db) if err != nil { return nil, fmt.Errorf("creating guarded transaction migration: %w", err) From c809003f09c3b3570d4a9490f3bde355f923e81e Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 26 Jun 2024 14:58:24 -0400 Subject: [PATCH 49/68] Add migration for lcm sequence index --- .../internal/db/sqlmigrations/03_ledgers_index.sql | 7 +++++++ 1 file changed, 7 insertions(+) create mode 100644 cmd/soroban-rpc/internal/db/sqlmigrations/03_ledgers_index.sql diff --git a/cmd/soroban-rpc/internal/db/sqlmigrations/03_ledgers_index.sql b/cmd/soroban-rpc/internal/db/sqlmigrations/03_ledgers_index.sql new file mode 100644 index 00000000..98a683f5 --- /dev/null +++ b/cmd/soroban-rpc/internal/db/sqlmigrations/03_ledgers_index.sql @@ -0,0 +1,7 @@ +-- +migrate Up + +-- creating an index in the ledgers table on the sequence number. +CREATE INDEX index_lcm_sequence ON ledger_close_meta(sequence); + +-- +migrate Down +DROP INDEX index_lcm_sequence; \ No newline at end of file From 29c67c2bc81e0898f294c432d002aa06ccba6c8e Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 26 Jun 2024 15:30:04 -0400 Subject: [PATCH 50/68] Revert transaction_test.go changes --- .../internal/db/transaction_test.go | 127 +++--------------- 1 file changed, 22 insertions(+), 105 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/transaction_test.go b/cmd/soroban-rpc/internal/db/transaction_test.go index 7c6aafe3..a4bf2342 100644 --- a/cmd/soroban-rpc/internal/db/transaction_test.go +++ b/cmd/soroban-rpc/internal/db/transaction_test.go @@ -17,28 +17,23 @@ import ( "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces" ) -const ( - txMetaV = 3 - ledgerCloseTimeConstant = 100 - feeCharged = 100 -) - func TestTransactionNotFound(t *testing.T) { db := NewTestDB(t) + log := log.DefaultLogger log.SetLevel(logrus.TraceLevel) - reader := NewTransactionReader(log.DefaultLogger, db, passphrase) + reader := NewTransactionReader(log, db, passphrase) _, _, err := reader.GetTransaction(context.TODO(), xdr.Hash{}) - require.ErrorIs(t, err, ErrNoTransaction) + require.Error(t, err, ErrNoTransaction) } func TestTransactionFound(t *testing.T) { db := NewTestDB(t) ctx := context.TODO() - logger := log.DefaultLogger + log := log.DefaultLogger log.SetLevel(logrus.TraceLevel) - writer := NewReadWriter(logger, db, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) + writer := NewReadWriter(log, db, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) write, err := writer.NewTx(ctx) require.NoError(t, err) @@ -57,17 +52,17 @@ func TestTransactionFound(t *testing.T) { require.NoError(t, write.Commit(lcms[len(lcms)-1].LedgerSequence())) // check 404 case - reader := NewTransactionReader(logger, db, passphrase) + reader := NewTransactionReader(log, db, passphrase) _, _, err = reader.GetTransaction(ctx, xdr.Hash{}) - require.ErrorIs(t, err, ErrNoTransaction) + require.Error(t, err, ErrNoTransaction) // check all 200 cases for _, lcm := range lcms { h := lcm.TransactionHash(0) tx, lRange, err := reader.GetTransaction(ctx, h) require.NoError(t, err, "failed to find txhash %s in db", hex.EncodeToString(h[:])) - assert.EqualValues(t, 1234, lRange.FirstLedger.Sequence) - assert.EqualValues(t, 1237, lRange.LastLedger.Sequence) + assert.EqualValues(t, 1234+100, lRange.FirstLedger.Sequence) + assert.EqualValues(t, 1237+100, lRange.LastLedger.Sequence) assert.EqualValues(t, 1, tx.ApplicationOrder) expectedEnvelope, err := lcm.TransactionEnvelopes()[0].MarshalBinary() @@ -76,67 +71,19 @@ func TestTransactionFound(t *testing.T) { } } -func TestGetLedgerRange_NonEmptyDB(t *testing.T) { - db := NewTestDB(t) - ctx := context.TODO() - logger := log.DefaultLogger - log.SetLevel(logrus.TraceLevel) - - writer := NewReadWriter(logger, db, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) - write, err := writer.NewTx(ctx) - require.NoError(t, err) - - lcms := []xdr.LedgerCloseMeta{ - txMeta(1234, true), - txMeta(1235, true), - txMeta(1236, true), - txMeta(1237, true), - } - - ledgerW, txW := write.LedgerWriter(), write.TransactionWriter() - for _, lcm := range lcms { - require.NoError(t, ledgerW.InsertLedger(lcm), "ingestion failed for ledger %+v", lcm.V1) - require.NoError(t, txW.InsertTransactions(lcm), "ingestion failed for ledger %+v", lcm.V1) - } - require.NoError(t, write.Commit(lcms[len(lcms)-1].LedgerSequence())) - - reader := NewTransactionReader(logger, db, passphrase) - ledgerRange, err := reader.GetLedgerRange(ctx) - require.NoError(t, err) - assert.Equal(t, uint32(1234), ledgerRange.FirstLedger.Sequence) - assert.Equal(t, ledgerCloseTime(1234), ledgerRange.FirstLedger.CloseTime) - assert.Equal(t, uint32(1237), ledgerRange.LastLedger.Sequence) - assert.Equal(t, ledgerCloseTime(1237), ledgerRange.LastLedger.CloseTime) -} - -func TestGetLedgerRange_EmptyDB(t *testing.T) { - db := NewTestDB(t) - ctx := context.TODO() - logger := log.DefaultLogger - log.SetLevel(logrus.TraceLevel) - - reader := NewTransactionReader(logger, db, passphrase) - ledgerRange, err := reader.GetLedgerRange(ctx) - require.NoError(t, err) - assert.Equal(t, uint32(0), ledgerRange.FirstLedger.Sequence) - assert.Equal(t, int64(0), ledgerRange.FirstLedger.CloseTime) - assert.Equal(t, uint32(0), ledgerRange.LastLedger.Sequence) - assert.Equal(t, int64(0), ledgerRange.LastLedger.CloseTime) -} - func BenchmarkTransactionFetch(b *testing.B) { db := NewTestDB(b) ctx := context.TODO() - logger := log.DefaultLogger + log := log.DefaultLogger - writer := NewReadWriter(logger, db, interfaces.MakeNoOpDeamon(), 100, 1_000_000, passphrase) + writer := NewReadWriter(log, db, interfaces.MakeNoOpDeamon(), 100, 1_000_000, passphrase) write, err := writer.NewTx(ctx) require.NoError(b, err) // ingest 100k tx rows lcms := make([]xdr.LedgerCloseMeta, 0, 100_000) - for i := range cap(lcms) { - lcms = append(lcms, txMeta(uint32(1234+i), i%2 == 0)) + for i := uint32(0); i < uint32(cap(lcms)); i++ { + lcms = append(lcms, txMeta(1234+i, i%2 == 0)) } ledgerW, txW := write.LedgerWriter(), write.TransactionWriter() @@ -145,15 +92,15 @@ func BenchmarkTransactionFetch(b *testing.B) { require.NoError(b, txW.InsertTransactions(lcm)) } require.NoError(b, write.Commit(lcms[len(lcms)-1].LedgerSequence())) - reader := NewTransactionReader(logger, db, passphrase) + reader := NewTransactionReader(log, db, passphrase) randoms := make([]int, b.N) - for i := range b.N { + for i := 0; i < b.N; i++ { randoms[i] = rand.Intn(len(lcms)) } b.ResetTimer() - for i := range b.N { + for i := 0; i < b.N; i++ { r := randoms[i] tx, _, err := reader.GetTransaction(ctx, lcms[r].TransactionHash(0)) require.NoError(b, err) @@ -161,36 +108,6 @@ func BenchmarkTransactionFetch(b *testing.B) { } } -func BenchmarkGetLedgerRange(b *testing.B) { - db := NewTestDB(b) - logger := log.DefaultLogger - writer := NewReadWriter(logger, db, interfaces.MakeNoOpDeamon(), 100, 1_000_000, passphrase) - write, err := writer.NewTx(context.TODO()) - require.NoError(b, err) - - // create 100k tx rows - lcms := make([]xdr.LedgerCloseMeta, 0, 100_000) - for i := range cap(lcms) { - lcms = append(lcms, txMeta(uint32(1234+i), i%2 == 0)) - } - - ledgerW, txW := write.LedgerWriter(), write.TransactionWriter() - for _, lcm := range lcms { - require.NoError(b, ledgerW.InsertLedger(lcm)) - require.NoError(b, txW.InsertTransactions(lcm)) - } - require.NoError(b, write.Commit(lcms[len(lcms)-1].LedgerSequence())) - reader := NewTransactionReader(logger, db, passphrase) - - b.ResetTimer() - for range b.N { - ledgerRange, err := reader.GetLedgerRange(context.TODO()) - require.NoError(b, err) - assert.Equal(b, ledgerRange.FirstLedger.Sequence, lcms[0].LedgerSequence()) - assert.Equal(b, ledgerRange.LastLedger.Sequence, lcms[len(lcms)-1].LedgerSequence()) - } -} - // // Structure creation methods below. // @@ -207,7 +124,7 @@ func txHash(acctSeq uint32) xdr.Hash { func txEnvelope(acctSeq uint32) xdr.TransactionEnvelope { envelope, err := xdr.NewTransactionEnvelope(xdr.EnvelopeTypeEnvelopeTypeTx, xdr.TransactionV1Envelope{ Tx: xdr.Transaction{ - Fee: feeCharged, + Fee: 1, SeqNum: xdr.SequenceNumber(acctSeq), SourceAccount: xdr.MustMuxedAddress("MA7QYNF7SOWQ3GLR2BGMZEHXAVIRZA4KVWLTJJFC7MGXUA74P7UJVAAAAAAAAAAAAAJLK"), }, @@ -225,7 +142,7 @@ func transactionResult(successful bool) xdr.TransactionResult { } opResults := []xdr.OperationResult{} return xdr.TransactionResult{ - FeeCharged: feeCharged, + FeeCharged: 100, Result: xdr.TransactionResultResult{ Code: code, Results: &opResults, @@ -238,7 +155,7 @@ func txMeta(acctSeq uint32, successful bool) xdr.LedgerCloseMeta { txProcessing := []xdr.TransactionResultMeta{ { TxApplyProcessing: xdr.TransactionMeta{ - V: txMetaV, + V: 3, Operations: &[]xdr.OperationMeta{}, V3: &xdr.TransactionMetaV3{}, }, @@ -264,9 +181,9 @@ func txMeta(acctSeq uint32, successful bool) xdr.LedgerCloseMeta { LedgerHeader: xdr.LedgerHeaderHistoryEntry{ Header: xdr.LedgerHeader{ ScpValue: xdr.StellarValue{ - CloseTime: xdr.TimePoint(ledgerCloseTime(acctSeq)), + CloseTime: xdr.TimePoint(ledgerCloseTime(acctSeq + 100)), }, - LedgerSeq: xdr.Uint32(acctSeq), + LedgerSeq: xdr.Uint32(acctSeq + 100), }, }, TxProcessing: txProcessing, @@ -285,5 +202,5 @@ func txMeta(acctSeq uint32, successful bool) xdr.LedgerCloseMeta { } func ledgerCloseTime(ledgerSequence uint32) int64 { - return int64(ledgerSequence)*25 + ledgerCloseTimeConstant + return int64(ledgerSequence)*25 + 100 } From b5b2598dee6f736fc74b9c4ee655478b0c926d66 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 26 Jun 2024 15:31:49 -0400 Subject: [PATCH 51/68] Add newline --- cmd/soroban-rpc/internal/db/sqlmigrations/03_ledgers_index.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/soroban-rpc/internal/db/sqlmigrations/03_ledgers_index.sql b/cmd/soroban-rpc/internal/db/sqlmigrations/03_ledgers_index.sql index 98a683f5..5752f64d 100644 --- a/cmd/soroban-rpc/internal/db/sqlmigrations/03_ledgers_index.sql +++ b/cmd/soroban-rpc/internal/db/sqlmigrations/03_ledgers_index.sql @@ -4,4 +4,4 @@ CREATE INDEX index_lcm_sequence ON ledger_close_meta(sequence); -- +migrate Down -DROP INDEX index_lcm_sequence; \ No newline at end of file +DROP INDEX index_lcm_sequence; From e88d3b6756e610869ffb2d4ba6861cc4596afc3e Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 26 Jun 2024 16:04:56 -0400 Subject: [PATCH 52/68] Add GetLedgerRange implementation in meta table --- cmd/soroban-rpc/internal/db/ledger.go | 34 +++++++++- cmd/soroban-rpc/internal/db/ledger_test.go | 74 ++++++++++++++++++++++ cmd/soroban-rpc/internal/db/mocks.go | 20 +++--- 3 files changed, 119 insertions(+), 9 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/ledger.go b/cmd/soroban-rpc/internal/db/ledger.go index 97887281..612f07a9 100644 --- a/cmd/soroban-rpc/internal/db/ledger.go +++ b/cmd/soroban-rpc/internal/db/ledger.go @@ -7,6 +7,8 @@ import ( sq "github.com/Masterminds/squirrel" "github.com/stellar/go/xdr" + + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" ) const ( @@ -18,6 +20,7 @@ type StreamLedgerFn func(xdr.LedgerCloseMeta) error type LedgerReader interface { GetLedger(ctx context.Context, sequence uint32) (xdr.LedgerCloseMeta, bool, error) StreamAllLedgers(ctx context.Context, f StreamLedgerFn) error + GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) } type LedgerWriter interface { @@ -65,10 +68,39 @@ func (r ledgerReader) GetLedger(ctx context.Context, sequence uint32) (xdr.Ledge case 1: return results[0], true, nil default: - return xdr.LedgerCloseMeta{}, false, fmt.Errorf("multiple lcm entries (%d) for sequence %d in table %q", len(results), sequence, ledgerCloseMetaTableName) + return xdr.LedgerCloseMeta{}, false, fmt.Errorf("multiple lcm entries (%d) for sequence %d in table %q", + len(results), sequence, ledgerCloseMetaTableName) } } +// GetLedgerRange pulls the min/max ledger sequence numbers from the meta table. +func (r ledgerReader) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { + var ledgerRange ledgerbucketwindow.LedgerRange + + query := sq.Select("lcm.meta"). + From(ledgerCloseMetaTableName + " as lcm"). + Where(sq.Expr("lcm.sequence = (SELECT MIN(sequence) FROM " + ledgerCloseMetaTableName + ") " + + "OR lcm.sequence = (SELECT MAX(sequence) FROM " + ledgerCloseMetaTableName + ")")) + + var lcms []xdr.LedgerCloseMeta + if err := r.db.Select(ctx, &lcms, query); err != nil { + return ledgerRange, fmt.Errorf("couldn't query ledger range: %w", err) + } else if len(lcms) < getLedgerRangeMetaArrayCheckValue { + // There is almost certainly a row, but we want to avoid a race condition + // with ingestion as well as support test cases from an empty DB, so we need + // to sanity check that there is in fact a result. Note that no ledgers in + // the database isn't an error, it's just an empty range. + return ledgerRange, nil + } + + lcm1, lcm2 := lcms[0], lcms[1] + ledgerRange.FirstLedger.Sequence = lcm1.LedgerSequence() + ledgerRange.FirstLedger.CloseTime = lcm1.LedgerCloseTime() + ledgerRange.LastLedger.Sequence = lcm2.LedgerSequence() + ledgerRange.LastLedger.CloseTime = lcm2.LedgerCloseTime() + return ledgerRange, nil +} + type ledgerWriter struct { stmtCache *sq.StmtCache } diff --git a/cmd/soroban-rpc/internal/db/ledger_test.go b/cmd/soroban-rpc/internal/db/ledger_test.go index a40bc255..1955e0a8 100644 --- a/cmd/soroban-rpc/internal/db/ledger_test.go +++ b/cmd/soroban-rpc/internal/db/ledger_test.go @@ -106,6 +106,80 @@ func TestLedgers(t *testing.T) { assertLedgerRange(t, reader, 8, 12) } +func TestGetLedgerRange_NonEmptyDB(t *testing.T) { + db := NewTestDB(t) + ctx := context.TODO() + + writer := NewReadWriter(logger, db, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) + write, err := writer.NewTx(ctx) + require.NoError(t, err) + + lcms := []xdr.LedgerCloseMeta{ + txMeta(1234, true), + txMeta(1235, true), + txMeta(1236, true), + txMeta(1237, true), + } + + ledgerW, txW := write.LedgerWriter(), write.TransactionWriter() + for _, lcm := range lcms { + require.NoError(t, ledgerW.InsertLedger(lcm), "ingestion failed for ledger %+v", lcm.V1) + require.NoError(t, txW.InsertTransactions(lcm), "ingestion failed for ledger %+v", lcm.V1) + } + require.NoError(t, write.Commit(lcms[len(lcms)-1].LedgerSequence())) + + reader := NewLedgerReader(db) + ledgerRange, err := reader.GetLedgerRange(ctx) + require.NoError(t, err) + assert.Equal(t, uint32(1334), ledgerRange.FirstLedger.Sequence) + assert.Equal(t, ledgerCloseTime(1334), ledgerRange.FirstLedger.CloseTime) + assert.Equal(t, uint32(1337), ledgerRange.LastLedger.Sequence) + assert.Equal(t, ledgerCloseTime(1337), ledgerRange.LastLedger.CloseTime) +} + +func TestGetLedgerRange_EmptyDB(t *testing.T) { + db := NewTestDB(t) + ctx := context.TODO() + + reader := NewLedgerReader(db) + ledgerRange, err := reader.GetLedgerRange(ctx) + require.NoError(t, err) + assert.Equal(t, uint32(0), ledgerRange.FirstLedger.Sequence) + assert.Equal(t, int64(0), ledgerRange.FirstLedger.CloseTime) + assert.Equal(t, uint32(0), ledgerRange.LastLedger.Sequence) + assert.Equal(t, int64(0), ledgerRange.LastLedger.CloseTime) +} + +func BenchmarkGetLedgerRange(b *testing.B) { + db := NewTestDB(b) + logger := log.DefaultLogger + writer := NewReadWriter(logger, db, interfaces.MakeNoOpDeamon(), 100, 1_000_000, passphrase) + write, err := writer.NewTx(context.TODO()) + require.NoError(b, err) + + // create 100k tx rows + lcms := make([]xdr.LedgerCloseMeta, 0, 100_000) + for i := range cap(lcms) { + lcms = append(lcms, txMeta(uint32(1234+i), i%2 == 0)) + } + + ledgerW, txW := write.LedgerWriter(), write.TransactionWriter() + for _, lcm := range lcms { + require.NoError(b, ledgerW.InsertLedger(lcm)) + require.NoError(b, txW.InsertTransactions(lcm)) + } + require.NoError(b, write.Commit(lcms[len(lcms)-1].LedgerSequence())) + reader := NewLedgerReader(db) + + b.ResetTimer() + for range b.N { + ledgerRange, err := reader.GetLedgerRange(context.TODO()) + require.NoError(b, err) + assert.Equal(b, ledgerRange.FirstLedger.Sequence, lcms[0].LedgerSequence()) + assert.Equal(b, ledgerRange.LastLedger.Sequence, lcms[len(lcms)-1].LedgerSequence()) + } +} + func NewTestDB(tb testing.TB) *DB { tmp := tb.TempDir() dbPath := path.Join(tmp, "db.sqlite") diff --git a/cmd/soroban-rpc/internal/db/mocks.go b/cmd/soroban-rpc/internal/db/mocks.go index a5e97530..e9f3644b 100644 --- a/cmd/soroban-rpc/internal/db/mocks.go +++ b/cmd/soroban-rpc/internal/db/mocks.go @@ -66,19 +66,19 @@ func (txn *mockTransactionHandler) InsertTransactions(lcm xdr.LedgerCloseMeta) e } // GetLedgerRange pulls the min/max ledger sequence numbers from the database. -func (txn *mockTransactionHandler) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { +func (txn *mockTransactionHandler) GetLedgerRange(_ context.Context) (ledgerbucketwindow.LedgerRange, error) { return txn.ledgerRange, nil } -func (txn *mockTransactionHandler) GetTransaction(ctx context.Context, hash xdr.Hash) ( +func (txn *mockTransactionHandler) GetTransaction(_ context.Context, hash xdr.Hash) ( Transaction, ledgerbucketwindow.LedgerRange, error, ) { - if tx, ok := txn.txs[hash.HexString()]; !ok { + tx, ok := txn.txs[hash.HexString()] + if !ok { return Transaction{}, txn.ledgerRange, ErrNoTransaction - } else { - itx, err := ParseTransaction(*txn.txHashToMeta[hash.HexString()], tx) - return itx, txn.ledgerRange, err } + itx, err := ParseTransaction(*txn.txHashToMeta[hash.HexString()], tx) + return itx, txn.ledgerRange, err } func (txn *mockTransactionHandler) RegisterMetrics(_, _ prometheus.Observer) {} @@ -87,13 +87,17 @@ type mockLedgerReader struct { txn mockTransactionHandler } +func (m *mockLedgerReader) GetLedgerRange(_ context.Context) (ledgerbucketwindow.LedgerRange, error) { + return ledgerbucketwindow.LedgerRange{}, nil +} + func NewMockLedgerReader(txn *mockTransactionHandler) *mockLedgerReader { return &mockLedgerReader{ txn: *txn, } } -func (m *mockLedgerReader) GetLedger(ctx context.Context, sequence uint32) (xdr.LedgerCloseMeta, bool, error) { +func (m *mockLedgerReader) GetLedger(_ context.Context, sequence uint32) (xdr.LedgerCloseMeta, bool, error) { lcm, ok := m.txn.ledgerSeqToMeta[sequence] if !ok { return xdr.LedgerCloseMeta{}, false, nil @@ -101,7 +105,7 @@ func (m *mockLedgerReader) GetLedger(ctx context.Context, sequence uint32) (xdr. return *lcm, true, nil } -func (m *mockLedgerReader) StreamAllLedgers(ctx context.Context, f StreamLedgerFn) error { +func (m *mockLedgerReader) StreamAllLedgers(_ context.Context, _ StreamLedgerFn) error { return nil } From 335ca1022911e704c25374bd87552e1724face3b Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 26 Jun 2024 16:33:54 -0400 Subject: [PATCH 53/68] Use new GetLedgerRange for getHealth and getFeeStats --- cmd/soroban-rpc/internal/jsonrpc.go | 6 ++---- cmd/soroban-rpc/internal/methods/get_fee_stats.go | 2 +- cmd/soroban-rpc/internal/methods/health.go | 2 +- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/cmd/soroban-rpc/internal/jsonrpc.go b/cmd/soroban-rpc/internal/jsonrpc.go index 3757c4fb..00cbd820 100644 --- a/cmd/soroban-rpc/internal/jsonrpc.go +++ b/cmd/soroban-rpc/internal/jsonrpc.go @@ -143,11 +143,9 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler { } // Get the largest history window - var ledgerRangeGetter db.LedgerRangeGetter = params.EventStore retentionWindow := cfg.EventLedgerRetentionWindow if cfg.TransactionLedgerRetentionWindow > cfg.EventLedgerRetentionWindow { retentionWindow = cfg.TransactionLedgerRetentionWindow - ledgerRangeGetter = params.TransactionReader } handlers := []struct { @@ -160,7 +158,7 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler { { methodName: "getHealth", underlyingHandler: methods.NewHealthCheck( - retentionWindow, ledgerRangeGetter, cfg.MaxHealthyLedgerLatency), + retentionWindow, params.LedgerReader, cfg.MaxHealthyLedgerLatency), longName: "get_health", queueLimit: cfg.RequestBacklogGetHealthQueueLimit, requestDurationLimit: cfg.MaxGetHealthExecutionDuration, @@ -243,7 +241,7 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler { }, { methodName: "getFeeStats", - underlyingHandler: methods.NewGetFeeStatsHandler(params.FeeStatWindows, ledgerRangeGetter, params.Logger), + underlyingHandler: methods.NewGetFeeStatsHandler(params.FeeStatWindows, params.LedgerReader, params.Logger), longName: "get_fee_stats", queueLimit: cfg.RequestBacklogGetFeeStatsTransactionQueueLimit, requestDurationLimit: cfg.MaxGetFeeStatsExecutionDuration, diff --git a/cmd/soroban-rpc/internal/methods/get_fee_stats.go b/cmd/soroban-rpc/internal/methods/get_fee_stats.go index 89e29a16..b6c0107e 100644 --- a/cmd/soroban-rpc/internal/methods/get_fee_stats.go +++ b/cmd/soroban-rpc/internal/methods/get_fee_stats.go @@ -58,7 +58,7 @@ type GetFeeStatsResult struct { } // NewGetFeeStatsHandler returns a handler obtaining fee statistics -func NewGetFeeStatsHandler(windows *feewindow.FeeWindows, reader db.LedgerRangeGetter, +func NewGetFeeStatsHandler(windows *feewindow.FeeWindows, reader db.LedgerReader, logger *log.Entry, ) jrpc2.Handler { return NewHandler(func(ctx context.Context) (GetFeeStatsResult, error) { diff --git a/cmd/soroban-rpc/internal/methods/health.go b/cmd/soroban-rpc/internal/methods/health.go index 9feca178..c0cee2e9 100644 --- a/cmd/soroban-rpc/internal/methods/health.go +++ b/cmd/soroban-rpc/internal/methods/health.go @@ -20,7 +20,7 @@ type HealthCheckResult struct { // NewHealthCheck returns a health check json rpc handler func NewHealthCheck( retentionWindow uint32, - reader db.LedgerRangeGetter, + reader db.LedgerReader, maxHealthyLedgerLatency time.Duration, ) jrpc2.Handler { return NewHandler(func(ctx context.Context) (HealthCheckResult, error) { From f1e17952e229f8788a09d8c3d92c1c618497b6bd Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Wed, 26 Jun 2024 16:57:55 -0400 Subject: [PATCH 54/68] Add GetLedgerRange to ConstantLedgerReader --- cmd/soroban-rpc/internal/db/mocks.go | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/mocks.go b/cmd/soroban-rpc/internal/db/mocks.go index e9f3644b..f70ac997 100644 --- a/cmd/soroban-rpc/internal/db/mocks.go +++ b/cmd/soroban-rpc/internal/db/mocks.go @@ -65,20 +65,15 @@ func (txn *mockTransactionHandler) InsertTransactions(lcm xdr.LedgerCloseMeta) e return nil } -// GetLedgerRange pulls the min/max ledger sequence numbers from the database. -func (txn *mockTransactionHandler) GetLedgerRange(_ context.Context) (ledgerbucketwindow.LedgerRange, error) { - return txn.ledgerRange, nil -} - func (txn *mockTransactionHandler) GetTransaction(_ context.Context, hash xdr.Hash) ( - Transaction, ledgerbucketwindow.LedgerRange, error, + Transaction, error, ) { tx, ok := txn.txs[hash.HexString()] if !ok { - return Transaction{}, txn.ledgerRange, ErrNoTransaction + return Transaction{}, ErrNoTransaction } itx, err := ParseTransaction(*txn.txHashToMeta[hash.HexString()], tx) - return itx, txn.ledgerRange, err + return itx, err } func (txn *mockTransactionHandler) RegisterMetrics(_, _ prometheus.Observer) {} @@ -87,10 +82,6 @@ type mockLedgerReader struct { txn mockTransactionHandler } -func (m *mockLedgerReader) GetLedgerRange(_ context.Context) (ledgerbucketwindow.LedgerRange, error) { - return ledgerbucketwindow.LedgerRange{}, nil -} - func NewMockLedgerReader(txn *mockTransactionHandler) *mockLedgerReader { return &mockLedgerReader{ txn: *txn, @@ -109,6 +100,10 @@ func (m *mockLedgerReader) StreamAllLedgers(_ context.Context, _ StreamLedgerFn) return nil } +func (m *mockLedgerReader) GetLedgerRange(_ context.Context) (ledgerbucketwindow.LedgerRange, error) { + return ledgerbucketwindow.LedgerRange{}, nil +} + var ( _ TransactionReader = &mockTransactionHandler{} _ TransactionWriter = &mockTransactionHandler{} From 61a5b88a8076905ad3c9f1b10a6b6406f8cb5897 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Thu, 27 Jun 2024 11:48:41 -0400 Subject: [PATCH 55/68] Remove GetLedgerRange from transactions code and use the meta table one --- cmd/soroban-rpc/internal/db/db.go | 5 -- cmd/soroban-rpc/internal/db/mocks.go | 39 ++++++++------- cmd/soroban-rpc/internal/db/transaction.go | 50 ++----------------- .../internal/db/transaction_test.go | 18 +++---- cmd/soroban-rpc/internal/jsonrpc.go | 6 +-- .../internal/methods/get_fee_stats.go | 4 +- .../methods/get_latest_ledger_test.go | 5 ++ .../internal/methods/get_transaction.go | 17 +++++-- .../internal/methods/get_transaction_test.go | 22 ++++---- .../internal/methods/get_transactions.go | 14 +++--- .../internal/methods/get_transactions_test.go | 10 +--- cmd/soroban-rpc/internal/methods/health.go | 4 +- .../internal/methods/send_transaction.go | 4 +- 13 files changed, 80 insertions(+), 118 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/db.go b/cmd/soroban-rpc/internal/db/db.go index f905704a..fedd1b93 100644 --- a/cmd/soroban-rpc/internal/db/db.go +++ b/cmd/soroban-rpc/internal/db/db.go @@ -19,7 +19,6 @@ import ( "github.com/stellar/go/xdr" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/daemon/interfaces" - "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" ) //go:embed sqlmigrations/*.sql @@ -32,10 +31,6 @@ const ( latestLedgerSequenceMetaKey = "LatestLedgerSequence" ) -type LedgerRangeGetter interface { - GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) -} - type ReadWriter interface { NewTx(ctx context.Context) (WriteTx, error) GetLatestLedgerSequence(ctx context.Context) (uint32, error) diff --git a/cmd/soroban-rpc/internal/db/mocks.go b/cmd/soroban-rpc/internal/db/mocks.go index f70ac997..492d64bb 100644 --- a/cmd/soroban-rpc/internal/db/mocks.go +++ b/cmd/soroban-rpc/internal/db/mocks.go @@ -2,6 +2,7 @@ package db import ( "context" + "errors" "io" "github.com/prometheus/client_golang/prometheus" @@ -12,7 +13,7 @@ import ( "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" ) -type mockTransactionHandler struct { +type MockTransactionHandler struct { passphrase string ledgerRange ledgerbucketwindow.LedgerRange @@ -21,8 +22,8 @@ type mockTransactionHandler struct { ledgerSeqToMeta map[uint32]*xdr.LedgerCloseMeta } -func NewMockTransactionStore(passphrase string) *mockTransactionHandler { - return &mockTransactionHandler{ +func NewMockTransactionStore(passphrase string) *MockTransactionHandler { + return &MockTransactionHandler{ passphrase: passphrase, txs: make(map[string]ingest.LedgerTransaction), txHashToMeta: make(map[string]*xdr.LedgerCloseMeta), @@ -30,7 +31,7 @@ func NewMockTransactionStore(passphrase string) *mockTransactionHandler { } } -func (txn *mockTransactionHandler) InsertTransactions(lcm xdr.LedgerCloseMeta) error { +func (txn *MockTransactionHandler) InsertTransactions(lcm xdr.LedgerCloseMeta) error { txn.ledgerSeqToMeta[lcm.LedgerSequence()] = &lcm reader, err := ingest.NewLedgerTransactionReaderFromLedgerCloseMeta(txn.passphrase, lcm) @@ -40,7 +41,7 @@ func (txn *mockTransactionHandler) InsertTransactions(lcm xdr.LedgerCloseMeta) e for { tx, err := reader.Read() - if err == io.EOF { + if errors.Is(err, io.EOF) { break } else if err != nil { return err @@ -65,7 +66,7 @@ func (txn *mockTransactionHandler) InsertTransactions(lcm xdr.LedgerCloseMeta) e return nil } -func (txn *mockTransactionHandler) GetTransaction(_ context.Context, hash xdr.Hash) ( +func (txn *MockTransactionHandler) GetTransaction(_ context.Context, hash xdr.Hash) ( Transaction, error, ) { tx, ok := txn.txs[hash.HexString()] @@ -76,19 +77,19 @@ func (txn *mockTransactionHandler) GetTransaction(_ context.Context, hash xdr.Ha return itx, err } -func (txn *mockTransactionHandler) RegisterMetrics(_, _ prometheus.Observer) {} +func (txn *MockTransactionHandler) RegisterMetrics(_, _ prometheus.Observer) {} -type mockLedgerReader struct { - txn mockTransactionHandler +type MockLedgerReader struct { + txn *MockTransactionHandler } -func NewMockLedgerReader(txn *mockTransactionHandler) *mockLedgerReader { - return &mockLedgerReader{ - txn: *txn, +func NewMockLedgerReader(txn *MockTransactionHandler) *MockLedgerReader { + return &MockLedgerReader{ + txn: txn, } } -func (m *mockLedgerReader) GetLedger(_ context.Context, sequence uint32) (xdr.LedgerCloseMeta, bool, error) { +func (m *MockLedgerReader) GetLedger(_ context.Context, sequence uint32) (xdr.LedgerCloseMeta, bool, error) { lcm, ok := m.txn.ledgerSeqToMeta[sequence] if !ok { return xdr.LedgerCloseMeta{}, false, nil @@ -96,16 +97,16 @@ func (m *mockLedgerReader) GetLedger(_ context.Context, sequence uint32) (xdr.Le return *lcm, true, nil } -func (m *mockLedgerReader) StreamAllLedgers(_ context.Context, _ StreamLedgerFn) error { +func (m *MockLedgerReader) StreamAllLedgers(_ context.Context, _ StreamLedgerFn) error { return nil } -func (m *mockLedgerReader) GetLedgerRange(_ context.Context) (ledgerbucketwindow.LedgerRange, error) { - return ledgerbucketwindow.LedgerRange{}, nil +func (m *MockLedgerReader) GetLedgerRange(_ context.Context) (ledgerbucketwindow.LedgerRange, error) { + return m.txn.ledgerRange, nil } var ( - _ TransactionReader = &mockTransactionHandler{} - _ TransactionWriter = &mockTransactionHandler{} - _ LedgerReader = &mockLedgerReader{} + _ TransactionReader = &MockTransactionHandler{} + _ TransactionWriter = &MockTransactionHandler{} + _ LedgerReader = &MockLedgerReader{} ) diff --git a/cmd/soroban-rpc/internal/db/transaction.go b/cmd/soroban-rpc/internal/db/transaction.go index 35a96854..a51b6e34 100644 --- a/cmd/soroban-rpc/internal/db/transaction.go +++ b/cmd/soroban-rpc/internal/db/transaction.go @@ -45,8 +45,7 @@ type TransactionWriter interface { // TransactionReader provides all the public ways to read from the DB. type TransactionReader interface { - GetTransaction(ctx context.Context, hash xdr.Hash) (Transaction, ledgerbucketwindow.LedgerRange, error) - LedgerRangeGetter + GetTransaction(ctx context.Context, hash xdr.Hash) (Transaction, error) } type transactionHandler struct { @@ -139,40 +138,6 @@ func (txn *transactionHandler) trimTransactions(latestLedgerSeq uint32, retentio return err } -// GetLedgerRange pulls the min/max ledger sequence numbers from the transactions table. -func (txn *transactionHandler) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { - var ledgerRange ledgerbucketwindow.LedgerRange - - query := sq.Select("lcm.meta"). - FromSelect( - sq.Select("meta"). - From(ledgerCloseMetaTableName+" as lcm"). - Where(sq.Expr("lcm.sequence IN (SELECT MIN(ledger_sequence) FROM "+transactionTableName+ - " UNION ALL SELECT MAX(ledger_sequence) FROM "+transactionTableName+")")), - "lcm", - ) - - var lcms []xdr.LedgerCloseMeta - if err := txn.db.Select(ctx, &lcms, query); err != nil { - return ledgerRange, fmt.Errorf("couldn't query ledger range: %w", err) - } else if len(lcms) < getLedgerRangeMetaArrayCheckValue { - // There is almost certainly a row, but we want to avoid a race condition - // with ingestion as well as support test cases from an empty DB, so we need - // to sanity check that there is in fact a result. Note that no ledgers in - // the database isn't an error, it's just an empty range. - return ledgerRange, nil - } - - lcm1, lcm2 := lcms[0], lcms[1] - ledgerRange.FirstLedger.Sequence = lcm1.LedgerSequence() - ledgerRange.FirstLedger.CloseTime = lcm1.LedgerCloseTime() - ledgerRange.LastLedger.Sequence = lcm2.LedgerSequence() - ledgerRange.LastLedger.CloseTime = lcm2.LedgerCloseTime() - - txn.log.Debugf("Database ledger range: [%d, %d]", ledgerRange.FirstLedger.Sequence, ledgerRange.LastLedger.Sequence) - return ledgerRange, nil -} - // GetTransaction conforms to the interface in // methods/get_transaction.go#NewGetTransactionHandler so that it can be used // directly against the RPC handler. @@ -180,23 +145,18 @@ func (txn *transactionHandler) GetLedgerRange(ctx context.Context) (ledgerbucket // Errors occur if there are issues with the DB connection or the XDR is // corrupted somehow. If the transaction is not found, io.EOF is returned. func (txn *transactionHandler) GetTransaction(ctx context.Context, hash xdr.Hash) ( - Transaction, ledgerbucketwindow.LedgerRange, error, + Transaction, error, ) { start := time.Now() tx := Transaction{} - ledgerRange, err := txn.GetLedgerRange(ctx) - if err != nil && !errors.Is(err, ErrEmptyDB) { - return tx, ledgerRange, err - } - lcm, ingestTx, err := txn.getTransactionByHash(ctx, hash) if err != nil { - return tx, ledgerRange, err + return tx, err } tx, err = ParseTransaction(lcm, ingestTx) if err != nil { - return tx, ledgerRange, err + return tx, err } txn.log. @@ -204,7 +164,7 @@ func (txn *transactionHandler) GetTransaction(ctx context.Context, hash xdr.Hash WithField("duration", time.Since(start)). Debugf("Fetched and encoded transaction from ledger %d", lcm.LedgerSequence()) - return tx, ledgerRange, nil + return tx, nil } // getTransactionByHash actually performs the DB ops to cross-reference a diff --git a/cmd/soroban-rpc/internal/db/transaction_test.go b/cmd/soroban-rpc/internal/db/transaction_test.go index a4bf2342..e05671b9 100644 --- a/cmd/soroban-rpc/internal/db/transaction_test.go +++ b/cmd/soroban-rpc/internal/db/transaction_test.go @@ -23,8 +23,8 @@ func TestTransactionNotFound(t *testing.T) { log.SetLevel(logrus.TraceLevel) reader := NewTransactionReader(log, db, passphrase) - _, _, err := reader.GetTransaction(context.TODO(), xdr.Hash{}) - require.Error(t, err, ErrNoTransaction) + _, err := reader.GetTransaction(context.TODO(), xdr.Hash{}) + require.ErrorIs(t, err, ErrNoTransaction) } func TestTransactionFound(t *testing.T) { @@ -53,16 +53,14 @@ func TestTransactionFound(t *testing.T) { // check 404 case reader := NewTransactionReader(log, db, passphrase) - _, _, err = reader.GetTransaction(ctx, xdr.Hash{}) - require.Error(t, err, ErrNoTransaction) + _, err = reader.GetTransaction(ctx, xdr.Hash{}) + require.ErrorIs(t, err, ErrNoTransaction) // check all 200 cases for _, lcm := range lcms { h := lcm.TransactionHash(0) - tx, lRange, err := reader.GetTransaction(ctx, h) + tx, err := reader.GetTransaction(ctx, h) require.NoError(t, err, "failed to find txhash %s in db", hex.EncodeToString(h[:])) - assert.EqualValues(t, 1234+100, lRange.FirstLedger.Sequence) - assert.EqualValues(t, 1237+100, lRange.LastLedger.Sequence) assert.EqualValues(t, 1, tx.ApplicationOrder) expectedEnvelope, err := lcm.TransactionEnvelopes()[0].MarshalBinary() @@ -82,8 +80,8 @@ func BenchmarkTransactionFetch(b *testing.B) { // ingest 100k tx rows lcms := make([]xdr.LedgerCloseMeta, 0, 100_000) - for i := uint32(0); i < uint32(cap(lcms)); i++ { - lcms = append(lcms, txMeta(1234+i, i%2 == 0)) + for i := range cap(lcms) { + lcms = append(lcms, txMeta(uint32(1234+i), i%2 == 0)) } ledgerW, txW := write.LedgerWriter(), write.TransactionWriter() @@ -102,7 +100,7 @@ func BenchmarkTransactionFetch(b *testing.B) { b.ResetTimer() for i := 0; i < b.N; i++ { r := randoms[i] - tx, _, err := reader.GetTransaction(ctx, lcms[r].TransactionHash(0)) + tx, err := reader.GetTransaction(ctx, lcms[r].TransactionHash(0)) require.NoError(b, err) assert.Equal(b, r%2 == 0, tx.Successful) } diff --git a/cmd/soroban-rpc/internal/jsonrpc.go b/cmd/soroban-rpc/internal/jsonrpc.go index 00cbd820..8b9223cc 100644 --- a/cmd/soroban-rpc/internal/jsonrpc.go +++ b/cmd/soroban-rpc/internal/jsonrpc.go @@ -209,7 +209,7 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler { }, { methodName: "getTransaction", - underlyingHandler: methods.NewGetTransactionHandler(params.Logger, params.TransactionReader), + underlyingHandler: methods.NewGetTransactionHandler(params.Logger, params.TransactionReader, params.LedgerReader), longName: "get_transaction", queueLimit: cfg.RequestBacklogGetTransactionQueueLimit, requestDurationLimit: cfg.MaxGetTransactionExecutionDuration, @@ -217,7 +217,7 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler { { methodName: "getTransactions", underlyingHandler: methods.NewGetTransactionsHandler(params.Logger, params.LedgerReader, - params.TransactionReader, cfg.MaxTransactionsLimit, cfg.DefaultTransactionsLimit, cfg.NetworkPassphrase), + cfg.MaxTransactionsLimit, cfg.DefaultTransactionsLimit, cfg.NetworkPassphrase), longName: "get_transactions", queueLimit: cfg.RequestBacklogGetTransactionsQueueLimit, requestDurationLimit: cfg.MaxGetTransactionsExecutionDuration, @@ -225,7 +225,7 @@ func NewJSONRPCHandler(cfg *config.Config, params HandlerParams) Handler { { methodName: "sendTransaction", underlyingHandler: methods.NewSendTransactionHandler( - params.Daemon, params.Logger, params.TransactionReader, cfg.NetworkPassphrase), + params.Daemon, params.Logger, params.LedgerReader, cfg.NetworkPassphrase), longName: "send_transaction", queueLimit: cfg.RequestBacklogSendTransactionQueueLimit, requestDurationLimit: cfg.MaxSendTransactionExecutionDuration, diff --git a/cmd/soroban-rpc/internal/methods/get_fee_stats.go b/cmd/soroban-rpc/internal/methods/get_fee_stats.go index b6c0107e..9fd85d76 100644 --- a/cmd/soroban-rpc/internal/methods/get_fee_stats.go +++ b/cmd/soroban-rpc/internal/methods/get_fee_stats.go @@ -58,11 +58,11 @@ type GetFeeStatsResult struct { } // NewGetFeeStatsHandler returns a handler obtaining fee statistics -func NewGetFeeStatsHandler(windows *feewindow.FeeWindows, reader db.LedgerReader, +func NewGetFeeStatsHandler(windows *feewindow.FeeWindows, ledgerReader db.LedgerReader, logger *log.Entry, ) jrpc2.Handler { return NewHandler(func(ctx context.Context) (GetFeeStatsResult, error) { - ledgerRange, err := reader.GetLedgerRange(ctx) + ledgerRange, err := ledgerReader.GetLedgerRange(ctx) if err != nil { // still not fatal logger.WithError(err). Error("could not fetch ledger range") diff --git a/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go b/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go index 86a8e48a..c9a94eaa 100644 --- a/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go +++ b/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go @@ -10,6 +10,7 @@ import ( "github.com/stellar/go/xdr" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/db" + "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" ) const ( @@ -24,6 +25,10 @@ type ConstantLedgerEntryReaderTx struct{} type ConstantLedgerReader struct{} +func (ledgerReader *ConstantLedgerReader) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { + return ledgerbucketwindow.LedgerRange{}, nil +} + func (entryReader *ConstantLedgerEntryReader) GetLatestLedgerSequence(ctx context.Context) (uint32, error) { return expectedLatestLedgerSequence, nil } diff --git a/cmd/soroban-rpc/internal/methods/get_transaction.go b/cmd/soroban-rpc/internal/methods/get_transaction.go index 37ef5f3a..ac6ac8b2 100644 --- a/cmd/soroban-rpc/internal/methods/get_transaction.go +++ b/cmd/soroban-rpc/internal/methods/get_transaction.go @@ -72,6 +72,7 @@ func GetTransaction( ctx context.Context, log *log.Entry, reader db.TransactionReader, + ledgerReader db.LedgerReader, request GetTransactionRequest, ) (GetTransactionResponse, error) { // parse hash @@ -91,7 +92,15 @@ func GetTransaction( } } - tx, storeRange, err := reader.GetTransaction(ctx, txHash) + storeRange, err := ledgerReader.GetLedgerRange(ctx) + if err != nil { + return GetTransactionResponse{}, &jrpc2.Error{ + Code: jrpc2.InternalError, + Message: fmt.Sprintf("unable to get ledger range: %v", err), + } + } + + tx, err := reader.GetTransaction(ctx, txHash) response := GetTransactionResponse{ LatestLedger: storeRange.LastLedger.Sequence, @@ -130,8 +139,10 @@ func GetTransaction( } // NewGetTransactionHandler returns a get transaction json rpc handler -func NewGetTransactionHandler(logger *log.Entry, getter db.TransactionReader) jrpc2.Handler { +func NewGetTransactionHandler(logger *log.Entry, getter db.TransactionReader, + ledgerReader db.LedgerReader, +) jrpc2.Handler { return NewHandler(func(ctx context.Context, request GetTransactionRequest) (GetTransactionResponse, error) { - return GetTransaction(ctx, logger, getter, request) + return GetTransaction(ctx, logger, getter, ledgerReader, request) }) } diff --git a/cmd/soroban-rpc/internal/methods/get_transaction_test.go b/cmd/soroban-rpc/internal/methods/get_transaction_test.go index 65ceea29..1bc6f2b9 100644 --- a/cmd/soroban-rpc/internal/methods/get_transaction_test.go +++ b/cmd/soroban-rpc/internal/methods/get_transaction_test.go @@ -17,19 +17,21 @@ import ( func TestGetTransaction(t *testing.T) { var ( - ctx = context.TODO() - log = log.DefaultLogger - store = db.NewMockTransactionStore("passphrase") + ctx = context.TODO() + log = log.DefaultLogger + store = db.NewMockTransactionStore("passphrase") + ledgerReader = db.NewMockLedgerReader(store) ) log.SetLevel(logrus.DebugLevel) - _, err := GetTransaction(ctx, log, store, GetTransactionRequest{"ab"}) + _, err := GetTransaction(ctx, log, store, ledgerReader, GetTransactionRequest{"ab"}) require.EqualError(t, err, "[-32602] unexpected hash length (2)") - _, err = GetTransaction(ctx, log, store, GetTransactionRequest{"foo "}) + _, err = GetTransaction(ctx, log, store, ledgerReader, + GetTransactionRequest{"foo "}) require.EqualError(t, err, "[-32602] incorrect hash: encoding/hex: invalid byte: U+006F 'o'") hash := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" - tx, err := GetTransaction(ctx, log, store, GetTransactionRequest{hash}) + tx, err := GetTransaction(ctx, log, store, ledgerReader, GetTransactionRequest{hash}) require.NoError(t, err) require.Equal(t, GetTransactionResponse{Status: TransactionStatusNotFound}, tx) @@ -38,7 +40,7 @@ func TestGetTransaction(t *testing.T) { xdrHash := txHash(1) hash = hex.EncodeToString(xdrHash[:]) - tx, err = GetTransaction(ctx, log, store, GetTransactionRequest{hash}) + tx, err = GetTransaction(ctx, log, store, ledgerReader, GetTransactionRequest{hash}) require.NoError(t, err) expectedTxResult, err := xdr.MarshalBase64(meta.V1.TxProcessing[0].Result.Result) @@ -68,7 +70,7 @@ func TestGetTransaction(t *testing.T) { require.NoError(t, store.InsertTransactions(meta)) // the first transaction should still be there - tx, err = GetTransaction(ctx, log, store, GetTransactionRequest{hash}) + tx, err = GetTransaction(ctx, log, store, ledgerReader, GetTransactionRequest{hash}) require.NoError(t, err) require.Equal(t, GetTransactionResponse{ Status: TransactionStatusSuccess, @@ -97,7 +99,7 @@ func TestGetTransaction(t *testing.T) { expectedTxMeta, err = xdr.MarshalBase64(meta.V1.TxProcessing[0].TxApplyProcessing) require.NoError(t, err) - tx, err = GetTransaction(ctx, log, store, GetTransactionRequest{hash}) + tx, err = GetTransaction(ctx, log, store, ledgerReader, GetTransactionRequest{hash}) require.NoError(t, err) require.Equal(t, GetTransactionResponse{ Status: TransactionStatusFailed, @@ -134,7 +136,7 @@ func TestGetTransaction(t *testing.T) { expectedEventsMeta, err := xdr.MarshalBase64(diagnosticEvents[0]) require.NoError(t, err) - tx, err = GetTransaction(ctx, log, store, GetTransactionRequest{hash}) + tx, err = GetTransaction(ctx, log, store, ledgerReader, GetTransactionRequest{hash}) require.NoError(t, err) require.Equal(t, GetTransactionResponse{ Status: TransactionStatusSuccess, diff --git a/cmd/soroban-rpc/internal/methods/get_transactions.go b/cmd/soroban-rpc/internal/methods/get_transactions.go index 52408c38..1d16f259 100644 --- a/cmd/soroban-rpc/internal/methods/get_transactions.go +++ b/cmd/soroban-rpc/internal/methods/get_transactions.go @@ -87,7 +87,6 @@ type GetTransactionsResponse struct { type transactionsRPCHandler struct { ledgerReader db.LedgerReader - dbReader db.TransactionReader maxLimit uint defaultLimit uint logger *log.Entry @@ -97,7 +96,7 @@ type transactionsRPCHandler struct { // getTransactionsByLedgerSequence fetches transactions between the start and end ledgers, inclusive of both. // The number of ledgers returned can be tuned using the pagination options - cursor and limit. func (h transactionsRPCHandler) getTransactionsByLedgerSequence(ctx context.Context, request GetTransactionsRequest) (GetTransactionsResponse, error) { - ledgerRange, err := h.dbReader.GetLedgerRange(ctx) + ledgerRange, err := h.ledgerReader.GetLedgerRange(ctx) if err != nil { return GetTransactionsResponse{}, &jrpc2.Error{ Code: jrpc2.InternalError, @@ -184,7 +183,7 @@ LedgerLoop: ingestTx, err := reader.Read() if err != nil { - if err == io.EOF { + if errors.Is(err, io.EOF) { // No more transactions to read. Start from next ledger break } @@ -234,17 +233,16 @@ LedgerLoop: }, nil } -func NewGetTransactionsHandler(logger *log.Entry, ledgerReader db.LedgerReader, dbReader db.TransactionReader, maxLimit, defaultLimit uint, networkPassphrase string) jrpc2.Handler { +func NewGetTransactionsHandler(logger *log.Entry, ledgerReader db.LedgerReader, maxLimit, + defaultLimit uint, networkPassphrase string, +) jrpc2.Handler { transactionsHandler := transactionsRPCHandler{ ledgerReader: ledgerReader, - dbReader: dbReader, maxLimit: maxLimit, defaultLimit: defaultLimit, logger: logger, networkPassphrase: networkPassphrase, } - return handler.New(func(context context.Context, request GetTransactionsRequest) (GetTransactionsResponse, error) { - return transactionsHandler.getTransactionsByLedgerSequence(context, request) - }) + return handler.New(transactionsHandler.getTransactionsByLedgerSequence) } diff --git a/cmd/soroban-rpc/internal/methods/get_transactions_test.go b/cmd/soroban-rpc/internal/methods/get_transactions_test.go index b02cc750..78021f86 100644 --- a/cmd/soroban-rpc/internal/methods/get_transactions_test.go +++ b/cmd/soroban-rpc/internal/methods/get_transactions_test.go @@ -20,7 +20,7 @@ const ( // createTestLedger Creates a test ledger with 2 transactions func createTestLedger(sequence uint32) xdr.LedgerCloseMeta { - sequence = sequence - 100 + sequence -= 100 meta := txMeta(sequence, true) meta.V1.TxProcessing = append(meta.V1.TxProcessing, xdr.TransactionResultMeta{ TxApplyProcessing: xdr.TransactionMeta{ @@ -47,7 +47,6 @@ func TestGetTransactions_DefaultLimit(t *testing.T) { handler := transactionsRPCHandler{ ledgerReader: mockLedgerReader, - dbReader: mockDbReader, maxLimit: 100, defaultLimit: 10, networkPassphrase: NetworkPassphrase, @@ -82,7 +81,6 @@ func TestGetTransactions_DefaultLimitExceedsLatestLedger(t *testing.T) { handler := transactionsRPCHandler{ ledgerReader: mockLedgerReader, - dbReader: mockDBReader, maxLimit: 100, defaultLimit: 10, networkPassphrase: NetworkPassphrase, @@ -117,7 +115,6 @@ func TestGetTransactions_CustomLimit(t *testing.T) { handler := transactionsRPCHandler{ ledgerReader: mockLedgerReader, - dbReader: mockDBReader, maxLimit: 100, defaultLimit: 10, networkPassphrase: NetworkPassphrase, @@ -157,7 +154,6 @@ func TestGetTransactions_CustomLimitAndCursor(t *testing.T) { handler := transactionsRPCHandler{ ledgerReader: mockLedgerReader, - dbReader: mockDBReader, maxLimit: 100, defaultLimit: 10, networkPassphrase: NetworkPassphrase, @@ -198,7 +194,6 @@ func TestGetTransactions_InvalidStartLedger(t *testing.T) { handler := transactionsRPCHandler{ ledgerReader: mockLedgerReader, - dbReader: mockDbReader, maxLimit: 100, defaultLimit: 10, networkPassphrase: NetworkPassphrase, @@ -232,7 +227,6 @@ func TestGetTransactions_LedgerNotFound(t *testing.T) { handler := transactionsRPCHandler{ ledgerReader: mockLedgerReader, - dbReader: mockDbReader, maxLimit: 100, defaultLimit: 10, networkPassphrase: NetworkPassphrase, @@ -259,7 +253,6 @@ func TestGetTransactions_LimitGreaterThanMaxLimit(t *testing.T) { handler := transactionsRPCHandler{ ledgerReader: mockLedgerReader, - dbReader: mockDbReader, maxLimit: 100, defaultLimit: 10, networkPassphrase: NetworkPassphrase, @@ -288,7 +281,6 @@ func TestGetTransactions_InvalidCursorString(t *testing.T) { handler := transactionsRPCHandler{ ledgerReader: mockLedgerReader, - dbReader: mockDbReader, maxLimit: 100, defaultLimit: 10, networkPassphrase: NetworkPassphrase, diff --git a/cmd/soroban-rpc/internal/methods/health.go b/cmd/soroban-rpc/internal/methods/health.go index c0cee2e9..8de0767c 100644 --- a/cmd/soroban-rpc/internal/methods/health.go +++ b/cmd/soroban-rpc/internal/methods/health.go @@ -20,11 +20,11 @@ type HealthCheckResult struct { // NewHealthCheck returns a health check json rpc handler func NewHealthCheck( retentionWindow uint32, - reader db.LedgerReader, + ledgerReader db.LedgerReader, maxHealthyLedgerLatency time.Duration, ) jrpc2.Handler { return NewHandler(func(ctx context.Context) (HealthCheckResult, error) { - ledgerRange, err := reader.GetLedgerRange(ctx) + ledgerRange, err := ledgerReader.GetLedgerRange(ctx) if err != nil || ledgerRange.LastLedger.Sequence < 1 { extra := "" if err != nil { diff --git a/cmd/soroban-rpc/internal/methods/send_transaction.go b/cmd/soroban-rpc/internal/methods/send_transaction.go index 8bd92b07..9cdd6a20 100644 --- a/cmd/soroban-rpc/internal/methods/send_transaction.go +++ b/cmd/soroban-rpc/internal/methods/send_transaction.go @@ -49,7 +49,7 @@ type SendTransactionRequest struct { func NewSendTransactionHandler( daemon interfaces.Daemon, logger *log.Entry, - reader db.TransactionReader, + ledgerReader db.LedgerReader, passphrase string, ) jrpc2.Handler { submitter := daemon.CoreClient() @@ -73,7 +73,7 @@ func NewSendTransactionHandler( } txHash := hex.EncodeToString(hash[:]) - ledgerInfo, err := reader.GetLedgerRange(ctx) + ledgerInfo, err := ledgerReader.GetLedgerRange(ctx) if err != nil { // still not fatal logger.WithError(err). WithField("tx", request.Transaction). From 1bf294c9f7d6e5d0947e876ae8ced9b8931c6bc8 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Thu, 27 Jun 2024 12:13:50 -0400 Subject: [PATCH 56/68] Remove unnecessary file changes --- cmd/soroban-rpc/internal/db/db.go | 16 ++- cmd/soroban-rpc/internal/db/migration.go | 8 +- cmd/soroban-rpc/internal/events/events.go | 30 ++---- .../integrationtest/infrastructure/util.go | 102 ------------------ 4 files changed, 15 insertions(+), 141 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/db.go b/cmd/soroban-rpc/internal/db/db.go index fedd1b93..1767f610 100644 --- a/cmd/soroban-rpc/internal/db/db.go +++ b/cmd/soroban-rpc/internal/db/db.go @@ -61,8 +61,7 @@ func openSQLiteDB(dbFilePath string) (*db.Session, error) { // 2. Disable WAL auto-checkpointing (we will do the checkpointing ourselves with wal_checkpoint pragmas // after every write transaction). // 3. Use synchronous=NORMAL, which is faster and still safe in WAL mode. - session, err := db.Open("sqlite3", - fmt.Sprintf("file:%s?_journal_mode=WAL&_wal_autocheckpoint=0&_synchronous=NORMAL", dbFilePath)) + session, err := db.Open("sqlite3", fmt.Sprintf("file:%s?_journal_mode=WAL&_wal_autocheckpoint=0&_synchronous=NORMAL", dbFilePath)) if err != nil { return nil, fmt.Errorf("open failed: %w", err) } @@ -74,9 +73,7 @@ func openSQLiteDB(dbFilePath string) (*db.Session, error) { return session, nil } -func OpenSQLiteDBWithPrometheusMetrics(dbFilePath string, namespace string, sub db.Subservice, - registry *prometheus.Registry, -) (*DB, error) { +func OpenSQLiteDBWithPrometheusMetrics(dbFilePath string, namespace string, sub db.Subservice, registry *prometheus.Registry) (*DB, error) { session, err := openSQLiteDB(dbFilePath) if err != nil { return nil, err @@ -131,8 +128,7 @@ func getMetaValue(ctx context.Context, q db.SessionInterface, key string) (strin case 1: // expected length on an initialized DB default: - return "", fmt.Errorf("multiple entries (%d) for key %q in table %q", len(results), - latestLedgerSequenceMetaKey, metaTableName) + return "", fmt.Errorf("multiple entries (%d) for key %q in table %q", len(results), latestLedgerSequenceMetaKey, metaTableName) } return results[0], nil } @@ -329,11 +325,11 @@ func (w writeTx) Rollback() error { // errors.New("not in transaction") is returned when rolling back a transaction which has // already been committed or rolled back. We can ignore those errors // because we allow rolling back after commits in defer statements. - err := w.tx.Rollback() - if err == nil || err.Error() == "not in transaction" { + if err := w.tx.Rollback(); err == nil || err.Error() == "not in transaction" { return nil + } else { + return err } - return err } func runSQLMigrations(db *sql.DB, dialect string) error { diff --git a/cmd/soroban-rpc/internal/db/migration.go b/cmd/soroban-rpc/internal/db/migration.go index d2fad06d..dbe07ab5 100644 --- a/cmd/soroban-rpc/internal/db/migration.go +++ b/cmd/soroban-rpc/internal/db/migration.go @@ -117,9 +117,7 @@ type guardedMigration struct { alreadyMigrated bool } -func newGuardedDataMigration(ctx context.Context, uniqueMigrationName string, - factory migrationApplierFactory, db *DB, -) (Migration, error) { +func newGuardedDataMigration(ctx context.Context, uniqueMigrationName string, factory migrationApplierFactory, db *DB) (Migration, error) { migrationDB := &DB{ cache: db.cache, SessionInterface: db.SessionInterface.Clone(), @@ -134,7 +132,7 @@ func newGuardedDataMigration(ctx context.Context, uniqueMigrationName string, return nil, err } latestLedger, err := NewLedgerEntryReader(db).GetLatestLedgerSequence(ctx) - if err != nil && !errors.Is(err, ErrEmptyDB) { + if err != nil && err != ErrEmptyDB { err = errors.Join(err, migrationDB.Rollback()) return nil, fmt.Errorf("failed to get latest ledger sequence: %w", err) } @@ -179,7 +177,7 @@ func (g *guardedMigration) Commit(ctx context.Context) error { return g.db.Commit() } -func (g *guardedMigration) Rollback(_ context.Context) error { +func (g *guardedMigration) Rollback(ctx context.Context) error { return g.db.Rollback() } diff --git a/cmd/soroban-rpc/internal/events/events.go b/cmd/soroban-rpc/internal/events/events.go index 2ac071e5..d9c3aa65 100644 --- a/cmd/soroban-rpc/internal/events/events.go +++ b/cmd/soroban-rpc/internal/events/events.go @@ -1,7 +1,6 @@ package events import ( - "context" "errors" "io" "sort" @@ -17,15 +16,6 @@ import ( "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" ) -const ( - QuantileKey1 = 0.5 - QuantileValue1 = 0.05 - QuantileKey2 = 0.9 - QuantileValue2 = 0.01 - QuantileKey3 = 0.99 - QuantileValue3 = 0.001 -) - type event struct { diagnosticEventXDR []byte txIndex uint32 @@ -70,24 +60,16 @@ func NewMemoryStore(daemon interfaces.Daemon, networkPassphrase string, retentio // eventsDurationMetric is a metric for measuring latency of event store operations eventsDurationMetric := prometheus.NewSummaryVec(prometheus.SummaryOpts{ Namespace: daemon.MetricsNamespace(), Subsystem: "events", Name: "operation_duration_seconds", - Help: "event store operation durations, sliding window = 10m", - Objectives: map[float64]float64{ - QuantileKey1: QuantileValue1, - QuantileKey2: QuantileValue2, - QuantileKey3: QuantileValue3, - }, + Help: "event store operation durations, sliding window = 10m", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, }, []string{"operation"}, ) eventCountMetric := prometheus.NewSummary(prometheus.SummaryOpts{ Namespace: daemon.MetricsNamespace(), Subsystem: "events", Name: "count", - Help: "count of events ingested, sliding window = 10m", - Objectives: map[float64]float64{ - QuantileKey1: QuantileValue1, - QuantileKey2: QuantileValue2, - QuantileKey3: QuantileValue3, - }, + Help: "count of events ingested, sliding window = 10m", + Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, }) daemon.MetricsRegistry().MustRegister(eventCountMetric, eventsDurationMetric) return &MemoryStore{ @@ -254,7 +236,7 @@ func readEvents(networkPassphrase string, ledgerCloseMeta xdr.LedgerCloseMeta) ( for { var tx ingest.LedgerTransaction tx, err = txReader.Read() - if errors.Is(err, io.EOF) { + if err == io.EOF { err = nil break } @@ -288,7 +270,7 @@ func readEvents(networkPassphrase string, ledgerCloseMeta xdr.LedgerCloseMeta) ( } // GetLedgerRange returns the first and latest ledger available in the store. -func (m *MemoryStore) GetLedgerRange(_ context.Context) (ledgerbucketwindow.LedgerRange, error) { +func (m *MemoryStore) GetLedgerRange() (ledgerbucketwindow.LedgerRange, error) { m.lock.RLock() defer m.lock.RUnlock() return m.eventsByLedger.GetLedgerRange(), nil diff --git a/cmd/soroban-rpc/internal/integrationtest/infrastructure/util.go b/cmd/soroban-rpc/internal/integrationtest/infrastructure/util.go index db88fa96..1a34325a 100644 --- a/cmd/soroban-rpc/internal/integrationtest/infrastructure/util.go +++ b/cmd/soroban-rpc/internal/integrationtest/infrastructure/util.go @@ -7,16 +7,7 @@ import ( "github.com/stretchr/testify/require" - "github.com/stellar/go/network" "github.com/stellar/go/txnbuild" - "github.com/stellar/go/xdr" -) - -const ( - txMetaV = 3 - ledgerCloseTimeConstant = 100 - feeCharged = 100 - passphrase = network.FutureNetworkPassphrase ) //go:noinline @@ -36,99 +27,6 @@ func getFreeTCPPort(t require.TestingT) uint16 { return uint16(l.Addr().(*net.TCPAddr).Port) } -func txHash(acctSeq uint32) xdr.Hash { - envelope := txEnvelope(acctSeq) - hash, err := network.HashTransactionInEnvelope(envelope, passphrase) - if err != nil { - panic(err) - } - return hash -} - -func txEnvelope(acctSeq uint32) xdr.TransactionEnvelope { - envelope, err := xdr.NewTransactionEnvelope(xdr.EnvelopeTypeEnvelopeTypeTx, xdr.TransactionV1Envelope{ - Tx: xdr.Transaction{ - Fee: feeCharged, - SeqNum: xdr.SequenceNumber(acctSeq), - SourceAccount: xdr.MustMuxedAddress("MA7QYNF7SOWQ3GLR2BGMZEHXAVIRZA4KVWLTJJFC7MGXUA74P7UJVAAAAAAAAAAAAAJLK"), - }, - }) - if err != nil { - panic(err) - } - return envelope -} - -func transactionResult(successful bool) xdr.TransactionResult { - code := xdr.TransactionResultCodeTxBadSeq - if successful { - code = xdr.TransactionResultCodeTxSuccess - } - opResults := []xdr.OperationResult{} - return xdr.TransactionResult{ - FeeCharged: feeCharged, - Result: xdr.TransactionResultResult{ - Code: code, - Results: &opResults, - }, - } -} - -func txMeta(acctSeq uint32, successful bool) xdr.LedgerCloseMeta { - envelope := txEnvelope(acctSeq) - txProcessing := []xdr.TransactionResultMeta{ - { - TxApplyProcessing: xdr.TransactionMeta{ - V: txMetaV, - Operations: &[]xdr.OperationMeta{}, - V3: &xdr.TransactionMetaV3{}, - }, - Result: xdr.TransactionResultPair{ - TransactionHash: txHash(acctSeq), - Result: transactionResult(successful), - }, - }, - } - components := []xdr.TxSetComponent{ - { - Type: xdr.TxSetComponentTypeTxsetCompTxsMaybeDiscountedFee, - TxsMaybeDiscountedFee: &xdr.TxSetComponentTxsMaybeDiscountedFee{ - BaseFee: nil, - Txs: []xdr.TransactionEnvelope{envelope}, - }, - }, - } - - return xdr.LedgerCloseMeta{ - V: 1, - V1: &xdr.LedgerCloseMetaV1{ - LedgerHeader: xdr.LedgerHeaderHistoryEntry{ - Header: xdr.LedgerHeader{ - ScpValue: xdr.StellarValue{ - CloseTime: xdr.TimePoint(ledgerCloseTime(acctSeq)), - }, - LedgerSeq: xdr.Uint32(acctSeq), - }, - }, - TxProcessing: txProcessing, - TxSet: xdr.GeneralizedTransactionSet{ - V: 1, - V1TxSet: &xdr.TransactionSetV1{ - PreviousLedgerHash: xdr.Hash{1}, - Phases: []xdr.TransactionPhase{{ - V: 0, - V0Components: &components, - }}, - }, - }, - }, - } -} - -func ledgerCloseTime(ledgerSequence uint32) int64 { - return int64(ledgerSequence)*25 + ledgerCloseTimeConstant -} - func CreateTransactionParams(account txnbuild.Account, op txnbuild.Operation) txnbuild.TransactionParams { return txnbuild.TransactionParams{ SourceAccount: account, From 8ef9f0205288f2e23c0c6e8847f39651eb6024fe Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Thu, 27 Jun 2024 12:24:18 -0400 Subject: [PATCH 57/68] Fix linting errors --- .../internal/methods/get_latest_ledger_test.go | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go b/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go index c9a94eaa..d080003a 100644 --- a/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go +++ b/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go @@ -25,19 +25,19 @@ type ConstantLedgerEntryReaderTx struct{} type ConstantLedgerReader struct{} -func (ledgerReader *ConstantLedgerReader) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { +func (ledgerReader *ConstantLedgerReader) GetLedgerRange(_ context.Context) (ledgerbucketwindow.LedgerRange, error) { return ledgerbucketwindow.LedgerRange{}, nil } -func (entryReader *ConstantLedgerEntryReader) GetLatestLedgerSequence(ctx context.Context) (uint32, error) { +func (entryReader *ConstantLedgerEntryReader) GetLatestLedgerSequence(_ context.Context) (uint32, error) { return expectedLatestLedgerSequence, nil } -func (entryReader *ConstantLedgerEntryReader) NewTx(ctx context.Context) (db.LedgerEntryReadTx, error) { +func (entryReader *ConstantLedgerEntryReader) NewTx(_ context.Context) (db.LedgerEntryReadTx, error) { return ConstantLedgerEntryReaderTx{}, nil } -func (entryReader *ConstantLedgerEntryReader) NewCachedTx(ctx context.Context) (db.LedgerEntryReadTx, error) { +func (entryReader *ConstantLedgerEntryReader) NewCachedTx(_ context.Context) (db.LedgerEntryReadTx, error) { return ConstantLedgerEntryReaderTx{}, nil } @@ -45,7 +45,7 @@ func (entryReaderTx ConstantLedgerEntryReaderTx) GetLatestLedgerSequence() (uint return expectedLatestLedgerSequence, nil } -func (entryReaderTx ConstantLedgerEntryReaderTx) GetLedgerEntries(keys ...xdr.LedgerKey) ([]db.LedgerKeyAndEntry, error) { +func (entryReaderTx ConstantLedgerEntryReaderTx) GetLedgerEntries(_ ...xdr.LedgerKey) ([]db.LedgerKeyAndEntry, error) { return nil, nil } @@ -53,11 +53,11 @@ func (entryReaderTx ConstantLedgerEntryReaderTx) Done() error { return nil } -func (ledgerReader *ConstantLedgerReader) GetLedger(ctx context.Context, sequence uint32) (xdr.LedgerCloseMeta, bool, error) { +func (ledgerReader *ConstantLedgerReader) GetLedger(_ context.Context, sequence uint32) (xdr.LedgerCloseMeta, bool, error) { return createLedger(sequence, expectedLatestLedgerProtocolVersion, expectedLatestLedgerHashBytes), true, nil } -func (ledgerReader *ConstantLedgerReader) StreamAllLedgers(ctx context.Context, f db.StreamLedgerFn) error { +func (ledgerReader *ConstantLedgerReader) StreamAllLedgers(_ context.Context, f db.StreamLedgerFn) error { return nil } From 7293e6c9724c5e09667d7e6a9887acf4f39907a1 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Thu, 27 Jun 2024 12:25:39 -0400 Subject: [PATCH 58/68] Fix linting errors - 2 --- cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go b/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go index d080003a..f2ebcc42 100644 --- a/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go +++ b/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go @@ -6,6 +6,7 @@ import ( "github.com/creachadair/jrpc2" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/stellar/go/xdr" @@ -80,7 +81,7 @@ func TestGetLatestLedger(t *testing.T) { getLatestLedgerHandler := NewGetLatestLedgerHandler(&ConstantLedgerEntryReader{}, &ConstantLedgerReader{}) latestLedgerRespI, err := getLatestLedgerHandler(context.Background(), &jrpc2.Request{}) latestLedgerResp := latestLedgerRespI.(GetLatestLedgerResponse) - assert.NoError(t, err) + require.NoError(t, err) expectedLatestLedgerHashStr := xdr.Hash{expectedLatestLedgerHashBytes}.HexString() assert.Equal(t, expectedLatestLedgerHashStr, latestLedgerResp.Hash) From 9ea2af21096fe13ecca087fdc10840b85dead27a Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Thu, 27 Jun 2024 12:30:15 -0400 Subject: [PATCH 59/68] Fix linting errors - 3 --- cmd/soroban-rpc/internal/db/ledger_test.go | 34 +++++++++---------- .../methods/get_latest_ledger_test.go | 6 ++-- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/ledger_test.go b/cmd/soroban-rpc/internal/db/ledger_test.go index 1955e0a8..03208498 100644 --- a/cmd/soroban-rpc/internal/db/ledger_test.go +++ b/cmd/soroban-rpc/internal/db/ledger_test.go @@ -44,24 +44,24 @@ func assertLedgerRange(t *testing.T, reader LedgerReader, start, end uint32) { allLedgers = append(allLedgers, txmeta) return nil }) - assert.NoError(t, err) + require.NoError(t, err) for i := start - 1; i <= end+1; i++ { ledger, exists, err := reader.GetLedger(context.Background(), i) - assert.NoError(t, err) + require.NoError(t, err) if i < start || i > end { assert.False(t, exists) continue } assert.True(t, exists) ledgerBinary, err := ledger.MarshalBinary() - assert.NoError(t, err) + require.NoError(t, err) expected := createLedger(i) expectedBinary, err := expected.MarshalBinary() - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, expectedBinary, ledgerBinary) ledgerBinary, err = allLedgers[0].MarshalBinary() - assert.NoError(t, err) + require.NoError(t, err) assert.Equal(t, expectedBinary, ledgerBinary) allLedgers = allLedgers[1:] } @@ -74,34 +74,34 @@ func TestLedgers(t *testing.T) { reader := NewLedgerReader(db) _, exists, err := reader.GetLedger(context.Background(), 1) - assert.NoError(t, err) + require.NoError(t, err) assert.False(t, exists) for i := 1; i <= 10; i++ { ledgerSequence := uint32(i) tx, err := NewReadWriter(logger, db, daemon, 150, 15, passphrase).NewTx(context.Background()) - assert.NoError(t, err) - assert.NoError(t, tx.LedgerWriter().InsertLedger(createLedger(ledgerSequence))) - assert.NoError(t, tx.Commit(ledgerSequence)) + require.NoError(t, err) + require.NoError(t, tx.LedgerWriter().InsertLedger(createLedger(ledgerSequence))) + require.NoError(t, tx.Commit(ledgerSequence)) // rolling back after a commit is a no-op - assert.NoError(t, tx.Rollback()) + require.NoError(t, tx.Rollback()) } assertLedgerRange(t, reader, 1, 10) ledgerSequence := uint32(11) tx, err := NewReadWriter(logger, db, daemon, 150, 15, passphrase).NewTx(context.Background()) - assert.NoError(t, err) - assert.NoError(t, tx.LedgerWriter().InsertLedger(createLedger(ledgerSequence))) - assert.NoError(t, tx.Commit(ledgerSequence)) + require.NoError(t, err) + require.NoError(t, tx.LedgerWriter().InsertLedger(createLedger(ledgerSequence))) + require.NoError(t, tx.Commit(ledgerSequence)) assertLedgerRange(t, reader, 1, 11) ledgerSequence = uint32(12) tx, err = NewReadWriter(logger, db, daemon, 150, 5, passphrase).NewTx(context.Background()) - assert.NoError(t, err) - assert.NoError(t, tx.LedgerWriter().InsertLedger(createLedger(ledgerSequence))) - assert.NoError(t, tx.Commit(ledgerSequence)) + require.NoError(t, err) + require.NoError(t, tx.LedgerWriter().InsertLedger(createLedger(ledgerSequence))) + require.NoError(t, tx.Commit(ledgerSequence)) assertLedgerRange(t, reader, 8, 12) } @@ -186,7 +186,7 @@ func NewTestDB(tb testing.TB) *DB { db, err := OpenSQLiteDB(dbPath) require.NoError(tb, err) tb.Cleanup(func() { - assert.NoError(tb, db.Close()) + require.NoError(tb, db.Close()) }) return db } diff --git a/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go b/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go index f2ebcc42..affa0536 100644 --- a/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go +++ b/cmd/soroban-rpc/internal/methods/get_latest_ledger_test.go @@ -54,11 +54,13 @@ func (entryReaderTx ConstantLedgerEntryReaderTx) Done() error { return nil } -func (ledgerReader *ConstantLedgerReader) GetLedger(_ context.Context, sequence uint32) (xdr.LedgerCloseMeta, bool, error) { +func (ledgerReader *ConstantLedgerReader) GetLedger(_ context.Context, + sequence uint32, +) (xdr.LedgerCloseMeta, bool, error) { return createLedger(sequence, expectedLatestLedgerProtocolVersion, expectedLatestLedgerHashBytes), true, nil } -func (ledgerReader *ConstantLedgerReader) StreamAllLedgers(_ context.Context, f db.StreamLedgerFn) error { +func (ledgerReader *ConstantLedgerReader) StreamAllLedgers(_ context.Context, _ db.StreamLedgerFn) error { return nil } From beb0a93c50040b254168b84f55c06c2baf30ed93 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Thu, 27 Jun 2024 12:47:46 -0400 Subject: [PATCH 60/68] Fix linting errors - 4 --- .../internal/methods/get_transactions_test.go | 55 ++++++++++--------- 1 file changed, 28 insertions(+), 27 deletions(-) diff --git a/cmd/soroban-rpc/internal/methods/get_transactions_test.go b/cmd/soroban-rpc/internal/methods/get_transactions_test.go index 78021f86..76f833c4 100644 --- a/cmd/soroban-rpc/internal/methods/get_transactions_test.go +++ b/cmd/soroban-rpc/internal/methods/get_transactions_test.go @@ -7,6 +7,7 @@ import ( "github.com/creachadair/jrpc2" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/stellar/go/toid" "github.com/stellar/go/xdr" @@ -37,12 +38,12 @@ func createTestLedger(sequence uint32) xdr.LedgerCloseMeta { } func TestGetTransactions_DefaultLimit(t *testing.T) { - mockDbReader := db.NewMockTransactionStore(NetworkPassphrase) - mockLedgerReader := db.NewMockLedgerReader(mockDbReader) + mockDBReader := db.NewMockTransactionStore(NetworkPassphrase) + mockLedgerReader := db.NewMockLedgerReader(mockDBReader) for i := 1; i <= 10; i++ { meta := createTestLedger(uint32(i)) - err := mockDbReader.InsertTransactions(meta) - assert.NoError(t, err) + err := mockDBReader.InsertTransactions(meta) + require.NoError(t, err) } handler := transactionsRPCHandler{ @@ -57,7 +58,7 @@ func TestGetTransactions_DefaultLimit(t *testing.T) { } response, err := handler.getTransactionsByLedgerSequence(context.TODO(), request) - assert.NoError(t, err) + require.NoError(t, err) // assert latest ledger details assert.Equal(t, uint32(10), response.LatestLedger) @@ -76,7 +77,7 @@ func TestGetTransactions_DefaultLimitExceedsLatestLedger(t *testing.T) { for i := 1; i <= 3; i++ { meta := createTestLedger(uint32(i)) err := mockDBReader.InsertTransactions(meta) - assert.NoError(t, err) + require.NoError(t, err) } handler := transactionsRPCHandler{ @@ -91,7 +92,7 @@ func TestGetTransactions_DefaultLimitExceedsLatestLedger(t *testing.T) { } response, err := handler.getTransactionsByLedgerSequence(context.TODO(), request) - assert.NoError(t, err) + require.NoError(t, err) // assert latest ledger details assert.Equal(t, uint32(3), response.LatestLedger) @@ -110,7 +111,7 @@ func TestGetTransactions_CustomLimit(t *testing.T) { for i := 1; i <= 10; i++ { meta := createTestLedger(uint32(i)) err := mockDBReader.InsertTransactions(meta) - assert.NoError(t, err) + require.NoError(t, err) } handler := transactionsRPCHandler{ @@ -128,7 +129,7 @@ func TestGetTransactions_CustomLimit(t *testing.T) { } response, err := handler.getTransactionsByLedgerSequence(context.TODO(), request) - assert.NoError(t, err) + require.NoError(t, err) // assert latest ledger details assert.Equal(t, uint32(10), response.LatestLedger) @@ -149,7 +150,7 @@ func TestGetTransactions_CustomLimitAndCursor(t *testing.T) { for i := 1; i <= 10; i++ { meta := createTestLedger(uint32(i)) err := mockDBReader.InsertTransactions(meta) - assert.NoError(t, err) + require.NoError(t, err) } handler := transactionsRPCHandler{ @@ -167,7 +168,7 @@ func TestGetTransactions_CustomLimitAndCursor(t *testing.T) { } response, err := handler.getTransactionsByLedgerSequence(context.TODO(), request) - assert.NoError(t, err) + require.NoError(t, err) // assert latest ledger details assert.Equal(t, uint32(10), response.LatestLedger) @@ -184,12 +185,12 @@ func TestGetTransactions_CustomLimitAndCursor(t *testing.T) { } func TestGetTransactions_InvalidStartLedger(t *testing.T) { - mockDbReader := db.NewMockTransactionStore(NetworkPassphrase) - mockLedgerReader := db.NewMockLedgerReader(mockDbReader) + mockDBReader := db.NewMockTransactionStore(NetworkPassphrase) + mockLedgerReader := db.NewMockLedgerReader(mockDBReader) for i := 1; i <= 3; i++ { meta := createTestLedger(uint32(i)) - err := mockDbReader.InsertTransactions(meta) - assert.NoError(t, err) + err := mockDBReader.InsertTransactions(meta) + require.NoError(t, err) } handler := transactionsRPCHandler{ @@ -213,16 +214,16 @@ func TestGetTransactions_InvalidStartLedger(t *testing.T) { } func TestGetTransactions_LedgerNotFound(t *testing.T) { - mockDbReader := db.NewMockTransactionStore(NetworkPassphrase) - mockLedgerReader := db.NewMockLedgerReader(mockDbReader) + mockDBReader := db.NewMockTransactionStore(NetworkPassphrase) + mockLedgerReader := db.NewMockLedgerReader(mockDBReader) for i := 1; i <= 3; i++ { // Skip creation of ledger 2 if i == 2 { continue } meta := createTestLedger(uint32(i)) - err := mockDbReader.InsertTransactions(meta) - assert.NoError(t, err) + err := mockDBReader.InsertTransactions(meta) + require.NoError(t, err) } handler := transactionsRPCHandler{ @@ -243,12 +244,12 @@ func TestGetTransactions_LedgerNotFound(t *testing.T) { } func TestGetTransactions_LimitGreaterThanMaxLimit(t *testing.T) { - mockDbReader := db.NewMockTransactionStore(NetworkPassphrase) - mockLedgerReader := db.NewMockLedgerReader(mockDbReader) + mockDBReader := db.NewMockTransactionStore(NetworkPassphrase) + mockLedgerReader := db.NewMockLedgerReader(mockDBReader) for i := 1; i <= 3; i++ { meta := createTestLedger(uint32(i)) - err := mockDbReader.InsertTransactions(meta) - assert.NoError(t, err) + err := mockDBReader.InsertTransactions(meta) + require.NoError(t, err) } handler := transactionsRPCHandler{ @@ -271,12 +272,12 @@ func TestGetTransactions_LimitGreaterThanMaxLimit(t *testing.T) { } func TestGetTransactions_InvalidCursorString(t *testing.T) { - mockDbReader := db.NewMockTransactionStore(NetworkPassphrase) - mockLedgerReader := db.NewMockLedgerReader(mockDbReader) + mockDBReader := db.NewMockTransactionStore(NetworkPassphrase) + mockLedgerReader := db.NewMockLedgerReader(mockDBReader) for i := 1; i <= 3; i++ { meta := createTestLedger(uint32(i)) - err := mockDbReader.InsertTransactions(meta) - assert.NoError(t, err) + err := mockDBReader.InsertTransactions(meta) + require.NoError(t, err) } handler := transactionsRPCHandler{ From 8c54b8798c394e7a121e80f755f33e467faa8f73 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 28 Jun 2024 12:09:40 -0400 Subject: [PATCH 61/68] Remove unnecessary constants --- cmd/soroban-rpc/internal/db/ledger.go | 2 +- cmd/soroban-rpc/internal/db/transaction.go | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/ledger.go b/cmd/soroban-rpc/internal/db/ledger.go index 612f07a9..08322c3a 100644 --- a/cmd/soroban-rpc/internal/db/ledger.go +++ b/cmd/soroban-rpc/internal/db/ledger.go @@ -85,7 +85,7 @@ func (r ledgerReader) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.Le var lcms []xdr.LedgerCloseMeta if err := r.db.Select(ctx, &lcms, query); err != nil { return ledgerRange, fmt.Errorf("couldn't query ledger range: %w", err) - } else if len(lcms) < getLedgerRangeMetaArrayCheckValue { + } else if len(lcms) < 2 { // There is almost certainly a row, but we want to avoid a race condition // with ingestion as well as support test cases from an empty DB, so we need // to sanity check that there is in fact a result. Note that no ledgers in diff --git a/cmd/soroban-rpc/internal/db/transaction.go b/cmd/soroban-rpc/internal/db/transaction.go index a51b6e34..bc407486 100644 --- a/cmd/soroban-rpc/internal/db/transaction.go +++ b/cmd/soroban-rpc/internal/db/transaction.go @@ -19,9 +19,7 @@ import ( ) const ( - transactionTableName = "transactions" - firstLedgerToMigrate = 2 - getLedgerRangeMetaArrayCheckValue = 2 + transactionTableName = "transactions" ) var ErrNoTransaction = errors.New("no transaction with this hash exists") @@ -272,7 +270,7 @@ func newTransactionTableMigration(ctx context.Context, logger *log.Entry, retentionWindow uint32, passphrase string, ) migrationApplierFactory { return migrationApplierFactoryF(func(db *DB, latestLedger uint32) (MigrationApplier, error) { - firstLedgerToMigrate := uint32(firstLedgerToMigrate) + firstLedgerToMigrate := uint32(2) writer := &transactionHandler{ log: logger, db: db, From 8481cedc2e9f300b9f324681e68141b531de1cae Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 28 Jun 2024 12:16:21 -0400 Subject: [PATCH 62/68] Use sq.Select instead of sq.Expr --- cmd/soroban-rpc/internal/db/ledger.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/ledger.go b/cmd/soroban-rpc/internal/db/ledger.go index 08322c3a..def7cac4 100644 --- a/cmd/soroban-rpc/internal/db/ledger.go +++ b/cmd/soroban-rpc/internal/db/ledger.go @@ -79,8 +79,10 @@ func (r ledgerReader) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.Le query := sq.Select("lcm.meta"). From(ledgerCloseMetaTableName + " as lcm"). - Where(sq.Expr("lcm.sequence = (SELECT MIN(sequence) FROM " + ledgerCloseMetaTableName + ") " + - "OR lcm.sequence = (SELECT MAX(sequence) FROM " + ledgerCloseMetaTableName + ")")) + Where(sq.Or{ + sq.Expr("lcm.sequence = (?)", sq.Select("MIN(sequence)").From(ledgerCloseMetaTableName)), + sq.Expr("lcm.sequence = (?)", sq.Select("MAX(sequence)").From(ledgerCloseMetaTableName)), + }) var lcms []xdr.LedgerCloseMeta if err := r.db.Select(ctx, &lcms, query); err != nil { From 0db4c663219aed4fdfeaa7b1e805567f24f1a8c7 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 28 Jun 2024 13:42:00 -0400 Subject: [PATCH 63/68] Add nolint --- cmd/soroban-rpc/internal/db/ledger.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/soroban-rpc/internal/db/ledger.go b/cmd/soroban-rpc/internal/db/ledger.go index def7cac4..a129dd5a 100644 --- a/cmd/soroban-rpc/internal/db/ledger.go +++ b/cmd/soroban-rpc/internal/db/ledger.go @@ -87,7 +87,7 @@ func (r ledgerReader) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.Le var lcms []xdr.LedgerCloseMeta if err := r.db.Select(ctx, &lcms, query); err != nil { return ledgerRange, fmt.Errorf("couldn't query ledger range: %w", err) - } else if len(lcms) < 2 { + } else if len(lcms) < 2 { //nolint:mnd // There is almost certainly a row, but we want to avoid a race condition // with ingestion as well as support test cases from an empty DB, so we need // to sanity check that there is in fact a result. Note that no ledgers in From c9f98a8897d926d49ce65c1f26a04864ea7c3e4f Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 28 Jun 2024 13:45:47 -0400 Subject: [PATCH 64/68] Add nolint - 2 --- cmd/soroban-rpc/internal/db/transaction.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/soroban-rpc/internal/db/transaction.go b/cmd/soroban-rpc/internal/db/transaction.go index bc407486..0a7c85a9 100644 --- a/cmd/soroban-rpc/internal/db/transaction.go +++ b/cmd/soroban-rpc/internal/db/transaction.go @@ -270,7 +270,7 @@ func newTransactionTableMigration(ctx context.Context, logger *log.Entry, retentionWindow uint32, passphrase string, ) migrationApplierFactory { return migrationApplierFactoryF(func(db *DB, latestLedger uint32) (MigrationApplier, error) { - firstLedgerToMigrate := uint32(2) + firstLedgerToMigrate := uint32(2) //nolint:mnd writer := &transactionHandler{ log: logger, db: db, From e2cfaaab811180857535093b3c56577c9f6a337a Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Fri, 28 Jun 2024 15:44:04 -0400 Subject: [PATCH 65/68] Handle single row case in ledger range --- cmd/soroban-rpc/internal/db/ledger.go | 32 +++++++++++----------- cmd/soroban-rpc/internal/db/ledger_test.go | 28 +++++++++++++++++++ 2 files changed, 44 insertions(+), 16 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/ledger.go b/cmd/soroban-rpc/internal/db/ledger.go index a129dd5a..13a78205 100644 --- a/cmd/soroban-rpc/internal/db/ledger.go +++ b/cmd/soroban-rpc/internal/db/ledger.go @@ -5,7 +5,6 @@ import ( "fmt" sq "github.com/Masterminds/squirrel" - "github.com/stellar/go/xdr" "github.com/stellar/soroban-rpc/cmd/soroban-rpc/internal/ledgerbucketwindow" @@ -75,8 +74,6 @@ func (r ledgerReader) GetLedger(ctx context.Context, sequence uint32) (xdr.Ledge // GetLedgerRange pulls the min/max ledger sequence numbers from the meta table. func (r ledgerReader) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.LedgerRange, error) { - var ledgerRange ledgerbucketwindow.LedgerRange - query := sq.Select("lcm.meta"). From(ledgerCloseMetaTableName + " as lcm"). Where(sq.Or{ @@ -86,21 +83,24 @@ func (r ledgerReader) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.Le var lcms []xdr.LedgerCloseMeta if err := r.db.Select(ctx, &lcms, query); err != nil { - return ledgerRange, fmt.Errorf("couldn't query ledger range: %w", err) - } else if len(lcms) < 2 { //nolint:mnd - // There is almost certainly a row, but we want to avoid a race condition - // with ingestion as well as support test cases from an empty DB, so we need - // to sanity check that there is in fact a result. Note that no ledgers in - // the database isn't an error, it's just an empty range. - return ledgerRange, nil + return ledgerbucketwindow.LedgerRange{}, fmt.Errorf("couldn't query ledger range: %w", err) + } + + // Empty DB + if len(lcms) == 0 { + return ledgerbucketwindow.LedgerRange{}, nil } - lcm1, lcm2 := lcms[0], lcms[1] - ledgerRange.FirstLedger.Sequence = lcm1.LedgerSequence() - ledgerRange.FirstLedger.CloseTime = lcm1.LedgerCloseTime() - ledgerRange.LastLedger.Sequence = lcm2.LedgerSequence() - ledgerRange.LastLedger.CloseTime = lcm2.LedgerCloseTime() - return ledgerRange, nil + return ledgerbucketwindow.LedgerRange{ + FirstLedger: ledgerbucketwindow.LedgerInfo{ + Sequence: lcms[0].LedgerSequence(), + CloseTime: lcms[0].LedgerCloseTime(), + }, + LastLedger: ledgerbucketwindow.LedgerInfo{ + Sequence: lcms[len(lcms)-1].LedgerSequence(), + CloseTime: lcms[len(lcms)-1].LedgerCloseTime(), + }, + }, nil } type ledgerWriter struct { diff --git a/cmd/soroban-rpc/internal/db/ledger_test.go b/cmd/soroban-rpc/internal/db/ledger_test.go index 03208498..c4409788 100644 --- a/cmd/soroban-rpc/internal/db/ledger_test.go +++ b/cmd/soroban-rpc/internal/db/ledger_test.go @@ -137,6 +137,34 @@ func TestGetLedgerRange_NonEmptyDB(t *testing.T) { assert.Equal(t, ledgerCloseTime(1337), ledgerRange.LastLedger.CloseTime) } +func TestGetLedgerRange_SingleDBRow(t *testing.T) { + db := NewTestDB(t) + ctx := context.TODO() + + writer := NewReadWriter(logger, db, interfaces.MakeNoOpDeamon(), 10, 10, passphrase) + write, err := writer.NewTx(ctx) + require.NoError(t, err) + + lcms := []xdr.LedgerCloseMeta{ + txMeta(1234, true), + } + + ledgerW, txW := write.LedgerWriter(), write.TransactionWriter() + for _, lcm := range lcms { + require.NoError(t, ledgerW.InsertLedger(lcm), "ingestion failed for ledger %+v", lcm.V1) + require.NoError(t, txW.InsertTransactions(lcm), "ingestion failed for ledger %+v", lcm.V1) + } + require.NoError(t, write.Commit(lcms[len(lcms)-1].LedgerSequence())) + + reader := NewLedgerReader(db) + ledgerRange, err := reader.GetLedgerRange(ctx) + require.NoError(t, err) + assert.Equal(t, uint32(1334), ledgerRange.FirstLedger.Sequence) + assert.Equal(t, ledgerCloseTime(1334), ledgerRange.FirstLedger.CloseTime) + assert.Equal(t, uint32(1334), ledgerRange.LastLedger.Sequence) + assert.Equal(t, ledgerCloseTime(1334), ledgerRange.LastLedger.CloseTime) +} + func TestGetLedgerRange_EmptyDB(t *testing.T) { db := NewTestDB(t) ctx := context.TODO() From b21593c7a96042e64fbb80e82346004ec949b815 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 1 Jul 2024 10:34:55 -0400 Subject: [PATCH 66/68] Remove index migration --- .../internal/db/sqlmigrations/03_ledgers_index.sql | 7 ------- 1 file changed, 7 deletions(-) delete mode 100644 cmd/soroban-rpc/internal/db/sqlmigrations/03_ledgers_index.sql diff --git a/cmd/soroban-rpc/internal/db/sqlmigrations/03_ledgers_index.sql b/cmd/soroban-rpc/internal/db/sqlmigrations/03_ledgers_index.sql deleted file mode 100644 index 5752f64d..00000000 --- a/cmd/soroban-rpc/internal/db/sqlmigrations/03_ledgers_index.sql +++ /dev/null @@ -1,7 +0,0 @@ --- +migrate Up - --- creating an index in the ledgers table on the sequence number. -CREATE INDEX index_lcm_sequence ON ledger_close_meta(sequence); - --- +migrate Down -DROP INDEX index_lcm_sequence; From a865983c2b717e63b6b4468075fa592924d13407 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 1 Jul 2024 12:00:26 -0400 Subject: [PATCH 67/68] Order the ledger results in ASC order --- cmd/soroban-rpc/internal/db/ledger.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/soroban-rpc/internal/db/ledger.go b/cmd/soroban-rpc/internal/db/ledger.go index 13a78205..7c547693 100644 --- a/cmd/soroban-rpc/internal/db/ledger.go +++ b/cmd/soroban-rpc/internal/db/ledger.go @@ -79,7 +79,7 @@ func (r ledgerReader) GetLedgerRange(ctx context.Context) (ledgerbucketwindow.Le Where(sq.Or{ sq.Expr("lcm.sequence = (?)", sq.Select("MIN(sequence)").From(ledgerCloseMetaTableName)), sq.Expr("lcm.sequence = (?)", sq.Select("MAX(sequence)").From(ledgerCloseMetaTableName)), - }) + }).OrderBy("lcm.sequence ASC") var lcms []xdr.LedgerCloseMeta if err := r.db.Select(ctx, &lcms, query); err != nil { From db771228ae8e2f1d7e3453bd7a2c37ba45e81131 Mon Sep 17 00:00:00 2001 From: Aditya Vyas Date: Mon, 1 Jul 2024 12:05:10 -0400 Subject: [PATCH 68/68] exchange the expected and actual values in assert --- cmd/soroban-rpc/internal/db/ledger_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/soroban-rpc/internal/db/ledger_test.go b/cmd/soroban-rpc/internal/db/ledger_test.go index c4409788..519a1168 100644 --- a/cmd/soroban-rpc/internal/db/ledger_test.go +++ b/cmd/soroban-rpc/internal/db/ledger_test.go @@ -203,8 +203,8 @@ func BenchmarkGetLedgerRange(b *testing.B) { for range b.N { ledgerRange, err := reader.GetLedgerRange(context.TODO()) require.NoError(b, err) - assert.Equal(b, ledgerRange.FirstLedger.Sequence, lcms[0].LedgerSequence()) - assert.Equal(b, ledgerRange.LastLedger.Sequence, lcms[len(lcms)-1].LedgerSequence()) + assert.Equal(b, lcms[0].LedgerSequence(), ledgerRange.FirstLedger.Sequence) + assert.Equal(b, lcms[len(lcms)-1].LedgerSequence(), ledgerRange.LastLedger.Sequence) } }