Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

#748 High memory usage in indexer on gathering txs from Solana and parsing txs #751

Merged
merged 11 commits into from
Apr 17, 2022
Prev Previous commit
Next Next commit
Select limited range of transaction receipts. #748
afalaleev committed Apr 17, 2022

Unverified

This commit is not signed, but one or more authors requires that any commit attributed to them is signed.
commit 2790d7f07d2d3cf6e0ba6f6752d686902d2fcb96
40 changes: 28 additions & 12 deletions proxy/indexer/trx_receipts_storage.py
Original file line number Diff line number Diff line change
@@ -45,18 +45,34 @@ def contains(self, slot, signature):

def get_txs(self, start_slot=0):
with self._conn.cursor() as cur:
cur.execute(f'SELECT slot FROM {self._table_name}' +
f' WHERE slot >= {start_slot} ORDER BY slot ASC, tx_idx DESC' +
f' OFFSET {INDEXER_RECEIPTS_COUNT_LIMIT} LIMIT 1')
slot_row = cur.fetchone()

stop_slot_req_part = ''
if slot_row is not None:
stop_slot_req_part = f' AND slot <= {slot_row[0]}'

cur.execute(f'SELECT slot, signature, tx FROM {self._table_name}' +
f' WHERE slot >= {start_slot}' + stop_slot_req_part +
f' ORDER BY slot ASC, tx_idx DESC')
cur.execute(f'''
SELECT MIN(slot) FROM {self._table_name}' +
WHERE slot > %s'
''',
(start_slot,))
min_slot_row = cur.fetchone()
min_slot = (min_slot_row[0] if min_slot_row else 0)

cur.execute(f'''
SELECT MAX(t.slot) FROM (
SELECT slot FROM {self._table_name}
WHERE slot > %s
ORDER BY slot
LIMIT {INDEXER_RECEIPTS_COUNT_LIMIT}
) AS t
''',
(start_slot,))
limit_slot_row = cur.fetchone()
limit_slot = (limit_slot_row[0] if limit_slot_row else 0)

stop_slot = max(min_slot, limit_slot, start_slot + 1)

cur.execute(f'''
SELECT slot, signature, tx FROM {self._table_name}
WHERE slot >= %s AND slot <= %s
ORDER BY slot ASC, tx_idx DESC
''',
(start_slot, stop_slot,))
rows = cur.fetchall()

for row in rows: