diff --git a/yarn-project/archiver/src/archiver/archiver.ts b/yarn-project/archiver/src/archiver/archiver.ts index a2a10d8c7a2..cd89bbc63d4 100644 --- a/yarn-project/archiver/src/archiver/archiver.ts +++ b/yarn-project/archiver/src/archiver/archiver.ts @@ -2,6 +2,7 @@ import { type EncryptedL2Log, type FromLogType, type GetUnencryptedLogsResponse, + type InBlock, type InboxLeaf, type L1ToL2MessageSource, type L2Block, @@ -589,7 +590,7 @@ export class Archiver implements ArchiveSource { } } - public getTxEffect(txHash: TxHash): Promise { + public getTxEffect(txHash: TxHash) { return this.store.getTxEffect(txHash); } @@ -933,7 +934,7 @@ class ArchiverStoreHelper getBlockHeaders(from: number, limit: number): Promise { return this.store.getBlockHeaders(from, limit); } - getTxEffect(txHash: TxHash): Promise { + getTxEffect(txHash: TxHash): Promise | undefined> { return this.store.getTxEffect(txHash); } getSettledTxReceipt(txHash: TxHash): Promise { diff --git a/yarn-project/archiver/src/archiver/archiver_store.ts b/yarn-project/archiver/src/archiver/archiver_store.ts index 4eb2c80ccc0..5ca7e878d86 100644 --- a/yarn-project/archiver/src/archiver/archiver_store.ts +++ b/yarn-project/archiver/src/archiver/archiver_store.ts @@ -1,6 +1,7 @@ import { type FromLogType, type GetUnencryptedLogsResponse, + type InBlock, type InboxLeaf, type L2Block, type L2BlockL2Logs, @@ -79,7 +80,7 @@ export interface ArchiverDataStore { * @param txHash - The txHash of the tx corresponding to the tx effect. * @returns The requested tx effect (or undefined if not found). */ - getTxEffect(txHash: TxHash): Promise; + getTxEffect(txHash: TxHash): Promise | undefined>; /** * Gets a receipt of a settled tx. diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts index a4a3ba3281c..85bbacc0369 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/block_store.ts @@ -1,4 +1,4 @@ -import { Body, L2Block, type TxEffect, type TxHash, TxReceipt } from '@aztec/circuit-types'; +import { Body, type InBlock, L2Block, type TxEffect, type TxHash, TxReceipt } from '@aztec/circuit-types'; import { AppendOnlyTreeSnapshot, type AztecAddress, Header, INITIAL_L2_BLOCK_NUM } from '@aztec/circuits.js'; import { createDebugLogger } from '@aztec/foundation/log'; import { type AztecKVStore, type AztecMap, type AztecSingleton, type Range } from '@aztec/kv-store'; @@ -170,14 +170,22 @@ export class BlockStore { * @param txHash - The txHash of the tx corresponding to the tx effect. * @returns The requested tx effect (or undefined if not found). */ - getTxEffect(txHash: TxHash): TxEffect | undefined { + getTxEffect(txHash: TxHash): InBlock | undefined { const [blockNumber, txIndex] = this.getTxLocation(txHash) ?? []; if (typeof blockNumber !== 'number' || typeof txIndex !== 'number') { return undefined; } const block = this.getBlock(blockNumber); - return block?.data.body.txEffects[txIndex]; + if (!block) { + return undefined; + } + + return { + data: block.data.body.txEffects[txIndex], + l2BlockNumber: block.data.number, + l2BlockHash: block.data.hash().toString(), + }; } /** diff --git a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts index 8cbb627a5ce..890a827288c 100644 --- a/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts +++ b/yarn-project/archiver/src/archiver/kv_archiver_store/kv_archiver_store.ts @@ -6,7 +6,6 @@ import { type L2BlockL2Logs, type LogFilter, type LogType, - type TxEffect, type TxHash, type TxReceipt, type TxScopedL2Log, @@ -159,7 +158,7 @@ export class KVArchiverDataStore implements ArchiverDataStore { * @param txHash - The txHash of the tx corresponding to the tx effect. * @returns The requested tx effect (or undefined if not found). */ - getTxEffect(txHash: TxHash): Promise { + getTxEffect(txHash: TxHash) { return Promise.resolve(this.#blockStore.getTxEffect(txHash)); } diff --git a/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts b/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts index e49ab8eccc2..441af8aa256 100644 --- a/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts +++ b/yarn-project/archiver/src/archiver/memory_archiver_store/memory_archiver_store.ts @@ -6,6 +6,7 @@ import { ExtendedUnencryptedL2Log, type FromLogType, type GetUnencryptedLogsResponse, + type InBlock, type InboxLeaf, type L2Block, type L2BlockL2Logs, @@ -17,6 +18,7 @@ import { TxReceipt, TxScopedL2Log, type UnencryptedL2BlockL2Logs, + wrapInBlock, } from '@aztec/circuit-types'; import { type ContractClassPublic, @@ -50,7 +52,7 @@ export class MemoryArchiverStore implements ArchiverDataStore { /** * An array containing all the tx effects in the L2 blocks that have been fetched so far. */ - private txEffects: TxEffect[] = []; + private txEffects: InBlock[] = []; private noteEncryptedLogsPerBlock: Map = new Map(); @@ -181,7 +183,7 @@ export class MemoryArchiverStore implements ArchiverDataStore { this.lastL1BlockNewBlocks = blocks[blocks.length - 1].l1.blockNumber; this.l2Blocks.push(...blocks); - this.txEffects.push(...blocks.flatMap(b => b.data.body.txEffects)); + this.txEffects.push(...blocks.flatMap(b => b.data.body.txEffects.map(txEffect => wrapInBlock(txEffect, b.data)))); return Promise.resolve(true); } @@ -365,8 +367,8 @@ export class MemoryArchiverStore implements ArchiverDataStore { * @param txHash - The txHash of the tx effect. * @returns The requested tx effect. */ - public getTxEffect(txHash: TxHash): Promise { - const txEffect = this.txEffects.find(tx => tx.txHash.equals(txHash)); + public getTxEffect(txHash: TxHash): Promise | undefined> { + const txEffect = this.txEffects.find(tx => tx.data.txHash.equals(txHash)); return Promise.resolve(txEffect); } diff --git a/yarn-project/archiver/src/test/mock_l2_block_source.ts b/yarn-project/archiver/src/test/mock_l2_block_source.ts index 86ffce4ae50..2ab843cb42a 100644 --- a/yarn-project/archiver/src/test/mock_l2_block_source.ts +++ b/yarn-project/archiver/src/test/mock_l2_block_source.ts @@ -119,8 +119,14 @@ export class MockL2BlockSource implements L2BlockSource { * @returns The requested tx effect. */ public getTxEffect(txHash: TxHash) { - const txEffect = this.l2Blocks.flatMap(b => b.body.txEffects).find(tx => tx.txHash.equals(txHash)); - return Promise.resolve(txEffect); + const match = this.l2Blocks + .flatMap(b => b.body.txEffects.map(tx => [tx, b] as const)) + .find(([tx]) => tx.txHash.equals(txHash)); + if (!match) { + return Promise.resolve(undefined); + } + const [txEffect, block] = match; + return Promise.resolve({ data: txEffect, l2BlockNumber: block.number, l2BlockHash: block.hash().toString() }); } /** diff --git a/yarn-project/aztec-node/src/aztec-node/server.ts b/yarn-project/aztec-node/src/aztec-node/server.ts index eec371b96fb..c746ba30534 100644 --- a/yarn-project/aztec-node/src/aztec-node/server.ts +++ b/yarn-project/aztec-node/src/aztec-node/server.ts @@ -6,6 +6,7 @@ import { type EpochProofQuote, type FromLogType, type GetUnencryptedLogsResponse, + type InBlock, type L1ToL2MessageSource, type L2Block, type L2BlockL2Logs, @@ -117,14 +118,18 @@ export class AztecNodeService implements AztecNode { this.log.info(message); } - addEpochProofQuote(quote: EpochProofQuote): Promise { + public addEpochProofQuote(quote: EpochProofQuote): Promise { return Promise.resolve(this.p2pClient.addEpochProofQuote(quote)); } - getEpochProofQuotes(epoch: bigint): Promise { + public getEpochProofQuotes(epoch: bigint): Promise { return this.p2pClient.getEpochProofQuotes(epoch); } + public getL2Tips() { + return this.blockSource.getL2Tips(); + } + /** * initializes the Aztec Node, wait for component to sync. * @param config - The configuration to be used by the aztec node. @@ -372,7 +377,7 @@ export class AztecNodeService implements AztecNode { return txReceipt; } - public getTxEffect(txHash: TxHash): Promise { + public getTxEffect(txHash: TxHash): Promise | undefined> { return this.blockSource.getTxEffect(txHash); } @@ -708,7 +713,7 @@ export class AztecNodeService implements AztecNode { * Returns the currently committed block header, or the initial header if no blocks have been produced. * @returns The current committed block header. */ - public async getHeader(blockNumber: L2BlockNumber = 'latest'): Promise
{ + public async getBlockHeader(blockNumber: L2BlockNumber = 'latest'): Promise
{ return ( (await this.getBlock(blockNumber === 'latest' ? -1 : blockNumber))?.header ?? this.worldStateSynchronizer.getCommitted().getInitialHeader() diff --git a/yarn-project/aztec.js/src/contract/sent_tx.ts b/yarn-project/aztec.js/src/contract/sent_tx.ts index 7b85b3fd853..d86f80528fb 100644 --- a/yarn-project/aztec.js/src/contract/sent_tx.ts +++ b/yarn-project/aztec.js/src/contract/sent_tx.ts @@ -78,7 +78,7 @@ export class SentTx { } if (opts?.debug) { const txHash = await this.getTxHash(); - const tx = (await this.pxe.getTxEffect(txHash))!; + const { data: tx } = (await this.pxe.getTxEffect(txHash))!; receipt.debugInfo = { noteHashes: tx.noteHashes, nullifiers: tx.nullifiers, diff --git a/yarn-project/aztec.js/src/wallet/base_wallet.ts b/yarn-project/aztec.js/src/wallet/base_wallet.ts index 00cc9127a38..94d7e479504 100644 --- a/yarn-project/aztec.js/src/wallet/base_wallet.ts +++ b/yarn-project/aztec.js/src/wallet/base_wallet.ts @@ -13,7 +13,6 @@ import { type SiblingPath, type SyncStatus, type Tx, - type TxEffect, type TxExecutionRequest, type TxHash, type TxProvingResult, @@ -122,7 +121,7 @@ export abstract class BaseWallet implements Wallet { sendTx(tx: Tx): Promise { return this.pxe.sendTx(tx); } - getTxEffect(txHash: TxHash): Promise { + getTxEffect(txHash: TxHash) { return this.pxe.getTxEffect(txHash); } getTxReceipt(txHash: TxHash): Promise { diff --git a/yarn-project/circuit-types/src/in_block.ts b/yarn-project/circuit-types/src/in_block.ts new file mode 100644 index 00000000000..5e5205b4237 --- /dev/null +++ b/yarn-project/circuit-types/src/in_block.ts @@ -0,0 +1,36 @@ +import { Fr } from '@aztec/circuits.js'; +import { schemas } from '@aztec/foundation/schemas'; + +import { type ZodTypeAny, z } from 'zod'; + +import { type L2Block } from './l2_block.js'; + +export type InBlock = { + l2BlockNumber: number; + l2BlockHash: string; + data: T; +}; + +export function randomInBlock(data: T): InBlock { + return { + data, + l2BlockNumber: Math.floor(Math.random() * 1000), + l2BlockHash: Fr.random().toString(), + }; +} + +export function wrapInBlock(data: T, block: L2Block): InBlock { + return { + data, + l2BlockNumber: block.number, + l2BlockHash: block.hash().toString(), + }; +} + +export function inBlockSchemaFor(schema: T) { + return z.object({ + data: schema, + l2BlockNumber: schemas.Integer, + l2BlockHash: z.string(), + }); +} diff --git a/yarn-project/circuit-types/src/index.ts b/yarn-project/circuit-types/src/index.ts index 86eade75e74..2040dedbbb3 100644 --- a/yarn-project/circuit-types/src/index.ts +++ b/yarn-project/circuit-types/src/index.ts @@ -22,3 +22,4 @@ export * from './simulation_error.js'; export * from './tx/index.js'; export * from './tx_effect.js'; export * from './tx_execution_request.js'; +export * from './in_block.js'; diff --git a/yarn-project/circuit-types/src/interfaces/archiver.test.ts b/yarn-project/circuit-types/src/interfaces/archiver.test.ts index b6b411c3177..70a7ab772f4 100644 --- a/yarn-project/circuit-types/src/interfaces/archiver.test.ts +++ b/yarn-project/circuit-types/src/interfaces/archiver.test.ts @@ -20,6 +20,7 @@ import { readFileSync } from 'fs'; import omit from 'lodash.omit'; import { resolve } from 'path'; +import { type InBlock } from '../in_block.js'; import { L2Block } from '../l2_block.js'; import { type L2Tips } from '../l2_block_source.js'; import { ExtendedUnencryptedL2Log } from '../logs/extended_unencrypted_l2_log.js'; @@ -106,7 +107,7 @@ describe('ArchiverApiSchema', () => { it('getTxEffect', async () => { const result = await context.client.getTxEffect(new TxHash(Buffer.alloc(32, 1))); - expect(result).toBeInstanceOf(TxEffect); + expect(result!.data).toBeInstanceOf(TxEffect); }); it('getSettledTxReceipt', async () => { @@ -261,9 +262,9 @@ class MockArchiver implements ArchiverApi { getBlocks(from: number, _limit: number, _proven?: boolean | undefined): Promise { return Promise.resolve([L2Block.random(from)]); } - getTxEffect(_txHash: TxHash): Promise { + getTxEffect(_txHash: TxHash): Promise | undefined> { expect(_txHash).toBeInstanceOf(TxHash); - return Promise.resolve(TxEffect.empty()); + return Promise.resolve({ l2BlockNumber: 1, l2BlockHash: '0x12', data: TxEffect.random() }); } getSettledTxReceipt(txHash: TxHash): Promise { expect(txHash).toBeInstanceOf(TxHash); diff --git a/yarn-project/circuit-types/src/interfaces/archiver.ts b/yarn-project/circuit-types/src/interfaces/archiver.ts index 0dc118875aa..686b512a65e 100644 --- a/yarn-project/circuit-types/src/interfaces/archiver.ts +++ b/yarn-project/circuit-types/src/interfaces/archiver.ts @@ -10,6 +10,7 @@ import { type ApiSchemaFor, optional, schemas } from '@aztec/foundation/schemas' import { z } from 'zod'; +import { inBlockSchemaFor } from '../in_block.js'; import { L2Block } from '../l2_block.js'; import { type L2BlockSource, L2TipsSchema } from '../l2_block_source.js'; import { GetUnencryptedLogsResponseSchema, TxScopedL2Log } from '../logs/get_logs_response.js'; @@ -42,7 +43,7 @@ export const ArchiverApiSchema: ApiSchemaFor = { .function() .args(schemas.Integer, schemas.Integer, optional(z.boolean())) .returns(z.array(L2Block.schema)), - getTxEffect: z.function().args(TxHash.schema).returns(TxEffect.schema.optional()), + getTxEffect: z.function().args(TxHash.schema).returns(inBlockSchemaFor(TxEffect.schema).optional()), getSettledTxReceipt: z.function().args(TxHash.schema).returns(TxReceipt.schema.optional()), getL2SlotNumber: z.function().args().returns(schemas.BigInt), getL2EpochNumber: z.function().args().returns(schemas.BigInt), diff --git a/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts b/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts index d415249c3f4..1c0f10fbf0f 100644 --- a/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts +++ b/yarn-project/circuit-types/src/interfaces/aztec-node.test.ts @@ -28,7 +28,9 @@ import omit from 'lodash.omit'; import times from 'lodash.times'; import { resolve } from 'path'; +import { type InBlock } from '../in_block.js'; import { L2Block } from '../l2_block.js'; +import { type L2Tips } from '../l2_block_source.js'; import { ExtendedUnencryptedL2Log } from '../logs/extended_unencrypted_l2_log.js'; import { type GetUnencryptedLogsResponse, TxScopedL2Log } from '../logs/get_logs_response.js'; import { @@ -80,6 +82,15 @@ describe('AztecNodeApiSchema', () => { expect([...tested].sort()).toEqual(all.sort()); }); + it('getL2Tips', async () => { + const result = await context.client.getL2Tips(); + expect(result).toEqual({ + latest: { number: 1, hash: `0x01` }, + proven: { number: 1, hash: `0x01` }, + finalized: { number: 1, hash: `0x01` }, + }); + }); + it('findLeavesIndexes', async () => { const response = await context.client.findLeavesIndexes(1, MerkleTreeId.ARCHIVE, [Fr.random(), Fr.random()]); expect(response).toEqual([1n, undefined]); @@ -231,7 +242,7 @@ describe('AztecNodeApiSchema', () => { it('getTxEffect', async () => { const response = await context.client.getTxEffect(TxHash.random()); - expect(response).toBeInstanceOf(TxEffect); + expect(response!.data).toBeInstanceOf(TxEffect); }); it('getPendingTxs', async () => { @@ -254,8 +265,8 @@ describe('AztecNodeApiSchema', () => { expect(response).toBeInstanceOf(Fr); }); - it('getHeader', async () => { - const response = await context.client.getHeader(); + it('getBlockHeader', async () => { + const response = await context.client.getBlockHeader(); expect(response).toBeInstanceOf(Header); }); @@ -318,6 +329,13 @@ describe('AztecNodeApiSchema', () => { class MockAztecNode implements AztecNode { constructor(private artifact: ContractArtifact) {} + getL2Tips(): Promise { + return Promise.resolve({ + latest: { number: 1, hash: `0x01` }, + proven: { number: 1, hash: `0x01` }, + finalized: { number: 1, hash: `0x01` }, + }); + } findLeavesIndexes( blockNumber: number | 'latest', treeId: MerkleTreeId, @@ -472,9 +490,9 @@ class MockAztecNode implements AztecNode { expect(txHash).toBeInstanceOf(TxHash); return Promise.resolve(TxReceipt.empty()); } - getTxEffect(txHash: TxHash): Promise { + getTxEffect(txHash: TxHash): Promise | undefined> { expect(txHash).toBeInstanceOf(TxHash); - return Promise.resolve(TxEffect.random()); + return Promise.resolve({ l2BlockNumber: 1, l2BlockHash: '0x12', data: TxEffect.random() }); } getPendingTxs(): Promise { return Promise.resolve([Tx.random()]); @@ -491,7 +509,7 @@ class MockAztecNode implements AztecNode { expect(slot).toBeInstanceOf(Fr); return Promise.resolve(Fr.random()); } - getHeader(_blockNumber?: number | 'latest' | undefined): Promise
{ + getBlockHeader(_blockNumber?: number | 'latest' | undefined): Promise
{ return Promise.resolve(Header.empty()); } simulatePublicCalls(tx: Tx): Promise { diff --git a/yarn-project/circuit-types/src/interfaces/aztec-node.ts b/yarn-project/circuit-types/src/interfaces/aztec-node.ts index 91b7dd8d0ca..252757521fe 100644 --- a/yarn-project/circuit-types/src/interfaces/aztec-node.ts +++ b/yarn-project/circuit-types/src/interfaces/aztec-node.ts @@ -21,7 +21,9 @@ import { type ApiSchemaFor, optional, schemas } from '@aztec/foundation/schemas' import { z } from 'zod'; +import { type InBlock, inBlockSchemaFor } from '../in_block.js'; import { L2Block } from '../l2_block.js'; +import { type L2BlockSource, type L2Tips, L2TipsSchema } from '../l2_block_source.js'; import { type FromLogType, type GetUnencryptedLogsResponse, @@ -48,7 +50,14 @@ import { type ProverCoordination, ProverCoordinationApiSchema } from './prover-c * The aztec node. * We will probably implement the additional interfaces by means other than Aztec Node as it's currently a privacy leak */ -export interface AztecNode extends ProverCoordination { +export interface AztecNode + extends ProverCoordination, + Pick { + /** + * Returns the tips of the L2 chain. + */ + getL2Tips(): Promise; + /** * Find the indexes of the given leaves in the given tree. * @param blockNumber - The block number at which to get the data or 'latest' for latest data @@ -298,7 +307,7 @@ export interface AztecNode extends ProverCoordination { * @param txHash - The hash of a transaction which resulted in the returned tx effect. * @returns The requested tx effect. */ - getTxEffect(txHash: TxHash): Promise; + getTxEffect(txHash: TxHash): Promise | undefined>; /** * Method to retrieve pending txs. @@ -336,7 +345,7 @@ export interface AztecNode extends ProverCoordination { * Returns the currently committed block header. * @returns The current committed block header. */ - getHeader(blockNumber?: L2BlockNumber): Promise
; + getBlockHeader(blockNumber?: L2BlockNumber): Promise
; /** * Simulates the public part of a transaction with the current state. @@ -396,6 +405,7 @@ export interface AztecNode extends ProverCoordination { export const AztecNodeApiSchema: ApiSchemaFor = { ...ProverCoordinationApiSchema, + getL2Tips: z.function().args().returns(L2TipsSchema), findLeavesIndexes: z .function() .args(L2BlockNumberSchema, z.nativeEnum(MerkleTreeId), z.array(schemas.Fr)) @@ -485,7 +495,7 @@ export const AztecNodeApiSchema: ApiSchemaFor = { getTxReceipt: z.function().args(TxHash.schema).returns(TxReceipt.schema), - getTxEffect: z.function().args(TxHash.schema).returns(TxEffect.schema.optional()), + getTxEffect: z.function().args(TxHash.schema).returns(inBlockSchemaFor(TxEffect.schema).optional()), getPendingTxs: z.function().returns(z.array(Tx.schema)), @@ -495,7 +505,7 @@ export const AztecNodeApiSchema: ApiSchemaFor = { getPublicStorageAt: z.function().args(schemas.AztecAddress, schemas.Fr, L2BlockNumberSchema).returns(schemas.Fr), - getHeader: z.function().args(optional(L2BlockNumberSchema)).returns(Header.schema), + getBlockHeader: z.function().args(optional(L2BlockNumberSchema)).returns(Header.schema), simulatePublicCalls: z.function().args(Tx.schema).returns(PublicSimulationOutput.schema), diff --git a/yarn-project/circuit-types/src/interfaces/pxe.test.ts b/yarn-project/circuit-types/src/interfaces/pxe.test.ts index 8aefa059edb..e2aa6c1cca5 100644 --- a/yarn-project/circuit-types/src/interfaces/pxe.test.ts +++ b/yarn-project/circuit-types/src/interfaces/pxe.test.ts @@ -30,6 +30,7 @@ import times from 'lodash.times'; import { resolve } from 'path'; import { AuthWitness } from '../auth_witness.js'; +import { type InBlock } from '../in_block.js'; import { L2Block } from '../l2_block.js'; import { ExtendedUnencryptedL2Log, type GetUnencryptedLogsResponse, type LogFilter } from '../logs/index.js'; import { type IncomingNotesFilter } from '../notes/incoming_notes_filter.js'; @@ -178,8 +179,10 @@ describe('PXESchema', () => { }); it('getTxEffect', async () => { - const result = await context.client.getTxEffect(TxHash.random()); - expect(result).toBeInstanceOf(TxEffect); + const { l2BlockHash, l2BlockNumber, data } = (await context.client.getTxEffect(TxHash.random()))!; + expect(data).toBeInstanceOf(TxEffect); + expect(l2BlockHash).toMatch(/0x[a-fA-F0-9]{64}/); + expect(l2BlockNumber).toBe(1); }); it('getPublicStorageAt', async () => { @@ -401,9 +404,9 @@ class MockPXE implements PXE { expect(txHash).toBeInstanceOf(TxHash); return Promise.resolve(TxReceipt.empty()); } - getTxEffect(txHash: TxHash): Promise { + getTxEffect(txHash: TxHash): Promise | undefined> { expect(txHash).toBeInstanceOf(TxHash); - return Promise.resolve(TxEffect.random()); + return Promise.resolve({ data: TxEffect.random(), l2BlockHash: Fr.random().toString(), l2BlockNumber: 1 }); } getPublicStorageAt(contract: AztecAddress, slot: Fr): Promise { expect(contract).toBeInstanceOf(AztecAddress); diff --git a/yarn-project/circuit-types/src/interfaces/pxe.ts b/yarn-project/circuit-types/src/interfaces/pxe.ts index 4bfa62f74bb..a2a6f6940fe 100644 --- a/yarn-project/circuit-types/src/interfaces/pxe.ts +++ b/yarn-project/circuit-types/src/interfaces/pxe.ts @@ -27,6 +27,7 @@ import { AbiDecodedSchema, type ApiSchemaFor, type ZodFor, optional, schemas } f import { z } from 'zod'; import { AuthWitness } from '../auth_witness.js'; +import { type InBlock, inBlockSchemaFor } from '../in_block.js'; import { L2Block } from '../l2_block.js'; import { type GetUnencryptedLogsResponse, @@ -216,7 +217,7 @@ export interface PXE { * @param txHash - The hash of a transaction which resulted in the returned tx effect. * @returns The requested tx effect. */ - getTxEffect(txHash: TxHash): Promise; + getTxEffect(txHash: TxHash): Promise | undefined>; /** * Gets the storage value at the given contract storage slot. @@ -500,7 +501,7 @@ export const PXESchema: ApiSchemaFor = { getTxEffect: z .function() .args(TxHash.schema) - .returns(z.union([TxEffect.schema, z.undefined()])), + .returns(z.union([inBlockSchemaFor(TxEffect.schema), z.undefined()])), getPublicStorageAt: z.function().args(schemas.AztecAddress, schemas.Fr).returns(schemas.Fr), getIncomingNotes: z.function().args(IncomingNotesFilterSchema).returns(z.array(UniqueNote.schema)), getL1ToL2MembershipWitness: z diff --git a/yarn-project/circuit-types/src/l2_block_downloader/l2_block_stream.ts b/yarn-project/circuit-types/src/l2_block_downloader/l2_block_stream.ts index 7a1d179dd16..50608b6c0ea 100644 --- a/yarn-project/circuit-types/src/l2_block_downloader/l2_block_stream.ts +++ b/yarn-project/circuit-types/src/l2_block_downloader/l2_block_stream.ts @@ -12,14 +12,15 @@ export class L2BlockStream { private readonly log = createDebugLogger('aztec:l2_block_stream'); constructor( - private l2BlockSource: L2BlockSource, + private l2BlockSource: Pick, private localData: L2BlockStreamLocalDataProvider, private handler: L2BlockStreamEventHandler, private opts: { proven?: boolean; pollIntervalMS?: number; batchSize?: number; - }, + startingBlock?: number; + } = {}, ) { this.runningPromise = new RunningPromise(() => this.work(), this.opts.pollIntervalMS ?? 1000); } @@ -70,6 +71,11 @@ export class L2BlockStream { await this.emitEvent({ type: 'chain-pruned', blockNumber: latestBlockNumber }); } + // If we are just starting, use the starting block number from the options. + if (latestBlockNumber === 0 && this.opts.startingBlock !== undefined) { + latestBlockNumber = Math.max(this.opts.startingBlock - 1, 0); + } + // Request new blocks from the source. while (latestBlockNumber < sourceTips.latest.number) { const from = latestBlockNumber + 1; diff --git a/yarn-project/circuit-types/src/l2_block_source.ts b/yarn-project/circuit-types/src/l2_block_source.ts index 4dac6da0db9..6f749b28189 100644 --- a/yarn-project/circuit-types/src/l2_block_source.ts +++ b/yarn-project/circuit-types/src/l2_block_source.ts @@ -2,6 +2,7 @@ import { type EthAddress, type Header } from '@aztec/circuits.js'; import { z } from 'zod'; +import { type InBlock } from './in_block.js'; import { type L2Block } from './l2_block.js'; import { type TxHash } from './tx/tx_hash.js'; import { type TxReceipt } from './tx/tx_receipt.js'; @@ -69,7 +70,7 @@ export interface L2BlockSource { * @param txHash - The hash of a transaction which resulted in the returned tx effect. * @returns The requested tx effect. */ - getTxEffect(txHash: TxHash): Promise; + getTxEffect(txHash: TxHash): Promise | undefined>; /** * Gets a receipt of a settled tx. @@ -121,6 +122,7 @@ export type L2Tips = Record; /** Identifies a block by number and hash. */ export type L2BlockId = z.infer; +// TODO(palla/schemas): This package should know what is the block hash of the genesis block 0. const L2BlockIdSchema = z.union([ z.object({ number: z.literal(0), diff --git a/yarn-project/cli/src/utils/inspect.ts b/yarn-project/cli/src/utils/inspect.ts index 445e48c3bd1..855c38b9ef5 100644 --- a/yarn-project/cli/src/utils/inspect.ts +++ b/yarn-project/cli/src/utils/inspect.ts @@ -38,22 +38,23 @@ export async function inspectTx( log: LogFn, opts: { includeBlockInfo?: boolean; artifactMap?: ArtifactMap } = {}, ) { - const [receipt, effects, notes] = await Promise.all([ + const [receipt, effectsInBlock, notes] = await Promise.all([ pxe.getTxReceipt(txHash), pxe.getTxEffect(txHash), pxe.getIncomingNotes({ txHash, status: NoteStatus.ACTIVE_OR_NULLIFIED }), ]); // Base tx data log(`Tx ${txHash.toString()}`); - log(` Status: ${receipt.status} ${effects ? `(${effects.revertCode.getDescription()})` : ''}`); + log(` Status: ${receipt.status} ${effectsInBlock ? `(${effectsInBlock.data.revertCode.getDescription()})` : ''}`); if (receipt.error) { log(` Error: ${receipt.error}`); } - if (!effects) { + if (!effectsInBlock) { return; } + const effects = effectsInBlock.data; const artifactMap = opts?.artifactMap ?? (await getKnownArtifacts(pxe)); if (opts.includeBlockInfo) { diff --git a/yarn-project/end-to-end/src/e2e_block_building.test.ts b/yarn-project/end-to-end/src/e2e_block_building.test.ts index 88bfd6a1b23..41a27b70a92 100644 --- a/yarn-project/end-to-end/src/e2e_block_building.test.ts +++ b/yarn-project/end-to-end/src/e2e_block_building.test.ts @@ -1,5 +1,4 @@ import { getSchnorrAccount } from '@aztec/accounts/schnorr'; -import { createAccount } from '@aztec/accounts/testing'; import { type AztecAddress, type AztecNode, @@ -23,7 +22,6 @@ import { poseidon2HashWithSeparator } from '@aztec/foundation/crypto'; import { StatefulTestContract, StatefulTestContractArtifact } from '@aztec/noir-contracts.js'; import { TestContract } from '@aztec/noir-contracts.js/Test'; import { TokenContract } from '@aztec/noir-contracts.js/Token'; -import { createPXEService, getPXEServiceConfig } from '@aztec/pxe'; import 'jest-extended'; @@ -439,49 +437,63 @@ describe('e2e_block_building', () => { await cheatCodes.rollup.advanceToNextEpoch(); await cheatCodes.rollup.markAsProven(); - // Send a tx to the contract that updates the public data tree, this should take the first slot + // Send a tx to the contract that creates a note. This tx will be reorgd but re-included, + // since it is being built against a proven block number. logger.info('Sending initial tx'); - const tx1 = await contract.methods.increment_public_value(ownerAddress, 20).send().wait(); + const tx1 = await contract.methods.create_note(ownerAddress, ownerAddress, 20).send().wait(); expect(tx1.blockNumber).toEqual(initialBlockNumber + 1); - expect(await contract.methods.get_public_value(ownerAddress).simulate()).toEqual(20n); + expect(await contract.methods.summed_values(ownerAddress).simulate()).toEqual(21n); + + // And send a second one, which won't be re-included. + logger.info('Sending second tx'); + const tx2 = await contract.methods.create_note(ownerAddress, ownerAddress, 30).send().wait(); + expect(tx2.blockNumber).toEqual(initialBlockNumber + 2); + expect(await contract.methods.summed_values(ownerAddress).simulate()).toEqual(51n); // Now move to a new epoch and past the proof claim window to cause a reorg logger.info('Advancing past the proof claim window'); await cheatCodes.rollup.advanceToNextEpoch(); await cheatCodes.rollup.advanceSlots(aztecEpochProofClaimWindowInL2Slots + 1); // off-by-one? - // Wait a bit before spawning a new pxe - await sleep(2000); + // Wait until the sequencer kicks out tx1 + logger.info(`Waiting for node to prune tx1`); + await retryUntil( + async () => (await aztecNode.getTxReceipt(tx1.txHash)).status === TxStatus.PENDING, + 'wait for pruning', + 15, + 1, + ); - // tx1 is valid because it was build against a proven block number - // the sequencer will bring it back on chain + // And wait until it is brought back tx1 + logger.info(`Waiting for node to re-include tx1`); await retryUntil( async () => (await aztecNode.getTxReceipt(tx1.txHash)).status === TxStatus.SUCCESS, 'wait for re-inclusion', - 60, + 15, 1, ); + // Tx1 should have been mined in a block with the same number but different hash now const newTx1Receipt = await aztecNode.getTxReceipt(tx1.txHash); expect(newTx1Receipt.blockNumber).toEqual(tx1.blockNumber); expect(newTx1Receipt.blockHash).not.toEqual(tx1.blockHash); - // Send another tx which should be mined a block that is built on the reorg'd chain - // We need to send it from a new pxe since pxe doesn't detect reorgs (yet) - logger.info(`Creating new PXE service`); - const pxeServiceConfig = { ...getPXEServiceConfig() }; - const newPxe = await createPXEService(aztecNode, pxeServiceConfig); - const newWallet = await createAccount(newPxe); + // PXE should have cleared out the 30-note from tx2, but reapplied the 20-note from tx1 + expect(await contract.methods.summed_values(ownerAddress).simulate()).toEqual(21n); - // TODO: Contract.at should automatically register the instance in the pxe - logger.info(`Registering contract at ${contract.address} in new pxe`); - await newPxe.registerContract({ instance: contract.instance, artifact: StatefulTestContractArtifact }); - const contractFromNewPxe = await StatefulTestContract.at(contract.address, newWallet); + // PXE should be synced to the block number on the new chain + await retryUntil( + async () => (await pxe.getSyncStatus()).blocks === newTx1Receipt.blockNumber, + 'wait for pxe block header sync', + 15, + 1, + ); + // And we should be able to send a new tx on the new chain logger.info('Sending new tx on reorgd chain'); - const tx2 = await contractFromNewPxe.methods.increment_public_value(ownerAddress, 10).send().wait(); - expect(await contractFromNewPxe.methods.get_public_value(ownerAddress).simulate()).toEqual(30n); - expect(tx2.blockNumber).toEqual(initialBlockNumber + 3); + const tx3 = await contract.methods.create_note(ownerAddress, ownerAddress, 10).send().wait(); + expect(await contract.methods.summed_values(ownerAddress).simulate()).toEqual(31n); + expect(tx3.blockNumber).toBeGreaterThanOrEqual(newTx1Receipt.blockNumber! + 1); }); }); }); diff --git a/yarn-project/end-to-end/src/e2e_event_logs.test.ts b/yarn-project/end-to-end/src/e2e_event_logs.test.ts index ab8d4f88785..162d53d6eb0 100644 --- a/yarn-project/end-to-end/src/e2e_event_logs.test.ts +++ b/yarn-project/end-to-end/src/e2e_event_logs.test.ts @@ -48,7 +48,7 @@ describe('Logs', () => { const txEffect = await node.getTxEffect(tx.txHash); - const encryptedLogs = txEffect!.encryptedLogs.unrollLogs(); + const encryptedLogs = txEffect!.data.encryptedLogs.unrollLogs(); expect(encryptedLogs.length).toBe(3); const decryptedEvent0 = L1EventPayload.decryptAsIncoming(encryptedLogs[0], wallets[0].getEncryptionSecret())!; diff --git a/yarn-project/kv-store/package.json b/yarn-project/kv-store/package.json index ae05345eeaa..f216507a2da 100644 --- a/yarn-project/kv-store/package.json +++ b/yarn-project/kv-store/package.json @@ -5,7 +5,8 @@ "exports": { ".": "./dest/interfaces/index.js", "./lmdb": "./dest/lmdb/index.js", - "./utils": "./dest/utils.js" + "./utils": "./dest/utils.js", + "./stores": "./dest/stores/index.js" }, "scripts": { "build": "yarn clean && tsc -b", @@ -55,10 +56,12 @@ ] }, "dependencies": { + "@aztec/circuit-types": "workspace:^", "@aztec/foundation": "workspace:^", "lmdb": "^3.0.6" }, "devDependencies": { + "@aztec/circuits.js": "workspace:^", "@jest/globals": "^29.5.0", "@types/jest": "^29.5.0", "@types/node": "^18.7.23", @@ -75,4 +78,4 @@ "engines": { "node": ">=18" } -} +} \ No newline at end of file diff --git a/yarn-project/kv-store/src/stores/index.ts b/yarn-project/kv-store/src/stores/index.ts new file mode 100644 index 00000000000..c279b4ba628 --- /dev/null +++ b/yarn-project/kv-store/src/stores/index.ts @@ -0,0 +1 @@ +export * from './l2_tips_store.js'; diff --git a/yarn-project/kv-store/src/stores/l2_tips_store.test.ts b/yarn-project/kv-store/src/stores/l2_tips_store.test.ts new file mode 100644 index 00000000000..2b820aaf432 --- /dev/null +++ b/yarn-project/kv-store/src/stores/l2_tips_store.test.ts @@ -0,0 +1,71 @@ +import { type L2Block } from '@aztec/circuit-types'; +import { Fr, type Header } from '@aztec/circuits.js'; +import { times } from '@aztec/foundation/collection'; +import { type AztecKVStore } from '@aztec/kv-store'; +import { openTmpStore } from '@aztec/kv-store/utils'; + +import { L2TipsStore } from './l2_tips_store.js'; + +describe('L2TipsStore', () => { + let kvStore: AztecKVStore; + let tipsStore: L2TipsStore; + + beforeEach(() => { + kvStore = openTmpStore(true); + tipsStore = new L2TipsStore(kvStore, 'test'); + }); + + const makeBlock = (number: number): L2Block => + ({ number, header: { hash: () => new Fr(number) } as Header } as L2Block); + + const makeTip = (number: number) => ({ number, hash: number === 0 ? undefined : new Fr(number).toString() }); + + const makeTips = (latest: number, proven: number, finalized: number) => ({ + latest: makeTip(latest), + proven: makeTip(proven), + finalized: makeTip(finalized), + }); + + it('returns zero if no tips are stored', async () => { + const tips = await tipsStore.getL2Tips(); + expect(tips).toEqual(makeTips(0, 0, 0)); + }); + + it('stores chain tips', async () => { + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: times(20, i => makeBlock(i + 1)) }); + + await tipsStore.handleBlockStreamEvent({ type: 'chain-finalized', blockNumber: 5 }); + await tipsStore.handleBlockStreamEvent({ type: 'chain-proven', blockNumber: 8 }); + await tipsStore.handleBlockStreamEvent({ type: 'chain-pruned', blockNumber: 10 }); + + const tips = await tipsStore.getL2Tips(); + expect(tips).toEqual(makeTips(10, 8, 5)); + }); + + it('sets latest tip from blocks added', async () => { + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: times(3, i => makeBlock(i + 1)) }); + + const tips = await tipsStore.getL2Tips(); + expect(tips).toEqual(makeTips(3, 0, 0)); + + expect(await tipsStore.getL2BlockHash(1)).toEqual(new Fr(1).toString()); + expect(await tipsStore.getL2BlockHash(2)).toEqual(new Fr(2).toString()); + expect(await tipsStore.getL2BlockHash(3)).toEqual(new Fr(3).toString()); + }); + + it('clears block hashes when setting finalized chain', async () => { + await tipsStore.handleBlockStreamEvent({ type: 'blocks-added', blocks: times(5, i => makeBlock(i + 1)) }); + await tipsStore.handleBlockStreamEvent({ type: 'chain-proven', blockNumber: 3 }); + await tipsStore.handleBlockStreamEvent({ type: 'chain-finalized', blockNumber: 3 }); + + const tips = await tipsStore.getL2Tips(); + expect(tips).toEqual(makeTips(5, 3, 3)); + + expect(await tipsStore.getL2BlockHash(1)).toBeUndefined(); + expect(await tipsStore.getL2BlockHash(2)).toBeUndefined(); + + expect(await tipsStore.getL2BlockHash(3)).toEqual(new Fr(3).toString()); + expect(await tipsStore.getL2BlockHash(4)).toEqual(new Fr(4).toString()); + expect(await tipsStore.getL2BlockHash(5)).toEqual(new Fr(5).toString()); + }); +}); diff --git a/yarn-project/kv-store/src/stores/l2_tips_store.ts b/yarn-project/kv-store/src/stores/l2_tips_store.ts new file mode 100644 index 00000000000..36180272967 --- /dev/null +++ b/yarn-project/kv-store/src/stores/l2_tips_store.ts @@ -0,0 +1,69 @@ +import { + type L2BlockId, + type L2BlockStreamEvent, + type L2BlockStreamEventHandler, + type L2BlockStreamLocalDataProvider, + type L2BlockTag, + type L2Tips, +} from '@aztec/circuit-types'; + +import { type AztecMap } from '../interfaces/map.js'; +import { type AztecKVStore } from '../interfaces/store.js'; + +/** Stores currently synced L2 tips and unfinalized block hashes. */ +export class L2TipsStore implements L2BlockStreamEventHandler, L2BlockStreamLocalDataProvider { + private readonly l2TipsStore: AztecMap; + private readonly l2BlockHashesStore: AztecMap; + + constructor(store: AztecKVStore, namespace: string) { + this.l2TipsStore = store.openMap([namespace, 'l2_tips'].join('_')); + this.l2BlockHashesStore = store.openMap([namespace, 'l2_block_hashes'].join('_')); + } + + public getL2BlockHash(number: number): Promise { + return Promise.resolve(this.l2BlockHashesStore.get(number)); + } + + public getL2Tips(): Promise { + return Promise.resolve({ + latest: this.getL2Tip('latest'), + finalized: this.getL2Tip('finalized'), + proven: this.getL2Tip('proven'), + }); + } + + private getL2Tip(tag: L2BlockTag): L2BlockId { + const blockNumber = this.l2TipsStore.get(tag); + if (blockNumber === undefined) { + return { number: 0, hash: undefined }; + } + const blockHash = this.l2BlockHashesStore.get(blockNumber); + if (!blockHash) { + throw new Error(`Block hash not found for block number ${blockNumber}`); + } + return { number: blockNumber, hash: blockHash }; + } + + public async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise { + switch (event.type) { + case 'blocks-added': + await this.l2TipsStore.set('latest', event.blocks.at(-1)!.number); + for (const block of event.blocks) { + await this.l2BlockHashesStore.set(block.number, block.header.hash().toString()); + } + break; + case 'chain-pruned': + await this.l2TipsStore.set('latest', event.blockNumber); + break; + case 'chain-proven': + await this.l2TipsStore.set('proven', event.blockNumber); + break; + case 'chain-finalized': + await this.l2TipsStore.set('finalized', event.blockNumber); + for (const key of this.l2BlockHashesStore.keys({ end: event.blockNumber })) { + await this.l2BlockHashesStore.delete(key); + } + break; + } + } +} diff --git a/yarn-project/kv-store/tsconfig.json b/yarn-project/kv-store/tsconfig.json index 63f8ab3e9f7..d2bc1df097f 100644 --- a/yarn-project/kv-store/tsconfig.json +++ b/yarn-project/kv-store/tsconfig.json @@ -6,8 +6,14 @@ "tsBuildInfoFile": ".tsbuildinfo" }, "references": [ + { + "path": "../circuit-types" + }, { "path": "../foundation" + }, + { + "path": "../circuits.js" } ], "include": ["src"] diff --git a/yarn-project/pxe/src/database/incoming_note_dao.test.ts b/yarn-project/pxe/src/database/incoming_note_dao.test.ts index a3cb0d5345c..1df9103e08f 100644 --- a/yarn-project/pxe/src/database/incoming_note_dao.test.ts +++ b/yarn-project/pxe/src/database/incoming_note_dao.test.ts @@ -1,38 +1,8 @@ -import { Note, randomTxHash } from '@aztec/circuit-types'; -import { AztecAddress, Fr, Point } from '@aztec/circuits.js'; -import { NoteSelector } from '@aztec/foundation/abi'; - import { IncomingNoteDao } from './incoming_note_dao.js'; -export const randomIncomingNoteDao = ({ - note = Note.random(), - contractAddress = AztecAddress.random(), - txHash = randomTxHash(), - storageSlot = Fr.random(), - noteTypeId = NoteSelector.random(), - nonce = Fr.random(), - noteHash = Fr.random(), - siloedNullifier = Fr.random(), - index = Fr.random().toBigInt(), - addressPoint = Point.random(), -}: Partial = {}) => { - return new IncomingNoteDao( - note, - contractAddress, - storageSlot, - noteTypeId, - txHash, - nonce, - noteHash, - siloedNullifier, - index, - addressPoint, - ); -}; - describe('Incoming Note DAO', () => { it('convert to and from buffer', () => { - const note = randomIncomingNoteDao(); + const note = IncomingNoteDao.random(); const buf = note.toBuffer(); expect(IncomingNoteDao.fromBuffer(buf)).toEqual(note); }); diff --git a/yarn-project/pxe/src/database/incoming_note_dao.ts b/yarn-project/pxe/src/database/incoming_note_dao.ts index cbd344b135c..d2dc2d38815 100644 --- a/yarn-project/pxe/src/database/incoming_note_dao.ts +++ b/yarn-project/pxe/src/database/incoming_note_dao.ts @@ -1,4 +1,4 @@ -import { type L1NotePayload, Note, TxHash } from '@aztec/circuit-types'; +import { type L1NotePayload, Note, TxHash, randomTxHash } from '@aztec/circuit-types'; import { AztecAddress, Fr, Point, type PublicKey } from '@aztec/circuits.js'; import { NoteSelector } from '@aztec/foundation/abi'; import { toBigIntBE } from '@aztec/foundation/bigint-buffer'; @@ -22,6 +22,10 @@ export class IncomingNoteDao implements NoteData { public noteTypeId: NoteSelector, /** The hash of the tx the note was created in. */ public txHash: TxHash, + /** The L2 block number in which the tx with this note was included. */ + public l2BlockNumber: number, + /** The L2 block hash in which the tx with this note was included. */ + public l2BlockHash: string, /** The nonce of the note. */ public nonce: Fr, /** @@ -44,6 +48,8 @@ export class IncomingNoteDao implements NoteData { note: Note, payload: L1NotePayload, noteInfo: NoteInfo, + l2BlockNumber: number, + l2BlockHash: string, dataStartIndexForTx: number, addressPoint: PublicKey, ) { @@ -54,6 +60,8 @@ export class IncomingNoteDao implements NoteData { payload.storageSlot, payload.noteTypeId, noteInfo.txHash, + l2BlockNumber, + l2BlockHash, noteInfo.nonce, noteInfo.noteHash, noteInfo.siloedNullifier, @@ -69,6 +77,8 @@ export class IncomingNoteDao implements NoteData { this.storageSlot, this.noteTypeId, this.txHash.buffer, + this.l2BlockNumber, + Fr.fromString(this.l2BlockHash), this.nonce, this.noteHash, this.siloedNullifier, @@ -76,6 +86,7 @@ export class IncomingNoteDao implements NoteData { this.addressPoint, ]); } + static fromBuffer(buffer: Buffer | BufferReader) { const reader = BufferReader.asReader(buffer); @@ -84,6 +95,8 @@ export class IncomingNoteDao implements NoteData { const storageSlot = Fr.fromBuffer(reader); const noteTypeId = reader.readObject(NoteSelector); const txHash = reader.readObject(TxHash); + const l2BlockNumber = reader.readNumber(); + const l2BlockHash = Fr.fromBuffer(reader).toString(); const nonce = Fr.fromBuffer(reader); const noteHash = Fr.fromBuffer(reader); const siloedNullifier = Fr.fromBuffer(reader); @@ -96,6 +109,8 @@ export class IncomingNoteDao implements NoteData { storageSlot, noteTypeId, txHash, + l2BlockNumber, + l2BlockHash, nonce, noteHash, siloedNullifier, @@ -122,4 +137,34 @@ export class IncomingNoteDao implements NoteData { const noteSize = 4 + this.note.items.length * Fr.SIZE_IN_BYTES; return noteSize + AztecAddress.SIZE_IN_BYTES + Fr.SIZE_IN_BYTES * 4 + TxHash.SIZE + Point.SIZE_IN_BYTES + indexSize; } + + static random({ + note = Note.random(), + contractAddress = AztecAddress.random(), + txHash = randomTxHash(), + storageSlot = Fr.random(), + noteTypeId = NoteSelector.random(), + nonce = Fr.random(), + l2BlockNumber = Math.floor(Math.random() * 1000), + l2BlockHash = Fr.random().toString(), + noteHash = Fr.random(), + siloedNullifier = Fr.random(), + index = Fr.random().toBigInt(), + addressPoint = Point.random(), + }: Partial = {}) { + return new IncomingNoteDao( + note, + contractAddress, + storageSlot, + noteTypeId, + txHash, + l2BlockNumber, + l2BlockHash, + nonce, + noteHash, + siloedNullifier, + index, + addressPoint, + ); + } } diff --git a/yarn-project/pxe/src/database/kv_pxe_database.ts b/yarn-project/pxe/src/database/kv_pxe_database.ts index 2fea8a0452b..90ccc3796e3 100644 --- a/yarn-project/pxe/src/database/kv_pxe_database.ts +++ b/yarn-project/pxe/src/database/kv_pxe_database.ts @@ -214,6 +214,41 @@ export class KVPxeDatabase implements PxeDatabase { }); } + public removeNotesAfter(blockNumber: number): Promise { + return this.db.transaction(() => { + for (const note of this.#notes.values()) { + const noteDao = IncomingNoteDao.fromBuffer(note); + if (noteDao.l2BlockNumber > blockNumber) { + const noteIndex = toBufferBE(noteDao.index, 32).toString('hex'); + void this.#notes.delete(noteIndex); + void this.#nullifierToNoteId.delete(noteDao.siloedNullifier.toString()); + for (const scope of this.#scopes.entries()) { + void this.#notesByAddressPointAndScope.get(scope)!.deleteValue(noteDao.addressPoint.toString(), noteIndex); + void this.#notesByTxHashAndScope.get(scope)!.deleteValue(noteDao.txHash.toString(), noteIndex); + void this.#notesByContractAndScope.get(scope)!.deleteValue(noteDao.contractAddress.toString(), noteIndex); + void this.#notesByStorageSlotAndScope.get(scope)!.deleteValue(noteDao.storageSlot.toString(), noteIndex); + } + } + } + + for (const note of this.#outgoingNotes.values()) { + const noteDao = OutgoingNoteDao.fromBuffer(note); + if (noteDao.l2BlockNumber > blockNumber) { + const noteIndex = toBufferBE(noteDao.index, 32).toString('hex'); + void this.#outgoingNotes.delete(noteIndex); + void this.#outgoingNotesByContract.deleteValue(noteDao.contractAddress.toString(), noteIndex); + void this.#outgoingNotesByStorageSlot.deleteValue(noteDao.storageSlot.toString(), noteIndex); + void this.#outgoingNotesByTxHash.deleteValue(noteDao.txHash.toString(), noteIndex); + void this.#outgoingNotesByOvpkM.deleteValue(noteDao.ovpkM.toString(), noteIndex); + } + } + }); + } + + public async unnullifyNotesAfter(_blockNumber: number): Promise { + // TODO: Implementme. Requires tracking when a note was nullified. + } + getIncomingNotes(filter: IncomingNotesFilter): Promise { const publicKey: PublicKey | undefined = filter.owner ? computePoint(filter.owner) : undefined; diff --git a/yarn-project/pxe/src/database/outgoing_note_dao.test.ts b/yarn-project/pxe/src/database/outgoing_note_dao.test.ts index 04b2b2201f7..0c293ba13eb 100644 --- a/yarn-project/pxe/src/database/outgoing_note_dao.test.ts +++ b/yarn-project/pxe/src/database/outgoing_note_dao.test.ts @@ -1,26 +1,8 @@ -import { Note, randomTxHash } from '@aztec/circuit-types'; -import { AztecAddress, Fr, Point } from '@aztec/circuits.js'; -import { NoteSelector } from '@aztec/foundation/abi'; - import { OutgoingNoteDao } from './outgoing_note_dao.js'; -export const randomOutgoingNoteDao = ({ - note = Note.random(), - contractAddress = AztecAddress.random(), - txHash = randomTxHash(), - storageSlot = Fr.random(), - noteTypeId = NoteSelector.random(), - nonce = Fr.random(), - noteHash = Fr.random(), - index = Fr.random().toBigInt(), - ovpkM = Point.random(), -}: Partial = {}) => { - return new OutgoingNoteDao(note, contractAddress, storageSlot, noteTypeId, txHash, nonce, noteHash, index, ovpkM); -}; - describe('Outgoing Note DAO', () => { it('convert to and from buffer', () => { - const note = randomOutgoingNoteDao(); + const note = OutgoingNoteDao.random(); const buf = note.toBuffer(); expect(OutgoingNoteDao.fromBuffer(buf)).toEqual(note); }); diff --git a/yarn-project/pxe/src/database/outgoing_note_dao.ts b/yarn-project/pxe/src/database/outgoing_note_dao.ts index 04bb7d4835c..386b23ecd57 100644 --- a/yarn-project/pxe/src/database/outgoing_note_dao.ts +++ b/yarn-project/pxe/src/database/outgoing_note_dao.ts @@ -1,4 +1,4 @@ -import { type L1NotePayload, Note, TxHash } from '@aztec/circuit-types'; +import { type L1NotePayload, Note, TxHash, randomTxHash } from '@aztec/circuit-types'; import { AztecAddress, Fr, Point, type PublicKey } from '@aztec/circuits.js'; import { NoteSelector } from '@aztec/foundation/abi'; import { toBigIntBE } from '@aztec/foundation/bigint-buffer'; @@ -21,6 +21,10 @@ export class OutgoingNoteDao { public noteTypeId: NoteSelector, /** The hash of the tx the note was created in. */ public txHash: TxHash, + /** The L2 block number in which the tx with this note was included. */ + public l2BlockNumber: number, + /** The L2 block hash in which the tx with this note was included. */ + public l2BlockHash: string, /** The nonce of the note. */ public nonce: Fr, /** @@ -38,6 +42,8 @@ export class OutgoingNoteDao { note: Note, payload: L1NotePayload, noteInfo: NoteInfo, + l2BlockNumber: number, + l2BlockHash: string, dataStartIndexForTx: number, ovpkM: PublicKey, ) { @@ -48,6 +54,8 @@ export class OutgoingNoteDao { payload.storageSlot, payload.noteTypeId, noteInfo.txHash, + l2BlockNumber, + l2BlockHash, noteInfo.nonce, noteInfo.noteHash, noteHashIndexInTheWholeTree, @@ -62,6 +70,8 @@ export class OutgoingNoteDao { this.storageSlot, this.noteTypeId, this.txHash.buffer, + this.l2BlockNumber, + Fr.fromString(this.l2BlockHash), this.nonce, this.noteHash, this.index, @@ -76,6 +86,8 @@ export class OutgoingNoteDao { const storageSlot = Fr.fromBuffer(reader); const noteTypeId = reader.readObject(NoteSelector); const txHash = new TxHash(reader.readBytes(TxHash.SIZE)); + const l2BlockNumber = reader.readNumber(); + const l2BlockHash = Fr.fromBuffer(reader).toString(); const nonce = Fr.fromBuffer(reader); const noteHash = Fr.fromBuffer(reader); const index = toBigIntBE(reader.readBytes(32)); @@ -87,6 +99,8 @@ export class OutgoingNoteDao { storageSlot, noteTypeId, txHash, + l2BlockNumber, + l2BlockHash, nonce, noteHash, index, @@ -111,4 +125,32 @@ export class OutgoingNoteDao { const noteSize = 4 + this.note.items.length * Fr.SIZE_IN_BYTES; return noteSize + AztecAddress.SIZE_IN_BYTES + Fr.SIZE_IN_BYTES * 2 + TxHash.SIZE + Point.SIZE_IN_BYTES; } + + static random({ + note = Note.random(), + contractAddress = AztecAddress.random(), + txHash = randomTxHash(), + storageSlot = Fr.random(), + noteTypeId = NoteSelector.random(), + nonce = Fr.random(), + l2BlockNumber = Math.floor(Math.random() * 1000), + l2BlockHash = Fr.random().toString(), + noteHash = Fr.random(), + index = Fr.random().toBigInt(), + ovpkM = Point.random(), + }: Partial = {}) { + return new OutgoingNoteDao( + note, + contractAddress, + storageSlot, + noteTypeId, + txHash, + l2BlockNumber, + l2BlockHash, + nonce, + noteHash, + index, + ovpkM, + ); + } } diff --git a/yarn-project/pxe/src/database/pxe_database.ts b/yarn-project/pxe/src/database/pxe_database.ts index d4cdb12bcec..58e39901da3 100644 --- a/yarn-project/pxe/src/database/pxe_database.ts +++ b/yarn-project/pxe/src/database/pxe_database.ts @@ -214,4 +214,16 @@ export interface PxeDatabase extends ContractArtifactDatabase, ContractInstanceD * @param appTaggingSecrets - The app siloed tagging secrets. */ setTaggingSecretsIndexesAsRecipient(indexedTaggingSecrets: IndexedTaggingSecret[]): Promise; + + /** + * Deletes all notes synched after this block number. + * @param blockNumber - All notes strictly after this block number are removed. + */ + removeNotesAfter(blockNumber: number): Promise; + + /** + * Restores notes nullified after the given block. + * @param blockNumber - All nullifiers strictly after this block are removed. + */ + unnullifyNotesAfter(blockNumber: number): Promise; } diff --git a/yarn-project/pxe/src/database/pxe_database_test_suite.ts b/yarn-project/pxe/src/database/pxe_database_test_suite.ts index 1fefa614bad..b0dfd5272d6 100644 --- a/yarn-project/pxe/src/database/pxe_database_test_suite.ts +++ b/yarn-project/pxe/src/database/pxe_database_test_suite.ts @@ -12,10 +12,8 @@ import { randomInt } from '@aztec/foundation/crypto'; import { Fr, Point } from '@aztec/foundation/fields'; import { BenchmarkingContractArtifact } from '@aztec/noir-contracts.js/Benchmarking'; -import { type IncomingNoteDao } from './incoming_note_dao.js'; -import { randomIncomingNoteDao } from './incoming_note_dao.test.js'; -import { type OutgoingNoteDao } from './outgoing_note_dao.js'; -import { randomOutgoingNoteDao } from './outgoing_note_dao.test.js'; +import { IncomingNoteDao } from './incoming_note_dao.js'; +import { OutgoingNoteDao } from './outgoing_note_dao.js'; import { type PxeDatabase } from './pxe_database.js'; /** @@ -121,11 +119,12 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { storageSlots = Array.from({ length: 2 }).map(() => Fr.random()); notes = Array.from({ length: 10 }).map((_, i) => - randomIncomingNoteDao({ + IncomingNoteDao.random({ contractAddress: contractAddresses[i % contractAddresses.length], storageSlot: storageSlots[i % storageSlots.length], addressPoint: computePoint(owners[i % owners.length].address), index: BigInt(i), + l2BlockNumber: i, }), ); @@ -260,6 +259,14 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { }), ).resolves.toEqual([]); }); + + it('removes notes after a given block', async () => { + await database.addNotes(notes, [], owners[0].address); + + await database.removeNotesAfter(5); + const result = await database.getIncomingNotes({ scopes: [owners[0].address] }); + expect(new Set(result)).toEqual(new Set(notes.slice(0, 6))); + }); }); describe('outgoing notes', () => { @@ -307,7 +314,7 @@ export function describePxeDatabase(getDatabase: () => PxeDatabase) { storageSlots = Array.from({ length: 2 }).map(() => Fr.random()); notes = Array.from({ length: 10 }).map((_, i) => - randomOutgoingNoteDao({ + OutgoingNoteDao.random({ contractAddress: contractAddresses[i % contractAddresses.length], storageSlot: storageSlots[i % storageSlots.length], ovpkM: owners[i % owners.length].publicKeys.masterOutgoingViewingPublicKey, diff --git a/yarn-project/pxe/src/kernel_oracle/index.ts b/yarn-project/pxe/src/kernel_oracle/index.ts index d2115d2cc40..a66ec8db465 100644 --- a/yarn-project/pxe/src/kernel_oracle/index.ts +++ b/yarn-project/pxe/src/kernel_oracle/index.ts @@ -70,7 +70,7 @@ export class KernelOracle implements ProvingDataOracle { } async getNoteHashTreeRoot(): Promise { - const header = await this.node.getHeader(this.blockNumber); + const header = await this.node.getBlockHeader(this.blockNumber); return header.state.partial.noteHashTree.root; } diff --git a/yarn-project/pxe/src/note_decryption_utils/produce_note_daos.ts b/yarn-project/pxe/src/note_decryption_utils/produce_note_daos.ts index 8e857d51330..fc3e1918ce1 100644 --- a/yarn-project/pxe/src/note_decryption_utils/produce_note_daos.ts +++ b/yarn-project/pxe/src/note_decryption_utils/produce_note_daos.ts @@ -34,6 +34,8 @@ export async function produceNoteDaos( ovpkM: PublicKey | undefined, payload: L1NotePayload, txHash: TxHash, + l2BlockNumber: number, + l2BlockHash: string, noteHashes: Fr[], dataStartIndexForTx: number, excludedIndices: Set, @@ -56,6 +58,8 @@ export async function produceNoteDaos( addressPoint, payload, txHash, + l2BlockNumber, + l2BlockHash, noteHashes, dataStartIndexForTx, excludedIndices, @@ -74,6 +78,8 @@ export async function produceNoteDaos( incomingNote.storageSlot, incomingNote.noteTypeId, incomingNote.txHash, + incomingNote.l2BlockNumber, + incomingNote.l2BlockHash, incomingNote.nonce, incomingNote.noteHash, incomingNote.index, @@ -86,6 +92,8 @@ export async function produceNoteDaos( ovpkM, payload, txHash, + l2BlockNumber, + l2BlockHash, noteHashes, dataStartIndexForTx, excludedIndices, diff --git a/yarn-project/pxe/src/note_decryption_utils/produce_note_daos_for_key.ts b/yarn-project/pxe/src/note_decryption_utils/produce_note_daos_for_key.ts index 9e530b387d1..eeeb6c9ee9e 100644 --- a/yarn-project/pxe/src/note_decryption_utils/produce_note_daos_for_key.ts +++ b/yarn-project/pxe/src/note_decryption_utils/produce_note_daos_for_key.ts @@ -13,6 +13,8 @@ export async function produceNoteDaosForKey( pkM: PublicKey, payload: L1NotePayload, txHash: TxHash, + l2BlockNumber: number, + l2BlockHash: string, noteHashes: Fr[], dataStartIndexForTx: number, excludedIndices: Set, @@ -21,6 +23,8 @@ export async function produceNoteDaosForKey( note: Note, payload: L1NotePayload, noteInfo: NoteInfo, + l2BlockNumber: number, + l2BlockHash: string, dataStartIndexForTx: number, pkM: PublicKey, ) => T, @@ -44,7 +48,7 @@ export async function produceNoteDaosForKey( ); excludedIndices?.add(noteInfo.noteHashIndex); - noteDao = daoConstructor(note, payload, noteInfo, dataStartIndexForTx, pkM); + noteDao = daoConstructor(note, payload, noteInfo, l2BlockNumber, l2BlockHash, dataStartIndexForTx, pkM); } catch (e) { logger.error(`Could not process note because of "${e}". Discarding note...`); } diff --git a/yarn-project/pxe/src/pxe_service/create_pxe_service.ts b/yarn-project/pxe/src/pxe_service/create_pxe_service.ts index bb608b93873..c4e4763e131 100644 --- a/yarn-project/pxe/src/pxe_service/create_pxe_service.ts +++ b/yarn-project/pxe/src/pxe_service/create_pxe_service.ts @@ -3,6 +3,7 @@ import { type AztecNode, type PrivateKernelProver } from '@aztec/circuit-types'; import { randomBytes } from '@aztec/foundation/crypto'; import { createDebugLogger } from '@aztec/foundation/log'; import { KeyStore } from '@aztec/key-store'; +import { L2TipsStore } from '@aztec/kv-store/stores'; import { createStore } from '@aztec/kv-store/utils'; import { type PXEServiceConfig } from '../config/index.js'; @@ -35,10 +36,12 @@ export async function createPXEService( const keyStore = new KeyStore( await createStore('pxe_key_store', storeConfig, createDebugLogger('aztec:pxe:keystore:lmdb')), ); - const db = new KVPxeDatabase(await createStore('pxe_data', storeConfig, createDebugLogger('aztec:pxe:data:lmdb'))); + const store = await createStore('pxe_data', storeConfig, createDebugLogger('aztec:pxe:data:lmdb')); + const db = new KVPxeDatabase(store); + const tips = new L2TipsStore(store, 'pxe'); const prover = proofCreator ?? (await createProver(config, logSuffix)); - const server = new PXEService(keyStore, aztecNode, db, prover, config, logSuffix); + const server = new PXEService(keyStore, aztecNode, db, tips, prover, config, logSuffix); await server.start(); return server; } diff --git a/yarn-project/pxe/src/pxe_service/pxe_service.ts b/yarn-project/pxe/src/pxe_service/pxe_service.ts index 62e96081780..cc73dfcd8dd 100644 --- a/yarn-project/pxe/src/pxe_service/pxe_service.ts +++ b/yarn-project/pxe/src/pxe_service/pxe_service.ts @@ -6,6 +6,7 @@ import { type ExtendedNote, type FunctionCall, type GetUnencryptedLogsResponse, + type InBlock, type IncomingNotesFilter, L1EventPayload, type L2Block, @@ -22,7 +23,6 @@ import { type SiblingPath, SimulationError, type Tx, - type TxEffect, type TxExecutionRequest, type TxHash, TxProvingResult, @@ -58,6 +58,7 @@ import { Fr, type Point } from '@aztec/foundation/fields'; import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; import { SerialQueue } from '@aztec/foundation/queue'; import { type KeyStore } from '@aztec/key-store'; +import { type L2TipsStore } from '@aztec/kv-store/stores'; import { ProtocolContractAddress, getCanonicalProtocolContract, @@ -93,12 +94,13 @@ export class PXEService implements PXE { private keyStore: KeyStore, private node: AztecNode, private db: PxeDatabase, + tipsStore: L2TipsStore, private proofCreator: PrivateKernelProver, - private config: PXEServiceConfig, + config: PXEServiceConfig, logSuffix?: string, ) { this.log = createDebugLogger(logSuffix ? `aztec:pxe_service_${logSuffix}` : `aztec:pxe_service`); - this.synchronizer = new Synchronizer(node, db, this.jobQueue, logSuffix); + this.synchronizer = new Synchronizer(node, db, tipsStore, config, logSuffix); this.contractDataOracle = new ContractDataOracle(db); this.simulator = getAcirSimulator(db, node, keyStore, this.contractDataOracle); this.packageVersion = getPackageInfo().version; @@ -112,8 +114,7 @@ export class PXEService implements PXE { * @returns A promise that resolves when the server has started successfully. */ public async start() { - const { l2BlockPollingIntervalMS } = this.config; - await this.synchronizer.start(1, l2BlockPollingIntervalMS); + await this.synchronizer.start(); await this.#registerProtocolContracts(); const info = await this.getNodeInfo(); this.log.info(`Started PXE connected to chain ${info.l1ChainId} version ${info.protocolVersion}`); @@ -350,7 +351,7 @@ export class PXEService implements PXE { throw new Error(`Unknown account: ${note.owner.toString()}`); } - const nonces = await this.#getNoteNonces(note); + const { data: nonces, l2BlockNumber, l2BlockHash } = await this.#getNoteNonces(note); if (nonces.length === 0) { throw new Error(`Cannot find the note in tx: ${note.txHash}.`); } @@ -385,6 +386,8 @@ export class PXEService implements PXE { note.storageSlot, note.noteTypeId, note.txHash, + l2BlockNumber, + l2BlockHash, nonce, noteHash, siloedNullifier, @@ -397,7 +400,7 @@ export class PXEService implements PXE { } public async addNullifiedNote(note: ExtendedNote) { - const nonces = await this.#getNoteNonces(note); + const { data: nonces, l2BlockHash, l2BlockNumber } = await this.#getNoteNonces(note); if (nonces.length === 0) { throw new Error(`Cannot find the note in tx: ${note.txHash}.`); } @@ -428,6 +431,8 @@ export class PXEService implements PXE { note.storageSlot, note.noteTypeId, note.txHash, + l2BlockNumber, + l2BlockHash, nonce, noteHash, Fr.ZERO, // We are not able to derive @@ -444,15 +449,15 @@ export class PXEService implements PXE { * @returns The nonces of the note. * @remarks More than a single nonce may be returned since there might be more than one nonce for a given note. */ - async #getNoteNonces(note: ExtendedNote): Promise { + async #getNoteNonces(note: ExtendedNote): Promise> { const tx = await this.node.getTxEffect(note.txHash); if (!tx) { throw new Error(`Unknown tx: ${note.txHash}`); } const nonces: Fr[] = []; - const firstNullifier = tx.nullifiers[0]; - const hashes = tx.noteHashes; + const firstNullifier = tx.data.nullifiers[0]; + const hashes = tx.data.noteHashes; for (let i = 0; i < hashes.length; ++i) { const hash = hashes[i]; if (hash.equals(Fr.ZERO)) { @@ -473,7 +478,7 @@ export class PXEService implements PXE { } } - return nonces; + return { l2BlockHash: tx.l2BlockHash, l2BlockNumber: tx.l2BlockNumber, data: nonces }; } public async getBlock(blockNumber: number): Promise { @@ -587,7 +592,7 @@ export class PXEService implements PXE { return this.node.getTxReceipt(txHash); } - public getTxEffect(txHash: TxHash): Promise { + public getTxEffect(txHash: TxHash) { return this.node.getTxEffect(txHash); } diff --git a/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts b/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts index ca0d00de0db..90d3bb2ce0d 100644 --- a/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts +++ b/yarn-project/pxe/src/pxe_service/test/pxe_service.test.ts @@ -1,8 +1,9 @@ -import { type AztecNode, type PXE, TxEffect, mockTx } from '@aztec/circuit-types'; +import { type AztecNode, type PXE, TxEffect, mockTx, randomInBlock } from '@aztec/circuit-types'; import { INITIAL_L2_BLOCK_NUM } from '@aztec/circuits.js/constants'; import { type L1ContractAddresses } from '@aztec/ethereum'; import { EthAddress } from '@aztec/foundation/eth-address'; import { KeyStore } from '@aztec/key-store'; +import { L2TipsStore } from '@aztec/kv-store/stores'; import { openTmpStore } from '@aztec/kv-store/utils'; import { type MockProxy, mock } from 'jest-mock-extended'; @@ -19,6 +20,7 @@ function createPXEService(): Promise { const keyStore = new KeyStore(kvStore); const node = mock(); const db = new KVPxeDatabase(kvStore); + const tips = new L2TipsStore(kvStore, 'pxe'); const config: PXEServiceConfig = { l2BlockPollingIntervalMS: 100, l2StartingBlock: INITIAL_L2_BLOCK_NUM }; // Setup the relevant mocks @@ -39,7 +41,7 @@ function createPXEService(): Promise { }; node.getL1ContractAddresses.mockResolvedValue(mockedContracts); - return Promise.resolve(new PXEService(keyStore, node, db, new TestPrivateKernelProver(), config)); + return Promise.resolve(new PXEService(keyStore, node, db, tips, new TestPrivateKernelProver(), config)); } pxeTestSuite('PXEService', createPXEService); @@ -49,11 +51,13 @@ describe('PXEService', () => { let node: MockProxy; let db: PxeDatabase; let config: PXEServiceConfig; + let tips: L2TipsStore; beforeEach(() => { const kvStore = openTmpStore(); keyStore = new KeyStore(kvStore); node = mock(); + tips = new L2TipsStore(kvStore, 'pxe'); db = new KVPxeDatabase(kvStore); config = { l2BlockPollingIntervalMS: 100, l2StartingBlock: INITIAL_L2_BLOCK_NUM, proverEnabled: false }; }); @@ -62,9 +66,9 @@ describe('PXEService', () => { const settledTx = TxEffect.random(); const duplicateTx = mockTx(); - node.getTxEffect.mockResolvedValue(settledTx); + node.getTxEffect.mockResolvedValue(randomInBlock(settledTx)); - const pxe = new PXEService(keyStore, node, db, new TestPrivateKernelProver(), config); + const pxe = new PXEService(keyStore, node, db, tips, new TestPrivateKernelProver(), config); await expect(pxe.sendTx(duplicateTx)).rejects.toThrow(/A settled tx with equal hash/); }); }); diff --git a/yarn-project/pxe/src/simulator_oracle/index.ts b/yarn-project/pxe/src/simulator_oracle/index.ts index 73cf685b016..ed3e32df19d 100644 --- a/yarn-project/pxe/src/simulator_oracle/index.ts +++ b/yarn-project/pxe/src/simulator_oracle/index.ts @@ -1,5 +1,6 @@ import { type AztecNode, + InBlock, L1NotePayload, type L2Block, type L2BlockNumber, @@ -469,7 +470,7 @@ export class SimulatorOracle implements DBOracle { const incomingNotes: IncomingNoteDao[] = []; const outgoingNotes: OutgoingNoteDao[] = []; - const txEffectsCache = new Map(); + const txEffectsCache = new Map | undefined>(); for (const scopedLog of scopedLogs) { const incomingNotePayload = L1NotePayload.decryptAsIncoming( @@ -513,8 +514,10 @@ export class SimulatorOracle implements DBOracle { incomingNotePayload ? computePoint(recipient) : undefined, outgoingNotePayload ? recipientCompleteAddress.publicKeys.masterOutgoingViewingPublicKey : undefined, payload!, - txEffect.txHash, - txEffect.noteHashes, + txEffect.data.txHash, + txEffect.l2BlockNumber, + txEffect.l2BlockHash, + txEffect.data.noteHashes, scopedLog.dataStartIndexForTx, excludedIndices.get(scopedLog.txHash.toString())!, this.log, diff --git a/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts b/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts index f4630610b6e..bd51c6a7a2f 100644 --- a/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts +++ b/yarn-project/pxe/src/simulator_oracle/simulator_oracle.test.ts @@ -7,6 +7,7 @@ import { type TxEffect, TxHash, TxScopedL2Log, + randomInBlock, } from '@aztec/circuit-types'; import { AztecAddress, @@ -423,7 +424,7 @@ describe('Simulator oracle', () => { }); aztecNode.getTxEffect.mockImplementation(txHash => { - return Promise.resolve(txEffectsMap[txHash.toString()] as TxEffect); + return Promise.resolve(randomInBlock(txEffectsMap[txHash.toString()] as TxEffect)); }); aztecNode.findLeavesIndexes.mockImplementation((_blockNumber, _treeId, leafValues) => Promise.resolve( diff --git a/yarn-project/pxe/src/synchronizer/synchronizer.test.ts b/yarn-project/pxe/src/synchronizer/synchronizer.test.ts index 8b0c6810eda..ffde1cef571 100644 --- a/yarn-project/pxe/src/synchronizer/synchronizer.test.ts +++ b/yarn-project/pxe/src/synchronizer/synchronizer.test.ts @@ -1,94 +1,54 @@ -import { type AztecNode, L2Block } from '@aztec/circuit-types'; -import { type Header } from '@aztec/circuits.js'; -import { makeHeader } from '@aztec/circuits.js/testing'; -import { randomInt } from '@aztec/foundation/crypto'; -import { SerialQueue } from '@aztec/foundation/queue'; +import { type AztecNode, L2Block, type L2BlockStream } from '@aztec/circuit-types'; +import { L2TipsStore } from '@aztec/kv-store/stores'; import { openTmpStore } from '@aztec/kv-store/utils'; +import { jest } from '@jest/globals'; import { type MockProxy, mock } from 'jest-mock-extended'; +import times from 'lodash.times'; import { type PxeDatabase } from '../database/index.js'; import { KVPxeDatabase } from '../database/kv_pxe_database.js'; import { Synchronizer } from './synchronizer.js'; describe('Synchronizer', () => { - let aztecNode: MockProxy; let database: PxeDatabase; - let synchronizer: TestSynchronizer; - let jobQueue: SerialQueue; - const initialSyncBlockNumber = 3; - let headerBlock3: Header; - - beforeEach(() => { - headerBlock3 = makeHeader(randomInt(1000), initialSyncBlockNumber, initialSyncBlockNumber); + let synchronizer: Synchronizer; + let tipsStore: L2TipsStore; // eslint-disable-line @typescript-eslint/no-unused-vars - aztecNode = mock(); - database = new KVPxeDatabase(openTmpStore()); - jobQueue = new SerialQueue(); - synchronizer = new TestSynchronizer(aztecNode, database, jobQueue); - }); - - it('sets header from aztec node on initial sync', async () => { - aztecNode.getBlockNumber.mockResolvedValue(initialSyncBlockNumber); - aztecNode.getHeader.mockResolvedValue(headerBlock3); + let aztecNode: MockProxy; + let blockStream: MockProxy; - await synchronizer.initialSync(); + const TestSynchronizer = class extends Synchronizer { + protected override createBlockStream(): L2BlockStream { + return blockStream; + } + }; - expect(database.getHeader()).toEqual(headerBlock3); + beforeEach(() => { + const store = openTmpStore(); + blockStream = mock(); + aztecNode = mock(); + database = new KVPxeDatabase(store); + tipsStore = new L2TipsStore(store, 'pxe'); + synchronizer = new TestSynchronizer(aztecNode, database, tipsStore); }); it('sets header from latest block', async () => { const block = L2Block.random(1, 4); - aztecNode.getLogs.mockResolvedValueOnce([block.body.encryptedLogs]).mockResolvedValue([block.body.unencryptedLogs]); - aztecNode.getBlocks.mockResolvedValue([block]); - - await synchronizer.work(); + await synchronizer.handleBlockStreamEvent({ type: 'blocks-added', blocks: [block] }); const obtainedHeader = database.getHeader(); expect(obtainedHeader).toEqual(block.header); }); - it('overrides header from initial sync once current block number is larger', async () => { - // Initial sync is done on block with height 3 - aztecNode.getBlockNumber.mockResolvedValue(initialSyncBlockNumber); - aztecNode.getHeader.mockResolvedValue(headerBlock3); - - await synchronizer.initialSync(); - const header0 = database.getHeader(); - expect(header0).toEqual(headerBlock3); - - // We then process block with height 1, this should not change the header - const block1 = L2Block.random(1, 4); - - aztecNode.getLogs - .mockResolvedValueOnce([block1.body.encryptedLogs]) - .mockResolvedValue([block1.body.unencryptedLogs]); - - aztecNode.getBlocks.mockResolvedValue([block1]); + it('removes notes from db on a reorg', async () => { + const removeNotesAfter = jest.spyOn(database, 'removeNotesAfter').mockImplementation(() => Promise.resolve()); + const unnullifyNotesAfter = jest.spyOn(database, 'unnullifyNotesAfter').mockImplementation(() => Promise.resolve()); - await synchronizer.work(); - const header1 = database.getHeader(); - expect(header1).toEqual(headerBlock3); - expect(header1).not.toEqual(block1.header); + await synchronizer.handleBlockStreamEvent({ type: 'blocks-added', blocks: times(5, L2Block.random) }); + await synchronizer.handleBlockStreamEvent({ type: 'chain-pruned', blockNumber: 3 }); - // But they should change when we process block with height 5 - const block5 = L2Block.random(5, 4); - - aztecNode.getBlocks.mockResolvedValue([block5]); - - await synchronizer.work(); - const header5 = database.getHeader(); - expect(header5).not.toEqual(headerBlock3); - expect(header5).toEqual(block5.header); + expect(removeNotesAfter).toHaveBeenCalledWith(3); + expect(unnullifyNotesAfter).toHaveBeenCalledWith(3); }); }); - -class TestSynchronizer extends Synchronizer { - public override work(limit = 1) { - return super.work(limit); - } - - public override initialSync(): Promise { - return super.initialSync(); - } -} diff --git a/yarn-project/pxe/src/synchronizer/synchronizer.ts b/yarn-project/pxe/src/synchronizer/synchronizer.ts index 3ed458b2db1..63fdc36843e 100644 --- a/yarn-project/pxe/src/synchronizer/synchronizer.ts +++ b/yarn-project/pxe/src/synchronizer/synchronizer.ts @@ -1,9 +1,14 @@ -import { type AztecNode, type L2Block } from '@aztec/circuit-types'; +import { + type AztecNode, + L2BlockStream, + type L2BlockStreamEvent, + type L2BlockStreamEventHandler, +} from '@aztec/circuit-types'; import { INITIAL_L2_BLOCK_NUM } from '@aztec/circuits.js'; import { type DebugLogger, createDebugLogger } from '@aztec/foundation/log'; -import { type SerialQueue } from '@aztec/foundation/queue'; -import { RunningPromise } from '@aztec/foundation/running-promise'; +import { type L2TipsStore } from '@aztec/kv-store/stores'; +import { type PXEConfig } from '../config/index.js'; import { type PxeDatabase } from '../database/index.js'; /** @@ -13,14 +18,48 @@ import { type PxeDatabase } from '../database/index.js'; * details, and fetch transactions by hash. The Synchronizer ensures that it maintains the note processors * in sync with the blockchain while handling retries and errors gracefully. */ -export class Synchronizer { - private runningPromise?: RunningPromise; +export class Synchronizer implements L2BlockStreamEventHandler { private running = false; private initialSyncBlockNumber = INITIAL_L2_BLOCK_NUM - 1; private log: DebugLogger; + protected readonly blockStream: L2BlockStream; - constructor(private node: AztecNode, private db: PxeDatabase, private jobQueue: SerialQueue, logSuffix = '') { + constructor( + private node: AztecNode, + private db: PxeDatabase, + private l2TipsStore: L2TipsStore, + config: Partial> = {}, + logSuffix?: string, + ) { this.log = createDebugLogger(logSuffix ? `aztec:pxe_synchronizer_${logSuffix}` : 'aztec:pxe_synchronizer'); + this.blockStream = this.createBlockStream(config); + } + + protected createBlockStream(config: Partial>) { + return new L2BlockStream(this.node, this.l2TipsStore, this, { + pollIntervalMS: config.l2BlockPollingIntervalMS, + startingBlock: config.l2StartingBlock, + }); + } + + /** Handle events emitted by the block stream. */ + public async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise { + await this.l2TipsStore.handleBlockStreamEvent(event); + + switch (event.type) { + case 'blocks-added': + this.log.verbose(`Processing blocks ${event.blocks[0].number} to ${event.blocks.at(-1)!.number}`); + await this.db.setHeader(event.blocks.at(-1)!.header); + break; + case 'chain-pruned': + this.log.info(`Pruning data after block ${event.blockNumber} due to reorg`); + // We first unnullify and then remove so that unnullified notes that were created after the block number end up deleted. + await this.db.unnullifyNotesAfter(event.blockNumber); + await this.db.removeNotesAfter(event.blockNumber); + // Update the header to the last block. + await this.db.setHeader(await this.node.getBlockHeader(event.blockNumber)); + break; + } } /** @@ -31,79 +70,21 @@ export class Synchronizer { * @param limit - The maximum number of encrypted, unencrypted logs and blocks to fetch in each iteration. * @param retryInterval - The time interval (in ms) to wait before retrying if no data is available. */ - public async start(limit = 1, retryInterval = 1000) { + public async start() { if (this.running) { return; } this.running = true; - await this.jobQueue.put(() => this.initialSync()); + // REFACTOR: We should know the header of the genesis block without having to request it from the node. + await this.db.setHeader(await this.node.getBlockHeader(0)); + + await this.trigger(); this.log.info('Initial sync complete'); - this.runningPromise = new RunningPromise(() => this.sync(limit), retryInterval); - this.runningPromise.start(); + this.blockStream.start(); this.log.debug('Started loop'); } - protected async initialSync() { - // fast forward to the latest block - const latestHeader = await this.node.getHeader(); - this.initialSyncBlockNumber = Number(latestHeader.globalVariables.blockNumber.toBigInt()); - await this.db.setHeader(latestHeader); - } - - /** - * Fetches encrypted logs and blocks from the Aztec node and processes them for all note processors. - * If needed, catches up note processors that are lagging behind the main sync, e.g. because we just added a new account. - * - * Uses the job queue to ensure that - * - sync does not overlap with pxe simulations. - * - one sync is running at a time. - * - * @param limit - The maximum number of encrypted, unencrypted logs and blocks to fetch in each iteration. - * @returns a promise that resolves when the sync is complete - */ - protected sync(limit: number) { - return this.jobQueue.put(async () => { - let moreWork = true; - // keep external this.running flag to interrupt greedy sync - while (moreWork && this.running) { - moreWork = await this.work(limit); - } - }); - } - - /** - * Fetches encrypted logs and blocks from the Aztec node and processes them for all note processors. - * - * @param limit - The maximum number of encrypted, unencrypted logs and blocks to fetch in each iteration. - * @returns true if there could be more work, false if we're caught up or there was an error. - */ - protected async work(limit = 1): Promise { - const from = this.getSynchedBlockNumber() + 1; - try { - const blocks = await this.node.getBlocks(from, limit); - if (blocks.length === 0) { - return false; - } - - // Update latest tree roots from the most recent block - const latestBlock = blocks[blocks.length - 1]; - await this.setHeaderFromBlock(latestBlock); - return true; - } catch (err) { - this.log.error(`Error in synchronizer work`, err); - return false; - } - } - - private async setHeaderFromBlock(latestBlock: L2Block) { - if (latestBlock.number < this.initialSyncBlockNumber) { - return; - } - - await this.db.setHeader(latestBlock.header); - } - /** * Stops the synchronizer gracefully, interrupting any ongoing sleep and waiting for the current * iteration to complete before setting the running state to false. Once stopped, the synchronizer @@ -113,10 +94,15 @@ export class Synchronizer { */ public async stop() { this.running = false; - await this.runningPromise?.stop(); + await this.blockStream.stop(); this.log.info('Stopped'); } + /** Triggers a single run. */ + public async trigger() { + await this.blockStream.sync(); + } + private getSynchedBlockNumber() { return this.db.getBlockNumber() ?? this.initialSyncBlockNumber; } diff --git a/yarn-project/yarn.lock b/yarn-project/yarn.lock index 449d4424f8e..68d0b7b6269 100644 --- a/yarn-project/yarn.lock +++ b/yarn-project/yarn.lock @@ -763,6 +763,8 @@ __metadata: version: 0.0.0-use.local resolution: "@aztec/kv-store@workspace:kv-store" dependencies: + "@aztec/circuit-types": "workspace:^" + "@aztec/circuits.js": "workspace:^" "@aztec/foundation": "workspace:^" "@jest/globals": ^29.5.0 "@types/jest": ^29.5.0