From ccbff99386b5ed8b9432e452f43778953f8c47dd Mon Sep 17 00:00:00 2001 From: just-mitch <68168980+just-mitch@users.noreply.github.com> Date: Tue, 9 Jan 2024 15:37:40 -0500 Subject: [PATCH] fix: reprocess notes in pxe when a new contract is added (#3867) Add notion of a Deferred Note: a note that we can *decrypt*, but not decode because we don't have the contract artifact in our PXE database. We hang on to these deferred notes in a new store in our PXE database, and reprocess them when their contract is made available. When we reprocess and successfully decode, we also need to search if we have already nullified the newly decoded note. --- .../acir-simulator/src/client/db_oracle.ts | 11 +- .../archiver/src/archiver/archiver.ts | 1 - .../end-to-end/src/e2e_2_pxes.test.ts | 114 ++++++++++- .../pxe/src/contract_data_oracle/index.ts | 31 +-- .../src/database/deferred_note_dao.test.ts | 34 ++++ .../pxe/src/database/deferred_note_dao.ts | 55 +++++ .../pxe/src/database/kv_pxe_database.ts | 63 ++++++ yarn-project/pxe/src/database/memory_db.ts | 23 +++ yarn-project/pxe/src/database/note_dao.ts | 3 +- yarn-project/pxe/src/database/pxe_database.ts | 20 ++ .../pxe/src/note_processor/note_processor.ts | 190 +++++++++--------- .../src/note_processor/produce_note_dao.ts | 131 ++++++++++++ .../pxe/src/pxe_service/pxe_service.ts | 13 +- .../pxe/src/synchronizer/synchronizer.ts | 151 ++++++++++---- yarn-project/types/src/contract_dao.ts | 22 +- yarn-project/types/src/stats/stats.ts | 2 + yarn-project/types/src/tx/tx_hash.ts | 20 +- 17 files changed, 725 insertions(+), 159 deletions(-) create mode 100644 yarn-project/pxe/src/database/deferred_note_dao.test.ts create mode 100644 yarn-project/pxe/src/database/deferred_note_dao.ts create mode 100644 yarn-project/pxe/src/note_processor/produce_note_dao.ts diff --git a/yarn-project/acir-simulator/src/client/db_oracle.ts b/yarn-project/acir-simulator/src/client/db_oracle.ts index 17b9171ff7f..b389d423891 100644 --- a/yarn-project/acir-simulator/src/client/db_oracle.ts +++ b/yarn-project/acir-simulator/src/client/db_oracle.ts @@ -6,7 +6,16 @@ import { Fr } from '@aztec/foundation/fields'; import { L2Block, MerkleTreeId, NullifierMembershipWitness, PublicDataWitness } from '@aztec/types'; import { NoteData } from '../acvm/index.js'; -import { CommitmentsDB } from '../public/index.js'; +import { CommitmentsDB } from '../public/db.js'; + +/** + * Error thrown when a contract is not found in the database. + */ +export class ContractNotFoundError extends Error { + constructor(contractAddress: string) { + super(`DB has no contract with address ${contractAddress}`); + } +} /** * A function artifact with optional debug metadata diff --git a/yarn-project/archiver/src/archiver/archiver.ts b/yarn-project/archiver/src/archiver/archiver.ts index 1357d331861..51bbe2b3cad 100644 --- a/yarn-project/archiver/src/archiver/archiver.ts +++ b/yarn-project/archiver/src/archiver/archiver.ts @@ -59,7 +59,6 @@ export class Archiver implements L2BlockSource, L2LogsSource, ContractDataSource * @param inboxAddress - Ethereum address of the inbox contract. * @param registryAddress - Ethereum address of the registry contract. * @param contractDeploymentEmitterAddress - Ethereum address of the contractDeploymentEmitter contract. - * @param searchStartBlock - The L1 block from which to start searching for new blocks. * @param pollingIntervalMs - The interval for polling for L1 logs (in milliseconds). * @param store - An archiver data store for storage & retrieval of blocks, encrypted logs & contract data. * @param log - A logger. diff --git a/yarn-project/end-to-end/src/e2e_2_pxes.test.ts b/yarn-project/end-to-end/src/e2e_2_pxes.test.ts index a04170dd66e..e7022888367 100644 --- a/yarn-project/end-to-end/src/e2e_2_pxes.test.ts +++ b/yarn-project/end-to-end/src/e2e_2_pxes.test.ts @@ -211,8 +211,11 @@ describe('e2e_2_pxes', () => { await awaitServerSynchronized(pxeA); - const storedValue = await getChildStoredValue(childCompleteAddress, pxeB); - expect(storedValue).toEqual(newValueToSet); + const storedValueOnB = await getChildStoredValue(childCompleteAddress, pxeB); + expect(storedValueOnB).toEqual(newValueToSet); + + const storedValueOnA = await getChildStoredValue(childCompleteAddress, pxeA); + expect(storedValueOnA).toEqual(newValueToSet); }); it('private state is "zero" when Private eXecution Environment (PXE) does not have the account private key', async () => { @@ -270,4 +273,111 @@ describe('e2e_2_pxes', () => { // registering should wait for the account to be synchronized await expect(walletOnB.isAccountStateSynchronized(completeAddress.address)).resolves.toBe(true); }); + + it('permits sending funds to a user before they have registered the contract', async () => { + const initialBalance = 987n; + const transferAmount1 = 654n; + + const completeTokenAddress = await deployTokenContract(initialBalance, userA.address, pxeA); + const tokenAddress = completeTokenAddress.address; + + // Add account B to wallet A + await pxeA.registerRecipient(userB); + // Add account A to wallet B + await pxeB.registerRecipient(userA); + + // Check initial balances and logs are as expected + await expectTokenBalance(walletA, tokenAddress, userA.address, initialBalance); + // don't check userB yet + + await expectsNumOfEncryptedLogsInTheLastBlockToBe(aztecNode, 1); + + // Transfer funds from A to B via PXE A + const contractWithWalletA = await TokenContract.at(tokenAddress, walletA); + const receiptAToB = await contractWithWalletA.methods + .transfer(userA.address, userB.address, transferAmount1, 0) + .send() + .wait(); + expect(receiptAToB.status).toBe(TxStatus.MINED); + + // now add the contract and check balances + await pxeB.addContracts([ + { + artifact: TokenContract.artifact, + completeAddress: completeTokenAddress, + portalContract: EthAddress.ZERO, + }, + ]); + await expectTokenBalance(walletA, tokenAddress, userA.address, initialBalance - transferAmount1); + await expectTokenBalance(walletB, tokenAddress, userB.address, transferAmount1); + }); + + it('permits sending funds to a user, and spending them, before they have registered the contract', async () => { + const initialBalance = 987n; + const transferAmount1 = 654n; + const transferAmount2 = 323n; + + // setup an account that is shared across PXEs + const sharedPrivateKey = GrumpkinScalar.random(); + const sharedAccountOnA = getUnsafeSchnorrAccount(pxeA, sharedPrivateKey, Fr.random()); + const sharedAccountAddress = sharedAccountOnA.getCompleteAddress(); + const sharedWalletOnA = await sharedAccountOnA.waitDeploy(); + await expect(sharedWalletOnA.isAccountStateSynchronized(sharedAccountAddress.address)).resolves.toBe(true); + + const sharedAccountOnB = getUnsafeSchnorrAccount(pxeB, sharedPrivateKey, sharedAccountAddress); + await sharedAccountOnB.register(); + const sharedWalletOnB = await sharedAccountOnB.getWallet(); + + await pxeA.registerRecipient(userB); + + // deploy the contract on PXE A + const completeTokenAddress = await deployTokenContract(initialBalance, userA.address, pxeA); + const tokenAddress = completeTokenAddress.address; + + // Transfer funds from A to Shared Wallet via PXE A + const contractWithWalletA = await TokenContract.at(tokenAddress, walletA); + const receiptAToShared = await contractWithWalletA.methods + .transfer(userA.address, sharedAccountAddress.address, transferAmount1, 0) + .send() + .wait(); + expect(receiptAToShared.status).toBe(TxStatus.MINED); + + // Now send funds from Shared Wallet to B via PXE A + const contractWithSharedWalletA = await TokenContract.at(tokenAddress, sharedWalletOnA); + const receiptSharedToB = await contractWithSharedWalletA.methods + .transfer(sharedAccountAddress.address, userB.address, transferAmount2, 0) + .send() + .wait(); + expect(receiptSharedToB.status).toBe(TxStatus.MINED); + + // check balances from PXE-A's perspective + await expectTokenBalance(walletA, tokenAddress, userA.address, initialBalance - transferAmount1); + await expectTokenBalance( + sharedWalletOnA, + tokenAddress, + sharedAccountAddress.address, + transferAmount1 - transferAmount2, + ); + + // now add the contract and check balances from PXE-B's perspective. + // The process should be: + // PXE-B had previously deferred the notes from A -> Shared, and Shared -> B + // PXE-B adds the contract + // PXE-B reprocesses the deferred notes, and sees the nullifier for A -> Shared + await pxeB.addContracts([ + { + artifact: TokenContract.artifact, + completeAddress: completeTokenAddress, + portalContract: EthAddress.ZERO, + }, + ]); + await expectTokenBalance(walletB, tokenAddress, userB.address, transferAmount2); + await expect(sharedWalletOnB.isAccountStateSynchronized(sharedAccountAddress.address)).resolves.toBe(true); + await expectTokenBalance( + sharedWalletOnB, + tokenAddress, + sharedAccountAddress.address, + transferAmount1 - transferAmount2, + ); + }); }); diff --git a/yarn-project/pxe/src/contract_data_oracle/index.ts b/yarn-project/pxe/src/contract_data_oracle/index.ts index 56eeaf60c59..5c1f07e03f0 100644 --- a/yarn-project/pxe/src/contract_data_oracle/index.ts +++ b/yarn-project/pxe/src/contract_data_oracle/index.ts @@ -1,5 +1,6 @@ -import { AztecAddress, MembershipWitness, VK_TREE_HEIGHT } from '@aztec/circuits.js'; -import { FunctionDebugMetadata, FunctionSelector, getFunctionDebugMetadata } from '@aztec/foundation/abi'; +import { ContractNotFoundError } from '@aztec/acir-simulator'; +import { AztecAddress, ContractFunctionDao, MembershipWitness, VK_TREE_HEIGHT } from '@aztec/circuits.js'; +import { FunctionDebugMetadata, FunctionSelector } from '@aztec/foundation/abi'; import { ContractDatabase, StateInfoProvider } from '@aztec/types'; import { ContractTree } from '../contract_tree/index.js'; @@ -47,20 +48,25 @@ export class ContractDataOracle { /** * Retrieves the artifact of a specified function within a given contract. * The function is identified by its name, which is unique within a contract. + * Throws if the contract has not been added to the database. * * @param contractAddress - The AztecAddress representing the contract containing the function. * @param functionName - The name of the function. - * @returns The corresponding function's artifact as an object. + * @returns The corresponding function's artifact as an object */ - public async getFunctionArtifactByName(contractAddress: AztecAddress, functionName: string) { - const contract = await this.db.getContract(contractAddress); - return contract?.functions.find(f => f.name === functionName); + public async getFunctionArtifactByName( + contractAddress: AztecAddress, + functionName: string, + ): Promise { + const tree = await this.getTree(contractAddress); + return tree.contract.getFunctionArtifactByName(functionName); } /** * Retrieves the debug metadata of a specified function within a given contract. * The function is identified by its selector, which is a unique code generated from the function's signature. * Returns undefined if the debug metadata for the given function is not found. + * Throws if the contract has not been added to the database. * * @param contractAddress - The AztecAddress representing the contract containing the function. * @param selector - The function selector. @@ -70,14 +76,14 @@ export class ContractDataOracle { contractAddress: AztecAddress, selector: FunctionSelector, ): Promise { - const contract = await this.db.getContract(contractAddress); - const functionArtifact = contract?.functions.find(f => f.selector.equals(selector)); + const tree = await this.getTree(contractAddress); + const functionArtifact = tree.contract.getFunctionArtifact(selector); - if (!contract || !functionArtifact) { + if (!functionArtifact) { return undefined; } - return getFunctionDebugMetadata(contract, functionArtifact.name); + return tree.contract.getFunctionDebugMetadataByName(functionArtifact.name); } /** @@ -88,6 +94,7 @@ export class ContractDataOracle { * @param contractAddress - The contract's address. * @param selector - The function selector. * @returns A Promise that resolves to a Buffer containing the bytecode of the specified function. + * @throws Error if the contract address is unknown or not found. */ public async getBytecode(contractAddress: AztecAddress, selector: FunctionSelector) { const tree = await this.getTree(contractAddress); @@ -147,12 +154,12 @@ export class ContractDataOracle { * @returns A ContractTree instance associated with the specified contract address. * @throws An Error if the contract is not found in the ContractDatabase. */ - private async getTree(contractAddress: AztecAddress) { + private async getTree(contractAddress: AztecAddress): Promise { let tree = this.trees.find(t => t.contract.completeAddress.address.equals(contractAddress)); if (!tree) { const contract = await this.db.getContract(contractAddress); if (!contract) { - throw new Error(`Unknown contract: ${contractAddress}`); + throw new ContractNotFoundError(contractAddress.toString()); } tree = new ContractTree(contract, this.stateProvider); diff --git a/yarn-project/pxe/src/database/deferred_note_dao.test.ts b/yarn-project/pxe/src/database/deferred_note_dao.test.ts new file mode 100644 index 00000000000..2eb6201c0b1 --- /dev/null +++ b/yarn-project/pxe/src/database/deferred_note_dao.test.ts @@ -0,0 +1,34 @@ +import { AztecAddress, Fr, Point } from '@aztec/circuits.js'; +import { Note, randomTxHash } from '@aztec/types'; + +import { DeferredNoteDao } from './deferred_note_dao.js'; + +export const randomDeferredNoteDao = ({ + publicKey = Point.random(), + note = Note.random(), + contractAddress = AztecAddress.random(), + txHash = randomTxHash(), + storageSlot = Fr.random(), + txNullifier = Fr.random(), + newCommitments = [Fr.random(), Fr.random()], + dataStartIndexForTx = Math.floor(Math.random() * 100), +}: Partial = {}) => { + return new DeferredNoteDao( + publicKey, + note, + contractAddress, + storageSlot, + txHash, + txNullifier, + newCommitments, + dataStartIndexForTx, + ); +}; + +describe('Deferred Note DAO', () => { + it('convert to and from buffer', () => { + const deferredNote = randomDeferredNoteDao(); + const buf = deferredNote.toBuffer(); + expect(DeferredNoteDao.fromBuffer(buf)).toEqual(deferredNote); + }); +}); diff --git a/yarn-project/pxe/src/database/deferred_note_dao.ts b/yarn-project/pxe/src/database/deferred_note_dao.ts new file mode 100644 index 00000000000..034a192daad --- /dev/null +++ b/yarn-project/pxe/src/database/deferred_note_dao.ts @@ -0,0 +1,55 @@ +import { AztecAddress, Fr, Point, PublicKey, Vector } from '@aztec/circuits.js'; +import { serializeToBuffer } from '@aztec/circuits.js/utils'; +import { BufferReader, Note, TxHash } from '@aztec/types'; + +/** + * A note that is intended for us, but we cannot decode it yet because the contract is not yet in our database. + * + * So keep the state that we need to decode it later. + */ +export class DeferredNoteDao { + constructor( + /** The public key associated with this note */ + public publicKey: PublicKey, + /** The note as emitted from the Noir contract. */ + public note: Note, + /** The contract address this note is created in. */ + public contractAddress: AztecAddress, + /** The specific storage location of the note on the contract. */ + public storageSlot: Fr, + /** The hash of the tx the note was created in. */ + public txHash: TxHash, + /** The first nullifier emitted by the transaction */ + public txNullifier: Fr, + /** New commitments in this transaction, one of which belongs to this note */ + public newCommitments: Fr[], + /** The next available leaf index for the note hash tree for this transaction */ + public dataStartIndexForTx: number, + ) {} + + toBuffer(): Buffer { + return serializeToBuffer( + this.publicKey.toBuffer(), + this.note.toBuffer(), + this.contractAddress.toBuffer(), + this.storageSlot.toBuffer(), + this.txHash.toBuffer(), + this.txNullifier.toBuffer(), + new Vector(this.newCommitments), + this.dataStartIndexForTx, + ); + } + static fromBuffer(buffer: Buffer | BufferReader) { + const reader = BufferReader.asReader(buffer); + return new DeferredNoteDao( + reader.readObject(Point), + reader.readObject(Note), + reader.readObject(AztecAddress), + reader.readObject(Fr), + reader.readObject(TxHash), + reader.readObject(Fr), + reader.readVector(Fr), + reader.readNumber(), + ); + } +} diff --git a/yarn-project/pxe/src/database/kv_pxe_database.ts b/yarn-project/pxe/src/database/kv_pxe_database.ts index 402599f7d83..8cfc806c7bc 100644 --- a/yarn-project/pxe/src/database/kv_pxe_database.ts +++ b/yarn-project/pxe/src/database/kv_pxe_database.ts @@ -3,6 +3,7 @@ import { Fr, Point } from '@aztec/foundation/fields'; import { AztecArray, AztecKVStore, AztecMap, AztecMultiMap, AztecSingleton } from '@aztec/kv-store'; import { ContractDao, MerkleTreeId, NoteFilter, PublicKey } from '@aztec/types'; +import { DeferredNoteDao } from './deferred_note_dao.js'; import { NoteDao } from './note_dao.js'; import { PxeDatabase } from './pxe_database.js'; @@ -32,6 +33,8 @@ export class KVPxeDatabase implements PxeDatabase { #notesByStorageSlot: AztecMultiMap; #notesByTxHash: AztecMultiMap; #notesByOwner: AztecMultiMap; + #deferredNotes: AztecArray; + #deferredNotesByContract: AztecMultiMap; #syncedBlockPerPublicKey: AztecMap; #db: AztecKVStore; @@ -55,6 +58,9 @@ export class KVPxeDatabase implements PxeDatabase { this.#notesByStorageSlot = db.createMultiMap('notes_by_storage_slot'); this.#notesByTxHash = db.createMultiMap('notes_by_tx_hash'); this.#notesByOwner = db.createMultiMap('notes_by_owner'); + + this.#deferredNotes = db.createArray('deferred_notes'); + this.#deferredNotesByContract = db.createMultiMap('deferred_notes_by_contract'); } async addAuthWitness(messageHash: Fr, witness: Fr[]): Promise { @@ -95,6 +101,60 @@ export class KVPxeDatabase implements PxeDatabase { } } + async addDeferredNotes(deferredNotes: DeferredNoteDao[]): Promise { + const newLength = await this.#deferredNotes.push(...deferredNotes.map(note => note.toBuffer())); + for (const [index, note] of deferredNotes.entries()) { + const noteId = newLength - deferredNotes.length + index; + await this.#deferredNotesByContract.set(note.contractAddress.toString(), noteId); + } + } + + getDeferredNotesByContract(contractAddress: AztecAddress): Promise { + const noteIds = this.#deferredNotesByContract.getValues(contractAddress.toString()); + const notes: DeferredNoteDao[] = []; + for (const noteId of noteIds) { + const serializedNote = this.#deferredNotes.at(noteId); + if (!serializedNote) { + continue; + } + + const note = DeferredNoteDao.fromBuffer(serializedNote); + notes.push(note); + } + + return Promise.resolve(notes); + } + + /** + * Removes all deferred notes for a given contract address. + * @param contractAddress - the contract address to remove deferred notes for + * @returns an array of the removed deferred notes + * + * @remarks We only remove indices from the deferred notes by contract map, but not the actual deferred notes. + * This is safe because our only getter for deferred notes is by contract address. + * If we should add a more general getter, we will need a delete vector for deferred notes as well, + * analogous to this.#nullifiedNotes. + */ + removeDeferredNotesByContract(contractAddress: AztecAddress): Promise { + return this.#db.transaction(() => { + const deferredNotes: DeferredNoteDao[] = []; + const indices = this.#deferredNotesByContract.getValues(contractAddress.toString()); + + for (const index of indices) { + const deferredNoteBuffer = this.#deferredNotes.at(index); + if (!deferredNoteBuffer) { + continue; + } else { + deferredNotes.push(DeferredNoteDao.fromBuffer(deferredNoteBuffer)); + } + + void this.#deferredNotesByContract.deleteValue(contractAddress.toString(), index); + } + + return deferredNotes; + }); + } + *#getAllNonNullifiedNotes(): IterableIterator { for (const [index, serialized] of this.#notes.entries()) { if (this.#nullifiedNotes.has(index)) { @@ -155,6 +215,9 @@ export class KVPxeDatabase implements PxeDatabase { } removeNullifiedNotes(nullifiers: Fr[], account: PublicKey): Promise { + if (nullifiers.length === 0) { + return Promise.resolve([]); + } const nullifierSet = new Set(nullifiers.map(n => n.toString())); return this.#db.transaction(() => { const notesIds = this.#notesByOwner.getValues(account.toString()); diff --git a/yarn-project/pxe/src/database/memory_db.ts b/yarn-project/pxe/src/database/memory_db.ts index e06ef3e8a5b..6e0462a4c7a 100644 --- a/yarn-project/pxe/src/database/memory_db.ts +++ b/yarn-project/pxe/src/database/memory_db.ts @@ -5,6 +5,7 @@ import { createDebugLogger } from '@aztec/foundation/log'; import { MerkleTreeId, NoteFilter } from '@aztec/types'; import { MemoryContractDatabase } from '../contract_database/index.js'; +import { DeferredNoteDao } from './deferred_note_dao.js'; import { NoteDao } from './note_dao.js'; import { PxeDatabase } from './pxe_database.js'; @@ -16,6 +17,7 @@ import { PxeDatabase } from './pxe_database.js'; */ export class MemoryDB extends MemoryContractDatabase implements PxeDatabase { private notesTable: NoteDao[] = []; + private deferredNotesTable: DeferredNoteDao[] = []; private treeRoots: Record | undefined; private globalVariablesHash: Fr | undefined; private blockNumber: number | undefined; @@ -54,6 +56,27 @@ export class MemoryDB extends MemoryContractDatabase implements PxeDatabase { return Promise.resolve(); } + public addDeferredNotes(notes: DeferredNoteDao[]): Promise { + this.deferredNotesTable.push(...notes); + return Promise.resolve(); + } + + public getDeferredNotesByContract(contractAddress: AztecAddress): Promise { + return Promise.resolve(this.deferredNotesTable.filter(note => note.contractAddress.equals(contractAddress))); + } + + public removeDeferredNotesByContract(contractAddress: AztecAddress): Promise { + const removed: DeferredNoteDao[] = []; + this.deferredNotesTable = this.deferredNotesTable.filter(note => { + if (note.contractAddress.equals(contractAddress)) { + removed.push(note); + return false; + } + return true; + }); + return Promise.resolve(removed); + } + public addCapsule(capsule: Fr[]): Promise { this.capsuleStack.push(capsule); return Promise.resolve(); diff --git a/yarn-project/pxe/src/database/note_dao.ts b/yarn-project/pxe/src/database/note_dao.ts index 9155e16536a..b531bcb058f 100644 --- a/yarn-project/pxe/src/database/note_dao.ts +++ b/yarn-project/pxe/src/database/note_dao.ts @@ -1,3 +1,4 @@ +import { NoteData } from '@aztec/acir-simulator'; import { AztecAddress, Fr, Point, PublicKey } from '@aztec/circuits.js'; import { toBigIntBE, toBufferBE } from '@aztec/foundation/bigint-buffer'; import { BufferReader, Note, TxHash } from '@aztec/types'; @@ -5,7 +6,7 @@ import { BufferReader, Note, TxHash } from '@aztec/types'; /** * A note with contextual data. */ -export class NoteDao { +export class NoteDao implements NoteData { constructor( /** The note as emitted from the Noir contract. */ public note: Note, diff --git a/yarn-project/pxe/src/database/pxe_database.ts b/yarn-project/pxe/src/database/pxe_database.ts index b2e9432a58b..134b56aa4c3 100644 --- a/yarn-project/pxe/src/database/pxe_database.ts +++ b/yarn-project/pxe/src/database/pxe_database.ts @@ -3,6 +3,7 @@ import { AztecAddress } from '@aztec/foundation/aztec-address'; import { Fr } from '@aztec/foundation/fields'; import { ContractDatabase, MerkleTreeId, NoteFilter } from '@aztec/types'; +import { DeferredNoteDao } from './deferred_note_dao.js'; import { NoteDao } from './note_dao.js'; /** @@ -60,6 +61,25 @@ export interface PxeDatabase extends ContractDatabase { */ addNotes(notes: NoteDao[]): Promise; + /** + * Add notes to the database that are intended for us, but we don't yet have the contract. + * @param deferredNotes - An array of deferred notes. + */ + addDeferredNotes(deferredNotes: DeferredNoteDao[]): Promise; + + /** + * Get deferred notes for a given contract address. + * @param contractAddress - The contract address to get the deferred notes for. + */ + getDeferredNotesByContract(contractAddress: AztecAddress): Promise; + + /** + * Remove deferred notes for a given contract address. + * @param contractAddress - The contract address to remove the deferred notes for. + * @returns an array of the removed deferred notes + */ + removeDeferredNotesByContract(contractAddress: AztecAddress): Promise; + /** * Remove nullified notes associated with the given account and nullifiers. * diff --git a/yarn-project/pxe/src/note_processor/note_processor.ts b/yarn-project/pxe/src/note_processor/note_processor.ts index 7a69fb2e007..a4ccdebb848 100644 --- a/yarn-project/pxe/src/note_processor/note_processor.ts +++ b/yarn-project/pxe/src/note_processor/note_processor.ts @@ -1,5 +1,5 @@ +import { ContractNotFoundError } from '@aztec/acir-simulator'; import { MAX_NEW_COMMITMENTS_PER_TX, MAX_NEW_NULLIFIERS_PER_TX, PublicKey } from '@aztec/circuits.js'; -import { computeCommitmentNonce, siloNullifier } from '@aztec/circuits.js/abis'; import { Grumpkin } from '@aztec/circuits.js/barretenberg'; import { Fr } from '@aztec/foundation/fields'; import { createDebugLogger } from '@aztec/foundation/log'; @@ -7,9 +7,11 @@ import { Timer } from '@aztec/foundation/timer'; import { AztecNode, INITIAL_L2_BLOCK_NUM, KeyStore, L1NotePayload, L2BlockContext, L2BlockL2Logs } from '@aztec/types'; import { NoteProcessorStats } from '@aztec/types/stats'; +import { DeferredNoteDao } from '../database/deferred_note_dao.js'; import { PxeDatabase } from '../database/index.js'; import { NoteDao } from '../database/note_dao.js'; import { getAcirSimulator } from '../simulator/index.js'; +import { produceNoteDao } from './produce_note_dao.js'; /** * Contains all the decrypted data in this array so that we can later batch insert it all into the database. @@ -34,7 +36,7 @@ export class NoteProcessor { public readonly timer: Timer = new Timer(); /** Stats accumulated for this processor. */ - public readonly stats: NoteProcessorStats = { seen: 0, decrypted: 0, failed: 0, blocks: 0, txs: 0 }; + public readonly stats: NoteProcessorStats = { seen: 0, decrypted: 0, deferred: 0, failed: 0, blocks: 0, txs: 0 }; constructor( /** @@ -92,8 +94,10 @@ export class NoteProcessor { return; } - const blocksAndNotes: ProcessedData[] = []; const curve = new Grumpkin(); + const blocksAndNotes: ProcessedData[] = []; + // Keep track of notes that we couldn't process because the contract was not found. + const deferredNoteDaos: DeferredNoteDao[] = []; // Iterate over both blocks and encrypted logs. for (let blockIndex = 0; blockIndex < encryptedL2BlockLogs.length; ++blockIndex) { @@ -130,32 +134,40 @@ export class NoteProcessor { const payload = L1NotePayload.fromEncryptedBuffer(logs, privateKey, curve); if (payload) { // We have successfully decrypted the data. + const txHash = blockContext.getTxHash(indexOfTxInABlock); + const txNullifier = newNullifiers[0]; try { - const { commitmentIndex, nonce, innerNoteHash, siloedNullifier } = await this.findNoteIndexAndNullifier( - newCommitments, - newNullifiers[0], + const noteDao = await produceNoteDao( + this.simulator, + this.publicKey, payload, + txHash, + txNullifier, + newCommitments, + dataStartIndexForTx, excludedIndices, ); - const index = BigInt(dataStartIndexForTx + commitmentIndex); - excludedIndices.add(commitmentIndex); - noteDaos.push( - new NoteDao( + noteDaos.push(noteDao); + this.stats.decrypted++; + } catch (e) { + if (e instanceof ContractNotFoundError) { + this.stats.deferred++; + this.log.warn(e.message); + const deferredNoteDao = new DeferredNoteDao( + this.publicKey, payload.note, payload.contractAddress, payload.storageSlot, - blockContext.getTxHash(indexOfTxInABlock), - nonce, - innerNoteHash, - siloedNullifier, - index, - this.publicKey, - ), - ); - this.stats.decrypted++; - } catch (e) { - this.stats.failed++; - this.log.warn(`Could not process note because of "${e}". Skipping note...`); + txHash, + txNullifier, + newCommitments, + dataStartIndexForTx, + ); + deferredNoteDaos.push(deferredNoteDao); + } else { + this.stats.failed++; + this.log.warn(`Could not process note because of "${e}". Discarding note...`); + } } } } @@ -169,6 +181,7 @@ export class NoteProcessor { } await this.processBlocksAndNotes(blocksAndNotes); + await this.processDeferredNotes(deferredNoteDaos); const syncedToBlock = l2BlockContexts[l2BlockContexts.length - 1].block.number; await this.db.setSynchedBlockNumberForPublicKey(this.publicKey, syncedToBlock); @@ -176,82 +189,6 @@ export class NoteProcessor { this.log(`Synched block ${syncedToBlock}`); } - /** - * Find the index of the note in the note hash tree by computing the note hash with different nonce and see which - * commitment for the current tx matches this value. - * Compute a nullifier for a given l1NotePayload. - * The nullifier is calculated using the private key of the account, - * contract address, and the note associated with the l1NotePayload. - * This method assists in identifying spent commitments in the private state. - * @param commitments - Commitments in the tx. One of them should be the note's commitment. - * @param firstNullifier - First nullifier in the tx. - * @param l1NotePayload - An instance of l1NotePayload. - * @param excludedIndices - Indices that have been assigned a note in the same tx. Notes in a tx can have the same - * l1NotePayload. We need to find a different index for each replicate. - * @returns Information for a decrypted note, including the index of its commitment, nonce, inner note - * hash, and the siloed nullifier. Throw if cannot find the nonce for the note. - */ - private async findNoteIndexAndNullifier( - commitments: Fr[], - firstNullifier: Fr, - { contractAddress, storageSlot, note }: L1NotePayload, - excludedIndices: Set, - ) { - let commitmentIndex = 0; - let nonce: Fr | undefined; - let innerNoteHash: Fr | undefined; - let siloedNoteHash: Fr | undefined; - let uniqueSiloedNoteHash: Fr | undefined; - let innerNullifier: Fr | undefined; - for (; commitmentIndex < commitments.length; ++commitmentIndex) { - if (excludedIndices.has(commitmentIndex)) { - continue; - } - - const commitment = commitments[commitmentIndex]; - if (commitment.equals(Fr.ZERO)) { - break; - } - - const expectedNonce = computeCommitmentNonce(firstNullifier, commitmentIndex); - ({ innerNoteHash, siloedNoteHash, uniqueSiloedNoteHash, innerNullifier } = - await this.simulator.computeNoteHashAndNullifier(contractAddress, expectedNonce, storageSlot, note)); - if (commitment.equals(uniqueSiloedNoteHash)) { - nonce = expectedNonce; - break; - } - } - - if (!nonce) { - let errorString; - if (siloedNoteHash == undefined) { - errorString = 'Cannot find a matching commitment for the note.'; - } else { - errorString = `We decrypted a log, but couldn't find a corresponding note in the tree. -This might be because the note was nullified in the same tx which created it. -In that case, everything is fine. To check whether this is the case, look back through -the logs for a notification -'important: chopped commitment for siloed inner hash note -${siloedNoteHash.toString()}'. -If you can see that notification. Everything's fine. -If that's not the case, and you can't find such a notification, something has gone wrong. -There could be a problem with the way you've defined a custom note, or with the way you're -serializing / deserializing / hashing / encrypting / decrypting that note. -Please see the following github issue to track an improvement that we're working on: -https://github.com/AztecProtocol/aztec-packages/issues/1641`; - } - - throw new Error(errorString); - } - - return { - commitmentIndex, - nonce, - innerNoteHash: innerNoteHash!, - siloedNullifier: siloNullifier(contractAddress, innerNullifier!), - }; - } - /** * Process the given blocks and their associated transaction auxiliary data. * This function updates the database with information about new transactions, @@ -284,4 +221,61 @@ https://github.com/AztecProtocol/aztec-packages/issues/1641`; ); }); } + + /** + * Store the given deferred notes in the database for later decoding. + * + * @param deferredNoteDaos - notes that are intended for us but we couldn't process because the contract was not found. + */ + private async processDeferredNotes(deferredNoteDaos: DeferredNoteDao[]) { + if (deferredNoteDaos.length) { + await this.db.addDeferredNotes(deferredNoteDaos); + deferredNoteDaos.forEach(noteDao => { + this.log( + `Deferred note for contract ${noteDao.contractAddress} at slot ${ + noteDao.storageSlot + } in tx ${noteDao.txHash.toString()}`, + ); + }); + } + } + + /** + * Retry decoding the given deferred notes because we now have the contract code. + * + * @param deferredNoteDaos - notes that we have previously deferred because the contract was not found + * @returns An array of NoteDaos that were successfully decoded. + * + * @remarks Caller is responsible for making sure that we have the contract for the + * deferred notes provided: we will not retry notes that fail again. + */ + public async decodeDeferredNotes(deferredNoteDaos: DeferredNoteDao[]): Promise { + const excludedIndices: Set = new Set(); + const noteDaos: NoteDao[] = []; + for (const deferredNote of deferredNoteDaos) { + const { note, contractAddress, storageSlot, txHash, txNullifier, newCommitments, dataStartIndexForTx } = + deferredNote; + const payload = new L1NotePayload(note, contractAddress, storageSlot); + + try { + const noteDao = await produceNoteDao( + this.simulator, + this.publicKey, + payload, + txHash, + txNullifier, + newCommitments, + dataStartIndexForTx, + excludedIndices, + ); + noteDaos.push(noteDao); + this.stats.decrypted++; + } catch (e) { + this.stats.failed++; + this.log.warn(`Could not process deferred note because of "${e}". Discarding note...`); + } + } + + return noteDaos; + } } diff --git a/yarn-project/pxe/src/note_processor/produce_note_dao.ts b/yarn-project/pxe/src/note_processor/produce_note_dao.ts new file mode 100644 index 00000000000..4a2dbec277a --- /dev/null +++ b/yarn-project/pxe/src/note_processor/produce_note_dao.ts @@ -0,0 +1,131 @@ +import { AcirSimulator } from '@aztec/acir-simulator'; +import { Fr, PublicKey } from '@aztec/circuits.js'; +import { computeCommitmentNonce, siloNullifier } from '@aztec/circuits.js/abis'; +import { L1NotePayload, TxHash } from '@aztec/types'; + +import { NoteDao } from '../database/note_dao.js'; + +/** + * Decodes a note from a transaction that we know was intended for us. + * Throws if we do not yet have the contract corresponding to the note in our database. + * Accepts a set of excluded indices, which are indices that have been assigned a note in the same tx. + * Inserts the index of the note into the excludedIndices set if the note is successfully decoded. + * + * @param publicKey - The public counterpart to the private key to be used in note decryption. + * @param payload - An instance of l1NotePayload. + * @param txHash - The hash of the transaction that created the note. + * @param txNullifier - The first nullifier emitted by the transaction. + * @param newCommitments - New commitments in this transaction, one of which belongs to this note. + * @param dataStartIndexForTx - The next available leaf index for the note hash tree for this transaction. + * @param excludedIndices - Indices that have been assigned a note in the same tx. Notes in a tx can have the same l1NotePayload, we need to find a different index for each replicate. + * @param simulator - An instance of AcirSimulator. + * @returns an instance of NoteDao, or throws. inserts the index of the note into the excludedIndices set. + */ +export async function produceNoteDao( + simulator: AcirSimulator, + publicKey: PublicKey, + payload: L1NotePayload, + txHash: TxHash, + txNullifier: Fr, + newCommitments: Fr[], + dataStartIndexForTx: number, + excludedIndices: Set, +): Promise { + const { commitmentIndex, nonce, innerNoteHash, siloedNullifier } = await findNoteIndexAndNullifier( + simulator, + newCommitments, + txNullifier, + payload, + excludedIndices, + ); + const index = BigInt(dataStartIndexForTx + commitmentIndex); + excludedIndices?.add(commitmentIndex); + return new NoteDao( + payload.note, + payload.contractAddress, + payload.storageSlot, + txHash, + nonce, + innerNoteHash, + siloedNullifier, + index, + publicKey, + ); +} + +/** + * Find the index of the note in the note hash tree by computing the note hash with different nonce and see which + * commitment for the current tx matches this value. + * Compute a nullifier for a given l1NotePayload. + * The nullifier is calculated using the private key of the account, + * contract address, and the note associated with the l1NotePayload. + * This method assists in identifying spent commitments in the private state. + * @param commitments - Commitments in the tx. One of them should be the note's commitment. + * @param firstNullifier - First nullifier in the tx. + * @param l1NotePayload - An instance of l1NotePayload. + * @param excludedIndices - Indices that have been assigned a note in the same tx. Notes in a tx can have the same + * l1NotePayload. We need to find a different index for each replicate. + * @returns Information for a decrypted note, including the index of its commitment, nonce, inner note + * hash, and the siloed nullifier. Throw if cannot find the nonce for the note. + */ +async function findNoteIndexAndNullifier( + simulator: AcirSimulator, + commitments: Fr[], + firstNullifier: Fr, + { contractAddress, storageSlot, note }: L1NotePayload, + excludedIndices: Set, +) { + let commitmentIndex = 0; + let nonce: Fr | undefined; + let innerNoteHash: Fr | undefined; + let siloedNoteHash: Fr | undefined; + let uniqueSiloedNoteHash: Fr | undefined; + let innerNullifier: Fr | undefined; + for (; commitmentIndex < commitments.length; ++commitmentIndex) { + if (excludedIndices.has(commitmentIndex)) { + continue; + } + + const commitment = commitments[commitmentIndex]; + if (commitment.equals(Fr.ZERO)) { + break; + } + + const expectedNonce = computeCommitmentNonce(firstNullifier, commitmentIndex); + ({ innerNoteHash, siloedNoteHash, uniqueSiloedNoteHash, innerNullifier } = + await simulator.computeNoteHashAndNullifier(contractAddress, expectedNonce, storageSlot, note)); + if (commitment.equals(uniqueSiloedNoteHash)) { + nonce = expectedNonce; + break; + } + } + + if (!nonce) { + let errorString; + if (siloedNoteHash == undefined) { + errorString = 'Cannot find a matching commitment for the note.'; + } else { + errorString = `We decrypted a log, but couldn't find a corresponding note in the tree. +This might be because the note was nullified in the same tx which created it. +In that case, everything is fine. To check whether this is the case, look back through +the logs for a notification +'important: chopped commitment for siloed inner hash note +${siloedNoteHash.toString()}'. +If you can see that notification. Everything's fine. +If that's not the case, and you can't find such a notification, something has gone wrong. +There could be a problem with the way you've defined a custom note, or with the way you're +serializing / deserializing / hashing / encrypting / decrypting that note. +Please see the following github issue to track an improvement that we're working on: +https://github.com/AztecProtocol/aztec-packages/issues/1641`; + } + + throw new Error(errorString); + } + + return { + commitmentIndex, + nonce, + innerNoteHash: innerNoteHash!, + siloedNullifier: siloNullifier(contractAddress, innerNullifier!), + }; +} diff --git a/yarn-project/pxe/src/pxe_service/pxe_service.ts b/yarn-project/pxe/src/pxe_service/pxe_service.ts index 651afaec957..636e0700fe0 100644 --- a/yarn-project/pxe/src/pxe_service/pxe_service.ts +++ b/yarn-project/pxe/src/pxe_service/pxe_service.ts @@ -88,8 +88,9 @@ export class PXEService implements PXE { this.synchronizer = new Synchronizer(node, db, this.jobQueue, logSuffix); this.contractDataOracle = new ContractDataOracle(db, node); this.simulator = getAcirSimulator(db, node, keyStore, this.contractDataOracle); - this.nodeVersion = getPackageInfo().version; + + this.jobQueue.start(); } /** @@ -99,11 +100,7 @@ export class PXEService implements PXE { */ public async start() { const { l2BlockPollingIntervalMS } = this.config; - this.synchronizer.start(1, l2BlockPollingIntervalMS); - this.jobQueue.start(); - this.log.info('Started Job Queue'); - await this.jobQueue.syncPoint(); - this.log.info('Synced Job Queue'); + await this.synchronizer.start(1, l2BlockPollingIntervalMS); await this.restoreNoteProcessors(); const info = await this.getNodeInfo(); this.log.info(`Started PXE connected to chain ${info.chainId} version ${info.protocolVersion}`); @@ -207,9 +204,11 @@ export class PXEService implements PXE { const contractDaos = contracts.map(c => new ContractDao(c.artifact, c.completeAddress, c.portalContract)); await Promise.all(contractDaos.map(c => this.db.addContract(c))); for (const contract of contractDaos) { + const contractAztecAddress = contract.completeAddress.address; const portalInfo = contract.portalContract && !contract.portalContract.isZero() ? ` with portal ${contract.portalContract}` : ''; - this.log.info(`Added contract ${contract.name} at ${contract.completeAddress.address}${portalInfo}`); + this.log.info(`Added contract ${contract.name} at ${contractAztecAddress}${portalInfo}`); + await this.synchronizer.reprocessDeferredNotesForContract(contractAztecAddress); } } diff --git a/yarn-project/pxe/src/synchronizer/synchronizer.ts b/yarn-project/pxe/src/synchronizer/synchronizer.ts index 266a7c48eb5..6a2e1256738 100644 --- a/yarn-project/pxe/src/synchronizer/synchronizer.ts +++ b/yarn-project/pxe/src/synchronizer/synchronizer.ts @@ -2,11 +2,22 @@ import { AztecAddress, BlockHeader, Fr, PublicKey } from '@aztec/circuits.js'; import { computeGlobalsHash } from '@aztec/circuits.js/abis'; import { SerialQueue } from '@aztec/foundation/fifo'; import { DebugLogger, createDebugLogger } from '@aztec/foundation/log'; -import { InterruptibleSleep } from '@aztec/foundation/sleep'; -import { AztecNode, INITIAL_L2_BLOCK_NUM, KeyStore, L2BlockContext, L2BlockL2Logs, LogType } from '@aztec/types'; +import { RunningPromise } from '@aztec/foundation/running-promise'; +import { + AztecNode, + INITIAL_L2_BLOCK_NUM, + KeyStore, + L2BlockContext, + L2BlockL2Logs, + LogType, + MerkleTreeId, + TxHash, +} from '@aztec/types'; import { NoteProcessorCaughtUpStats } from '@aztec/types/stats'; +import { DeferredNoteDao } from '../database/deferred_note_dao.js'; import { PxeDatabase } from '../database/index.js'; +import { NoteDao } from '../database/note_dao.js'; import { NoteProcessor } from '../note_processor/index.js'; /** @@ -17,9 +28,8 @@ import { NoteProcessor } from '../note_processor/index.js'; * in sync with the blockchain while handling retries and errors gracefully. */ export class Synchronizer { - private runningPromise?: Promise; + private runningPromise?: RunningPromise; private noteProcessors: NoteProcessor[] = []; - private interruptibleSleep = new InterruptibleSleep(); private running = false; private initialSyncBlockNumber = INITIAL_L2_BLOCK_NUM - 1; private log: DebugLogger; @@ -37,40 +47,17 @@ export class Synchronizer { * @param limit - The maximum number of encrypted, unencrypted logs and blocks to fetch in each iteration. * @param retryInterval - The time interval (in ms) to wait before retrying if no data is available. */ - public start(limit = 1, retryInterval = 1000) { + public async start(limit = 1, retryInterval = 1000) { if (this.running) { return; } this.running = true; - this.jobQueue - .put(() => this.initialSync()) - .catch(err => { - this.log.error(`Error in synchronizer initial sync`, err); - this.running = false; - throw err; - }); - - const run = async () => { - while (this.running) { - await this.jobQueue.put(async () => { - let moreWork = true; - while (moreWork && this.running) { - if (this.noteProcessorsToCatchUp.length > 0) { - // There is a note processor that needs to catch up. We hijack the main loop to catch up the note processor. - moreWork = await this.workNoteProcessorCatchUp(limit); - } else { - // No note processor needs to catch up. We continue with the normal flow. - moreWork = await this.work(limit); - } - } - }); - await this.interruptibleSleep.sleep(retryInterval); - } - }; - - this.runningPromise = run(); - this.log('Started'); + await this.jobQueue.put(() => this.initialSync()); + this.log('Initial sync complete'); + this.runningPromise = new RunningPromise(() => this.sync(limit), retryInterval); + this.runningPromise.start(); + this.log('Started loop'); } protected async initialSync() { @@ -83,6 +70,33 @@ export class Synchronizer { await this.db.setBlockData(latestBlockNumber, latestBlockHeader); } + /** + * Fetches encrypted logs and blocks from the Aztec node and processes them for all note processors. + * If needed, catches up note processors that are lagging behind the main sync, e.g. because we just added a new account. + * + * Uses the job queue to ensure that + * - sync does not overlap with pxe simulations. + * - one sync is running at a time. + * + * @param limit - The maximum number of encrypted, unencrypted logs and blocks to fetch in each iteration. + * @returns a promise that resolves when the sync is complete + */ + protected sync(limit: number) { + return this.jobQueue.put(async () => { + let moreWork = true; + // keep external this.running flag to interrupt greedy sync + while (moreWork && this.running) { + if (this.noteProcessorsToCatchUp.length > 0) { + // There is a note processor that needs to catch up. We hijack the main loop to catch up the note processor. + moreWork = await this.workNoteProcessorCatchUp(limit); + } else { + // No note processor needs to catch up. We continue with the normal flow. + moreWork = await this.work(limit); + } + } + }); + } + /** * Fetches encrypted logs and blocks from the Aztec node and processes them for all note processors. * @@ -92,6 +106,7 @@ export class Synchronizer { protected async work(limit = 1): Promise { const from = this.getSynchedBlockNumber() + 1; try { + // Possibly improve after https://github.com/AztecProtocol/aztec-packages/issues/3870 let encryptedLogs = await this.node.getLogs(from, limit, LogType.ENCRYPTED); if (!encryptedLogs.length) { return false; @@ -246,8 +261,7 @@ export class Synchronizer { */ public async stop() { this.running = false; - this.interruptibleSleep.interrupt(); - await this.runningPromise; + await this.runningPromise?.stop(); this.log('Stopped'); } @@ -320,4 +334,71 @@ export class Synchronizer { notes: Object.fromEntries(this.noteProcessors.map(n => [n.publicKey.toString(), n.status.syncedToBlock])), }; } + + /** + * Retry decoding any deferred notes for the specified contract address. + * @param contractAddress - the contract address that has just been added + */ + public reprocessDeferredNotesForContract(contractAddress: AztecAddress): Promise { + return this.jobQueue.put(() => this.#reprocessDeferredNotesForContract(contractAddress)); + } + + async #reprocessDeferredNotesForContract(contractAddress: AztecAddress): Promise { + const deferredNotes = await this.db.getDeferredNotesByContract(contractAddress); + + // group deferred notes by txHash to properly deal with possible duplicates + const txHashToDeferredNotes: Map = new Map(); + for (const note of deferredNotes) { + const notesForTx = txHashToDeferredNotes.get(note.txHash) ?? []; + notesForTx.push(note); + txHashToDeferredNotes.set(note.txHash, notesForTx); + } + + // keep track of decoded notes + const newNotes: NoteDao[] = []; + // now process each txHash + for (const deferredNotes of txHashToDeferredNotes.values()) { + // to be safe, try each note processor in case the deferred notes are for different accounts. + for (const processor of this.noteProcessors) { + const decodedNotes = await processor.decodeDeferredNotes( + deferredNotes.filter(n => n.publicKey.equals(processor.publicKey)), + ); + newNotes.push(...decodedNotes); + } + } + + // now drop the deferred notes, and add the decoded notes + await this.db.removeDeferredNotesByContract(contractAddress); + await this.db.addNotes(newNotes); + + newNotes.forEach(noteDao => { + this.log( + `Decoded deferred note for contract ${noteDao.contractAddress} at slot ${ + noteDao.storageSlot + } with nullifier ${noteDao.siloedNullifier.toString()}`, + ); + }); + + // now group the decoded notes by public key + const publicKeyToNotes: Map = new Map(); + for (const noteDao of newNotes) { + const notesForPublicKey = publicKeyToNotes.get(noteDao.publicKey) ?? []; + notesForPublicKey.push(noteDao); + publicKeyToNotes.set(noteDao.publicKey, notesForPublicKey); + } + + // now for each group, look for the nullifiers in the nullifier tree + for (const [publicKey, notes] of publicKeyToNotes.entries()) { + const nullifiers = notes.map(n => n.siloedNullifier); + const relevantNullifiers: Fr[] = []; + for (const nullifier of nullifiers) { + // NOTE: this leaks information about the nullifiers I'm interested in to the node. + const found = await this.node.findLeafIndex('latest', MerkleTreeId.NULLIFIER_TREE, nullifier); + if (found) { + relevantNullifiers.push(nullifier); + } + } + await this.db.removeNullifiedNotes(relevantNullifiers, publicKey); + } + } } diff --git a/yarn-project/types/src/contract_dao.ts b/yarn-project/types/src/contract_dao.ts index 10cb955d743..4d70d0aca4d 100644 --- a/yarn-project/types/src/contract_dao.ts +++ b/yarn-project/types/src/contract_dao.ts @@ -1,5 +1,13 @@ import { CompleteAddress, ContractFunctionDao } from '@aztec/circuits.js'; -import { ContractArtifact, DebugMetadata, EventAbi, FunctionSelector, FunctionType } from '@aztec/foundation/abi'; +import { + ContractArtifact, + DebugMetadata, + EventAbi, + FunctionDebugMetadata, + FunctionSelector, + FunctionType, + getFunctionDebugMetadata, +} from '@aztec/foundation/abi'; import { EthAddress } from '@aztec/foundation/eth-address'; import { prefixBufferWithLength } from '@aztec/foundation/serialize'; @@ -42,6 +50,18 @@ export class ContractDao implements ContractArtifact { return this.contractArtifact.debug; } + getFunctionArtifact(selector: FunctionSelector): ContractFunctionDao | undefined { + return this.functions.find(f => f.selector.equals(selector)); + } + + getFunctionArtifactByName(functionName: string): ContractFunctionDao | undefined { + return this.functions.find(f => f.name === functionName); + } + + getFunctionDebugMetadataByName(functionName: string): FunctionDebugMetadata | undefined { + return getFunctionDebugMetadata(this, functionName); + } + toBuffer(): Buffer { // the contract artifact was originally emitted to a JSON file by Noir // should be safe to JSON.stringify it (i.e. it doesn't contain BigInts) diff --git a/yarn-project/types/src/stats/stats.ts b/yarn-project/types/src/stats/stats.ts index 19a0535c922..b78b98ea0be 100644 --- a/yarn-project/types/src/stats/stats.ts +++ b/yarn-project/types/src/stats/stats.ts @@ -108,6 +108,8 @@ export type NoteProcessorCaughtUpStats = { export type NoteProcessorStats = { /** How many notes have been seen and trial-decrypted. */ seen: number; + /** How many notes had decryption deferred due to a missing contract */ + deferred: number; /** How many notes were successfully decrypted. */ decrypted: number; /** How many notes failed processing. */ diff --git a/yarn-project/types/src/tx/tx_hash.ts b/yarn-project/types/src/tx/tx_hash.ts index 00003456ee3..514097b3d6b 100644 --- a/yarn-project/types/src/tx/tx_hash.ts +++ b/yarn-project/types/src/tx/tx_hash.ts @@ -1,4 +1,4 @@ -import { deserializeBigInt, serializeBigInt } from '@aztec/foundation/serialize'; +import { BufferReader, deserializeBigInt, serializeBigInt } from '@aztec/foundation/serialize'; /** * A class representing hash of Aztec transaction. @@ -25,6 +25,24 @@ export class TxHash { } } + /** + * Returns the raw buffer of the hash. + * @returns The buffer containing the hash. + */ + public toBuffer() { + return this.buffer; + } + + /** + * Creates a TxHash from a buffer. + * @param buffer - The buffer to create from. + * @returns A new TxHash object. + */ + public static fromBuffer(buffer: Buffer | BufferReader) { + const reader = BufferReader.asReader(buffer); + return new TxHash(reader.readBytes(TxHash.SIZE)); + } + /** * Checks if this hash and another hash are equal. * @param hash - A hash to compare with.