diff --git a/packages/api/src/beacon/routes/beacon/block.ts b/packages/api/src/beacon/routes/beacon/block.ts index 3e8cd18df0db..47002fd9c28f 100644 --- a/packages/api/src/beacon/routes/beacon/block.ts +++ b/packages/api/src/beacon/routes/beacon/block.ts @@ -187,18 +187,43 @@ export type Api = { > >; /** - * Get block BlobsSidecar - * Retrieves BlobsSidecar included in requested block. + * Get block BlobSidecar + * Retrieves BlobSidecar included in requested block. * @param blockId Block identifier. * Can be one of: "head" (canonical head in node's view), "genesis", "finalized", \, \. */ - getBlobsSidecar( + getBlobSidecars( blockId: BlockId ): Promise< ApiClientResponse<{ - [HttpStatusCode.OK]: {executionOptimistic: ExecutionOptimistic; data: deneb.BlobsSidecar}; + [HttpStatusCode.OK]: {executionOptimistic: ExecutionOptimistic; data: deneb.BlobSidecars}; }> >; + /** + * Publish a signed blob. + */ + publishBlob( + blob: deneb.SignedBlobSidecar + ): Promise< + ApiClientResponse< + { + [HttpStatusCode.OK]: void; + [HttpStatusCode.ACCEPTED]: void; + }, + HttpStatusCode.BAD_REQUEST | HttpStatusCode.SERVICE_UNAVAILABLE + > + >; + publishBlindedBlob( + blob: deneb.SignedBlindedBlobSidecar + ): Promise< + ApiClientResponse< + { + [HttpStatusCode.OK]: void; + [HttpStatusCode.ACCEPTED]: void; + }, + HttpStatusCode.BAD_REQUEST | HttpStatusCode.SERVICE_UNAVAILABLE + > + >; }; /** @@ -213,7 +238,9 @@ export const routesData: RoutesData = { getBlockRoot: {url: "/eth/v1/beacon/blocks/{block_id}/root", method: "GET"}, publishBlock: {url: "/eth/v1/beacon/blocks", method: "POST"}, publishBlindedBlock: {url: "/eth/v1/beacon/blinded_blocks", method: "POST"}, - getBlobsSidecar: {url: "/eth/v1/beacon/blobs_sidecars/{block_id}", method: "GET"}, + getBlobSidecars: {url: "/eth/v1/beacon/blob_sidecars/{block_id}", method: "GET"}, + publishBlob: {url: "/eth/v1/beacon/blob_sidecars", method: "POST"}, + publishBlindedBlob: {url: "/eth/v1/beacon/blinded_blob_sidecars", method: "POST"}, }; /* eslint-disable @typescript-eslint/naming-convention */ @@ -229,7 +256,9 @@ export type ReqTypes = { getBlockRoot: BlockIdOnlyReq; publishBlock: {body: unknown}; publishBlindedBlock: {body: unknown}; - getBlobsSidecar: BlockIdOnlyReq; + getBlobSidecars: BlockIdOnlyReq; + publishBlob: {body: unknown}; + publishBlindedBlob: {body: unknown}; }; export function getReqSerializers(config: ChainForkConfig): ReqSerializers { @@ -272,7 +301,9 @@ export function getReqSerializers(config: ChainForkConfig): ReqSerializers { getBlockHeader: ContainerDataExecutionOptimistic(BeaconHeaderResType), getBlockHeaders: ContainerDataExecutionOptimistic(ArrayOf(BeaconHeaderResType)), getBlockRoot: ContainerDataExecutionOptimistic(RootContainer), - getBlobsSidecar: ContainerDataExecutionOptimistic(ssz.deneb.BlobsSidecar), + getBlobSidecars: ContainerDataExecutionOptimistic(ssz.deneb.BlobSidecars), }; } diff --git a/packages/api/src/beacon/routes/validator.ts b/packages/api/src/beacon/routes/validator.ts index 12f8fb97139f..53c7efdb37bf 100644 --- a/packages/api/src/beacon/routes/validator.ts +++ b/packages/api/src/beacon/routes/validator.ts @@ -17,6 +17,7 @@ import { RootHex, StringType, Wei, + deneb, } from "@lodestar/types"; import {ApiClientResponse} from "../../interfaces.js"; import {HttpStatusCode} from "../../utils/client/httpStatusCode.js"; @@ -209,6 +210,26 @@ export type Api = { > >; + getBlob( + blockRoot: Root, + index: number + ): Promise< + ApiClientResponse< + {[HttpStatusCode.OK]: {data: deneb.BlobSidecar}}, + HttpStatusCode.BAD_REQUEST | HttpStatusCode.SERVICE_UNAVAILABLE + > + >; + + getBlindedBlob( + blockRoot: Root, + index: number + ): Promise< + ApiClientResponse< + {[HttpStatusCode.OK]: {data: deneb.BlindedBlobSidecar}}, + HttpStatusCode.BAD_REQUEST | HttpStatusCode.SERVICE_UNAVAILABLE + > + >; + /** * Produce an attestation data * Requests that the beacon node produce an AttestationData. @@ -323,6 +344,8 @@ export const routesData: RoutesData = { getSyncCommitteeDuties: {url: "/eth/v1/validator/duties/sync/{epoch}", method: "POST"}, produceBlock: {url: "/eth/v1/validator/blocks/{slot}", method: "GET"}, produceBlockV2: {url: "/eth/v2/validator/blocks/{slot}", method: "GET"}, + getBlob: {url: "/eth/v1/validator/blob/{root}/${index}", method: "GET"}, + getBlindedBlob: {url: "/eth/v1/validator/blinded_blob/{root}/${index}", method: "GET"}, produceBlindedBlock: {url: "/eth/v1/validator/blinded_blocks/{slot}", method: "GET"}, produceAttestationData: {url: "/eth/v1/validator/attestation_data", method: "GET"}, produceSyncCommitteeContribution: {url: "/eth/v1/validator/sync_committee_contribution", method: "GET"}, @@ -344,6 +367,8 @@ export type ReqTypes = { produceBlock: {params: {slot: number}; query: {randao_reveal: string; graffiti: string}}; produceBlockV2: {params: {slot: number}; query: {randao_reveal: string; graffiti: string}}; produceBlindedBlock: {params: {slot: number}; query: {randao_reveal: string; graffiti: string}}; + getBlob: {params: {root: string; index: number}}; + getBlindedBlob: {params: {root: string; index: number}}; produceAttestationData: {query: {slot: number; committee_index: number}}; produceSyncCommitteeContribution: {query: {slot: number; subcommittee_index: number; beacon_block_root: string}}; getAggregatedAttestation: {query: {attestation_data_root: string; slot: number}}; @@ -389,6 +414,14 @@ export function getReqSerializers(): ReqSerializers { }, }; + const getBlobReqSerializer: ReqSerializers["getBlob"] = { + writeReq: (root, index) => ({ + params: {root: toHexString(root), index}, + }), + parseReq: ({params}) => [fromHexString(params.root), params.index], + schema: {params: {root: Schema.StringRequired, index: Schema.UintRequired}}, + }; + return { getAttesterDuties: { writeReq: (epoch, indexes) => ({params: {epoch}, body: indexes.map((i) => toU64Str(i))}), @@ -420,6 +453,9 @@ export function getReqSerializers(): ReqSerializers { produceBlockV2: produceBlock, produceBlindedBlock: produceBlock, + getBlob: getBlobReqSerializer, + getBlindedBlob: getBlobReqSerializer, + produceAttestationData: { writeReq: (index, slot) => ({query: {slot, committee_index: index}}), parseReq: ({query}) => [query.committee_index, query.slot], @@ -529,6 +565,8 @@ export function getReturnTypes(): ReturnTypes { return ssz[fork].BlindedBeaconBlock; }) ), + getBlob: ContainerData(ssz.deneb.BlobSidecar), + getBlindedBlob: ContainerData(ssz.deneb.BlindedBlobSidecar), produceAttestationData: ContainerData(ssz.phase0.AttestationData), produceSyncCommitteeContribution: ContainerData(ssz.altair.SyncCommitteeContribution), getAggregatedAttestation: ContainerData(ssz.phase0.Attestation), diff --git a/packages/api/src/builder/routes.ts b/packages/api/src/builder/routes.ts index a594643366b6..2e5888ad8247 100644 --- a/packages/api/src/builder/routes.ts +++ b/packages/api/src/builder/routes.ts @@ -1,6 +1,6 @@ import {ssz, allForks, bellatrix, Slot, Root, BLSPubkey} from "@lodestar/types"; import {fromHexString, toHexString} from "@chainsafe/ssz"; -import {ForkName, isForkExecution, isForkBlobs} from "@lodestar/params"; +import {ForkName, isForkExecution} from "@lodestar/params"; import {ChainForkConfig} from "@lodestar/config"; import { @@ -42,14 +42,6 @@ export type Api = { HttpStatusCode.SERVICE_UNAVAILABLE > >; - submitBlindedBlockV2( - signedBlock: allForks.SignedBlindedBeaconBlock - ): Promise< - ApiClientResponse< - {[HttpStatusCode.OK]: {data: allForks.SignedBeaconBlockAndBlobsSidecar; version: ForkName}}, - HttpStatusCode.SERVICE_UNAVAILABLE - > - >; }; /** @@ -60,7 +52,6 @@ export const routesData: RoutesData = { registerValidator: {url: "/eth/v1/builder/validators", method: "POST"}, getHeader: {url: "/eth/v1/builder/header/{slot}/{parent_hash}/{pubkey}", method: "GET"}, submitBlindedBlock: {url: "/eth/v1/builder/blinded_blocks", method: "POST"}, - submitBlindedBlockV2: {url: "/eth/v2/builder/blinded_blocks", method: "POST"}, }; /* eslint-disable @typescript-eslint/naming-convention */ @@ -69,7 +60,6 @@ export type ReqTypes = { registerValidator: {body: unknown}; getHeader: {params: {slot: Slot; parent_hash: string; pubkey: string}}; submitBlindedBlock: {body: unknown}; - submitBlindedBlockV2: {body: unknown}; }; export function getReqSerializers(config: ChainForkConfig): ReqSerializers { @@ -86,7 +76,6 @@ export function getReqSerializers(config: ChainForkConfig): ReqSerializers { submitBlindedBlock: WithVersion((fork: ForkName) => isForkExecution(fork) ? ssz.allForksExecution[fork].ExecutionPayload : ssz.bellatrix.ExecutionPayload ), - submitBlindedBlockV2: WithVersion((fork: ForkName) => - isForkBlobs(fork) - ? ssz.allForksBlobs[fork].SignedBeaconBlockAndBlobsSidecar - : ssz.deneb.SignedBeaconBlockAndBlobsSidecar - ), }; } diff --git a/packages/api/test/unit/beacon/testData/beacon.ts b/packages/api/test/unit/beacon/testData/beacon.ts index 41f4b1d84f08..98a4ab639bd4 100644 --- a/packages/api/test/unit/beacon/testData/beacon.ts +++ b/packages/api/test/unit/beacon/testData/beacon.ts @@ -56,9 +56,17 @@ export const testData: GenericServerTestCases = { args: [getDefaultBlindedBlock(64)], res: undefined, }, - getBlobsSidecar: { + getBlobSidecars: { args: ["head"], - res: {executionOptimistic: true, data: ssz.deneb.BlobsSidecar.defaultValue()}, + res: {executionOptimistic: true, data: ssz.deneb.BlobSidecars.defaultValue()}, + }, + publishBlob: { + args: [ssz.deneb.SignedBlobSidecar.defaultValue()], + res: undefined, + }, + publishBlindedBlob: { + args: [ssz.deneb.SignedBlindedBlobSidecar.defaultValue()], + res: undefined, }, // pool diff --git a/packages/api/test/unit/beacon/testData/validator.ts b/packages/api/test/unit/beacon/testData/validator.ts index e6939614ec73..92f3ca6b68a9 100644 --- a/packages/api/test/unit/beacon/testData/validator.ts +++ b/packages/api/test/unit/beacon/testData/validator.ts @@ -98,4 +98,12 @@ export const testData: GenericServerTestCases = { args: [[ssz.bellatrix.SignedValidatorRegistrationV1.defaultValue()]], res: undefined, }, + getBlob: { + args: [ZERO_HASH, 0], + res: {data: ssz.deneb.BlobSidecar.defaultValue()}, + }, + getBlindedBlob: { + args: [ZERO_HASH, 0], + res: {data: ssz.deneb.BlindedBlobSidecar.defaultValue()}, + }, }; diff --git a/packages/api/test/unit/builder/testData.ts b/packages/api/test/unit/builder/testData.ts index 443522269fa2..dc7a28fc9d2b 100644 --- a/packages/api/test/unit/builder/testData.ts +++ b/packages/api/test/unit/builder/testData.ts @@ -26,8 +26,4 @@ export const testData: GenericServerTestCases = { args: [ssz.deneb.SignedBlindedBeaconBlock.defaultValue()], res: {version: ForkName.bellatrix, data: ssz.bellatrix.ExecutionPayload.defaultValue()}, }, - submitBlindedBlockV2: { - args: [ssz.deneb.SignedBlindedBeaconBlock.defaultValue()], - res: {version: ForkName.deneb, data: ssz.deneb.SignedBeaconBlockAndBlobsSidecar.defaultValue()}, - }, }; diff --git a/packages/beacon-node/package.json b/packages/beacon-node/package.json index b152e923944b..6bb2bc2ff937 100644 --- a/packages/beacon-node/package.json +++ b/packages/beacon-node/package.json @@ -131,7 +131,7 @@ "@multiformats/multiaddr": "^11.0.0", "@types/datastore-level": "^3.0.0", "buffer-xor": "^2.0.2", - "c-kzg": "^1.0.9", + "c-kzg": "^2.0.0", "cross-fetch": "^3.1.4", "datastore-core": "^8.0.1", "datastore-level": "^9.0.1", @@ -157,9 +157,6 @@ "varint": "^6.0.0", "xxhash-wasm": "1.0.1" }, - "peerDependencies": { - "c-kzg": "^1.0.7" - }, "devDependencies": { "@types/bl": "^5.0.1", "@types/eventsource": "^1.1.5", diff --git a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts index 02ae7f658fd9..cdc2fbaf5038 100644 --- a/packages/beacon-node/src/api/impl/beacon/blocks/index.ts +++ b/packages/beacon-node/src/api/impl/beacon/blocks/index.ts @@ -2,16 +2,15 @@ import {routes, ServerApi} from "@lodestar/api"; import {computeTimeAtSlot} from "@lodestar/state-transition"; import {ForkSeq, SLOTS_PER_HISTORICAL_ROOT} from "@lodestar/params"; import {sleep} from "@lodestar/utils"; -import {deneb, allForks} from "@lodestar/types"; +import {allForks} from "@lodestar/types"; import {fromHexString, toHexString} from "@chainsafe/ssz"; -import {getBlockInput} from "../../../../chain/blocks/types.js"; +import {getBlockInput, GossipedInputType} from "../../../../chain/blocks/types.js"; import {promiseAllMaybeAsync} from "../../../../util/promises.js"; import {isOptimisticBlock} from "../../../../util/forkChoice.js"; import {BlockError, BlockErrorCode} from "../../../../chain/errors/index.js"; import {OpSource} from "../../../../metrics/validatorMonitor.js"; import {NetworkEvent} from "../../../../network/index.js"; import {ApiModules} from "../../types.js"; -import {ckzg} from "../../../../util/kzg.js"; import {resolveBlockId, toBeaconHeaderResponse} from "./utils.js"; /** @@ -183,17 +182,9 @@ export function getBeaconBlockApi({ const executionBuilder = chain.executionBuilder; if (!executionBuilder) throw Error("exeutionBuilder required to publish SignedBlindedBeaconBlock"); let signedBlock: allForks.SignedBeaconBlock; + // Mechanism for blobs & blocks on builder is not yet finalized if (config.getForkSeq(signedBlindedBlock.message.slot) >= ForkSeq.deneb) { - const {beaconBlock, blobsSidecar} = await executionBuilder.submitBlindedBlockV2(signedBlindedBlock); - signedBlock = beaconBlock; - // add this blobs to the map for access & broadcasting in publishBlock - const {blockHash} = signedBlindedBlock.message.body.executionPayloadHeader; - chain.producedBlobsSidecarCache.set(toHexString(blockHash), blobsSidecar); - // TODO: Do we need to prune here ? prune will anyway be called in local execution flow - // pruneSetToMax( - // chain.producedBlobsSidecarCache, - // chain.opts.maxCachedBlobsSidecar ?? DEFAULT_MAX_CACHED_BLOBS_SIDECAR - // ); + throw Error("exeutionBuilder not yet implemented for deneb+ forks"); } else { signedBlock = await executionBuilder.submitBlindedBlock(signedBlindedBlock); } @@ -212,54 +203,83 @@ export function getBeaconBlockApi({ } // TODO: Validate block - metrics?.registerBeaconBlock(OpSource.api, seenTimestampSec, signedBlock.message); - - // TODO Deneb: Open question if broadcast to both block topic + block_and_blobs topic const blockForImport = config.getForkSeq(signedBlock.message.slot) >= ForkSeq.deneb - ? getBlockInput.postDeneb( - config, - signedBlock, - chain.getBlobsSidecar(signedBlock.message as deneb.BeaconBlock) - ) + ? getBlockInput.getFullBlockInput(config, {type: GossipedInputType.block, signedBlock}).blockInput : getBlockInput.preDeneb(config, signedBlock); - await promiseAllMaybeAsync([ + const publishPromises = [ // Send the block, regardless of whether or not it is valid. The API // specification is very clear that this is the desired behaviour. - () => network.publishBeaconBlockMaybeBlobs(blockForImport), + () => network.gossip.publishBeaconBlock(signedBlock), + ]; + if (blockForImport !== null) { + publishPromises.push(() => + chain.processBlock(blockForImport).catch((e) => { + if (e instanceof BlockError && e.type.code === BlockErrorCode.PARENT_UNKNOWN) { + network.events.emit(NetworkEvent.unknownBlockParent, blockForImport, network.peerId.toString()); + } + throw e; + }) + ); + } + await promiseAllMaybeAsync(publishPromises); + }, + + async publishBlob(signedBlob) { + const seenTimestampSec = Date.now() / 1000; - () => + // Simple implementation of a pending block queue. Keeping the block here recycles the API logic, and keeps the + // REST request promise without any extra infrastructure. + const msToBlockSlot = computeTimeAtSlot(config, signedBlob.message.slot, chain.genesisTime) * 1000 - Date.now(); + if (msToBlockSlot <= MAX_API_CLOCK_DISPARITY_MS && msToBlockSlot > 0) { + // If block is a bit early, hold it in a promise. Equivalent to a pending queue. + await sleep(msToBlockSlot); + } + metrics?.registerBlobSideCar(OpSource.api, seenTimestampSec, signedBlob.message); + const blockForImport = getBlockInput.getFullBlockInput(config, {type: GossipedInputType.blob, signedBlob}) + .blockInput; + + const publishPromises = [ + // Send the block, regardless of whether or not it is valid. The API + // specification is very clear that this is the desired behaviour. + () => network.gossip.publishBlobSidecar(signedBlob), + ]; + if (blockForImport !== null) { + publishPromises.push(() => chain.processBlock(blockForImport).catch((e) => { if (e instanceof BlockError && e.type.code === BlockErrorCode.PARENT_UNKNOWN) { network.events.emit(NetworkEvent.unknownBlockParent, blockForImport, network.peerId.toString()); } throw e; - }), - ]); + }) + ); + } + + await promiseAllMaybeAsync(publishPromises); }, - async getBlobsSidecar(blockId) { - const {block, executionOptimistic} = await resolveBlockId(chain.forkChoice, db, blockId); + async publishBlindedBlob(_signedBlob) { + // TODO: freetheblobs + throw Error("publishBlindedBlob for builder not implemented"); + }, + async getBlobSidecars(blockId) { + const {block, executionOptimistic} = await resolveBlockId(chain.forkChoice, db, blockId); const blockRoot = config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message); - let blobsSidecar = await db.blobsSidecar.get(blockRoot); - if (!blobsSidecar) { - blobsSidecar = await db.blobsSidecarArchive.get(block.message.slot); - if (!blobsSidecar) { - blobsSidecar = { - beaconBlockRoot: blockRoot, - beaconBlockSlot: block.message.slot, - blobs: [] as deneb.Blobs, - kzgAggregatedProof: ckzg.computeAggregateKzgProof([]), - }; - } + let {blobSidecars} = (await db.blobSidecars.get(blockRoot)) ?? {}; + if (!blobSidecars) { + ({blobSidecars} = (await db.blobSidecarsArchive.get(block.message.slot)) ?? {}); + } + + if (!blobSidecars) { + throw Error("Not found in db"); } return { executionOptimistic, - data: blobsSidecar, + data: blobSidecars, }; }, }; diff --git a/packages/beacon-node/src/api/impl/validator/index.ts b/packages/beacon-node/src/api/impl/validator/index.ts index 53441c898a81..c1e8a204766e 100644 --- a/packages/beacon-node/src/api/impl/validator/index.ts +++ b/packages/beacon-node/src/api/impl/validator/index.ts @@ -272,6 +272,27 @@ export function getValidatorApi({ produceBlockV2: produceBlock, produceBlindedBlock, + async getBlob(blockRoot, index) { + const blockRootHex = toHex(blockRoot); + const blob = (chain.producedBlobSidecarsCache.get(blockRootHex)?.blobSidecars ?? [])[index]; + if (blob === undefined) { + throw Error(`Blob not found for blockRoot=${blockRootHex} index=${index}`); + } + return {data: blob}; + }, + + async getBlindedBlob(blockRoot, index) { + const blockRootHex = toHex(blockRoot); + const blindedBlob = (chain.producedBlindedBlobSidecarsCache.get(blockRootHex)?.blobSidecars ?? []).slice( + index, + index + 1 + )[0]; + if (blindedBlob === undefined) { + throw Error(`Blob not found for blockRoot=${blockRootHex} index=${index}`); + } + return {data: blindedBlob}; + }, + async produceAttestationData(committeeIndex, slot) { notWhileSyncing(); diff --git a/packages/beacon-node/src/chain/archiver/archiveBlocks.ts b/packages/beacon-node/src/chain/archiver/archiveBlocks.ts index 41442c6b0064..c728b9c2a1eb 100644 --- a/packages/beacon-node/src/chain/archiver/archiveBlocks.ts +++ b/packages/beacon-node/src/chain/archiver/archiveBlocks.ts @@ -58,7 +58,7 @@ export async function archiveBlocks( if (finalizedPostDeneb) { await migrateBlobsSidecarFromHotToColdDb(config, db, finalizedCanonicalBlockRoots); - logger.verbose("Migrated blobsSidecar from hot DB to cold DB"); + logger.verbose("Migrated blobSidecars from hot DB to cold DB"); } } @@ -73,7 +73,7 @@ export async function archiveBlocks( }); if (finalizedPostDeneb) { - await db.blobsSidecar.batchDelete(nonCanonicalBlockRoots); + await db.blobSidecars.batchDelete(nonCanonicalBlockRoots); logger.verbose("Deleted non canonical blobsSider from hot DB"); } } @@ -81,16 +81,16 @@ export async function archiveBlocks( // Delete expired blobs // Keep only `[max(GENESIS_EPOCH, current_epoch - MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS), current_epoch]` if (finalizedPostDeneb) { - const blobsSidecarMinEpoch = currentEpoch - config.MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS; + const blobsSidecarMinEpoch = currentEpoch - config.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS; if (blobsSidecarMinEpoch >= config.DENEB_FORK_EPOCH) { - const slotsToDelete = await db.blobsSidecarArchive.keys({lt: computeStartSlotAtEpoch(blobsSidecarMinEpoch)}); + const slotsToDelete = await db.blobSidecarsArchive.keys({lt: computeStartSlotAtEpoch(blobsSidecarMinEpoch)}); if (slotsToDelete.length > 0) { - await db.blobsSidecarArchive.batchDelete(slotsToDelete); + await db.blobSidecarsArchive.batchDelete(slotsToDelete); logger.verbose( - `blobsSidecar prune: batchDelete range ${slotsToDelete[0]}..${slotsToDelete[slotsToDelete.length - 1]}` + `blobSidecars prune: batchDelete range ${slotsToDelete[0]}..${slotsToDelete[slotsToDelete.length - 1]}` ); } else { - logger.verbose(`blobsSidecar prune: no entries before epoch ${blobsSidecarMinEpoch}`); + logger.verbose(`blobSidecars prune: no entries before epoch ${blobsSidecarMinEpoch}`); } } } @@ -160,13 +160,13 @@ async function migrateBlobsSidecarFromHotToColdDb( if (canonicalBlocks.length === 0) return; // load Buffer instead of ssz deserialized to improve performance - const canonicalBlobsSidecarEntries: KeyValue[] = await Promise.all( + const canonicalBlobSidecarsEntries: KeyValue[] = await Promise.all( canonicalBlocks .filter((block) => config.getForkSeq(block.slot) >= ForkSeq.deneb) .map(async (block) => { - const bytes = await db.blobsSidecar.getBinary(block.root); + const bytes = await db.blobSidecars.getBinary(block.root); if (!bytes) { - throw Error(`No blobsSidecar found for slot ${block.slot} root ${toHex(block.root)}`); + throw Error(`No blobSidecars found for slot ${block.slot} root ${toHex(block.root)}`); } return {key: block.slot, value: bytes}; }) @@ -174,8 +174,8 @@ async function migrateBlobsSidecarFromHotToColdDb( // put to blockArchive db and delete block db await Promise.all([ - db.blobsSidecarArchive.batchPutBinary(canonicalBlobsSidecarEntries), - db.blobsSidecar.batchDelete(canonicalBlocks.map((block) => block.root)), + db.blobSidecarsArchive.batchPutBinary(canonicalBlobSidecarsEntries), + db.blobSidecars.batchDelete(canonicalBlocks.map((block) => block.root)), ]); } } diff --git a/packages/beacon-node/src/chain/blocks/importBlock.ts b/packages/beacon-node/src/chain/blocks/importBlock.ts index ec9f0d68c3cd..b33ceafeba9d 100644 --- a/packages/beacon-node/src/chain/blocks/importBlock.ts +++ b/packages/beacon-node/src/chain/blocks/importBlock.ts @@ -68,13 +68,13 @@ export async function importBlock( }); if (blockInput.type === BlockInputType.postDeneb) { - const {blobs} = blockInput; + const {blobs: blobSidecars} = blockInput; // NOTE: Old blobs are pruned on archive - await this.db.blobsSidecar.add(blobs); - this.logger.debug("Persisted blobsSidecar to hot DB", { - blobsLen: blobs.blobs.length, - slot: blobs.beaconBlockSlot, - root: toHexString(blobs.beaconBlockRoot), + await this.db.blobSidecars.add({blockRoot, slot: block.message.slot, blobSidecars}); + this.logger.debug("Persisted blobSidecars to hot DB", { + blobsLen: blobSidecars.length, + slot: block.message.slot, + root: blockRootHex, }); } diff --git a/packages/beacon-node/src/chain/blocks/types.ts b/packages/beacon-node/src/chain/blocks/types.ts index 9e6a71bae1f8..1e41fb0b8f20 100644 --- a/packages/beacon-node/src/chain/blocks/types.ts +++ b/packages/beacon-node/src/chain/blocks/types.ts @@ -1,8 +1,10 @@ import {CachedBeaconStateAllForks, computeEpochAtSlot} from "@lodestar/state-transition"; import {MaybeValidExecutionStatus} from "@lodestar/fork-choice"; -import {allForks, deneb, Slot} from "@lodestar/types"; +import {allForks, deneb, Slot, RootHex} from "@lodestar/types"; import {ForkSeq} from "@lodestar/params"; import {ChainForkConfig} from "@lodestar/config"; +import {toHexString} from "@chainsafe/ssz"; +import {pruneSetToMax} from "@lodestar/utils"; export enum BlockInputType { preDeneb = "preDeneb", @@ -11,17 +13,97 @@ export enum BlockInputType { export type BlockInput = | {type: BlockInputType.preDeneb; block: allForks.SignedBeaconBlock} - | {type: BlockInputType.postDeneb; block: allForks.SignedBeaconBlock; blobs: deneb.BlobsSidecar}; + | {type: BlockInputType.postDeneb; block: allForks.SignedBeaconBlock; blobs: deneb.BlobSidecars}; export function blockRequiresBlobs(config: ChainForkConfig, blockSlot: Slot, clockSlot: Slot): boolean { return ( config.getForkSeq(blockSlot) >= ForkSeq.deneb && // Only request blobs if they are recent enough - computeEpochAtSlot(blockSlot) >= computeEpochAtSlot(clockSlot) - config.MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS + computeEpochAtSlot(blockSlot) >= computeEpochAtSlot(clockSlot) - config.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS ); } +export enum GossipedInputType { + block = "block", + blob = "blob", +} +type GossipedBlockInput = + | {type: GossipedInputType.block; signedBlock: allForks.SignedBeaconBlock} + | {type: GossipedInputType.blob; signedBlob: deneb.SignedBlobSidecar}; +type BlockInputCacheType = {block?: allForks.SignedBeaconBlock; blobs: Map}; + +const MAX_GOSSIPINPUT_CACHE = 5; + export const getBlockInput = { + blockInputCache: new Map(), + + getFullBlockInput( + config: ChainForkConfig, + gossipedInput: GossipedBlockInput + ): + | {blockInput: BlockInput; blockInputMeta: {pending: null; haveBlobs: number; expectedBlobs: number}} + | {blockInput: null; blockInputMeta: {pending: GossipedInputType.block; haveBlobs: number; expectedBlobs: null}} + | {blockInput: null; blockInputMeta: {pending: GossipedInputType.blob; haveBlobs: number; expectedBlobs: number}} { + let blockHex; + let blockCache; + if (gossipedInput.type === GossipedInputType.block) { + const {signedBlock} = gossipedInput; + blockHex = toHexString( + config.getForkTypes(signedBlock.message.slot).BeaconBlock.hashTreeRoot(signedBlock.message) + ); + blockCache = this.blockInputCache.get(blockHex) ?? {blobs: new Map()}; + blockCache.block = signedBlock; + } else { + const {signedBlob} = gossipedInput; + blockHex = toHexString(signedBlob.message.blockRoot); + blockCache = this.blockInputCache.get(blockHex); + + // If a new entry is going to be inserted, prune out old ones + if (blockCache === undefined) { + pruneSetToMax(this.blockInputCache, MAX_GOSSIPINPUT_CACHE); + blockCache = {blobs: new Map()}; + } + // TODO: freetheblobs check if its the same blob or a duplicate and throw/take actions + blockCache.blobs.set(signedBlob.message.index, signedBlob.message); + } + this.blockInputCache.set(blockHex, blockCache); + const {block: signedBlock} = blockCache; + if (signedBlock !== undefined) { + const {blobKzgCommitments} = (signedBlock as deneb.SignedBeaconBlock).message.body; + if (blobKzgCommitments.length < blockCache.blobs.size) { + throw Error(`Received more blobs=${blockCache.blobs.size} than commitments=${blobKzgCommitments.length}`); + } + if (blobKzgCommitments.length === blockCache.blobs.size) { + const blobSidecars = []; + for (let index = 0; index < blobKzgCommitments.length; index++) { + const blobSidecar = blockCache.blobs.get(index); + if (blobSidecar === undefined) { + throw Error("Missing blobSidecar"); + } + blobSidecars.push(blobSidecar); + } + return { + blockInput: getBlockInput.postDeneb(config, signedBlock, blobSidecars), + blockInputMeta: {pending: null, haveBlobs: blockCache.blobs.size, expectedBlobs: blobKzgCommitments.length}, + }; + } else { + return { + blockInput: null, + blockInputMeta: { + pending: GossipedInputType.blob, + haveBlobs: blockCache.blobs.size, + expectedBlobs: blobKzgCommitments.length, + }, + }; + } + } else { + return { + blockInput: null, + blockInputMeta: {pending: GossipedInputType.block, haveBlobs: blockCache.blobs.size, expectedBlobs: null}, + }; + } + }, + preDeneb(config: ChainForkConfig, block: allForks.SignedBeaconBlock): BlockInput { if (config.getForkSeq(block.message.slot) >= ForkSeq.deneb) { throw Error(`Post Deneb block slot ${block.message.slot}`); @@ -32,7 +114,7 @@ export const getBlockInput = { }; }, - postDeneb(config: ChainForkConfig, block: allForks.SignedBeaconBlock, blobs: deneb.BlobsSidecar): BlockInput { + postDeneb(config: ChainForkConfig, block: allForks.SignedBeaconBlock, blobs: deneb.BlobSidecars): BlockInput { if (config.getForkSeq(block.message.slot) < ForkSeq.deneb) { throw Error(`Pre Deneb block slot ${block.message.slot}`); } @@ -82,8 +164,8 @@ export type ImportBlockOpts = { * Metadata: `true` if all the signatures including the proposer signature have been verified */ validSignatures?: boolean; - /** Set to true if already run `validateBlobsSidecar()` sucessfully on the blobs */ - validBlobsSidecar?: boolean; + /** Set to true if already run `validateBlobSidecars()` sucessfully on the blobs */ + validBlobSidecars?: boolean; /** Seen timestamp seconds */ seenTimestampSec?: number; }; diff --git a/packages/beacon-node/src/chain/blocks/verifyBlocksSanityChecks.ts b/packages/beacon-node/src/chain/blocks/verifyBlocksSanityChecks.ts index 6bddba07c5eb..540ff696c619 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlocksSanityChecks.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlocksSanityChecks.ts @@ -5,7 +5,7 @@ import {Slot, deneb} from "@lodestar/types"; import {toHexString} from "@lodestar/utils"; import {BeaconClock} from "../clock/interface.js"; import {BlockError, BlockErrorCode} from "../errors/index.js"; -import {validateBlobsSidecar} from "../validation/blobsSidecar.js"; +import {validateBlobSidecars} from "../validation/blobSidecar.js"; import {BlockInput, BlockInputType, ImportBlockOpts} from "./types.js"; /** @@ -123,16 +123,19 @@ function maybeValidateBlobs( // TODO Deneb: Make switch verify it's exhaustive switch (blockInput.type) { case BlockInputType.postDeneb: { - if (opts.validBlobsSidecar) { + if (opts.validBlobSidecars) { return DataAvailableStatus.available; } const {block, blobs} = blockInput; const blockSlot = block.message.slot; - const {blobKzgCommitments} = (block as deneb.SignedBeaconBlock).message.body; + const { + blobKzgCommitments, + executionPayload: {transactions}, + } = (block as deneb.SignedBeaconBlock).message.body; const beaconBlockRoot = config.getForkTypes(blockSlot).BeaconBlock.hashTreeRoot(block.message); // TODO Deneb: This function throws un-typed errors - validateBlobsSidecar(blockSlot, beaconBlockRoot, blobKzgCommitments, blobs); + validateBlobSidecars(blockSlot, beaconBlockRoot, transactions, blobKzgCommitments, blobs); return DataAvailableStatus.available; } diff --git a/packages/beacon-node/src/chain/blocks/verifyBlocksStateTransitionOnly.ts b/packages/beacon-node/src/chain/blocks/verifyBlocksStateTransitionOnly.ts index 46b48ef8f339..455ba6ad26d4 100644 --- a/packages/beacon-node/src/chain/blocks/verifyBlocksStateTransitionOnly.ts +++ b/packages/beacon-node/src/chain/blocks/verifyBlocksStateTransitionOnly.ts @@ -30,8 +30,8 @@ export async function verifyBlocksStateTransitionOnly( const {block} = blocks[i]; const preState = i === 0 ? preState0 : postStates[i - 1]; - // TODO Deneb: Is the best place here to call validateBlobsSidecar()? - // TODO Deneb: Gossip may already call validateBlobsSidecar, add some flag to de-dup from here + // TODO Deneb: Is the best place here to call validateBlobSidecars()? + // TODO Deneb: Gossip may already call validateBlobSidecars, add some flag to de-dup from here // TODO Deneb: For sync if this function is expensive, consider adding sleep(0) if metrics show it // STFN - per_slot_processing() + per_block_processing() diff --git a/packages/beacon-node/src/chain/chain.ts b/packages/beacon-node/src/chain/chain.ts index 7c9c333e897b..2746d94c2bcc 100644 --- a/packages/beacon-node/src/chain/chain.ts +++ b/packages/beacon-node/src/chain/chain.ts @@ -24,7 +24,6 @@ import {IBeaconDb} from "../db/index.js"; import {Metrics} from "../metrics/index.js"; import {bytesToData, numToQuantity} from "../eth1/provider/utils.js"; import {wrapError} from "../util/wrapError.js"; -import {ckzg} from "../util/kzg.js"; import {IEth1ForBlockProduction} from "../eth1/index.js"; import {IExecutionEngine, IExecutionBuilder, TransitionConfigurationV1} from "../execution/index.js"; import {ensureDir, writeIfNotExist} from "../util/file.js"; @@ -69,7 +68,7 @@ import {BlockInput} from "./blocks/types.js"; * Arbitrary constants, blobs should be consumed immediately in the same slot they are produced. * A value of 1 would probably be sufficient. However it's sensible to allow some margin if the node overloads. */ -const DEFAULT_MAX_CACHED_BLOBS_SIDECAR = 8; +const DEFAULT_MAX_CACHED_BLOB_SIDECARS = 8; const MAX_RETAINED_SLOTS_CACHED_BLOBS_SIDECAR = 8; export class BeaconChain implements IBeaconChain { @@ -120,7 +119,8 @@ export class BeaconChain implements IBeaconChain { readonly checkpointBalancesCache: CheckpointBalancesCache; // TODO DENEB: Prune data structure every time period, for both old entries /** Map keyed by executionPayload.blockHash of the block for those blobs */ - readonly producedBlobsSidecarCache = new Map(); + readonly producedBlobSidecarsCache = new Map(); + readonly producedBlindedBlobSidecarsCache = new Map(); readonly opts: IChainOptions; protected readonly blockProcessor: BlockProcessor; @@ -391,17 +391,20 @@ export class BeaconChain implements IBeaconChain { // blinded blobs will be fetched and added to this cache later before finally // publishing the blinded block's full version if (blobs.type === BlobsResultType.produced) { - // TODO DENEB: Prune data structure for max entries - this.producedBlobsSidecarCache.set(blobs.blockHash, { - // TODO DENEB: Optimize, hashing the full block is not free. - beaconBlockRoot: this.config.getForkTypes(block.slot).BeaconBlock.hashTreeRoot(block), - beaconBlockSlot: block.slot, - blobs: blobs.blobs, - kzgAggregatedProof: ckzg.computeAggregateKzgProof(blobs.blobs), - }); + const blockRoot = this.config.getForkTypes(slot).BeaconBlock.hashTreeRoot(block); + const blockRootHex = toHex(blockRoot); + const blobSidecars = blobs.blobSidecars.map((blobSidecar) => ({ + ...blobSidecar, + blockRoot, + slot, + blockParentRoot: parentBlockRoot, + proposerIndex, + })); + + this.producedBlobSidecarsCache.set(blockRootHex, {blobSidecars, slot}); pruneSetToMax( - this.producedBlobsSidecarCache, - this.opts.maxCachedBlobsSidecar ?? DEFAULT_MAX_CACHED_BLOBS_SIDECAR + this.producedBlobSidecarsCache, + this.opts.maxCachedBlobSidecars ?? DEFAULT_MAX_CACHED_BLOB_SIDECARS ); } @@ -410,22 +413,22 @@ export class BeaconChain implements IBeaconChain { /** * https://github.com/ethereum/consensus-specs/blob/dev/specs/eip4844/validator.md#sidecar - * def get_blobs_sidecar(block: BeaconBlock, blobs: Sequence[Blob]) -> BlobsSidecar: - * return BlobsSidecar( + * def get_blobs_sidecar(block: BeaconBlock, blobs: Sequence[Blob]) -> BlobSidecars: + * return BlobSidecars( * beacon_block_root=hash_tree_root(block), * beacon_block_slot=block.slot, * blobs=blobs, * kzg_aggregated_proof=compute_proof_from_blobs(blobs), * ) */ - getBlobsSidecar(beaconBlock: deneb.BeaconBlock): deneb.BlobsSidecar { + getBlobSidecars(beaconBlock: deneb.BeaconBlock): deneb.BlobSidecars { const blockHash = toHex(beaconBlock.body.executionPayload.blockHash); - const blobsSidecar = this.producedBlobsSidecarCache.get(blockHash); - if (!blobsSidecar) { - throw Error(`No blobsSidecar for executionPayload.blockHash ${blockHash}`); + const {blobSidecars} = this.producedBlobSidecarsCache.get(blockHash) ?? {}; + if (!blobSidecars) { + throw Error(`No blobSidecars for executionPayload.blockHash ${blockHash}`); } - return blobsSidecar; + return blobSidecars; } async processBlock(block: BlockInput, opts?: ImportBlockOpts): Promise { @@ -641,11 +644,21 @@ export class BeaconChain implements IBeaconChain { }); } - // Prune old blobsSidecar for block production, those are only useful on their slot - if (this.config.getForkSeq(slot) >= ForkSeq.deneb && this.producedBlobsSidecarCache.size > 0) { - for (const [key, blobsSidecar] of this.producedBlobsSidecarCache) { - if (slot > blobsSidecar.beaconBlockSlot + MAX_RETAINED_SLOTS_CACHED_BLOBS_SIDECAR) { - this.producedBlobsSidecarCache.delete(key); + // Prune old blobSidecars for block production, those are only useful on their slot + if (this.config.getForkSeq(slot) >= ForkSeq.deneb) { + if (this.producedBlobSidecarsCache.size > 0) { + for (const [key, {slot: blobSlot}] of this.producedBlobSidecarsCache) { + if (slot > blobSlot + MAX_RETAINED_SLOTS_CACHED_BLOBS_SIDECAR) { + this.producedBlobSidecarsCache.delete(key); + } + } + } + + if (this.producedBlindedBlobSidecarsCache.size > 0) { + for (const [key, {slot: blobSlot}] of this.producedBlindedBlobSidecarsCache) { + if (slot > blobSlot + MAX_RETAINED_SLOTS_CACHED_BLOBS_SIDECAR) { + this.producedBlindedBlobSidecarsCache.delete(key); + } } } } diff --git a/packages/beacon-node/src/chain/errors/blobsSidecarError.ts b/packages/beacon-node/src/chain/errors/blobSidecarError.ts similarity index 54% rename from packages/beacon-node/src/chain/errors/blobsSidecarError.ts rename to packages/beacon-node/src/chain/errors/blobSidecarError.ts index 628a2cdc0640..f0ad69167c19 100644 --- a/packages/beacon-node/src/chain/errors/blobsSidecarError.ts +++ b/packages/beacon-node/src/chain/errors/blobSidecarError.ts @@ -1,7 +1,8 @@ import {Slot} from "@lodestar/types"; import {GossipActionError} from "./gossipValidation.js"; -export enum BlobsSidecarErrorCode { +export enum BlobSidecarErrorCode { + INVALID_INDEX = "BLOBS_SIDECAR_ERROR_INVALID_INDEX", /** !bls.KeyValidate(block.body.blob_kzg_commitments[i]) */ INVALID_KZG = "BLOBS_SIDECAR_ERROR_INVALID_KZG", /** !verify_kzg_commitments_against_transactions(block.body.execution_payload.transactions, block.body.blob_kzg_commitments) */ @@ -14,11 +15,12 @@ export enum BlobsSidecarErrorCode { INVALID_KZG_PROOF = "BLOBS_SIDECAR_ERROR_INVALID_KZG_PROOF", } -export type BlobsSidecarErrorType = - | {code: BlobsSidecarErrorCode.INVALID_KZG; kzgIdx: number} - | {code: BlobsSidecarErrorCode.INVALID_KZG_TXS} - | {code: BlobsSidecarErrorCode.INCORRECT_SLOT; blockSlot: Slot; blobSlot: Slot} - | {code: BlobsSidecarErrorCode.INVALID_BLOB; blobIdx: number} - | {code: BlobsSidecarErrorCode.INVALID_KZG_PROOF}; +export type BlobSidecarErrorType = + | {code: BlobSidecarErrorCode.INVALID_INDEX; blobIdx: number; gossipIndex: number} + | {code: BlobSidecarErrorCode.INVALID_KZG; blobIdx: number} + | {code: BlobSidecarErrorCode.INVALID_KZG_TXS} + | {code: BlobSidecarErrorCode.INCORRECT_SLOT; blockSlot: Slot; blobSlot: Slot; blobIdx: number} + | {code: BlobSidecarErrorCode.INVALID_BLOB; blobIdx: number} + | {code: BlobSidecarErrorCode.INVALID_KZG_PROOF; blobIdx: number}; -export class BlobsSidecarError extends GossipActionError {} +export class BlobSidecarError extends GossipActionError {} diff --git a/packages/beacon-node/src/chain/errors/index.ts b/packages/beacon-node/src/chain/errors/index.ts index 9ce6c8109749..1bd8f8577305 100644 --- a/packages/beacon-node/src/chain/errors/index.ts +++ b/packages/beacon-node/src/chain/errors/index.ts @@ -1,6 +1,6 @@ export * from "./attestationError.js"; export * from "./attesterSlashingError.js"; -export * from "./blobsSidecarError.js"; +export * from "./blobSidecarError.js"; export * from "./blockError.js"; export * from "./gossipValidation.js"; export * from "./proposerSlashingError.js"; diff --git a/packages/beacon-node/src/chain/interface.ts b/packages/beacon-node/src/chain/interface.ts index d087d2b2a3ce..23e93aea56f7 100644 --- a/packages/beacon-node/src/chain/interface.ts +++ b/packages/beacon-node/src/chain/interface.ts @@ -88,7 +88,8 @@ export interface IBeaconChain { readonly beaconProposerCache: BeaconProposerCache; readonly checkpointBalancesCache: CheckpointBalancesCache; - readonly producedBlobsSidecarCache: Map; + readonly producedBlobSidecarsCache: Map; + readonly producedBlindedBlobSidecarsCache: Map; readonly opts: IChainOptions; /** Stop beacon chain processing */ @@ -111,7 +112,7 @@ export interface IBeaconChain { */ getCanonicalBlockAtSlot(slot: Slot): Promise; - getBlobsSidecar(beaconBlock: deneb.BeaconBlock): deneb.BlobsSidecar; + getBlobSidecars(beaconBlock: deneb.BeaconBlock): deneb.BlobSidecars; produceBlock(blockAttributes: BlockAttributes): Promise<{block: allForks.BeaconBlock; blockValue: Wei}>; produceBlindedBlock(blockAttributes: BlockAttributes): Promise<{block: allForks.BlindedBeaconBlock; blockValue: Wei}>; diff --git a/packages/beacon-node/src/chain/options.ts b/packages/beacon-node/src/chain/options.ts index cd8b01113494..87dcff9c3940 100644 --- a/packages/beacon-node/src/chain/options.ts +++ b/packages/beacon-node/src/chain/options.ts @@ -19,7 +19,7 @@ export type IChainOptions = BlockProcessOpts & /** Ensure blobs returned by the execution engine are valid */ sanityCheckExecutionEngineBlobs?: boolean; /** Max number of produced blobs by local validators to cache */ - maxCachedBlobsSidecar?: number; + maxCachedBlobSidecars?: number; }; export type BlockProcessOpts = { diff --git a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts index 7541c2a19062..9659def3329a 100644 --- a/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts +++ b/packages/beacon-node/src/chain/produceBlock/produceBlockBody.ts @@ -35,6 +35,7 @@ import {PayloadId, IExecutionEngine, IExecutionBuilder, PayloadAttributes} from import {ZERO_HASH, ZERO_HASH_HEX} from "../../constants/index.js"; import {IEth1ForBlockProduction} from "../../eth1/index.js"; import {numToQuantity} from "../../eth1/provider/utils.js"; +import {ckzg} from "../../util/kzg.js"; import {validateBlobsAndKzgCommitments} from "./validateBlobsAndKzgCommitments.js"; // Time to provide the EL to generate a payload from new payload id @@ -70,7 +71,7 @@ export enum BlobsResultType { export type BlobsResult = | {type: BlobsResultType.preDeneb | BlobsResultType.blinded} - | {type: BlobsResultType.produced; blobs: deneb.Blobs; blockHash: RootHex}; + | {type: BlobsResultType.produced; blobSidecars: deneb.BlobSidecars; blockHash: RootHex}; export async function produceBlockBody( this: BeaconChain, @@ -260,7 +261,16 @@ export async function produceBlockBody( } (blockBody as deneb.BeaconBlockBody).blobKzgCommitments = blobsBundle.kzgs; - blobsResult = {type: BlobsResultType.produced, blobs: blobsBundle.blobs, blockHash}; + const blobSidecars = Array.from({length: blobsBundle.blobs.length}, (_v, index) => { + const blobSidecar = { + index, + blob: blobsBundle.blobs[index], + kzgProof: ckzg.computeBlobKzgProof(blobsBundle.blobs[index]), + kzgCommitment: blobsBundle.kzgs[index], + }; + return blobSidecar; + }) as deneb.BlobSidecars; + blobsResult = {type: BlobsResultType.produced, blobSidecars, blockHash}; } else { blobsResult = {type: BlobsResultType.preDeneb}; } diff --git a/packages/beacon-node/src/chain/regen/interface.ts b/packages/beacon-node/src/chain/regen/interface.ts index 845868074bd2..f245428fe604 100644 --- a/packages/beacon-node/src/chain/regen/interface.ts +++ b/packages/beacon-node/src/chain/regen/interface.ts @@ -6,6 +6,7 @@ export enum RegenCaller { processBlock = "processBlock", produceBlock = "produceBlock", validateGossipBlock = "validateGossipBlock", + validateGossipBlob = "validateGossipBlob", precomputeEpoch = "precomputeEpoch", produceAttestationData = "produceAttestationData", processBlocksInEpoch = "processBlocksInEpoch", diff --git a/packages/beacon-node/src/chain/validation/blobSidecar.ts b/packages/beacon-node/src/chain/validation/blobSidecar.ts new file mode 100644 index 000000000000..9fd7195fdf15 --- /dev/null +++ b/packages/beacon-node/src/chain/validation/blobSidecar.ts @@ -0,0 +1,211 @@ +import {ChainForkConfig} from "@lodestar/config"; +import {deneb, Root, Slot, bellatrix} from "@lodestar/types"; +import {toHex} from "@lodestar/utils"; +import { + verifyKzgCommitmentsAgainstTransactions, + getBlobProposerSignatureSet, + computeStartSlotAtEpoch, +} from "@lodestar/state-transition"; + +import {BlobSidecarError, BlobSidecarErrorCode} from "../errors/blobSidecarError.js"; +import {GossipAction} from "../errors/gossipValidation.js"; +import {ckzg} from "../../util/kzg.js"; +import {IBeaconChain} from "../interface.js"; +import {RegenCaller} from "../regen/index.js"; + +// TODO: freetheblobs define blobs own gossip error +import {BlockGossipError, BlockErrorCode} from "../errors/index.js"; + +export async function validateGossipBlobSidecar( + config: ChainForkConfig, + chain: IBeaconChain, + signedBlob: deneb.SignedBlobSidecar, + gossipIndex: number +): Promise { + const blobSidecar = signedBlob.message; + const blobSlot = blobSidecar.slot; + + // [REJECT] The sidecar is for the correct topic -- i.e. sidecar.index matches the topic {index}. + if (blobSidecar.index !== gossipIndex) { + throw new BlobSidecarError(GossipAction.REJECT, { + code: BlobSidecarErrorCode.INVALID_INDEX, + blobIdx: blobSidecar.index, + gossipIndex, + }); + } + + // [IGNORE] The sidecar is not from a future slot (with a MAXIMUM_GOSSIP_CLOCK_DISPARITY allowance) -- + // i.e. validate that sidecar.slot <= current_slot (a client MAY queue future blocks for processing at + // the appropriate slot). + const currentSlotWithGossipDisparity = chain.clock.currentSlotWithGossipDisparity; + if (currentSlotWithGossipDisparity < blobSlot) { + throw new BlockGossipError(GossipAction.IGNORE, { + code: BlockErrorCode.FUTURE_SLOT, + currentSlot: currentSlotWithGossipDisparity, + blockSlot: blobSlot, + }); + } + + // [IGNORE] The sidecar is from a slot greater than the latest finalized slot -- i.e. validate that + // sidecar.slot > compute_start_slot_at_epoch(state.finalized_checkpoint.epoch) + const finalizedCheckpoint = chain.forkChoice.getFinalizedCheckpoint(); + const finalizedSlot = computeStartSlotAtEpoch(finalizedCheckpoint.epoch); + if (blobSlot <= finalizedSlot) { + throw new BlockGossipError(GossipAction.IGNORE, { + code: BlockErrorCode.WOULD_REVERT_FINALIZED_SLOT, + blockSlot: blobSlot, + finalizedSlot, + }); + } + + // Check if the block is already known. We know it is post-finalization, so it is sufficient to check the fork choice. + // + // In normal operation this isn't necessary, however it is useful immediately after a + // reboot if the `observed_block_producers` cache is empty. In that case, without this + // check, we will load the parent and state from disk only to find out later that we + // already know this block. + const blockRoot = toHex(blobSidecar.blockRoot); + if (chain.forkChoice.getBlockHex(blockRoot) !== null) { + throw new BlockGossipError(GossipAction.IGNORE, {code: BlockErrorCode.ALREADY_KNOWN, root: blockRoot}); + } + + // TODO: freetheblobs - check for badblock + // TODO: freetheblobs - check that its first blob with valid signature + + // _[IGNORE]_ The blob's block's parent (defined by `sidecar.block_parent_root`) has been seen (via both + // gossip and non-gossip sources) (a client MAY queue blocks for processing once the parent block is + // retrieved). + const parentRoot = toHex(blobSidecar.blockParentRoot); + const parentBlock = chain.forkChoice.getBlockHex(parentRoot); + if (parentBlock === null) { + // If fork choice does *not* consider the parent to be a descendant of the finalized block, + // then there are two more cases: + // + // 1. We have the parent stored in our database. Because fork-choice has confirmed the + // parent is *not* in our post-finalization DAG, all other blocks must be either + // pre-finalization or conflicting with finalization. + // 2. The parent is unknown to us, we probably want to download it since it might actually + // descend from the finalized root. + // (Non-Lighthouse): Since we prune all blocks non-descendant from finalized checking the `db.block` database won't be useful to guard + // against known bad fork blocks, so we throw PARENT_UNKNOWN for cases (1) and (2) + throw new BlockGossipError(GossipAction.IGNORE, {code: BlockErrorCode.PARENT_UNKNOWN, parentRoot}); + } + + // [REJECT] The blob is from a higher slot than its parent. + if (parentBlock.slot >= blobSlot) { + throw new BlockGossipError(GossipAction.IGNORE, { + code: BlockErrorCode.NOT_LATER_THAN_PARENT, + parentSlot: parentBlock.slot, + slot: blobSlot, + }); + } + + // getBlockSlotState also checks for whether the current finalized checkpoint is an ancestor of the block. + // As a result, we throw an IGNORE (whereas the spec says we should REJECT for this scenario). + // this is something we should change this in the future to make the code airtight to the spec. + // _[IGNORE]_ The blob's block's parent (defined by `sidecar.block_parent_root`) has been seen (via both + // gossip and non-gossip sources) // _[REJECT]_ The blob's block's parent (defined by `sidecar.block_parent_root`) passes validation + // The above validation will happen while importing + const blockState = await chain.regen + .getBlockSlotState(parentRoot, blobSlot, {dontTransferCache: true}, RegenCaller.validateGossipBlob) + .catch(() => { + throw new BlockGossipError(GossipAction.IGNORE, {code: BlockErrorCode.PARENT_UNKNOWN, parentRoot}); + }); + + // _[REJECT]_ The proposer signature, `signed_blob_sidecar.signature`, is valid with respect to the + // `sidecar.proposer_index` pubkey. + const signatureSet = getBlobProposerSignatureSet(blockState, signedBlob); + // Don't batch so verification is not delayed + if (!(await chain.bls.verifySignatureSets([signatureSet], {verifyOnMainThread: true}))) { + throw new BlockGossipError(GossipAction.REJECT, { + code: BlockErrorCode.PROPOSAL_SIGNATURE_INVALID, + }); + } + + // _[IGNORE]_ The sidecar is the only sidecar with valid signature received for the tuple + // `(sidecar.block_root, sidecar.index)` + // + // This is already taken care of by the way we group the blobs in getFullBlockInput helper + // but may be an error can be thrown there for this + + // _[REJECT]_ The sidecar is proposed by the expected `proposer_index` for the block's slot in the + // context of the current shuffling (defined by `block_parent_root`/`slot`) + // If the `proposer_index` cannot immediately be verified against the expected shuffling, the sidecar + // MAY be queued for later processing while proposers for the block's branch are calculated -- in such + // a case _do not_ `REJECT`, instead `IGNORE` this message. + const proposerIndex = blobSidecar.proposerIndex; + if (blockState.epochCtx.getBeaconProposer(blobSlot) !== proposerIndex) { + throw new BlockGossipError(GossipAction.REJECT, {code: BlockErrorCode.INCORRECT_PROPOSER, proposerIndex}); + } + + // blob, proof and commitment as a valid BLS G1 point gets verified in batch validation + validateBlobsAndProofs([blobSidecar.kzgCommitment], [blobSidecar.blob], [blobSidecar.kzgProof]); +} + +// https://github.com/ethereum/consensus-specs/blob/dev/specs/eip4844/beacon-chain.md#validate_blobs_sidecar +export function validateBlobSidecars( + blockSlot: Slot, + blockRoot: Root, + transactions: bellatrix.Transactions, + expectedKzgCommitments: deneb.BlobKzgCommitments, + blobSidecars: deneb.BlobSidecars +): void { + // TODO: freetheblobs + if (!verifyKzgCommitmentsAgainstTransactions(transactions, expectedKzgCommitments)) { + throw new Error("Invalid block transactions and commitments"); + } + + // assert len(expected_kzg_commitments) == len(blobs) + if (expectedKzgCommitments.length !== blobSidecars.length) { + throw new Error( + `blobSidecars length to commitments length mismatch. Blob length: ${blobSidecars.length}, Expected commitments length ${expectedKzgCommitments.length}` + ); + } + + // No need to verify the aggregate proof of zero blobs + if (blobSidecars.length > 0) { + // Verify the blob slot and root matches + const blobs = []; + const proofs = []; + for (let index = 0; index < blobSidecars.length; index++) { + const blobSidecar = blobSidecars[index]; + // TODO: freetheblobs + // + // Cleaning up commitment match check as the batch verify of commitments blobs and proofs + // is should be matching this + // if ( + // blobSidecar.slot !== blockSlot || + // !byteArrayEquals(blobSidecar.blockRoot, blockRoot) || + // blobSidecar.index !== index || + // !byteArrayEquals(expectedKzgCommitments[index], blobSidecar.kzgCommitment) + // ) { + // throw new Error( + // `Invalid blob with slot=${blobSidecar.slot} blockRoot=${toHex(blockRoot)} index=${ + // blobSidecar.index + // } for the block root=${toHex(blockRoot)} slot=${blockSlot} index=${index}` + // ); + // } + blobs.push(blobSidecar.blob); + proofs.push(blobSidecar.kzgProof); + } + validateBlobsAndProofs(expectedKzgCommitments, blobs, proofs); + } +} + +function validateBlobsAndProofs( + expectedKzgCommitments: deneb.BlobKzgCommitments, + blobs: deneb.Blobs, + proofs: deneb.KZGProofs +): void { + // assert verify_aggregate_kzg_proof(blobs, expected_kzg_commitments, kzg_aggregated_proof) + let isProofValid: boolean; + try { + isProofValid = ckzg.verifyBlobKzgProofBatch(blobs, expectedKzgCommitments, proofs); + } catch (e) { + (e as Error).message = `Error on verifyBlobKzgProofBatch: ${(e as Error).message}`; + throw e; + } + if (!isProofValid) { + throw Error("Invalid verifyBlobKzgProofBatch"); + } +} diff --git a/packages/beacon-node/src/chain/validation/blobsSidecar.ts b/packages/beacon-node/src/chain/validation/blobsSidecar.ts deleted file mode 100644 index bb46c384e508..000000000000 --- a/packages/beacon-node/src/chain/validation/blobsSidecar.ts +++ /dev/null @@ -1,150 +0,0 @@ -import bls from "@chainsafe/bls"; -import {CoordType} from "@chainsafe/bls/types"; -import {deneb, Root, ssz} from "@lodestar/types"; -import {bytesToBigInt} from "@lodestar/utils"; -import {BYTES_PER_FIELD_ELEMENT, FIELD_ELEMENTS_PER_BLOB} from "@lodestar/params"; -import {verifyKzgCommitmentsAgainstTransactions} from "@lodestar/state-transition"; -import {BlobsSidecarError, BlobsSidecarErrorCode} from "../errors/blobsSidecarError.js"; -import {GossipAction} from "../errors/gossipValidation.js"; -import {byteArrayEquals} from "../../util/bytes.js"; -import {ckzg} from "../../util/kzg.js"; - -const BLS_MODULUS = BigInt("52435875175126190479447740508185965837690552500527637822603658699938581184513"); - -export function validateGossipBlobsSidecar( - signedBlock: deneb.SignedBeaconBlock, - blobsSidecar: deneb.BlobsSidecar -): void { - const block = signedBlock.message; - - // Spec: https://github.com/ethereum/consensus-specs/blob/4cb6fd1c8c8f190d147d15b182c2510d0423ec61/specs/eip4844/p2p-interface.md#beacon_block_and_blobs_sidecar - // [REJECT] The KZG commitments of the blobs are all correctly encoded compressed BLS G1 Points. - // -- i.e. all(bls.KeyValidate(commitment) for commitment in block.body.blob_kzg_commitments) - const {blobKzgCommitments} = block.body; - for (let i = 0; i < blobKzgCommitments.length; i++) { - if (!blsKeyValidate(blobKzgCommitments[i])) { - throw new BlobsSidecarError(GossipAction.REJECT, {code: BlobsSidecarErrorCode.INVALID_KZG, kzgIdx: i}); - } - } - - // [REJECT] The KZG commitments correspond to the versioned hashes in the transactions list. - // -- i.e. verify_kzg_commitments_against_transactions(block.body.execution_payload.transactions, block.body.blob_kzg_commitments) - if ( - !verifyKzgCommitmentsAgainstTransactions(block.body.executionPayload.transactions, block.body.blobKzgCommitments) - ) { - throw new BlobsSidecarError(GossipAction.REJECT, {code: BlobsSidecarErrorCode.INVALID_KZG_TXS}); - } - - // [IGNORE] the sidecar.beacon_block_slot is for the current slot (with a MAXIMUM_GOSSIP_CLOCK_DISPARITY allowance) - // -- i.e. sidecar.beacon_block_slot == block.slot. - if (blobsSidecar.beaconBlockSlot !== block.slot) { - throw new BlobsSidecarError(GossipAction.IGNORE, { - code: BlobsSidecarErrorCode.INCORRECT_SLOT, - blobSlot: blobsSidecar.beaconBlockSlot, - blockSlot: block.slot, - }); - } - - // [REJECT] the sidecar.blobs are all well formatted, i.e. the BLSFieldElement in valid range (x < BLS_MODULUS). - for (let i = 0; i < blobsSidecar.blobs.length; i++) { - if (!blobIsValidRange(blobsSidecar.blobs[i])) { - throw new BlobsSidecarError(GossipAction.REJECT, {code: BlobsSidecarErrorCode.INVALID_BLOB, blobIdx: i}); - } - } - - // [REJECT] The KZG proof is a correctly encoded compressed BLS G1 Point - // -- i.e. blsKeyValidate(blobs_sidecar.kzg_aggregated_proof) - if (!blsKeyValidate(blobsSidecar.kzgAggregatedProof)) { - throw new BlobsSidecarError(GossipAction.REJECT, {code: BlobsSidecarErrorCode.INVALID_KZG_PROOF}); - } - - // [REJECT] The KZG commitments in the block are valid against the provided blobs sidecar. -- i.e. - // validate_blobs_sidecar(block.slot, hash_tree_root(block), block.body.blob_kzg_commitments, sidecar) - validateBlobsSidecar( - block.slot, - ssz.bellatrix.BeaconBlock.hashTreeRoot(block), - block.body.blobKzgCommitments, - blobsSidecar - ); -} - -// https://github.com/ethereum/consensus-specs/blob/dev/specs/eip4844/beacon-chain.md#validate_blobs_sidecar -export function validateBlobsSidecar( - slot: number, - beaconBlockRoot: Root, - expectedKzgCommitments: deneb.KZGCommitment[], - blobsSidecar: deneb.BlobsSidecar -): void { - // assert slot == blobs_sidecar.beacon_block_slot - if (slot != blobsSidecar.beaconBlockSlot) { - throw new Error(`slot mismatch. Block slot: ${slot}, Blob slot ${blobsSidecar.beaconBlockSlot}`); - } - - // assert beacon_block_root == blobs_sidecar.beacon_block_root - if (!byteArrayEquals(beaconBlockRoot, blobsSidecar.beaconBlockRoot)) { - throw new Error( - `beacon block root mismatch. Block root: ${beaconBlockRoot}, Blob root ${blobsSidecar.beaconBlockRoot}` - ); - } - - // blobs = blobs_sidecar.blobs - // kzg_aggregated_proof = blobs_sidecar.kzg_aggregated_proof - const {blobs, kzgAggregatedProof} = blobsSidecar; - - // assert len(expected_kzg_commitments) == len(blobs) - if (expectedKzgCommitments.length !== blobs.length) { - throw new Error( - `blobs length to commitments length mismatch. Blob length: ${blobs.length}, Expected commitments length ${expectedKzgCommitments.length}` - ); - } - - // No need to verify the aggregate proof of zero blobs. Also c-kzg throws. - // https://github.com/dankrad/c-kzg/pull/12/files#r1025851956 - if (blobs.length > 0) { - // assert verify_aggregate_kzg_proof(blobs, expected_kzg_commitments, kzg_aggregated_proof) - let isProofValid: boolean; - try { - isProofValid = ckzg.verifyAggregateKzgProof(blobs, expectedKzgCommitments, kzgAggregatedProof); - } catch (e) { - // TODO DENEB: TEMP Nov17: May always throw error -- we need to fix Geth's KZG to match C-KZG and the trusted setup used here - (e as Error).message = `Error on verifyAggregateKzgProof: ${(e as Error).message}`; - throw e; - } - - // TODO DENEB: TEMP Nov17: May always throw error -- we need to fix Geth's KZG to match C-KZG and the trusted setup used here - if (!isProofValid) { - throw Error("Invalid AggregateKzgProof"); - } - } -} - -/** - * From https://datatracker.ietf.org/doc/html/draft-irtf-cfrg-bls-signature-04#section-2.5 - * KeyValidate = valid, non-identity point that is in the correct subgroup - */ -function blsKeyValidate(g1Point: Uint8Array): boolean { - try { - bls.PublicKey.fromBytes(g1Point, CoordType.jacobian, true); - return true; - } catch (e) { - return false; - } -} - -/** - * ``` - * Blob = new ByteVectorType(BYTES_PER_FIELD_ELEMENT * FIELD_ELEMENTS_PER_BLOB); - * ``` - * Check that each FIELD_ELEMENT as a uint256 < BLS_MODULUS - */ -function blobIsValidRange(blob: deneb.Blob): boolean { - for (let i = 0; i < FIELD_ELEMENTS_PER_BLOB; i++) { - const fieldElement = blob.subarray(i * BYTES_PER_FIELD_ELEMENT, (i + 1) * BYTES_PER_FIELD_ELEMENT); - const fieldElementBN = bytesToBigInt(fieldElement, "be"); - if (fieldElementBN >= BLS_MODULUS) { - return false; - } - } - - return true; -} diff --git a/packages/beacon-node/src/chain/validation/block.ts b/packages/beacon-node/src/chain/validation/block.ts index 67f7736b85f3..8a7b4c659dc8 100644 --- a/packages/beacon-node/src/chain/validation/block.ts +++ b/packages/beacon-node/src/chain/validation/block.ts @@ -6,7 +6,7 @@ import { isExecutionBlockBodyType, isExecutionStateType, isExecutionEnabled, - getProposerSignatureSet, + getBlockProposerSignatureSet, } from "@lodestar/state-transition"; import {sleep} from "@lodestar/utils"; import {ForkName} from "@lodestar/params"; @@ -141,7 +141,7 @@ export async function validateGossipBlock( } // [REJECT] The proposer signature, signed_beacon_block.signature, is valid with respect to the proposer_index pubkey. - const signatureSet = getProposerSignatureSet(blockState, signedBlock); + const signatureSet = getBlockProposerSignatureSet(blockState, signedBlock); // Don't batch so verification is not delayed if (!(await chain.bls.verifySignatureSets([signatureSet], {verifyOnMainThread: true}))) { throw new BlockGossipError(GossipAction.REJECT, { diff --git a/packages/beacon-node/src/db/beacon.ts b/packages/beacon-node/src/db/beacon.ts index ef7dc00a144f..cb34561e91b2 100644 --- a/packages/beacon-node/src/db/beacon.ts +++ b/packages/beacon-node/src/db/beacon.ts @@ -15,17 +15,17 @@ import { SyncCommitteeRepository, SyncCommitteeWitnessRepository, BackfilledRanges, - BlobsSidecarRepository, - BlobsSidecarArchiveRepository, + BlobSidecarsRepository, + BlobSidecarsArchiveRepository, BLSToExecutionChangeRepository, } from "./repositories/index.js"; import {PreGenesisState, PreGenesisStateLastProcessedBlock} from "./single/index.js"; export class BeaconDb extends DatabaseService implements IBeaconDb { block: BlockRepository; - blobsSidecar: BlobsSidecarRepository; + blobSidecars: BlobSidecarsRepository; blockArchive: BlockArchiveRepository; - blobsSidecarArchive: BlobsSidecarArchiveRepository; + blobSidecarsArchive: BlobSidecarsArchiveRepository; stateArchive: StateArchiveRepository; voluntaryExit: VoluntaryExitRepository; @@ -52,9 +52,9 @@ export class BeaconDb extends DatabaseService implements IBeaconDb { // Warning: If code is ever run in the constructor, must change this stub to not extend 'packages/beacon-node/test/utils/stub/beaconDb.ts' - this.block = new BlockRepository(this.config, this.db); - this.blobsSidecar = new BlobsSidecarRepository(this.config, this.db); + this.blobSidecars = new BlobSidecarsRepository(this.config, this.db); this.blockArchive = new BlockArchiveRepository(this.config, this.db); - this.blobsSidecarArchive = new BlobsSidecarArchiveRepository(this.config, this.db); + this.blobSidecarsArchive = new BlobSidecarsArchiveRepository(this.config, this.db); this.stateArchive = new StateArchiveRepository(this.config, this.db); this.voluntaryExit = new VoluntaryExitRepository(this.config, this.db); this.blsToExecutionChange = new BLSToExecutionChangeRepository(this.config, this.db); @@ -81,7 +81,7 @@ export class BeaconDb extends DatabaseService implements IBeaconDb { async pruneHotDb(): Promise { // Prune all hot blobs - await this.blobsSidecar.batchDelete(await this.blobsSidecar.keys()); + await this.blobSidecars.batchDelete(await this.blobSidecars.keys()); // Prune all hot blocks // TODO: Enable once it's deemed safe // await this.block.batchDelete(await this.block.keys()); diff --git a/packages/beacon-node/src/db/interface.ts b/packages/beacon-node/src/db/interface.ts index 3e0b41d6fe47..dc8b648f4cb9 100644 --- a/packages/beacon-node/src/db/interface.ts +++ b/packages/beacon-node/src/db/interface.ts @@ -14,8 +14,8 @@ import { SyncCommitteeRepository, SyncCommitteeWitnessRepository, BackfilledRanges, - BlobsSidecarRepository, - BlobsSidecarArchiveRepository, + BlobSidecarsRepository, + BlobSidecarsArchiveRepository, BLSToExecutionChangeRepository, } from "./repositories/index.js"; import {PreGenesisState, PreGenesisStateLastProcessedBlock} from "./single/index.js"; @@ -28,11 +28,11 @@ import {PreGenesisState, PreGenesisStateLastProcessedBlock} from "./single/index export interface IBeaconDb { // unfinalized blocks block: BlockRepository; - blobsSidecar: BlobsSidecarRepository; + blobSidecars: BlobSidecarsRepository; // finalized blocks blockArchive: BlockArchiveRepository; - blobsSidecarArchive: BlobsSidecarArchiveRepository; + blobSidecarsArchive: BlobSidecarsArchiveRepository; // finalized states stateArchive: StateArchiveRepository; diff --git a/packages/beacon-node/src/db/repositories/blobSidecars.ts b/packages/beacon-node/src/db/repositories/blobSidecars.ts new file mode 100644 index 000000000000..a29f98dbb151 --- /dev/null +++ b/packages/beacon-node/src/db/repositories/blobSidecars.ts @@ -0,0 +1,26 @@ +import {ChainForkConfig} from "@lodestar/config"; +import {Bucket, Db, Repository} from "@lodestar/db"; +import {deneb, ssz} from "@lodestar/types"; + +export const BLOB_SIDECARS_IN_WRAPPER_INDEX = 44; +// ssz.deneb.BlobSidecars.elementType.fixedSize; +export const BLOBSIDECAR_FIXED_SIZE = 131256; + +/** + * blobSidecarsWrapper by block root (= hash_tree_root(SignedBeaconBlockAndBlobsSidecar.beacon_block.message)) + * + * Used to store unfinalized BlobsSidecar + */ +export class BlobSidecarsRepository extends Repository { + constructor(config: ChainForkConfig, db: Db) { + super(config, db, Bucket.allForks_blobSidecars, ssz.deneb.BlobSidecarsWrapper); + } + + /** + * Id is hashTreeRoot of unsigned BeaconBlock + */ + getId(value: deneb.BlobSidecarsWrapper): Uint8Array { + const {blockRoot} = value; + return blockRoot; + } +} diff --git a/packages/beacon-node/src/db/repositories/blobSidecarsArchive.ts b/packages/beacon-node/src/db/repositories/blobSidecarsArchive.ts new file mode 100644 index 000000000000..403bd32acf94 --- /dev/null +++ b/packages/beacon-node/src/db/repositories/blobSidecarsArchive.ts @@ -0,0 +1,25 @@ +import {ChainForkConfig} from "@lodestar/config"; +import {Bucket, Db, Repository} from "@lodestar/db"; +import {deneb, ssz, Slot} from "@lodestar/types"; +import {bytesToInt} from "@lodestar/utils"; + +/** + * blobSidecarsWrapper by slot + * + * Used to store unfinalized BlobsSidecar + */ +export class BlobSidecarsArchiveRepository extends Repository { + constructor(config: ChainForkConfig, db: Db) { + super(config, db, Bucket.allForks_blobSidecarsArchive, ssz.deneb.BlobSidecarsWrapper); + } + + // Handle key as slot + + getId(value: deneb.BlobSidecarsWrapper): Slot { + return value.slot; + } + + decodeKey(data: Uint8Array): number { + return bytesToInt((super.decodeKey(data) as unknown) as Uint8Array, "be"); + } +} diff --git a/packages/beacon-node/src/db/repositories/blobsSidecar.ts b/packages/beacon-node/src/db/repositories/blobsSidecar.ts deleted file mode 100644 index fd575a0ddf14..000000000000 --- a/packages/beacon-node/src/db/repositories/blobsSidecar.ts +++ /dev/null @@ -1,21 +0,0 @@ -import {ChainForkConfig} from "@lodestar/config"; -import {Bucket, Db, Repository} from "@lodestar/db"; -import {deneb, ssz} from "@lodestar/types"; - -/** - * BlobsSidecar by block root (= hash_tree_root(SignedBeaconBlockAndBlobsSidecar.beacon_block.message)) - * - * Used to store unfinalized BlobsSidecar - */ -export class BlobsSidecarRepository extends Repository { - constructor(config: ChainForkConfig, db: Db) { - super(config, db, Bucket.allForks_blobsSidecar, ssz.deneb.BlobsSidecar); - } - - /** - * Id is hashTreeRoot of unsigned BeaconBlock - */ - getId(value: deneb.BlobsSidecar): Uint8Array { - return value.beaconBlockRoot; - } -} diff --git a/packages/beacon-node/src/db/repositories/blobsSidecarArchive.ts b/packages/beacon-node/src/db/repositories/blobsSidecarArchive.ts deleted file mode 100644 index 64c86dc9ef58..000000000000 --- a/packages/beacon-node/src/db/repositories/blobsSidecarArchive.ts +++ /dev/null @@ -1,33 +0,0 @@ -import {ChainForkConfig} from "@lodestar/config"; -import {Db, Repository, KeyValue, FilterOptions, Bucket} from "@lodestar/db"; -import {Slot, Root, ssz, deneb} from "@lodestar/types"; -import {bytesToInt} from "@lodestar/utils"; - -export interface BlockFilterOptions extends FilterOptions { - step?: number; -} - -export type BlockArchiveBatchPutBinaryItem = KeyValue & { - slot: Slot; - blockRoot: Root; - parentRoot: Root; -}; - -/** - * Stores finalized blocks. Block slot is identifier. - */ -export class BlobsSidecarArchiveRepository extends Repository { - constructor(config: ChainForkConfig, db: Db) { - super(config, db, Bucket.allForks_blobsSidecarArchive, ssz.deneb.BlobsSidecar); - } - - // Handle key as slot - - getId(value: deneb.BlobsSidecar): Slot { - return value.beaconBlockSlot; - } - - decodeKey(data: Uint8Array): number { - return bytesToInt((super.decodeKey(data) as unknown) as Uint8Array, "be"); - } -} diff --git a/packages/beacon-node/src/db/repositories/index.ts b/packages/beacon-node/src/db/repositories/index.ts index 0cd99e2fadf9..bb0bbba07e99 100644 --- a/packages/beacon-node/src/db/repositories/index.ts +++ b/packages/beacon-node/src/db/repositories/index.ts @@ -1,5 +1,5 @@ -export {BlobsSidecarRepository} from "./blobsSidecar.js"; -export {BlobsSidecarArchiveRepository} from "./blobsSidecarArchive.js"; +export {BlobSidecarsRepository} from "./blobSidecars.js"; +export {BlobSidecarsArchiveRepository} from "./blobSidecarsArchive.js"; export {BlockRepository} from "./block.js"; export {BlockArchiveBatchPutBinaryItem, BlockArchiveRepository, BlockFilterOptions} from "./blockArchive.js"; export {StateArchiveRepository} from "./stateArchive.js"; diff --git a/packages/beacon-node/src/execution/builder/http.ts b/packages/beacon-node/src/execution/builder/http.ts index 5b088742c74d..b56ecca64087 100644 --- a/packages/beacon-node/src/execution/builder/http.ts +++ b/packages/beacon-node/src/execution/builder/http.ts @@ -5,7 +5,6 @@ import {byteArrayEquals, toHexString} from "@chainsafe/ssz"; import {SLOTS_PER_EPOCH} from "@lodestar/params"; import {ApiError} from "@lodestar/api"; -import {validateBlobsAndKzgCommitments} from "../../chain/produceBlock/validateBlobsAndKzgCommitments.js"; import {Metrics} from "../../metrics/metrics.js"; import {IExecutionBuilder} from "./interface.js"; @@ -114,48 +113,4 @@ export class ExecutionBuilderHttp implements IExecutionBuilder { }; return fullySignedBlock; } - - async submitBlindedBlockV2( - signedBlock: allForks.SignedBlindedBeaconBlock - ): Promise { - const res = await this.api.submitBlindedBlockV2(signedBlock); - ApiError.assert(res, "execution.builder.submitBlindedBlockV2"); - const signedBeaconBlockAndBlobsSidecar = res.response.data; - // Since we get the full block back, we can just just compare the hash of blinded to returned - const {beaconBlock, blobsSidecar} = signedBeaconBlockAndBlobsSidecar; - - // Verify if the transactions and withdrawals match with their corresponding roots - // since we get the full signed block back, its easy to validate response consistency - // if the signed blinded and signed full root simply match - const signedBlockRoot = this.config - .getBlindedForkTypes(signedBlock.message.slot) - .SignedBeaconBlock.hashTreeRoot(signedBlock); - const beaconBlockRoot = this.config - .getForkTypes(beaconBlock.message.slot) - .SignedBeaconBlock.hashTreeRoot(beaconBlock); - if (!byteArrayEquals(signedBlockRoot, beaconBlockRoot)) { - throw Error( - `Invalid SignedBeaconBlock of the builder submitBlindedBlockV2 response, expected=${toHexString( - signedBlockRoot - )}, actual=${toHexString(beaconBlockRoot)}` - ); - } - - // Sanity check consistency between payload and blobs bundle still needs to be done - const payload = beaconBlock.message.body.executionPayload; - const blockHash = toHexString(payload.blockHash); - const blobsBlockHash = toHexString(blobsSidecar.beaconBlockRoot); - if (blockHash !== blobsBlockHash) { - throw Error(`blobsSidecar incorrect blockHash expected=${blockHash}, actual=${blobsBlockHash}`); - } - // Sanity-check that the KZG commitments match the versioned hashes in the transactions - const {blobKzgCommitments: kzgs} = beaconBlock.message.body as deneb.BeaconBlockBody; - if (kzgs === undefined) { - throw Error("Missing blobKzgCommitments on beaconBlock's body"); - } - const {blobs} = blobsSidecar; - validateBlobsAndKzgCommitments(payload, {blockHash, kzgs, blobs}); - - return signedBeaconBlockAndBlobsSidecar; - } } diff --git a/packages/beacon-node/src/execution/builder/interface.ts b/packages/beacon-node/src/execution/builder/interface.ts index a52c029a46a6..05cc426b4b3f 100644 --- a/packages/beacon-node/src/execution/builder/interface.ts +++ b/packages/beacon-node/src/execution/builder/interface.ts @@ -26,7 +26,4 @@ export interface IExecutionBuilder { blobKzgCommitments?: deneb.BlobKzgCommitments; }>; submitBlindedBlock(signedBlock: allForks.SignedBlindedBeaconBlock): Promise; - submitBlindedBlockV2( - signedBlock: allForks.SignedBlindedBeaconBlock - ): Promise; } diff --git a/packages/beacon-node/src/metrics/validatorMonitor.ts b/packages/beacon-node/src/metrics/validatorMonitor.ts index 5f425876851a..a8dbf1927057 100644 --- a/packages/beacon-node/src/metrics/validatorMonitor.ts +++ b/packages/beacon-node/src/metrics/validatorMonitor.ts @@ -1,6 +1,6 @@ import {computeEpochAtSlot, AttesterStatus, parseAttesterFlags} from "@lodestar/state-transition"; import {Logger} from "@lodestar/utils"; -import {allForks, altair} from "@lodestar/types"; +import {allForks, altair, deneb} from "@lodestar/types"; import {ChainForkConfig} from "@lodestar/config"; import {MIN_ATTESTATION_INCLUSION_DELAY, SLOTS_PER_EPOCH} from "@lodestar/params"; import {Epoch, Slot, ValidatorIndex} from "@lodestar/types"; @@ -23,6 +23,7 @@ export type ValidatorMonitor = { registerLocalValidatorInSyncCommittee(index: number, untilEpoch: Epoch): void; registerValidatorStatuses(currentEpoch: Epoch, statuses: AttesterStatus[], balances?: number[]): void; registerBeaconBlock(src: OpSource, seenTimestampSec: Seconds, block: allForks.BeaconBlock): void; + registerBlobSideCar(src: OpSource, seenTimestampSec: Seconds, blob: deneb.BlobSidecar): void; registerImportedBlock(block: allForks.BeaconBlock, data: {proposerBalanceDelta: number}): void; submitUnaggregatedAttestation( seenTimestampSec: number, @@ -299,6 +300,10 @@ export function createValidatorMonitor( } }, + registerBlobSideCar(_src, _seenTimestampSec, _blob) { + //TODO: freetheblobs + }, + registerImportedBlock(block, {proposerBalanceDelta}) { if (validators.has(block.proposerIndex)) { metrics.validatorMonitor.proposerBalanceDeltaKnown.observe(proposerBalanceDelta); diff --git a/packages/beacon-node/src/network/gossip/gossipsub.ts b/packages/beacon-node/src/network/gossip/gossipsub.ts index bd66d4952c07..8a2a4f056cfc 100644 --- a/packages/beacon-node/src/network/gossip/gossipsub.ts +++ b/packages/beacon-node/src/network/gossip/gossipsub.ts @@ -207,13 +207,10 @@ export class Eth2Gossipsub extends GossipSub implements GossipBeaconNode { }); } - async publishSignedBeaconBlockAndBlobsSidecar(item: deneb.SignedBeaconBlockAndBlobsSidecar): Promise { - const fork = this.config.getForkName(item.beaconBlock.message.slot); - await this.publishObject( - {type: GossipType.beacon_block_and_blobs_sidecar, fork}, - item, - {ignoreDuplicatePublishError: true} - ); + async publishBlobSidecar(item: deneb.SignedBlobSidecar): Promise { + const fork = this.config.getForkName(item.message.slot); + const {index} = item.message; + await this.publishObject({type: GossipType.blob_sidecar, index, fork}, item); } async publishBeaconAggregateAndProof(aggregateAndProof: phase0.SignedAggregateAndProof): Promise { diff --git a/packages/beacon-node/src/network/gossip/handlers/index.ts b/packages/beacon-node/src/network/gossip/handlers/index.ts index 8f04057b5c9f..fc54cc941f38 100644 --- a/packages/beacon-node/src/network/gossip/handlers/index.ts +++ b/packages/beacon-node/src/network/gossip/handlers/index.ts @@ -2,7 +2,7 @@ import {peerIdFromString} from "@libp2p/peer-id"; import {toHexString} from "@chainsafe/ssz"; import {BeaconConfig} from "@lodestar/config"; import {Logger, prettyBytes} from "@lodestar/utils"; -import {phase0, Root, Slot, ssz} from "@lodestar/types"; +import {phase0, Root, Slot, ssz, allForks, deneb} from "@lodestar/types"; import {ForkName, ForkSeq} from "@lodestar/params"; import {Metrics} from "../../../metrics/index.js"; import {OpSource} from "../../../metrics/validatorMonitor.js"; @@ -33,8 +33,8 @@ import {NetworkEvent, NetworkEventBus} from "../../events.js"; import {PeerAction, PeerRpcScoreStore} from "../../peers/index.js"; import {validateLightClientFinalityUpdate} from "../../../chain/validation/lightClientFinalityUpdate.js"; import {validateLightClientOptimisticUpdate} from "../../../chain/validation/lightClientOptimisticUpdate.js"; -import {validateGossipBlobsSidecar} from "../../../chain/validation/blobsSidecar.js"; -import {BlockInput, getBlockInput} from "../../../chain/blocks/types.js"; +import {validateGossipBlobSidecar} from "../../../chain/validation/blobSidecar.js"; +import {BlockInput, getBlockInput, GossipedInputType} from "../../../chain/blocks/types.js"; import {AttnetsService} from "../../subnets/attnetsService.js"; /** @@ -82,17 +82,28 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH const {attnetsService, chain, config, metrics, networkEventBus, peerRpcScores, logger} = modules; async function validateBeaconBlock( - blockInput: BlockInput, + signedBlock: allForks.SignedBeaconBlock, fork: ForkName, peerIdStr: string, seenTimestampSec: number - ): Promise { - const signedBlock = blockInput.block; + ): Promise { const slot = signedBlock.message.slot; const forkTypes = config.getForkTypes(slot); const blockHex = prettyBytes(forkTypes.BeaconBlock.hashTreeRoot(signedBlock.message)); const delaySec = chain.clock.secFromSlot(slot, seenTimestampSec); const recvToVal = Date.now() / 1000 - seenTimestampSec; + + let blockInput; + let blockInputMeta; + if (config.getForkSeq(signedBlock.message.slot) >= ForkSeq.deneb) { + const blockInputRes = getBlockInput.getFullBlockInput(config, {type: GossipedInputType.block, signedBlock}); + blockInput = blockInputRes.blockInput; + blockInputMeta = blockInputRes.blockInputMeta; + } else { + blockInput = getBlockInput.preDeneb(config, signedBlock); + blockInputMeta = {}; + } + metrics?.gossipBlock.receivedToGossipValidate.observe(recvToVal); logger.verbose("Received gossip block", { slot: slot, @@ -101,13 +112,16 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH peerId: peerIdStr, delaySec, recvToVal, + ...blockInputMeta, }); try { await validateGossipBlock(config, chain, signedBlock, fork); + return blockInput; } catch (e) { if (e instanceof BlockGossipError) { - if (e instanceof BlockGossipError && e.type.code === BlockErrorCode.PARENT_UNKNOWN) { + // Don't trigger this yet if full block and blobs haven't arrived yet + if (e instanceof BlockGossipError && e.type.code === BlockErrorCode.PARENT_UNKNOWN && blockInput !== null) { logger.debug("Gossip block has error", {slot, root: blockHex, code: e.type.code}); networkEventBus.emit(NetworkEvent.unknownBlockParent, blockInput, peerIdStr); } @@ -121,6 +135,55 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH } } + async function validateBeaconBlob( + signedBlob: deneb.SignedBlobSidecar, + gossipIndex: number, + peerIdStr: string, + seenTimestampSec: number + ): Promise { + const slot = signedBlob.message.slot; + const blockHex = prettyBytes(signedBlob.message.blockRoot); + const delaySec = chain.clock.secFromSlot(slot, seenTimestampSec); + const recvToVal = Date.now() / 1000 - seenTimestampSec; + + const {blockInput, blockInputMeta} = getBlockInput.getFullBlockInput(config, { + type: GossipedInputType.blob, + signedBlob, + }); + + // TODO: freetheblobs + // metrics?.gossipBlock.receivedToGossipValidate.observe(recvToVal); + logger.verbose("Received gossip blob", { + slot: slot, + root: blockHex, + curentSlot: chain.clock.currentSlot, + peerId: peerIdStr, + delaySec, + recvToVal, + gossipIndex, + ...blockInputMeta, + }); + + try { + await validateGossipBlobSidecar(config, chain, signedBlob, gossipIndex); + return blockInput; + } catch (e) { + if (e instanceof BlockGossipError) { + // Don't trigger this yet if full block and blobs haven't arrived yet + if (e instanceof BlockGossipError && e.type.code === BlockErrorCode.PARENT_UNKNOWN && blockInput !== null) { + logger.debug("Gossip block has error", {slot, root: blockHex, code: e.type.code}); + events.emit(NetworkEvent.unknownBlockParent, blockInput, peerIdStr); + } + } + + if (e instanceof BlockGossipError && e.action === GossipAction.REJECT) { + chain.persistInvalidSszValue(ssz.deneb.SignedBlobSidecar, signedBlob, `gossip_reject_slot_${slot}`); + } + + throw e; + } + } + function handleValidBeaconBlock(blockInput: BlockInput, peerIdStr: string, seenTimestampSec: number): void { const signedBlock = blockInput.block; @@ -132,8 +195,8 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH .processBlock(blockInput, { // proposer signature already checked in validateBeaconBlock() validProposerSignature: true, - // blobsSidecar already checked in validateGossipBlobsSidecar() - validBlobsSidecar: true, + // blobsSidecar needs to be validated for block + validBlobSidecars: true, // It's critical to keep a good number of mesh peers. // To do that, the Gossip Job Wait Time should be consistently <3s to avoid the behavior penalties in gossip // Gossip Job Wait Time depends on the BLS Job Wait Time @@ -167,28 +230,21 @@ export function getGossipHandlers(modules: ValidatorFnsModules, options: GossipH return { [GossipType.beacon_block]: async (signedBlock, topic, peerIdStr, seenTimestampSec) => { - // TODO Deneb: Can blocks be received by this topic? - if (config.getForkSeq(signedBlock.message.slot) >= ForkSeq.deneb) { - throw new GossipActionError(GossipAction.REJECT, {code: "POST_DENEB_BLOCK"}); + const blockInput = await validateBeaconBlock(signedBlock, topic.fork, peerIdStr, seenTimestampSec); + if (blockInput !== null) { + handleValidBeaconBlock(blockInput, peerIdStr, seenTimestampSec); } - - const blockInput = getBlockInput.preDeneb(config, signedBlock); - await validateBeaconBlock(blockInput, topic.fork, peerIdStr, seenTimestampSec); - handleValidBeaconBlock(blockInput, peerIdStr, seenTimestampSec); }, - [GossipType.beacon_block_and_blobs_sidecar]: async (blockAndBlocks, topic, peerIdStr, seenTimestampSec) => { - const {beaconBlock, blobsSidecar} = blockAndBlocks; - // TODO Deneb: Should throw for pre fork blocks? - if (config.getForkSeq(beaconBlock.message.slot) < ForkSeq.deneb) { + [GossipType.blob_sidecar]: async (signedBlob, topic, peerIdStr, seenTimestampSec) => { + if (config.getForkSeq(signedBlob.message.slot) < ForkSeq.deneb) { throw new GossipActionError(GossipAction.REJECT, {code: "PRE_DENEB_BLOCK"}); } - // Validate block + blob. Then forward, then handle both - const blockInput = getBlockInput.postDeneb(config, beaconBlock, blobsSidecar); - await validateBeaconBlock(blockInput, topic.fork, peerIdStr, seenTimestampSec); - validateGossipBlobsSidecar(beaconBlock, blobsSidecar); - handleValidBeaconBlock(blockInput, peerIdStr, seenTimestampSec); + const blockInput = await validateBeaconBlob(signedBlob, topic.index, peerIdStr, seenTimestampSec); + if (blockInput !== null) { + handleValidBeaconBlock(blockInput, peerIdStr, seenTimestampSec); + } }, [GossipType.beacon_aggregate_and_proof]: async (signedAggregateAndProof, _topic, _peer, seenTimestampSec) => { diff --git a/packages/beacon-node/src/network/gossip/interface.ts b/packages/beacon-node/src/network/gossip/interface.ts index 4fbc550f457e..419ae8886833 100644 --- a/packages/beacon-node/src/network/gossip/interface.ts +++ b/packages/beacon-node/src/network/gossip/interface.ts @@ -13,7 +13,7 @@ import {JobItemQueue} from "../../util/queue/index.js"; export enum GossipType { beacon_block = "beacon_block", - beacon_block_and_blobs_sidecar = "beacon_block_and_blobs_sidecar", + blob_sidecar = "blob_sidecar", beacon_aggregate_and_proof = "beacon_aggregate_and_proof", beacon_attestation = "beacon_attestation", voluntary_exit = "voluntary_exit", @@ -41,7 +41,7 @@ export interface IGossipTopic { export type GossipTopicTypeMap = { [GossipType.beacon_block]: {type: GossipType.beacon_block}; - [GossipType.beacon_block_and_blobs_sidecar]: {type: GossipType.beacon_block_and_blobs_sidecar}; + [GossipType.blob_sidecar]: {type: GossipType.blob_sidecar; index: number}; [GossipType.beacon_aggregate_and_proof]: {type: GossipType.beacon_aggregate_and_proof}; [GossipType.beacon_attestation]: {type: GossipType.beacon_attestation; subnet: number}; [GossipType.voluntary_exit]: {type: GossipType.voluntary_exit}; @@ -67,7 +67,7 @@ export type GossipTopic = GossipTopicMap[keyof GossipTopicMap]; export type GossipTypeMap = { [GossipType.beacon_block]: allForks.SignedBeaconBlock; - [GossipType.beacon_block_and_blobs_sidecar]: deneb.SignedBeaconBlockAndBlobsSidecar; + [GossipType.blob_sidecar]: deneb.SignedBlobSidecar; [GossipType.beacon_aggregate_and_proof]: phase0.SignedAggregateAndProof; [GossipType.beacon_attestation]: phase0.Attestation; [GossipType.voluntary_exit]: phase0.SignedVoluntaryExit; @@ -82,9 +82,7 @@ export type GossipTypeMap = { export type GossipFnByType = { [GossipType.beacon_block]: (signedBlock: allForks.SignedBeaconBlock) => Promise | void; - [GossipType.beacon_block_and_blobs_sidecar]: ( - signedBeaconBlockAndBlobsSidecar: deneb.SignedBeaconBlockAndBlobsSidecar - ) => Promise | void; + [GossipType.blob_sidecar]: (signedBlobSidecar: deneb.SignedBlobSidecar) => Promise | void; [GossipType.beacon_aggregate_and_proof]: (aggregateAndProof: phase0.SignedAggregateAndProof) => Promise | void; [GossipType.beacon_attestation]: (attestation: phase0.Attestation) => Promise | void; [GossipType.voluntary_exit]: (voluntaryExit: phase0.SignedVoluntaryExit) => Promise | void; @@ -124,7 +122,7 @@ export type GossipModules = { export type GossipBeaconNode = { publishBeaconBlock(signedBlock: allForks.SignedBeaconBlock): Promise; - publishSignedBeaconBlockAndBlobsSidecar(item: deneb.SignedBeaconBlockAndBlobsSidecar): Promise; + publishBlobSidecar(item: deneb.SignedBlobSidecar): Promise; publishBeaconAggregateAndProof(aggregateAndProof: phase0.SignedAggregateAndProof): Promise; publishBeaconAttestation(attestation: phase0.Attestation, subnet: number): Promise; publishVoluntaryExit(voluntaryExit: phase0.SignedVoluntaryExit): Promise; diff --git a/packages/beacon-node/src/network/gossip/topic.ts b/packages/beacon-node/src/network/gossip/topic.ts index bc3491990415..9576c4f4be63 100644 --- a/packages/beacon-node/src/network/gossip/topic.ts +++ b/packages/beacon-node/src/network/gossip/topic.ts @@ -6,6 +6,7 @@ import { ForkSeq, SYNC_COMMITTEE_SUBNET_COUNT, isForkLightClient, + MAX_BLOBS_PER_BLOCK, } from "@lodestar/params"; import {GossipEncoding, GossipTopic, GossipType, GossipTopicTypeMap} from "./interface.js"; @@ -59,7 +60,6 @@ export function stringifyGossipTopic(forkDigestContext: ForkDigestContext, topic function stringifyGossipTopicType(topic: GossipTopic): string { switch (topic.type) { case GossipType.beacon_block: - case GossipType.beacon_block_and_blobs_sidecar: case GossipType.beacon_aggregate_and_proof: case GossipType.voluntary_exit: case GossipType.proposer_slashing: @@ -72,6 +72,8 @@ function stringifyGossipTopicType(topic: GossipTopic): string { case GossipType.beacon_attestation: case GossipType.sync_committee: return `${topic.type}_${topic.subnet}`; + case GossipType.blob_sidecar: + return `${topic.type}_${topic.index}`; } } @@ -81,8 +83,8 @@ export function getGossipSSZType(topic: GossipTopic) { case GossipType.beacon_block: // beacon_block is updated in altair to support the updated SignedBeaconBlock type return ssz[topic.fork].SignedBeaconBlock; - case GossipType.beacon_block_and_blobs_sidecar: - return ssz.deneb.SignedBeaconBlockAndBlobsSidecar; + case GossipType.blob_sidecar: + return ssz.deneb.SignedBlobSidecar; case GossipType.beacon_aggregate_and_proof: return ssz.phase0.SignedAggregateAndProof; case GossipType.beacon_attestation: @@ -136,7 +138,6 @@ export function parseGossipTopic(forkDigestContext: ForkDigestContext, topicStr: // Inline-d the parseGossipTopicType() function since spreading the resulting object x4 the time to parse a topicStr switch (gossipTypeStr) { case GossipType.beacon_block: - case GossipType.beacon_block_and_blobs_sidecar: case GossipType.beacon_aggregate_and_proof: case GossipType.voluntary_exit: case GossipType.proposer_slashing: @@ -157,6 +158,13 @@ export function parseGossipTopic(forkDigestContext: ForkDigestContext, topicStr: } } + if (gossipTypeStr.startsWith(GossipType.blob_sidecar)) { + const indexStr = gossipTypeStr.slice(GossipType.blob_sidecar.length + 1); // +1 for '_' concatenating the topic name and the index + const index = parseInt(indexStr, 10); + if (Number.isNaN(index)) throw Error(`index ${indexStr} is not a number`); + return {type: GossipType.blob_sidecar, index, fork, encoding}; + } + throw Error(`Unknown gossip type ${gossipTypeStr}`); } catch (e) { (e as Error).message = `Invalid gossip topic ${topicStr}: ${(e as Error).message}`; @@ -178,13 +186,14 @@ export function getCoreTopicsAtFork( {type: GossipType.voluntary_exit}, {type: GossipType.proposer_slashing}, {type: GossipType.attester_slashing}, + {type: GossipType.beacon_block}, ]; - // After Deneb only track beacon_block_and_blobs_sidecar topic - if (ForkSeq[fork] < ForkSeq.deneb) { - topics.push({type: GossipType.beacon_block}); - } else { - topics.push({type: GossipType.beacon_block_and_blobs_sidecar}); + // deneb + if (ForkSeq[fork] >= ForkSeq.deneb) { + for (let index = 0; index < MAX_BLOBS_PER_BLOCK; index++) { + topics.push({type: GossipType.blob_sidecar, index}); + } } // capella diff --git a/packages/beacon-node/src/network/gossip/validation/queue.ts b/packages/beacon-node/src/network/gossip/validation/queue.ts index fbc43dadfca0..db9af5b6af58 100644 --- a/packages/beacon-node/src/network/gossip/validation/queue.ts +++ b/packages/beacon-node/src/network/gossip/validation/queue.ts @@ -11,8 +11,8 @@ const gossipQueueOpts: { } = { // validation gossip block asap [GossipType.beacon_block]: {maxLength: 1024, type: QueueType.FIFO, noYieldIfOneItem: true}, - // TODO DENEB: What's a good queue max given that now blocks are much bigger? - [GossipType.beacon_block_and_blobs_sidecar]: {maxLength: 32, type: QueueType.FIFO, noYieldIfOneItem: true}, + // gossip length for blob is beacon block length * max blobs per block = 4096 + [GossipType.blob_sidecar]: {maxLength: 4096, type: QueueType.FIFO, noYieldIfOneItem: true}, // lighthoue has aggregate_queue 4096 and unknown_block_aggregate_queue 1024, we use single queue [GossipType.beacon_aggregate_and_proof]: {maxLength: 5120, type: QueueType.LIFO, maxConcurrency: 16}, // lighthouse has attestation_queue 16384 and unknown_block_attestation_queue 8192, we use single queue diff --git a/packages/beacon-node/src/network/interface.ts b/packages/beacon-node/src/network/interface.ts index 5ba8998ba748..f38189262bdc 100644 --- a/packages/beacon-node/src/network/interface.ts +++ b/packages/beacon-node/src/network/interface.ts @@ -35,7 +35,6 @@ export interface INetwork { getConnectedPeers(): PeerId[]; getConnectedPeerCount(): number; - publishBeaconBlockMaybeBlobs(signedBlock: BlockInput): Promise; beaconBlocksMaybeBlobsByRange(peerId: PeerId, request: phase0.BeaconBlocksByRangeRequest): Promise; beaconBlocksMaybeBlobsByRoot(peerId: PeerId, request: phase0.BeaconBlocksByRootRequest): Promise; diff --git a/packages/beacon-node/src/network/network.ts b/packages/beacon-node/src/network/network.ts index f7d8ce534c7c..43721d5564ae 100644 --- a/packages/beacon-node/src/network/network.ts +++ b/packages/beacon-node/src/network/network.ts @@ -3,15 +3,21 @@ import {PeerId} from "@libp2p/interface-peer-id"; import {Multiaddr} from "@multiformats/multiaddr"; import {BeaconConfig} from "@lodestar/config"; import {Logger, sleep} from "@lodestar/utils"; -import {ATTESTATION_SUBNET_COUNT, ForkName, ForkSeq, SYNC_COMMITTEE_SUBNET_COUNT} from "@lodestar/params"; +import { + ATTESTATION_SUBNET_COUNT, + ForkName, + ForkSeq, + SYNC_COMMITTEE_SUBNET_COUNT, + MAX_BLOBS_PER_BLOCK, +} from "@lodestar/params"; import {SignableENR} from "@chainsafe/discv5"; import {computeEpochAtSlot, computeTimeAtSlot} from "@lodestar/state-transition"; -import {deneb, Epoch, phase0, allForks, altair} from "@lodestar/types"; +import {Epoch, phase0, allForks, altair} from "@lodestar/types"; import {routes} from "@lodestar/api"; import {PeerScoreStatsDump} from "@chainsafe/libp2p-gossipsub/score"; import {Metrics} from "../metrics/index.js"; import {ChainEvent, IBeaconChain, BeaconClock} from "../chain/index.js"; -import {BlockInput, BlockInputType} from "../chain/blocks/types.js"; +import {BlockInput} from "../chain/blocks/types.js"; import {isValidBlsToExecutionChangeForBlockInclusion} from "../chain/opPools/utils.js"; import {formatNodePeer} from "../api/impl/node/utils.js"; import {NetworkOptions} from "./options.js"; @@ -333,19 +339,6 @@ export class Network implements INetwork { return this.peerManager.getConnectedPeerIds().length; } - publishBeaconBlockMaybeBlobs(blockInput: BlockInput): Promise { - switch (blockInput.type) { - case BlockInputType.preDeneb: - return this.gossip.publishBeaconBlock(blockInput.block); - - case BlockInputType.postDeneb: - return this.gossip.publishSignedBeaconBlockAndBlobsSidecar({ - beaconBlock: blockInput.block as deneb.SignedBeaconBlock, - blobsSidecar: blockInput.blobs, - }); - } - } - async beaconBlocksMaybeBlobsByRange( peerId: PeerId, request: phase0.BeaconBlocksByRangeRequest @@ -354,14 +347,7 @@ export class Network implements INetwork { } async beaconBlocksMaybeBlobsByRoot(peerId: PeerId, request: phase0.BeaconBlocksByRootRequest): Promise { - return beaconBlocksMaybeBlobsByRoot( - this.config, - this.reqResp, - peerId, - request, - this.clock.currentSlot, - this.chain.forkChoice.getFinalizedBlock().slot - ); + return beaconBlocksMaybeBlobsByRoot(this.config, this.reqResp, peerId, request); } /** @@ -563,18 +549,18 @@ export class Network implements INetwork { private coreTopicsAtFork(fork: ForkName): GossipTopicTypeMap[keyof GossipTopicTypeMap][] { // Common topics for all forks const topics: GossipTopicTypeMap[keyof GossipTopicTypeMap][] = [ - // {type: GossipType.beacon_block}, // Handled below + {type: GossipType.beacon_block}, {type: GossipType.beacon_aggregate_and_proof}, {type: GossipType.voluntary_exit}, {type: GossipType.proposer_slashing}, {type: GossipType.attester_slashing}, ]; - // After Deneb only track beacon_block_and_blobs_sidecar topic - if (ForkSeq[fork] < ForkSeq.deneb) { - topics.push({type: GossipType.beacon_block}); - } else { - topics.push({type: GossipType.beacon_block_and_blobs_sidecar}); + // After Deneb also track blob_sidecar_{index} + if (ForkSeq[fork] >= ForkSeq.deneb) { + for (let index = 0; index < MAX_BLOBS_PER_BLOCK; index++) { + topics.push({type: GossipType.blob_sidecar, index}); + } } // capella diff --git a/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts b/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts index 6ecd1308dfd3..028da94f4112 100644 --- a/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts +++ b/packages/beacon-node/src/network/reqresp/ReqRespBeaconNode.ts @@ -252,14 +252,11 @@ export class ReqRespBeaconNode extends ReqResp implements IReqRespBeaconNode { ); } - async blobsSidecarsByRange( - peerId: PeerId, - request: deneb.BlobsSidecarsByRangeRequest - ): Promise { + async blobSidecarsByRange(peerId: PeerId, request: deneb.BlobSidecarsByRangeRequest): Promise { return collectMaxResponse( - this.sendRequest( + this.sendRequest( peerId, - ReqRespMethod.BlobsSidecarsByRange, + ReqRespMethod.BlobSidecarsByRange, [Version.V1], request ), @@ -267,14 +264,11 @@ export class ReqRespBeaconNode extends ReqResp implements IReqRespBeaconNode { ); } - async beaconBlockAndBlobsSidecarByRoot( - peerId: PeerId, - request: deneb.BeaconBlockAndBlobsSidecarByRootRequest - ): Promise { + async blobSidecarsByRoot(peerId: PeerId, request: deneb.BlobSidecarsByRootRequest): Promise { return collectMaxResponse( - this.sendRequest( + this.sendRequest( peerId, - ReqRespMethod.BeaconBlockAndBlobsSidecarByRoot, + ReqRespMethod.BlobSidecarsByRoot, [Version.V1], request ), @@ -321,11 +315,8 @@ export class ReqRespBeaconNode extends ReqResp implements IReqRespBeaconNode { if (ForkSeq[fork] >= ForkSeq.deneb) { protocols.push( - reqRespProtocols.BeaconBlockAndBlobsSidecarByRoot( - modules, - this.reqRespHandlers.onBeaconBlockAndBlobsSidecarByRoot - ), - reqRespProtocols.BlobsSidecarsByRange(modules, this.reqRespHandlers.onBlobsSidecarsByRange) + reqRespProtocols.BlobSidecarsByRoot(modules, this.reqRespHandlers.onBlobSidecarsByRoot), + reqRespProtocols.BlobSidecarsByRange(modules, this.reqRespHandlers.onBlobSidecarsByRange) ); } diff --git a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts index a7989f6eef93..17bc7a20eda3 100644 --- a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts +++ b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRange.ts @@ -1,11 +1,10 @@ import {PeerId} from "@libp2p/interface-peer-id"; import {BeaconConfig} from "@lodestar/config"; -import {deneb, Epoch, phase0} from "@lodestar/types"; +import {deneb, Epoch, phase0, allForks, Slot} from "@lodestar/types"; import {ForkSeq} from "@lodestar/params"; import {computeEpochAtSlot} from "@lodestar/state-transition"; import {BlockInput, getBlockInput} from "../../chain/blocks/types.js"; -import {getEmptyBlobsSidecar} from "../../util/blobs.js"; import {IReqRespBeaconNode} from "./interface.js"; export async function beaconBlocksMaybeBlobsByRange( @@ -38,63 +37,78 @@ export async function beaconBlocksMaybeBlobsByRange( } // Only request blobs if they are recent enough - else if (computeEpochAtSlot(startSlot) >= currentEpoch - config.MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS) { - const [blocks, blobsSidecars] = await Promise.all([ + else if (computeEpochAtSlot(startSlot) >= currentEpoch - config.MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS) { + const [allBlocks, allBlobSidecars] = await Promise.all([ reqResp.beaconBlocksByRange(peerId, request), - reqResp.blobsSidecarsByRange(peerId, request), + reqResp.blobSidecarsByRange(peerId, request), ]); - const blockInputs: BlockInput[] = []; - let blobSideCarIndex = 0; - let lastMatchedSlot = -1; + return matchBlockWithBlobs(config, allBlocks, allBlobSidecars, endSlot); + } + + // Post Deneb but old blobs + else { + throw Error("Cannot sync blobs outside of blobs prune window"); + } +} + +// Assumes that the blobs are in the same sequence as blocks, doesn't require block to be sorted +export function matchBlockWithBlobs( + config: BeaconConfig, + allBlocks: allForks.SignedBeaconBlock[], + allBlobSidecars: deneb.BlobSidecar[], + endSlot: Slot +): BlockInput[] { + const blockInputs: BlockInput[] = []; + let blobSideCarIndex = 0; + let lastMatchedSlot = -1; - // Match blobSideCar with the block as some blocks would have no blobs and hence - // would be omitted from the response. If there are any inconsitencies in the - // response, the validations during import will reject the block and hence this - // entire segment. - // - // Assuming that the blocks and blobs will come in same sorted order - for (let i = 0; i < blocks.length; i++) { - const block = blocks[i]; - let blobsSidecar: deneb.BlobsSidecar; + // Match blobSideCar with the block as some blocks would have no blobs and hence + // would be omitted from the response. If there are any inconsitencies in the + // response, the validations during import will reject the block and hence this + // entire segment. + // + // Assuming that the blocks and blobs will come in same sorted order + for (let i = 0; i < allBlocks.length; i++) { + const block = allBlocks[i]; + if (config.getForkSeq(block.message.slot) < ForkSeq.deneb) { + blockInputs.push(getBlockInput.preDeneb(config, block)); + } else { + const blobSidecars: deneb.BlobSidecar[] = []; - if (blobsSidecars[blobSideCarIndex]?.beaconBlockSlot === block.message.slot) { - blobsSidecar = blobsSidecars[blobSideCarIndex]; + let blobSidecar: deneb.BlobSidecar; + while ((blobSidecar = allBlobSidecars[blobSideCarIndex])?.slot === block.message.slot) { + blobSidecars.push(blobSidecar); lastMatchedSlot = block.message.slot; blobSideCarIndex++; - } else { - // Quick inspect if the blobsSidecar was expected - const blobKzgCommitmentsLen = (block.message.body as deneb.BeaconBlockBody).blobKzgCommitments.length; - if (blobKzgCommitmentsLen !== 0) { - throw Error( - `Missing blobsSidecar for blockSlot=${block.message.slot} with blobKzgCommitmentsLen=${blobKzgCommitmentsLen}` - ); - } - blobsSidecar = getEmptyBlobsSidecar(config, block as deneb.SignedBeaconBlock); } - blockInputs.push(getBlockInput.postDeneb(config, block, blobsSidecar)); - } - // If there are still unconsumed blobs this means that the response was inconsistent - // and matching was wrong and hence we should throw error - if ( - blobsSidecars[blobSideCarIndex] !== undefined && - // If there are no blobs, the blobs request can give 1 block outside the requested range - blobsSidecars[blobSideCarIndex].beaconBlockSlot <= endSlot - ) { - throw Error( - `Unmatched blobsSidecars, blocks=${blocks.length}, blobs=${ - blobsSidecars.length - } lastMatchedSlot=${lastMatchedSlot}, pending blobsSidecars slots=${blobsSidecars - .slice(blobSideCarIndex) - .map((blb) => blb.beaconBlockSlot)}` - ); + // Quick inspect how many blobSidecars was expected + const blobKzgCommitmentsLen = (block.message.body as deneb.BeaconBlockBody).blobKzgCommitments.length; + if (blobKzgCommitmentsLen !== blobSidecars.length) { + throw Error( + `Missing blobSidecars for blockSlot=${block.message.slot} with blobKzgCommitmentsLen=${blobKzgCommitmentsLen} blobSidecars=${blobSidecars.length}` + ); + } + + blockInputs.push(getBlockInput.postDeneb(config, block, blobSidecars)); } - return blockInputs; } - // Post Deneb but old blobs - else { - throw Error("Cannot sync blobs outside of blobs prune window"); + // If there are still unconsumed blobs this means that the response was inconsistent + // and matching was wrong and hence we should throw error + if ( + allBlobSidecars[blobSideCarIndex] !== undefined && + // If there are no blobs, the blobs request can give 1 block outside the requested range + allBlobSidecars[blobSideCarIndex].slot <= endSlot + ) { + throw Error( + `Unmatched blobSidecars, blocks=${allBlocks.length}, blobs=${ + allBlobSidecars.length + } lastMatchedSlot=${lastMatchedSlot}, pending blobsSidecars slots=${allBlobSidecars + .slice(blobSideCarIndex) + .map((blb) => blb.slot)}` + ); } + return blockInputs; } diff --git a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts index e648dc7b0ae3..f4f2fad67ef2 100644 --- a/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts +++ b/packages/beacon-node/src/network/reqresp/beaconBlocksMaybeBlobsByRoot.ts @@ -1,83 +1,35 @@ import {PeerId} from "@libp2p/interface-peer-id"; import {BeaconConfig} from "@lodestar/config"; -import {RequestError, RequestErrorCode} from "@lodestar/reqresp"; -import {Epoch, phase0, Root, Slot} from "@lodestar/types"; -import {toHex} from "@lodestar/utils"; -import {ForkSeq} from "@lodestar/params"; -import {BlockInput, getBlockInput} from "../../chain/blocks/types.js"; -import {wrapError} from "../../util/wrapError.js"; +import {phase0, deneb} from "@lodestar/types"; +import {BlockInput} from "../../chain/blocks/types.js"; import {IReqRespBeaconNode} from "./interface.js"; +import {matchBlockWithBlobs} from "./beaconBlocksMaybeBlobsByRange.js"; export async function beaconBlocksMaybeBlobsByRoot( config: BeaconConfig, reqResp: IReqRespBeaconNode, peerId: PeerId, - request: phase0.BeaconBlocksByRootRequest, - currentSlot: Epoch, - finalizedSlot: Slot + request: phase0.BeaconBlocksByRootRequest ): Promise { - // Assume all requests are post Deneb - if (config.getForkSeq(finalizedSlot) >= ForkSeq.deneb) { - const blocksAndBlobs = await reqResp.beaconBlockAndBlobsSidecarByRoot(peerId, request); - return blocksAndBlobs.map(({beaconBlock, blobsSidecar}) => - getBlockInput.postDeneb(config, beaconBlock, blobsSidecar) - ); - } - - // Assume all request are pre EIP-4844 - else if (config.getForkSeq(currentSlot) < ForkSeq.deneb) { - const blocks = await reqResp.beaconBlocksByRoot(peerId, request); - return blocks.map((block) => getBlockInput.preDeneb(config, block)); - } - - // We don't know if a requested root is after the deneb fork or not. - // Thus some sort of retry is necessary while deneb is not finalized - else { - return Promise.all( - request.map(async (beaconBlockRoot) => - beaconBlockAndBlobsSidecarByRootFallback(config, reqResp, peerId, beaconBlockRoot) - ) - ); - } -} - -async function beaconBlockAndBlobsSidecarByRootFallback( - config: BeaconConfig, - reqResp: IReqRespBeaconNode, - peerId: PeerId, - beaconBlockRoot: Root -): Promise { - const resBlockBlobs = await wrapError(reqResp.beaconBlockAndBlobsSidecarByRoot(peerId, [beaconBlockRoot])); - - if (resBlockBlobs.err) { - // From the spec, if the block is from before the fork, errors with 3: ResourceUnavailable - // > Clients MUST support requesting blocks and sidecars since minimum_request_epoch, where - // minimum_request_epoch = max(finalized_epoch, current_epoch - MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS, DENEB_FORK_EPOCH). - // If any root in the request content references a block earlier than minimum_request_epoch, - // peers SHOULD respond with error code 3: ResourceUnavailable. - // Ref: https://github.com/ethereum/consensus-specs/blob/aede132f4999ed54b98d35e27aca9451042a1ee9/specs/eip4844/p2p-interface.md#beaconblockandblobssidecarbyroot-v1 - if ( - resBlockBlobs.err instanceof RequestError && - resBlockBlobs.err.type.code === RequestErrorCode.RESOURCE_UNAVAILABLE - ) { - // retry with blocks - } else { - // Unexpected error, throw - throw resBlockBlobs.err; + const allBlocks = await reqResp.beaconBlocksByRoot(peerId, request); + const blobIdentifiers: deneb.BlobIdentifier[] = []; + + for (const block of allBlocks) { + const blockRoot = config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message); + const blobKzgCommitmentsLen = (block.message.body as deneb.BeaconBlockBody).blobKzgCommitments.length ?? 0; + for (let index = 0; index < blobKzgCommitmentsLen; index++) { + blobIdentifiers.push({blockRoot, index}); } - } else { - if (resBlockBlobs.result.length < 1) { - throw Error(`beaconBlockAndBlobsSidecarByRoot return empty for block root ${toHex(beaconBlockRoot)}`); - } - - const {beaconBlock, blobsSidecar} = resBlockBlobs.result[0]; - return getBlockInput.postDeneb(config, beaconBlock, blobsSidecar); } - const resBlocks = await reqResp.beaconBlocksByRoot(peerId, [beaconBlockRoot]); - if (resBlocks.length < 1) { - throw Error(`beaconBlocksByRoot return empty for block root ${toHex(beaconBlockRoot)}`); + let allBlobSidecars: deneb.BlobSidecar[]; + if (blobIdentifiers.length > 0) { + allBlobSidecars = await reqResp.blobSidecarsByRoot(peerId, blobIdentifiers); + } else { + allBlobSidecars = []; } - return getBlockInput.preDeneb(config, resBlocks[0]); + // The last arg is to provide slot to which all blobs should be exausted in matching + // and here it should be infinity since all bobs should match + return matchBlockWithBlobs(config, allBlocks, allBlobSidecars, Infinity); } diff --git a/packages/beacon-node/src/network/reqresp/handlers/beaconBlockAndBlobsSidecarByRoot.ts b/packages/beacon-node/src/network/reqresp/handlers/beaconBlockAndBlobsSidecarByRoot.ts deleted file mode 100644 index 13147a47fde4..000000000000 --- a/packages/beacon-node/src/network/reqresp/handlers/beaconBlockAndBlobsSidecarByRoot.ts +++ /dev/null @@ -1,77 +0,0 @@ -import {ContextBytesType, EncodedPayload, EncodedPayloadType} from "@lodestar/reqresp"; -import {deneb} from "@lodestar/types"; -import {toHex} from "@lodestar/utils"; -import {IBeaconChain} from "../../../chain/index.js"; -import {IBeaconDb} from "../../../db/index.js"; -import {getSlotFromBytes} from "../../../util/multifork.js"; - -export async function* onBeaconBlockAndBlobsSidecarByRoot( - requestBody: deneb.BeaconBlockAndBlobsSidecarByRootRequest, - chain: IBeaconChain, - db: IBeaconDb -): AsyncIterable> { - const finalizedSlot = chain.forkChoice.getFinalizedBlock().slot; - - for (const blockRoot of requestBody) { - const blockRootHex = toHex(blockRoot); - const summary = chain.forkChoice.getBlockHex(blockRootHex); - - // NOTE: Only support non-finalized blocks. - // SPEC: Clients MUST support requesting blocks and sidecars since the latest finalized epoch. - // https://github.com/ethereum/consensus-specs/blob/11a037fd9227e29ee809c9397b09f8cc3383a8c0/specs/eip4844/p2p-interface.md#beaconblockandblobssidecarbyroot-v1 - if (!summary || summary.slot <= finalizedSlot) { - // TODO: Should accept the finalized block? Is the finalized block in the archive DB or hot DB? - continue; - } - - // finalized block has summary in forkchoice but it stays in blockArchive db - const blockBytes = await db.block.getBinary(blockRoot); - if (!blockBytes) { - throw Error(`Inconsistent state, block known to fork-choice not in db ${blockRootHex}`); - } - - const blobsSidecarBytes = await db.blobsSidecar.getBinary(blockRoot); - if (!blobsSidecarBytes) { - throw Error(`Inconsistent state, blobsSidecar known to fork-choice not in db ${blockRootHex}`); - } - - yield { - type: EncodedPayloadType.bytes, - bytes: signedBeaconBlockAndBlobsSidecarFromBytes(blockBytes, blobsSidecarBytes), - contextBytes: { - type: ContextBytesType.ForkDigest, - forkSlot: getSlotFromBytes(blockBytes), - }, - }; - } -} - -/** - * Construct a valid SSZ serialized container from its properties also serialized. - * ``` - * class SignedBeaconBlockAndBlobsSidecar(Container): - * beacon_block: SignedBeaconBlock - * blobs_sidecar: BlobsSidecar - * ``` - */ -export function signedBeaconBlockAndBlobsSidecarFromBytes( - blockBytes: Uint8Array, - blobsSidecarBytes: Uint8Array -): Uint8Array { - const totalLen = 4 + 4 + blockBytes.length + blobsSidecarBytes.length; - const arrayBuffer = new ArrayBuffer(totalLen); - const dataView = new DataView(arrayBuffer); - const uint8Array = new Uint8Array(arrayBuffer); - - const blockOffset = 8; - const blobsOffset = 8 + blockBytes.length; - - // Write offsets - dataView.setUint32(0, blockOffset, true); - dataView.setUint32(4, blobsOffset, true); - - uint8Array.set(blockBytes, blockOffset); - uint8Array.set(blobsSidecarBytes, blobsOffset); - - return uint8Array; -} diff --git a/packages/beacon-node/src/network/reqresp/handlers/beaconBlocksByRange.ts b/packages/beacon-node/src/network/reqresp/handlers/beaconBlocksByRange.ts index d7d6312c1e31..ac4745a4bfd4 100644 --- a/packages/beacon-node/src/network/reqresp/handlers/beaconBlocksByRange.ts +++ b/packages/beacon-node/src/network/reqresp/handlers/beaconBlocksByRange.ts @@ -12,14 +12,14 @@ export function onBeaconBlocksByRange( chain: IBeaconChain, db: IBeaconDb ): AsyncIterable { - return onBlocksOrBlobsSidecarsByRange(request, chain, { + return onBlocksOrBlobSidecarsByRange(request, chain, { finalized: db.blockArchive, unfinalized: db.block, }); } -export async function* onBlocksOrBlobsSidecarsByRange( - request: deneb.BlobsSidecarsByRangeRequest, +export async function* onBlocksOrBlobSidecarsByRange( + request: phase0.BeaconBlocksByRangeRequest, chain: IBeaconChain, db: { finalized: Pick; @@ -96,8 +96,8 @@ export async function* onBlocksOrBlobsSidecarsByRange( } export function validateBeaconBlocksByRangeRequest( - request: deneb.BlobsSidecarsByRangeRequest -): deneb.BlobsSidecarsByRangeRequest { + request: deneb.BlobSidecarsByRangeRequest +): deneb.BlobSidecarsByRangeRequest { const {startSlot} = request; let {count} = request; diff --git a/packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRange.ts b/packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRange.ts new file mode 100644 index 000000000000..fbf6bf980740 --- /dev/null +++ b/packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRange.ts @@ -0,0 +1,111 @@ +import {GENESIS_SLOT, MAX_REQUEST_BLOB_SIDECARS} from "@lodestar/params"; +import {ContextBytesType, EncodedPayloadBytes, EncodedPayloadType, ResponseError, RespStatus} from "@lodestar/reqresp"; +import {deneb, Slot} from "@lodestar/types"; +import {fromHex} from "@lodestar/utils"; +import {IBeaconChain} from "../../../chain/index.js"; +import {IBeaconDb} from "../../../db/index.js"; +import {BLOB_SIDECARS_IN_WRAPPER_INDEX, BLOBSIDECAR_FIXED_SIZE} from "../../../db/repositories/blobSidecars.js"; + +export async function* onBlobSidecarsByRange( + request: deneb.BlobSidecarsByRangeRequest, + chain: IBeaconChain, + db: IBeaconDb +): AsyncIterable { + // Non-finalized range of blobs + const {startSlot, count} = validateBlobSidecarsByRangeRequest(request); + const endSlot = startSlot + count; + const finalizedSlot = chain.forkChoice.getFinalizedBlock().slot; + + // Finalized range of blobs + + if (startSlot <= finalizedSlot) { + // Chain of blobs won't change + for await (const {key, value: blobSideCarsBytesWrapped} of db.blobSidecarsArchive.binaryEntriesStream({ + gte: startSlot, + lt: endSlot, + })) { + yield* iterateBlobBytesFromWrapper(blobSideCarsBytesWrapped, db.blobSidecarsArchive.decodeKey(key)); + } + } + + if (endSlot > finalizedSlot) { + const headRoot = chain.forkChoice.getHeadRoot(); + // TODO DENEB: forkChoice should mantain an array of canonical blocks, and change only on reorg + const headChain = chain.forkChoice.getAllAncestorBlocks(headRoot); + + // Iterate head chain with ascending block numbers + for (let i = headChain.length - 1; i >= 0; i--) { + const block = headChain[i]; + + // Must include only blobs in the range requested + if (block.slot >= startSlot && block.slot < endSlot) { + // Note: Here the forkChoice head may change due to a re-org, so the headChain reflects the cannonical chain + // at the time of the start of the request. Spec is clear the chain of blobs must be consistent, but on + // re-org there's no need to abort the request + // Spec: https://github.com/ethereum/consensus-specs/blob/a1e46d1ae47dd9d097725801575b46907c12a1f8/specs/eip4844/p2p-interface.md#blobssidecarsbyrange-v1 + + const blobSideCarsBytesWrapped = await db.blobSidecars.getBinary(fromHex(block.blockRoot)); + if (!blobSideCarsBytesWrapped) { + // Handle the same to onBeaconBlocksByRange + throw new ResponseError(RespStatus.SERVER_ERROR, `No item for root ${block.blockRoot} slot ${block.slot}`); + } + yield* iterateBlobBytesFromWrapper(blobSideCarsBytesWrapped, block.slot); + } + + // If block is after endSlot, stop iterating + else if (block.slot >= endSlot) { + break; + } + } + } +} + +export function* iterateBlobBytesFromWrapper( + blobSideCarsBytesWrapped: Uint8Array, + blockSlot: Slot +): Iterable { + const blobSideCarsBytes = blobSideCarsBytesWrapped.slice(BLOB_SIDECARS_IN_WRAPPER_INDEX); + const blobsLen = blobSideCarsBytes.length / BLOBSIDECAR_FIXED_SIZE; + + for (let index = 0; index < blobsLen; index++) { + const blobSideCarBytes = blobSideCarsBytes.slice( + index * BLOBSIDECAR_FIXED_SIZE, + (index + 1) * BLOBSIDECAR_FIXED_SIZE + ); + if (blobSideCarBytes.length !== BLOBSIDECAR_FIXED_SIZE) { + throw new ResponseError( + RespStatus.SERVER_ERROR, + `Invalid blobSidecar index=${index} bytes length=${blobSideCarBytes.length} expected=${BLOBSIDECAR_FIXED_SIZE} for slot ${blockSlot} blobsLen=${blobsLen}` + ); + } + yield { + type: EncodedPayloadType.bytes, + bytes: blobSideCarBytes, + contextBytes: { + type: ContextBytesType.ForkDigest, + forkSlot: blockSlot, + }, + }; + } +} + +export function validateBlobSidecarsByRangeRequest( + request: deneb.BlobSidecarsByRangeRequest +): deneb.BlobSidecarsByRangeRequest { + const {startSlot} = request; + let {count} = request; + + if (count < 1) { + throw new ResponseError(RespStatus.INVALID_REQUEST, "count < 1"); + } + // TODO: validate against MIN_EPOCHS_FOR_BLOCK_REQUESTS + if (startSlot < GENESIS_SLOT) { + throw new ResponseError(RespStatus.INVALID_REQUEST, "startSlot < genesis"); + } + + if (count > MAX_REQUEST_BLOB_SIDECARS) { + count = MAX_REQUEST_BLOB_SIDECARS; + } + + return {startSlot, count}; +} diff --git a/packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRoot.ts b/packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRoot.ts new file mode 100644 index 000000000000..1b1cbfce7e24 --- /dev/null +++ b/packages/beacon-node/src/network/reqresp/handlers/blobSidecarsByRoot.ts @@ -0,0 +1,63 @@ +import {ContextBytesType, EncodedPayload, EncodedPayloadType, ResponseError, RespStatus} from "@lodestar/reqresp"; +import {deneb, RootHex} from "@lodestar/types"; +import {toHex, fromHex} from "@lodestar/utils"; +import {IBeaconChain} from "../../../chain/index.js"; +import {IBeaconDb} from "../../../db/index.js"; +import {BLOB_SIDECARS_IN_WRAPPER_INDEX, BLOBSIDECAR_FIXED_SIZE} from "../../../db/repositories/blobSidecars.js"; + +export async function* onBlobSidecarsByRoot( + requestBody: deneb.BlobSidecarsByRootRequest, + chain: IBeaconChain, + db: IBeaconDb +): AsyncIterable> { + const finalizedSlot = chain.forkChoice.getFinalizedBlock().slot; + + // In sidecars by root request, it can be expected that sidecar requests will be come + // clustured by blockroots, and this helps us save db lookups once we load sidecars + // for a root + let lastFetchedSideCars: {blockRoot: RootHex; bytes: Uint8Array} | null = null; + + for (const blobIdentifier of requestBody) { + const {blockRoot, index} = blobIdentifier; + const blockRootHex = toHex(blockRoot); + const block = chain.forkChoice.getBlockHex(blockRootHex); + + // NOTE: Only support non-finalized blocks. + // SPEC: Clients MUST support requesting blocks and sidecars since the latest finalized epoch. + // https://github.com/ethereum/consensus-specs/blob/11a037fd9227e29ee809c9397b09f8cc3383a8c0/specs/eip4844/p2p-interface.md#beaconblockandblobssidecarbyroot-v1 + if (!block || block.slot <= finalizedSlot) { + continue; + } + + // Check if we need to load sidecars for a new block root + if (lastFetchedSideCars === null || lastFetchedSideCars.blockRoot !== blockRootHex) { + const blobSideCarsBytesWrapped = await db.blobSidecars.getBinary(fromHex(block.blockRoot)); + if (!blobSideCarsBytesWrapped) { + // Handle the same to onBeaconBlocksByRange + throw new ResponseError(RespStatus.SERVER_ERROR, `No item for root ${block.blockRoot} slot ${block.slot}`); + } + const blobSideCarsBytes = blobSideCarsBytesWrapped.slice(BLOB_SIDECARS_IN_WRAPPER_INDEX); + + lastFetchedSideCars = {blockRoot: blockRootHex, bytes: blobSideCarsBytes}; + } + + const blobSidecarBytes = lastFetchedSideCars.bytes.slice( + index * BLOBSIDECAR_FIXED_SIZE, + (index + 1) * BLOBSIDECAR_FIXED_SIZE + ); + if (blobSidecarBytes.length !== BLOBSIDECAR_FIXED_SIZE) { + throw Error( + `Inconsistent state, blobSidecar blockRoot=${blockRootHex} index=${index} blobSidecarBytes=${blobSidecarBytes.length} expected=${BLOBSIDECAR_FIXED_SIZE}` + ); + } + + yield { + type: EncodedPayloadType.bytes, + bytes: blobSidecarBytes, + contextBytes: { + type: ContextBytesType.ForkDigest, + forkSlot: block.slot, + }, + }; + } +} diff --git a/packages/beacon-node/src/network/reqresp/handlers/blobsSidecarsByRange.ts b/packages/beacon-node/src/network/reqresp/handlers/blobsSidecarsByRange.ts deleted file mode 100644 index dbf389ba2134..000000000000 --- a/packages/beacon-node/src/network/reqresp/handlers/blobsSidecarsByRange.ts +++ /dev/null @@ -1,18 +0,0 @@ -import {deneb} from "@lodestar/types"; -import {EncodedPayloadBytes} from "@lodestar/reqresp"; -import {IBeaconChain} from "../../../chain/index.js"; -import {IBeaconDb} from "../../../db/index.js"; -import {onBlocksOrBlobsSidecarsByRange} from "./beaconBlocksByRange.js"; - -// TODO DENEB: Unit test - -export function onBlobsSidecarsByRange( - request: deneb.BlobsSidecarsByRangeRequest, - chain: IBeaconChain, - db: IBeaconDb -): AsyncIterable { - return onBlocksOrBlobsSidecarsByRange(request, chain, { - finalized: db.blobsSidecarArchive, - unfinalized: db.blobsSidecar, - }); -} diff --git a/packages/beacon-node/src/network/reqresp/handlers/index.ts b/packages/beacon-node/src/network/reqresp/handlers/index.ts index af22ea6209da..3042379cff11 100644 --- a/packages/beacon-node/src/network/reqresp/handlers/index.ts +++ b/packages/beacon-node/src/network/reqresp/handlers/index.ts @@ -4,8 +4,8 @@ import {IBeaconChain} from "../../../chain/index.js"; import {IBeaconDb} from "../../../db/index.js"; import {onBeaconBlocksByRange} from "./beaconBlocksByRange.js"; import {onBeaconBlocksByRoot} from "./beaconBlocksByRoot.js"; -import {onBeaconBlockAndBlobsSidecarByRoot} from "./beaconBlockAndBlobsSidecarByRoot.js"; -import {onBlobsSidecarsByRange} from "./blobsSidecarsByRange.js"; +import {onBlobSidecarsByRoot} from "./blobSidecarsByRoot.js"; +import {onBlobSidecarsByRange} from "./blobSidecarsByRange.js"; import {onLightClientBootstrap} from "./lightClientBootstrap.js"; import {onLightClientFinalityUpdate} from "./lightClientFinalityUpdate.js"; import {onLightClientOptimisticUpdate} from "./lightClientOptimisticUpdate.js"; @@ -16,8 +16,8 @@ export interface ReqRespHandlers { onStatus: HandlerTypeFromMessage; onBeaconBlocksByRange: HandlerTypeFromMessage; onBeaconBlocksByRoot: HandlerTypeFromMessage; - onBeaconBlockAndBlobsSidecarByRoot: HandlerTypeFromMessage; - onBlobsSidecarsByRange: HandlerTypeFromMessage; + onBlobSidecarsByRoot: HandlerTypeFromMessage; + onBlobSidecarsByRange: HandlerTypeFromMessage; onLightClientBootstrap: HandlerTypeFromMessage; onLightClientUpdatesByRange: HandlerTypeFromMessage; onLightClientFinalityUpdate: HandlerTypeFromMessage; @@ -38,11 +38,11 @@ export function getReqRespHandlers({db, chain}: {db: IBeaconDb; chain: IBeaconCh async *onBeaconBlocksByRoot(req) { yield* onBeaconBlocksByRoot(req, chain, db); }, - async *onBeaconBlockAndBlobsSidecarByRoot(req) { - yield* onBeaconBlockAndBlobsSidecarByRoot(req, chain, db); + async *onBlobSidecarsByRoot(req) { + yield* onBlobSidecarsByRoot(req, chain, db); }, - async *onBlobsSidecarsByRange(req) { - yield* onBlobsSidecarsByRange(req, chain, db); + async *onBlobSidecarsByRange(req) { + yield* onBlobSidecarsByRange(req, chain, db); }, async *onLightClientBootstrap(req) { yield* onLightClientBootstrap(req, chain); diff --git a/packages/beacon-node/src/network/reqresp/interface.ts b/packages/beacon-node/src/network/reqresp/interface.ts index bcaf456d6dff..93875e06a23d 100644 --- a/packages/beacon-node/src/network/reqresp/interface.ts +++ b/packages/beacon-node/src/network/reqresp/interface.ts @@ -11,11 +11,8 @@ export interface IReqRespBeaconNode { request: phase0.BeaconBlocksByRangeRequest ): Promise; beaconBlocksByRoot(peerId: PeerId, request: phase0.BeaconBlocksByRootRequest): Promise; - blobsSidecarsByRange(peerId: PeerId, request: deneb.BlobsSidecarsByRangeRequest): Promise; - beaconBlockAndBlobsSidecarByRoot( - peerId: PeerId, - request: deneb.BeaconBlockAndBlobsSidecarByRootRequest - ): Promise; + blobSidecarsByRange(peerId: PeerId, request: deneb.BlobSidecarsByRangeRequest): Promise; + blobSidecarsByRoot(peerId: PeerId, request: deneb.BlobSidecarsByRootRequest): Promise; lightClientBootstrap(peerId: PeerId, request: Uint8Array): Promise; lightClientOptimisticUpdate(peerId: PeerId): Promise; lightClientFinalityUpdate(peerId: PeerId): Promise; diff --git a/packages/beacon-node/src/network/reqresp/types.ts b/packages/beacon-node/src/network/reqresp/types.ts index 66cedd61abc1..84f425dffcc8 100644 --- a/packages/beacon-node/src/network/reqresp/types.ts +++ b/packages/beacon-node/src/network/reqresp/types.ts @@ -9,8 +9,8 @@ export enum ReqRespMethod { Metadata = "metadata", BeaconBlocksByRange = "beacon_blocks_by_range", BeaconBlocksByRoot = "beacon_blocks_by_root", - BlobsSidecarsByRange = "blobs_sidecars_by_range", - BeaconBlockAndBlobsSidecarByRoot = "beacon_block_and_blobs_sidecar_by_root", + BlobSidecarsByRange = "blob_sidecars_by_range", + BlobSidecarsByRoot = "blob_sidecars_by_root", LightClientBootstrap = "light_client_bootstrap", LightClientUpdatesByRange = "light_client_updates_by_range", LightClientFinalityUpdate = "light_client_finality_update", @@ -26,8 +26,8 @@ type RequestBodyByMethod = { // Do not matter [ReqRespMethod.BeaconBlocksByRange]: unknown; [ReqRespMethod.BeaconBlocksByRoot]: unknown; - [ReqRespMethod.BlobsSidecarsByRange]: unknown; - [ReqRespMethod.BeaconBlockAndBlobsSidecarByRoot]: unknown; + [ReqRespMethod.BlobSidecarsByRange]: unknown; + [ReqRespMethod.BlobSidecarsByRoot]: unknown; [ReqRespMethod.LightClientBootstrap]: unknown; [ReqRespMethod.LightClientUpdatesByRange]: unknown; [ReqRespMethod.LightClientFinalityUpdate]: unknown; diff --git a/packages/beacon-node/src/sync/backfill/verify.ts b/packages/beacon-node/src/sync/backfill/verify.ts index 0ac52b80a953..36a00bea6833 100644 --- a/packages/beacon-node/src/sync/backfill/verify.ts +++ b/packages/beacon-node/src/sync/backfill/verify.ts @@ -1,4 +1,4 @@ -import {CachedBeaconStateAllForks, ISignatureSet, getProposerSignatureSet} from "@lodestar/state-transition"; +import {CachedBeaconStateAllForks, ISignatureSet, getBlockProposerSignatureSet} from "@lodestar/state-transition"; import {BeaconConfig} from "@lodestar/config"; import {allForks, Root, allForks as allForkTypes, ssz, Slot} from "@lodestar/types"; import {GENESIS_SLOT} from "@lodestar/params"; @@ -48,7 +48,7 @@ export async function verifyBlockProposerSignature( if (blocks.length === 1 && blocks[0].message.slot === GENESIS_SLOT) return; const signatures = blocks.reduce((sigs: ISignatureSet[], block) => { // genesis block doesn't have valid signature - if (block.message.slot !== GENESIS_SLOT) sigs.push(getProposerSignatureSet(state, block)); + if (block.message.slot !== GENESIS_SLOT) sigs.push(getBlockProposerSignatureSet(state, block)); return sigs; }, []); diff --git a/packages/beacon-node/src/sync/range/chain.ts b/packages/beacon-node/src/sync/range/chain.ts index 79d4e533c847..8897edfe9a9e 100644 --- a/packages/beacon-node/src/sync/range/chain.ts +++ b/packages/beacon-node/src/sync/range/chain.ts @@ -3,7 +3,7 @@ import {Epoch, Root, Slot, phase0} from "@lodestar/types"; import {ErrorAborted, Logger} from "@lodestar/utils"; import {ChainForkConfig} from "@lodestar/config"; import {toHexString} from "@chainsafe/ssz"; -import {BlockInput} from "../../chain/blocks/types.js"; +import {BlockInput, BlockInputType} from "../../chain/blocks/types.js"; import {PeerAction} from "../../network/index.js"; import {ItTrigger} from "../../util/itTrigger.js"; import {PeerMap} from "../../util/peerMap.js"; @@ -401,6 +401,15 @@ export class SyncChain { if (!res.err) { batch.downloadingSuccess(res.result); + const blobs = res.result.reduce( + (acc, blockInput) => acc + (blockInput.type === BlockInputType.postDeneb ? blockInput.blobs.length : 0), + 0 + ); + const downloadInfo = {blocks: res.result.length}; + if (blobs > 0) { + Object.assign(downloadInfo, {blobs}); + } + this.logger.debug("Downloaded batch", {id: this.logId, ...batch.getMetadata(), ...downloadInfo}); this.triggerBatchProcessor(); } else { this.logger.verbose("Batch download error", {id: this.logId, ...batch.getMetadata()}, res.err); diff --git a/packages/beacon-node/src/util/blobs.ts b/packages/beacon-node/src/util/blobs.ts deleted file mode 100644 index f8db4b02a581..000000000000 --- a/packages/beacon-node/src/util/blobs.ts +++ /dev/null @@ -1,24 +0,0 @@ -import {ChainForkConfig} from "@lodestar/config"; -import {deneb} from "@lodestar/types"; -import {ckzg} from "./kzg.js"; - -// Cache empty KZG proof, compute once lazily if needed -let emptyKzgAggregatedProof: Uint8Array | null = null; -function getEmptyKzgAggregatedProof(): Uint8Array { - if (!emptyKzgAggregatedProof) { - emptyKzgAggregatedProof = ckzg.computeAggregateKzgProof([]); - } - return emptyKzgAggregatedProof; -} - -/** - * Construct a valid BlobsSidecar for a SignedBeaconBlock that references 0 commitments - */ -export function getEmptyBlobsSidecar(config: ChainForkConfig, block: deneb.SignedBeaconBlock): deneb.BlobsSidecar { - return { - beaconBlockRoot: config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message), - beaconBlockSlot: block.message.slot, - blobs: [], - kzgAggregatedProof: getEmptyKzgAggregatedProof(), - }; -} diff --git a/packages/beacon-node/src/util/kzg.ts b/packages/beacon-node/src/util/kzg.ts index 38aa0ff0a86f..3760021fd3cb 100644 --- a/packages/beacon-node/src/util/kzg.ts +++ b/packages/beacon-node/src/util/kzg.ts @@ -16,18 +16,16 @@ export let ckzg: { freeTrustedSetup(): void; loadTrustedSetup(filePath: string): void; blobToKzgCommitment(blob: Uint8Array): Uint8Array; - computeAggregateKzgProof(blobs: Uint8Array[]): Uint8Array; - verifyAggregateKzgProof( - blobs: Uint8Array[], - expectedKzgCommitments: Uint8Array[], - kzgAggregatedProof: Uint8Array - ): boolean; + computeBlobKzgProof(blob: Uint8Array): Uint8Array; + verifyBlobKzgProof(blob: Uint8Array, expectedKzgCommitments: Uint8Array, kzgAggregatedProof: Uint8Array): boolean; + verifyBlobKzgProofBatch(blobs: Uint8Array[], expectedKzgCommitments: Uint8Array[], kzgProofs: Uint8Array[]): boolean; } = { freeTrustedSetup: ckzgNotLoaded, loadTrustedSetup: ckzgNotLoaded, blobToKzgCommitment: ckzgNotLoaded, - computeAggregateKzgProof: ckzgNotLoaded, - verifyAggregateKzgProof: ckzgNotLoaded, + computeBlobKzgProof: ckzgNotLoaded, + verifyBlobKzgProof: ckzgNotLoaded, + verifyBlobKzgProofBatch: ckzgNotLoaded, }; // Global variable __dirname no longer available in ES6 modules. diff --git a/packages/beacon-node/test/e2e/api/impl/getBlobsSidecar.test.ts b/packages/beacon-node/test/e2e/api/impl/getBlobsSidecar.test.ts index d95c776b5b30..268c82e8332f 100644 --- a/packages/beacon-node/test/e2e/api/impl/getBlobsSidecar.test.ts +++ b/packages/beacon-node/test/e2e/api/impl/getBlobsSidecar.test.ts @@ -5,26 +5,23 @@ import {GENESIS_SLOT} from "@lodestar/params"; import {setupApiImplTestServer, ApiImplTestModules} from "../../../unit/api/impl/index.test.js"; -describe("getBlobsSideCar", function () { +describe("getBlobSideCar", function () { let server: ApiImplTestModules; before(function () { server = setupApiImplTestServer(); }); - it("getBlobsSideCar", async () => { + it("getBlobSideCar From BlobSidecars", async () => { const block = config.getForkTypes(GENESIS_SLOT).SignedBeaconBlock.defaultValue(); - const blobsSidecar = ssz.deneb.BlobsSidecar.defaultValue(); + const blobSidecars = ssz.deneb.BlobSidecarsWrapper.defaultValue(); block.message.slot = GENESIS_SLOT; server.dbStub.blockArchive.get.resolves(block); - blobsSidecar.beaconBlockRoot = config.getForkTypes(GENESIS_SLOT).BeaconBlock.hashTreeRoot(block.message); + server.dbStub.blobSidecars.get.resolves(blobSidecars); - server.dbStub.blobsSidecar.get.resolves(blobsSidecar); - //server.dbStub.blobsSidecarArchive.get.resolves(blobsSidecar); + const returnedBlobSideCars = await server.blockApi.getBlobSidecars("genesis"); - const returnedBlobSideCar = await server.blockApi.getBlobsSidecar("genesis"); - - expect(returnedBlobSideCar.data).to.equal(blobsSidecar); + expect(returnedBlobSideCars.data).to.equal(blobSidecars.blobSidecars); }); }); diff --git a/packages/beacon-node/test/e2e/network/deneb.test.ts b/packages/beacon-node/test/e2e/network/deneb.test.ts deleted file mode 100644 index af99e916c3de..000000000000 --- a/packages/beacon-node/test/e2e/network/deneb.test.ts +++ /dev/null @@ -1,29 +0,0 @@ -import {expect} from "chai"; -import {deneb, ssz} from "@lodestar/types"; -import {toHex} from "@lodestar/utils"; -import {signedBeaconBlockAndBlobsSidecarFromBytes} from "../../../src/network/reqresp/handlers/beaconBlockAndBlobsSidecarByRoot.js"; - -describe("signedBeaconBlockAndBlobsSidecarFromBytes", () => { - it("signedBeaconBlockAndBlobsSidecarFromBytes", () => { - const beaconBlock = ssz.deneb.SignedBeaconBlock.defaultValue(); - const blobsSidecar = ssz.deneb.BlobsSidecar.defaultValue(); - - const signedBeaconBlockAndBlobsSidecarBytes = signedBeaconBlockAndBlobsSidecarFromBytes( - ssz.deneb.SignedBeaconBlock.serialize(beaconBlock), - ssz.deneb.BlobsSidecar.serialize(blobsSidecar) - ); - - const signedBeaconBlockAndBlobsSidecar: deneb.SignedBeaconBlockAndBlobsSidecar = { - beaconBlock, - blobsSidecar, - }; - - expect(toHex(signedBeaconBlockAndBlobsSidecarBytes)).equals( - toHex(ssz.deneb.SignedBeaconBlockAndBlobsSidecar.serialize(signedBeaconBlockAndBlobsSidecar)), - "Wrong signedBeaconBlockAndBlobsSidecarBytes" - ); - - // Ensure deserialize does not throw - ssz.deneb.SignedBeaconBlockAndBlobsSidecar.deserialize(signedBeaconBlockAndBlobsSidecarBytes); - }); -}); diff --git a/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts b/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts index 6a15f56f92cd..3307c834d7a1 100644 --- a/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts +++ b/packages/beacon-node/test/e2e/network/peers/peerManager.test.ts @@ -109,8 +109,8 @@ describe("network / peers / PeerManager", function () { ping = sinon.stub(); beaconBlocksByRange = sinon.stub(); beaconBlocksByRoot = sinon.stub(); - blobsSidecarsByRange = sinon.stub(); - beaconBlockAndBlobsSidecarByRoot = sinon.stub(); + blobSidecarsByRange = sinon.stub(); + blobSidecarsByRoot = sinon.stub(); lightClientBootstrap = sinon.stub(); lightClientOptimisticUpdate = sinon.stub(); lightClientFinalityUpdate = sinon.stub(); diff --git a/packages/beacon-node/test/e2e/network/reqresp.test.ts b/packages/beacon-node/test/e2e/network/reqresp.test.ts index 62d462b13918..4d870b8d7fa8 100644 --- a/packages/beacon-node/test/e2e/network/reqresp.test.ts +++ b/packages/beacon-node/test/e2e/network/reqresp.test.ts @@ -96,8 +96,8 @@ describe("network / ReqResp", function () { } as HandlerTypeFromMessage, onBeaconBlocksByRange: notImplemented, onBeaconBlocksByRoot: notImplemented, - onBlobsSidecarsByRange: notImplemented, - onBeaconBlockAndBlobsSidecarByRoot: notImplemented, + onBlobSidecarsByRoot: notImplemented, + onBlobSidecarsByRange: notImplemented, onLightClientBootstrap: notImplemented, onLightClientUpdatesByRange: notImplemented, onLightClientOptimisticUpdate: notImplemented, diff --git a/packages/beacon-node/test/spec/presets/fork_choice.ts b/packages/beacon-node/test/spec/presets/fork_choice.ts index 02d3ea55fd9a..8522fd051ae4 100644 --- a/packages/beacon-node/test/spec/presets/fork_choice.ts +++ b/packages/beacon-node/test/spec/presets/fork_choice.ts @@ -3,7 +3,7 @@ import {BeaconStateAllForks, isExecutionStateType} from "@lodestar/state-transit import {InputType} from "@lodestar/spec-test-util"; import {toHexString} from "@chainsafe/ssz"; import {CheckpointWithHex, ForkChoice} from "@lodestar/fork-choice"; -import {phase0, allForks, bellatrix, ssz, RootHex, deneb} from "@lodestar/types"; +import {phase0, allForks, bellatrix, ssz, RootHex} from "@lodestar/types"; import {bnToNum} from "@lodestar/utils"; import {createBeaconConfig} from "@lodestar/config"; import {ForkSeq, isForkBlobs} from "@lodestar/params"; @@ -20,7 +20,6 @@ import {defaultChainOptions} from "../../../src/chain/options.js"; import {getStubbedBeaconDb} from "../../utils/mocks/db.js"; import {ClockStopped} from "../../utils/mocks/clock.js"; import {getBlockInput, AttestationImportOpt} from "../../../src/chain/blocks/types.js"; -import {getEmptyBlobsSidecar} from "../../../src/util/blobs.js"; import {ZERO_HASH_HEX} from "../../../src/constants/constants.js"; import {PowMergeBlock} from "../../../src/eth1/interface.js"; import {assertCorrectProgressiveBalances} from "../config.js"; @@ -156,16 +155,12 @@ export const forkChoiceTest = (opts: {onlyPredefinedResponses: boolean}): TestRu const blockImport = config.getForkSeq(slot) < ForkSeq.deneb ? getBlockInput.preDeneb(config, signedBlock) - : getBlockInput.postDeneb( - config, - signedBlock, - getEmptyBlobsSidecar(config, signedBlock as deneb.SignedBeaconBlock) - ); + : getBlockInput.postDeneb(config, signedBlock, ssz.deneb.BlobSidecars.defaultValue()); try { await chain.processBlock(blockImport, { seenTimestampSec: tickTime, - validBlobsSidecar: true, + validBlobSidecars: true, importAttestations: AttestationImportOpt.Force, }); if (!isValid) throw Error("Expect error since this is a negative test"); diff --git a/packages/beacon-node/test/spec/presets/index.test.ts b/packages/beacon-node/test/spec/presets/index.test.ts index db7afce6fc3e..0ff8eefcd977 100644 --- a/packages/beacon-node/test/spec/presets/index.test.ts +++ b/packages/beacon-node/test/spec/presets/index.test.ts @@ -30,8 +30,6 @@ import {transition} from "./transition.js"; // ], // ``` const skipOpts: SkipOpts = { - // To be enabled in decouple blobs PR: https://github.com/ChainSafe/lodestar/pull/5181 - skippedForks: ["deneb"], // TODO: capella // BeaconBlockBody proof in lightclient is the new addition in v1.3.0-rc.2-hotfix // Skip them for now to enable subsequently diff --git a/packages/beacon-node/test/unit/network/beaconBlocksMaybeBlobsByRange.test.ts b/packages/beacon-node/test/unit/network/beaconBlocksMaybeBlobsByRange.test.ts index a8c00ab25961..e9673f85da0e 100644 --- a/packages/beacon-node/test/unit/network/beaconBlocksMaybeBlobsByRange.test.ts +++ b/packages/beacon-node/test/unit/network/beaconBlocksMaybeBlobsByRange.test.ts @@ -6,7 +6,7 @@ import {createBeaconConfig, createChainForkConfig, defaultChainConfig} from "@lo import {beaconBlocksMaybeBlobsByRange, ReqRespBeaconNode} from "../../../src/network/reqresp/index.js"; import {BlockInputType} from "../../../src/chain/blocks/types.js"; -import {ckzg, initCKZG, loadEthereumTrustedSetup} from "../../../src/util/kzg.js"; +import {initCKZG, loadEthereumTrustedSetup} from "../../../src/util/kzg.js"; describe("beaconBlocksMaybeBlobsByRange", () => { before(async function () { @@ -35,43 +35,64 @@ describe("beaconBlocksMaybeBlobsByRange", () => { const block1 = ssz.deneb.SignedBeaconBlock.defaultValue(); block1.message.slot = 1; + block1.message.body.blobKzgCommitments.push(ssz.deneb.KZGCommitment.defaultValue()); + const blobSidecar1 = ssz.deneb.BlobSidecar.defaultValue(); + blobSidecar1.slot = 1; + const block2 = ssz.deneb.SignedBeaconBlock.defaultValue(); block2.message.slot = 2; + block2.message.body.blobKzgCommitments.push(ssz.deneb.KZGCommitment.defaultValue()); + const blobSidecar2 = ssz.deneb.BlobSidecar.defaultValue(); + blobSidecar2.slot = 2; + + const block3 = ssz.deneb.SignedBeaconBlock.defaultValue(); + block3.message.slot = 3; + // no blobsidecar for block3 - const blobsSidecar1 = ssz.deneb.BlobsSidecar.defaultValue(); - blobsSidecar1.beaconBlockSlot = 1; - const blobsSidecar2 = ssz.deneb.BlobsSidecar.defaultValue(); - blobsSidecar2.beaconBlockSlot = 2; + const block4 = ssz.deneb.SignedBeaconBlock.defaultValue(); + block4.message.slot = 4; + // two blobsidecars + block4.message.body.blobKzgCommitments.push(ssz.deneb.KZGCommitment.defaultValue()); + block4.message.body.blobKzgCommitments.push(ssz.deneb.KZGCommitment.defaultValue()); + const blobSidecar41 = ssz.deneb.BlobSidecar.defaultValue(); + blobSidecar41.slot = 4; + const blobSidecar42 = ssz.deneb.BlobSidecar.defaultValue(); + blobSidecar42.slot = 4; + blobSidecar42.index = 1; // Array of testcases which are array of matched blocks with/without (if empty) sidecars - const testCases: [string, [deneb.SignedBeaconBlock, deneb.BlobsSidecar | undefined][]][] = [ - ["one block with sidecar", [[block1, blobsSidecar1]]], + const testCases: [string, [deneb.SignedBeaconBlock, deneb.BlobSidecar[] | undefined][]][] = [ + ["one block with sidecar", [[block1, [blobSidecar1]]]], [ "two blocks with sidecar", [ - [block1, blobsSidecar1], - [block2, blobsSidecar2], + [block1, [blobSidecar1]], + [block2, [blobSidecar2]], + ], + ], + ["block with skipped sidecar", [[block3, undefined]]], + ["multiple blob sidecars per block", [[block4, [blobSidecar41, blobSidecar42]]]], + [ + "all blocks together", + [ + [block1, [blobSidecar1]], + [block2, [blobSidecar2]], + [block3, undefined], + [block4, [blobSidecar41, blobSidecar42]], ], ], - ["block with skipped sidecar", [[block1, undefined]]], ]; testCases.map(([testName, blocksWithBlobs]) => { it(testName, async () => { const blocks = blocksWithBlobs.map(([block, _blobs]) => block as deneb.SignedBeaconBlock); - const blobsSidecars = blocksWithBlobs - .map(([_block, blobs]) => blobs as deneb.BlobsSidecar) - .filter((blobs) => blobs !== undefined); - const emptyKzgAggregatedProof = ckzg.computeAggregateKzgProof([]); - const expectedResponse = blocksWithBlobs.map(([block, blobsSidecar]) => { - const blobs = - blobsSidecar !== undefined - ? blobsSidecar - : { - beaconBlockRoot: ssz.deneb.BeaconBlock.hashTreeRoot(block.message), - beaconBlockSlot: block.message.slot, - blobs: [], - kzgAggregatedProof: emptyKzgAggregatedProof, - }; + + const blobSidecars = blocksWithBlobs + .map(([_block, blobs]) => blobs as deneb.BlobSidecars) + .filter((blobs) => blobs !== undefined) + .reduce((acc, elem) => acc.concat(elem), []); + + const expectedResponse = blocksWithBlobs.map(([block, blobSidecars]) => { + const blobs = blobSidecars !== undefined ? blobSidecars : []; return { type: BlockInputType.postDeneb, block, @@ -79,7 +100,7 @@ describe("beaconBlocksMaybeBlobsByRange", () => { }; }); reqResp.beaconBlocksByRange.resolves(blocks); - reqResp.blobsSidecarsByRange.resolves(blobsSidecars); + reqResp.blobSidecarsByRange.resolves(blobSidecars); const response = await beaconBlocksMaybeBlobsByRange(config, reqResp, peerId, rangeRequest, 0); expect(response).to.be.deep.equal(expectedResponse); diff --git a/packages/beacon-node/test/unit/network/gossip/topic.test.ts b/packages/beacon-node/test/unit/network/gossip/topic.test.ts index 7eb037a8fe43..eee3f2fa3ff9 100644 --- a/packages/beacon-node/test/unit/network/gossip/topic.test.ts +++ b/packages/beacon-node/test/unit/network/gossip/topic.test.ts @@ -15,10 +15,10 @@ describe("network / gossip / topic", function () { topicStr: "/eth2/18ae4ccb/beacon_block/ssz_snappy", }, ], - [GossipType.beacon_block_and_blobs_sidecar]: [ + [GossipType.blob_sidecar]: [ { - topic: {type: GossipType.beacon_block_and_blobs_sidecar, fork: ForkName.deneb, encoding}, - topicStr: "/eth2/46acb19a/beacon_block_and_blobs_sidecar/ssz_snappy", + topic: {type: GossipType.blob_sidecar, index: 1, fork: ForkName.deneb, encoding}, + topicStr: "/eth2/46acb19a/blob_sidecar_1/ssz_snappy", }, ], [GossipType.beacon_aggregate_and_proof]: [ diff --git a/packages/beacon-node/test/unit/util/kzg.test.ts b/packages/beacon-node/test/unit/util/kzg.test.ts index 6bb8c17d41c5..6538031d858d 100644 --- a/packages/beacon-node/test/unit/util/kzg.test.ts +++ b/packages/beacon-node/test/unit/util/kzg.test.ts @@ -6,10 +6,20 @@ import { OPAQUE_TX_BLOB_VERSIONED_HASHES_OFFSET, OPAQUE_TX_MESSAGE_OFFSET, } from "@lodestar/state-transition"; + import {loadEthereumTrustedSetup, initCKZG, ckzg, FIELD_ELEMENTS_PER_BLOB_MAINNET} from "../../../src/util/kzg.js"; -import {validateBlobsSidecar, validateGossipBlobsSidecar} from "../../../src/chain/validation/blobsSidecar.js"; +import {validateBlobSidecars, validateGossipBlobSidecar} from "../../../src/chain/validation/blobSidecar.js"; +import {getDevBeaconNode} from "../../utils/node/beacon.js"; + +describe("C-KZG", async () => { + const afterEachCallbacks: (() => Promise | void)[] = []; + afterEach(async () => { + while (afterEachCallbacks.length > 0) { + const callback = afterEachCallbacks.pop(); + if (callback) await callback(); + } + }); -describe("C-KZG", () => { before(async function () { this.timeout(10000); // Loading trusted setup is slow await initCKZG(); @@ -22,34 +32,78 @@ describe("C-KZG", () => { // ==================== const blobs = new Array(2).fill(0).map(generateRandomBlob); const commitments = blobs.map((blob) => ckzg.blobToKzgCommitment(blob)); - const proof = ckzg.computeAggregateKzgProof(blobs); - expect(ckzg.verifyAggregateKzgProof(blobs, commitments, proof)).to.equal(true); + const proofs = blobs.map((blob) => ckzg.computeBlobKzgProof(blob)); + expect(ckzg.verifyBlobKzgProofBatch(blobs, commitments, proofs)).to.equal(true); }); - it("BlobsSidecar", () => { + it("BlobsSidecar", async () => { + const bn = await getDevBeaconNode({ + params: { + // eslint-disable-next-line @typescript-eslint/naming-convention + ALTAIR_FORK_EPOCH: 0, + // eslint-disable-next-line @typescript-eslint/naming-convention + BELLATRIX_FORK_EPOCH: 0, + // eslint-disable-next-line @typescript-eslint/naming-convention + CAPELLA_FORK_EPOCH: 0, + // eslint-disable-next-line @typescript-eslint/naming-convention + DENEB_FORK_EPOCH: 0, + }, + }); + + afterEachCallbacks.push(() => bn.close()); + const slot = 0; const blobs = [generateRandomBlob(), generateRandomBlob()]; const kzgCommitments = blobs.map((blob) => ckzg.blobToKzgCommitment(blob)); const signedBeaconBlock = ssz.deneb.SignedBeaconBlock.defaultValue(); + for (const kzgCommitment of kzgCommitments) { signedBeaconBlock.message.body.executionPayload.transactions.push(transactionForKzgCommitment(kzgCommitment)); signedBeaconBlock.message.body.blobKzgCommitments.push(kzgCommitment); } - const beaconBlockRoot = ssz.deneb.BeaconBlock.hashTreeRoot(signedBeaconBlock.message); + const blockRoot = ssz.deneb.BeaconBlock.hashTreeRoot(signedBeaconBlock.message); + + const blobSidecars: deneb.BlobSidecars = blobs.map((blob, index) => { + return { + blockRoot, + index, + slot, + blob, + kzgProof: ckzg.computeBlobKzgProof(blob), + kzgCommitment: kzgCommitments[index], + blockParentRoot: Buffer.alloc(32), + proposerIndex: 0, + }; + }); - const blobsSidecar: deneb.BlobsSidecar = { - beaconBlockRoot, - beaconBlockSlot: 0, - blobs, - kzgAggregatedProof: ckzg.computeAggregateKzgProof(blobs), - }; + const signedBlobSidecars: deneb.SignedBlobSidecar[] = blobs.map((blob, index) => { + const signedBlobSidecar = ssz.deneb.SignedBlobSidecar.defaultValue(); + signedBlobSidecar.message = { + blockRoot, + index, + slot, + blob, + kzgProof: ckzg.computeBlobKzgProof(blob), + kzgCommitment: kzgCommitments[index], + blockParentRoot: Buffer.alloc(32), + proposerIndex: 0, + }; + return signedBlobSidecar; + }); // Full validation - validateBlobsSidecar(slot, beaconBlockRoot, kzgCommitments, blobsSidecar); + validateBlobSidecars( + slot, + blockRoot, + signedBeaconBlock.message.body.executionPayload.transactions, + kzgCommitments, + blobSidecars + ); - // Gossip validation - validateGossipBlobsSidecar(signedBeaconBlock, blobsSidecar); + signedBlobSidecars.forEach(async (signedBlobSidecar) => { + await validateGossipBlobSidecar(bn.config, bn.chain, signedBlobSidecar, signedBlobSidecar.message.index); + }); }); }); diff --git a/packages/beacon-node/test/utils/mocks/chain/chain.ts b/packages/beacon-node/test/utils/mocks/chain/chain.ts index a73978029e0e..099def2406ab 100644 --- a/packages/beacon-node/test/utils/mocks/chain/chain.ts +++ b/packages/beacon-node/test/utils/mocks/chain/chain.ts @@ -106,7 +106,8 @@ export class MockBeaconChain implements IBeaconChain { private readonly state: CachedBeaconStateAllForks; private abortController: AbortController; - readonly producedBlobsSidecarCache = new Map(); + readonly producedBlobSidecarsCache = new Map(); + readonly producedBlindedBlobSidecarsCache = new Map(); constructor({genesisTime, chainId, networkId, state, config}: MockChainParams) { this.logger = testLogger(); @@ -185,7 +186,7 @@ export class MockBeaconChain implements IBeaconChain { throw Error("Not implemented"); } - getBlobsSidecar(): never { + getBlobSidecars(): never { throw Error("Not implemented"); } diff --git a/packages/beacon-node/test/utils/mocks/db.ts b/packages/beacon-node/test/utils/mocks/db.ts index 70e60e2234c2..b63f97bb4d1e 100644 --- a/packages/beacon-node/test/utils/mocks/db.ts +++ b/packages/beacon-node/test/utils/mocks/db.ts @@ -14,8 +14,8 @@ import { SyncCommitteeRepository, SyncCommitteeWitnessRepository, BackfilledRanges, - BlobsSidecarRepository, - BlobsSidecarArchiveRepository, + BlobSidecarsRepository, + BlobSidecarsArchiveRepository, BLSToExecutionChangeRepository, } from "../../../src/db/repositories/index.js"; import {PreGenesisState, PreGenesisStateLastProcessedBlock} from "../../../src/db/single/index.js"; @@ -30,11 +30,11 @@ export function getStubbedBeaconDb(): IBeaconDb { return { // unfinalized blocks block: createStubInstance(BlockRepository), - blobsSidecar: createStubInstance(BlobsSidecarRepository), + blobSidecars: createStubInstance(BlobSidecarsRepository), // finalized blocks blockArchive: createStubInstance(BlockArchiveRepository), - blobsSidecarArchive: createStubInstance(BlobsSidecarArchiveRepository), + blobSidecarsArchive: createStubInstance(BlobSidecarsArchiveRepository), // finalized states stateArchive: createStubInstance(StateArchiveRepository), diff --git a/packages/beacon-node/test/utils/node/beacon.ts b/packages/beacon-node/test/utils/node/beacon.ts index d948bc571b75..0f7be9ed2212 100644 --- a/packages/beacon-node/test/utils/node/beacon.ts +++ b/packages/beacon-node/test/utils/node/beacon.ts @@ -80,9 +80,9 @@ export async function getDevBeaconNode( await db.blockArchive.add(block); if (config.getForkSeq(GENESIS_SLOT) >= ForkSeq.deneb) { - const blobsSidecar = ssz.deneb.BlobsSidecar.defaultValue(); - blobsSidecar.beaconBlockRoot = config.getForkTypes(GENESIS_SLOT).BeaconBlock.hashTreeRoot(block.message); - await db.blobsSidecar.add(blobsSidecar); + const blobSidecars = ssz.deneb.BlobSidecarsWrapper.defaultValue(); + blobSidecars.blockRoot = config.getForkTypes(GENESIS_SLOT).BeaconBlock.hashTreeRoot(block.message); + await db.blobSidecars.add(blobSidecars); } } diff --git a/packages/beacon-node/test/utils/stub/beaconDb.ts b/packages/beacon-node/test/utils/stub/beaconDb.ts index cb7e2e8cea01..5101639ac816 100644 --- a/packages/beacon-node/test/utils/stub/beaconDb.ts +++ b/packages/beacon-node/test/utils/stub/beaconDb.ts @@ -14,8 +14,8 @@ import { StateArchiveRepository, VoluntaryExitRepository, BLSToExecutionChangeRepository, - BlobsSidecarRepository, - BlobsSidecarArchiveRepository, + BlobSidecarsRepository, + BlobSidecarsArchiveRepository, } from "../../../src/db/repositories/index.js"; import {createStubInstance} from "../types.js"; @@ -25,8 +25,8 @@ export class StubbedBeaconDb extends BeaconDb { block: SinonStubbedInstance & BlockRepository; blockArchive: SinonStubbedInstance & BlockArchiveRepository; - blobsSidecar: SinonStubbedInstance & BlobsSidecarRepository; - blobsSidecarArchive: SinonStubbedInstance & BlobsSidecarArchiveRepository; + blobSidecars: SinonStubbedInstance & BlobSidecarsRepository; + blobSidecarsArchive: SinonStubbedInstance & BlobSidecarsArchiveRepository; stateArchive: SinonStubbedInstance & StateArchiveRepository; @@ -54,7 +54,7 @@ export class StubbedBeaconDb extends BeaconDb { this.depositDataRoot = createStubInstance(DepositDataRootRepository); this.eth1Data = createStubInstance(Eth1DataRepository); - this.blobsSidecar = createStubInstance(BlobsSidecarRepository); - this.blobsSidecarArchive = createStubInstance(BlobsSidecarArchiveRepository); + this.blobSidecars = createStubInstance(BlobSidecarsRepository); + this.blobSidecarsArchive = createStubInstance(BlobSidecarsArchiveRepository); } } diff --git a/packages/cli/src/applyPreset.ts b/packages/cli/src/applyPreset.ts index 4824f766a241..78c87c93b42a 100644 --- a/packages/cli/src/applyPreset.ts +++ b/packages/cli/src/applyPreset.ts @@ -1,3 +1,4 @@ +import {setActivePreset, PresetName} from "@lodestar/params/setPreset"; // MUST import this file first before anything and not import any Lodestar code. // // ## Rationale @@ -30,6 +31,9 @@ else if (process.env.LODESTAR_PRESET) { else if (network) { if (network === "dev") { process.env.LODESTAR_PRESET = "minimal"; + // "c-kzg" has hardcoded the mainnet value, do not use presets + // eslint-disable-next-line @typescript-eslint/naming-convention + setActivePreset(PresetName.minimal, {FIELD_ELEMENTS_PER_BLOB: 4096}); } else if (network === "gnosis" || network === "chiado") { process.env.LODESTAR_PRESET = "gnosis"; } @@ -39,6 +43,9 @@ else if (network) { else if (process.argv[2] === "dev") { process.env.LODESTAR_PRESET = "minimal"; process.env.LODESTAR_NETWORK = "dev"; + // "c-kzg" has hardcoded the mainnet value, do not use presets + // eslint-disable-next-line @typescript-eslint/naming-convention + setActivePreset(PresetName.minimal, {FIELD_ELEMENTS_PER_BLOB: 4096}); } /** diff --git a/packages/config/src/chainConfig/presets/mainnet.ts b/packages/config/src/chainConfig/presets/mainnet.ts index b7d79948668b..dcbae44266d5 100644 --- a/packages/config/src/chainConfig/presets/mainnet.ts +++ b/packages/config/src/chainConfig/presets/mainnet.ts @@ -82,6 +82,5 @@ export const chainConfig: ChainConfig = { // Blobs // --------------------------------------------------------------- // https://github.com/ethereum/consensus-specs/blob/dev/specs/eip4844/p2p-interface.md#configuration - MAX_REQUEST_BLOBS_SIDECARS: 128, - MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS: 4096, + MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096, }; diff --git a/packages/config/src/chainConfig/presets/minimal.ts b/packages/config/src/chainConfig/presets/minimal.ts index d7d70342ea43..3ae3b2d938ed 100644 --- a/packages/config/src/chainConfig/presets/minimal.ts +++ b/packages/config/src/chainConfig/presets/minimal.ts @@ -81,6 +81,5 @@ export const chainConfig: ChainConfig = { // Blobs // --------------------------------------------------------------- // https://github.com/ethereum/consensus-specs/blob/dev/specs/eip4844/p2p-interface.md#configuration - MAX_REQUEST_BLOBS_SIDECARS: 128, - MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS: 4096, + MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: 4096, }; diff --git a/packages/config/src/chainConfig/types.ts b/packages/config/src/chainConfig/types.ts index 3a4b967b3005..4a5bf845c3e7 100644 --- a/packages/config/src/chainConfig/types.ts +++ b/packages/config/src/chainConfig/types.ts @@ -64,10 +64,8 @@ export type ChainConfig = { DEPOSIT_CONTRACT_ADDRESS: Uint8Array; // DENEB - // https://github.com/ethereum/consensus-specs/blob/11a037fd9227e29ee809c9397b09f8cc3383a8c0/specs/eip4844/p2p-interface.md#configuration - MAX_REQUEST_BLOBS_SIDECARS: number; /** The minimum epoch range over which a node must serve blobs sidecars */ - MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS: number; + MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: number; }; export const chainConfigTypes: SpecTypes = { @@ -122,8 +120,7 @@ export const chainConfigTypes: SpecTypes = { DEPOSIT_CONTRACT_ADDRESS: "bytes", // Blobs - MAX_REQUEST_BLOBS_SIDECARS: "number", - MIN_EPOCHS_FOR_BLOBS_SIDECARS_REQUESTS: "number", + MIN_EPOCHS_FOR_BLOB_SIDECARS_REQUESTS: "number", }; /** Allows values in a Spec file */ diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index bb42cb4a4786..7de701fde867 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -47,8 +47,8 @@ export enum Bucket { index_stateArchiveRootIndex = 26, // State Root -> slot - allForks_blobsSidecar = 27, // DENEB BeaconBlockRoot -> BlobsSidecar - allForks_blobsSidecarArchive = 28, // DENEB BeaconBlockSlot -> BlobsSidecar + allForks_blobSidecars = 27, // DENEB BeaconBlockRoot -> BlobsSidecars + allForks_blobSidecarsArchive = 28, // DENEB Slot -> BlobsSidecar // Lightclient server // altair_bestUpdatePerCommitteePeriod = 30, // DEPRECATED on v0.32.0 diff --git a/packages/params/src/index.ts b/packages/params/src/index.ts index 2a8742e83e95..f3351f0ccab1 100644 --- a/packages/params/src/index.ts +++ b/packages/params/src/index.ts @@ -134,6 +134,7 @@ export const DOMAIN_SYNC_COMMITTEE = Uint8Array.from([7, 0, 0, 0]); export const DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF = Uint8Array.from([8, 0, 0, 0]); export const DOMAIN_CONTRIBUTION_AND_PROOF = Uint8Array.from([9, 0, 0, 0]); export const DOMAIN_BLS_TO_EXECUTION_CHANGE = Uint8Array.from([10, 0, 0, 0]); +export const DOMAIN_BLOB_SIDECAR = Uint8Array.from([11, 0, 0, 0]); // Application specific domains @@ -180,7 +181,8 @@ export const SYNC_COMMITTEE_SUBNET_COUNT = 4; export const SYNC_COMMITTEE_SUBNET_SIZE = Math.floor(SYNC_COMMITTEE_SIZE / SYNC_COMMITTEE_SUBNET_COUNT); export const MAX_REQUEST_BLOCKS = 2 ** 10; // 1024 - +export const MAX_REQUEST_BLOCKS_DENEB = 2 ** 7; // 128 +export const MAX_REQUEST_BLOB_SIDECARS = 2 ** 7; // 128 // Lightclient pre-computed /** * ```ts diff --git a/packages/reqresp/src/protocols/BlobsSidecarsByRange.ts b/packages/reqresp/src/protocols/BlobSidecarsByRange.ts similarity index 68% rename from packages/reqresp/src/protocols/BlobsSidecarsByRange.ts rename to packages/reqresp/src/protocols/BlobSidecarsByRange.ts index 36233885fe2d..ffc445dc43db 100644 --- a/packages/reqresp/src/protocols/BlobsSidecarsByRange.ts +++ b/packages/reqresp/src/protocols/BlobSidecarsByRange.ts @@ -3,23 +3,23 @@ import {deneb, ssz} from "@lodestar/types"; import {ContextBytesType, Encoding, ProtocolDefinitionGenerator} from "../types.js"; // eslint-disable-next-line @typescript-eslint/naming-convention -export const BlobsSidecarsByRange: ProtocolDefinitionGenerator< - deneb.BlobsSidecarsByRangeRequest, - deneb.BlobsSidecar -> = (modules, handler) => { +export const BlobSidecarsByRange: ProtocolDefinitionGenerator = ( + modules, + handler +) => { return { - method: "blobs_sidecars_by_range", + method: "blob_sidecars_by_range", version: 1, encoding: Encoding.SSZ_SNAPPY, handler, - requestType: () => ssz.deneb.BlobsSidecarsByRangeRequest, + requestType: () => ssz.deneb.BlobSidecarsByRangeRequest, // TODO: Make it fork compliant - responseType: () => ssz.deneb.BlobsSidecar, + responseType: () => ssz.deneb.BlobSidecar, renderRequestBody: (req) => `${req.startSlot},${req.count}`, contextBytes: { type: ContextBytesType.ForkDigest, forkDigestContext: modules.config, - forkFromResponse: (blobsSidecar) => modules.config.getForkName(blobsSidecar.beaconBlockSlot), + forkFromResponse: (blobSidecar) => modules.config.getForkName(blobSidecar.slot), }, inboundRateLimits: { // TODO DENEB: For now same value as BeaconBlocksByRange https://github.com/sigp/lighthouse/blob/bf533c8e42cc73c35730e285c21df8add0195369/beacon_node/lighthouse_network/src/rpc/mod.rs#L118-L130 diff --git a/packages/reqresp/src/protocols/BeaconBlockAndBlobsSidecarByRoot.ts b/packages/reqresp/src/protocols/BlobSidecarsByRoot.ts similarity index 58% rename from packages/reqresp/src/protocols/BeaconBlockAndBlobsSidecarByRoot.ts rename to packages/reqresp/src/protocols/BlobSidecarsByRoot.ts index 03b8c566bfd5..00359398380b 100644 --- a/packages/reqresp/src/protocols/BeaconBlockAndBlobsSidecarByRoot.ts +++ b/packages/reqresp/src/protocols/BlobSidecarsByRoot.ts @@ -3,23 +3,23 @@ import {toHex} from "@lodestar/utils"; import {ContextBytesType, Encoding, ProtocolDefinitionGenerator} from "../types.js"; // eslint-disable-next-line @typescript-eslint/naming-convention -export const BeaconBlockAndBlobsSidecarByRoot: ProtocolDefinitionGenerator< - deneb.BeaconBlockAndBlobsSidecarByRootRequest, - deneb.SignedBeaconBlockAndBlobsSidecar -> = (modules, handler) => { +export const BlobSidecarsByRoot: ProtocolDefinitionGenerator = ( + modules, + handler +) => { return { - method: "beacon_block_and_blobs_sidecar_by_root", + method: "blob_sidecars_by_root", version: 1, encoding: Encoding.SSZ_SNAPPY, handler, - requestType: () => ssz.deneb.BeaconBlockAndBlobsSidecarByRootRequest, + requestType: () => ssz.deneb.BlobSidecarsByRootRequest, // TODO: Make it fork compliant - responseType: () => ssz.deneb.SignedBeaconBlockAndBlobsSidecar, - renderRequestBody: (req) => req.map((root) => toHex(root)).join(","), + responseType: () => ssz.deneb.BlobSidecar, + renderRequestBody: (req) => req.map(({blockRoot, index}) => `${toHex(blockRoot)}-${index}`).join(","), contextBytes: { type: ContextBytesType.ForkDigest, forkDigestContext: modules.config, - forkFromResponse: ({beaconBlock}) => modules.config.getForkName(beaconBlock.message.slot), + forkFromResponse: (blobsSidecar) => modules.config.getForkName(blobsSidecar.slot), }, inboundRateLimits: { // TODO DENEB: For now same value as BeaconBlocksByRoot https://github.com/sigp/lighthouse/blob/bf533c8e42cc73c35730e285c21df8add0195369/beacon_node/lighthouse_network/src/rpc/mod.rs#L118-L130 diff --git a/packages/reqresp/src/protocols/index.ts b/packages/reqresp/src/protocols/index.ts index 0531341c5271..d48ba83629e2 100644 --- a/packages/reqresp/src/protocols/index.ts +++ b/packages/reqresp/src/protocols/index.ts @@ -1,7 +1,7 @@ -export {BeaconBlockAndBlobsSidecarByRoot} from "./BeaconBlockAndBlobsSidecarByRoot.js"; +export {BlobSidecarsByRange} from "./BlobSidecarsByRange.js"; +export {BlobSidecarsByRoot} from "./BlobSidecarsByRoot.js"; export {BeaconBlocksByRoot, BeaconBlocksByRootV2} from "./BeaconBlocksByRoot.js"; export {BeaconBlocksByRange, BeaconBlocksByRangeV2} from "./BeaconBlocksByRange.js"; -export {BlobsSidecarsByRange} from "./BlobsSidecarsByRange.js"; export {Goodbye} from "./Goodbye.js"; export {LightClientBootstrap} from "./LightClientBootstrap.js"; export {LightClientFinalityUpdate} from "./LightClientFinalityUpdate.js"; diff --git a/packages/state-transition/src/signatureSets/index.ts b/packages/state-transition/src/signatureSets/index.ts index c1a3812bb61a..05e4ad4b197a 100644 --- a/packages/state-transition/src/signatureSets/index.ts +++ b/packages/state-transition/src/signatureSets/index.ts @@ -6,7 +6,7 @@ import {getSyncCommitteeSignatureSet} from "../block/processSyncCommittee.js"; import {getProposerSlashingsSignatureSets} from "./proposerSlashings.js"; import {getAttesterSlashingsSignatureSets} from "./attesterSlashings.js"; import {getAttestationsSignatureSets} from "./indexedAttestation.js"; -import {getProposerSignatureSet} from "./proposer.js"; +import {getBlockProposerSignatureSet} from "./proposer.js"; import {getRandaoRevealSignatureSet} from "./randao.js"; import {getVoluntaryExitsSignatureSets} from "./voluntaryExits.js"; import {getBlsToExecutionChangeSignatureSets} from "./blsToExecutionChange.js"; @@ -40,7 +40,7 @@ export function getBlockSignatureSets( ]; if (!opts?.skipProposerSignature) { - signatureSets.push(getProposerSignatureSet(state, signedBlock)); + signatureSets.push(getBlockProposerSignatureSet(state, signedBlock)); } // fork based validations diff --git a/packages/state-transition/src/signatureSets/proposer.ts b/packages/state-transition/src/signatureSets/proposer.ts index 4e76cacb42da..a00bcacc7c99 100644 --- a/packages/state-transition/src/signatureSets/proposer.ts +++ b/packages/state-transition/src/signatureSets/proposer.ts @@ -1,5 +1,5 @@ -import {DOMAIN_BEACON_PROPOSER} from "@lodestar/params"; -import {allForks, isBlindedBeaconBlock} from "@lodestar/types"; +import {DOMAIN_BEACON_PROPOSER, DOMAIN_BLOB_SIDECAR} from "@lodestar/params"; +import {allForks, isBlindedBeaconBlock, isBlindedBlobSidecar, ssz} from "@lodestar/types"; import {computeSigningRoot} from "../util/index.js"; import {ISignatureSet, SignatureSetType, verifySignatureSet} from "../util/signatureSets.js"; import {CachedBeaconStateAllForks} from "../types.js"; @@ -8,11 +8,11 @@ export function verifyProposerSignature( state: CachedBeaconStateAllForks, signedBlock: allForks.FullOrBlindedSignedBeaconBlock ): boolean { - const signatureSet = getProposerSignatureSet(state, signedBlock); + const signatureSet = getBlockProposerSignatureSet(state, signedBlock); return verifySignatureSet(signatureSet); } -export function getProposerSignatureSet( +export function getBlockProposerSignatureSet( state: CachedBeaconStateAllForks, signedBlock: allForks.FullOrBlindedSignedBeaconBlock ): ISignatureSet { @@ -30,3 +30,20 @@ export function getProposerSignatureSet( signature: signedBlock.signature, }; } + +export function getBlobProposerSignatureSet( + state: CachedBeaconStateAllForks, + signedBlob: allForks.FullOrBlindedSignedBlobSidecar +): ISignatureSet { + const {config, epochCtx} = state; + const domain = config.getDomain(state.slot, DOMAIN_BLOB_SIDECAR, signedBlob.message.slot); + + const blockType = isBlindedBlobSidecar(signedBlob.message) ? ssz.deneb.BlindedBlobSidecar : ssz.deneb.BlobSidecar; + + return { + type: SignatureSetType.single, + pubkey: epochCtx.index2pubkey[signedBlob.message.proposerIndex], + signingRoot: computeSigningRoot(blockType, signedBlob.message, domain), + signature: signedBlob.signature, + }; +} diff --git a/packages/state-transition/src/util/blindedBlock.ts b/packages/state-transition/src/util/blindedBlock.ts index 92d9f2d1e028..3b0ecc469597 100644 --- a/packages/state-transition/src/util/blindedBlock.ts +++ b/packages/state-transition/src/util/blindedBlock.ts @@ -1,5 +1,5 @@ import {ChainForkConfig} from "@lodestar/config"; -import {allForks, phase0, Root, isBlindedBeaconBlock} from "@lodestar/types"; +import {allForks, phase0, Root, isBlindedBeaconBlock, isBlindedBlobSidecar} from "@lodestar/types"; export function blindedOrFullBlockHashTreeRoot( config: ChainForkConfig, @@ -12,6 +12,17 @@ export function blindedOrFullBlockHashTreeRoot( config.getForkTypes(blindedOrFull.slot).BeaconBlock.hashTreeRoot(blindedOrFull); } +export function blindedOrFullBlobSidecarHashTreeRoot( + config: ChainForkConfig, + blindedOrFull: allForks.FullOrBlindedBlobSidecar +): Root { + return isBlindedBlobSidecar(blindedOrFull) + ? // Blinded + config.getBlobsForkTypes(blindedOrFull.slot).BlindedBlobSidecar.hashTreeRoot(blindedOrFull) + : // Full + config.getBlobsForkTypes(blindedOrFull.slot).BlobSidecar.hashTreeRoot(blindedOrFull); +} + export function blindedOrFullBlockToHeader( config: ChainForkConfig, blindedOrFull: allForks.FullOrBlindedBeaconBlock diff --git a/packages/types/src/allForks/sszTypes.ts b/packages/types/src/allForks/sszTypes.ts index a38fc8d63d75..0fd85c1250ea 100644 --- a/packages/types/src/allForks/sszTypes.ts +++ b/packages/types/src/allForks/sszTypes.ts @@ -154,6 +154,8 @@ export const allForksLightClient = { export const allForksBlobs = { deneb: { - SignedBeaconBlockAndBlobsSidecar: deneb.SignedBeaconBlockAndBlobsSidecar, + SignedBeaconBlockAndBlobSidecars: deneb.SignedBeaconBlockAndBlobSidecars, + BlobSidecar: deneb.BlobSidecar, + BlindedBlobSidecar: deneb.BlindedBlobSidecar, }, }; diff --git a/packages/types/src/allForks/types.ts b/packages/types/src/allForks/types.ts index 4f906926a5cb..0628792f98d6 100644 --- a/packages/types/src/allForks/types.ts +++ b/packages/types/src/allForks/types.ts @@ -68,6 +68,11 @@ export type FullOrBlindedBeaconBlockBody = BeaconBlockBody | BlindedBeaconBlockB export type FullOrBlindedBeaconBlock = BeaconBlock | BlindedBeaconBlock; export type FullOrBlindedSignedBeaconBlock = SignedBeaconBlock | SignedBlindedBeaconBlock; +export type FullOrBlindedBlobSidecar = deneb.BlobSidecar | deneb.BlindedBlobSidecar; +export type FullOrBlindedSignedBlobSidecar = deneb.SignedBlobSidecar | deneb.SignedBlindedBlobSidecar; + +export type FullOrBlindedBlobSidecars = deneb.BlobSidecars | deneb.BlindedBlobSidecars; + export type BuilderBid = bellatrix.BuilderBid | capella.BuilderBid | deneb.BuilderBid; export type SignedBuilderBid = bellatrix.SignedBuilderBid | capella.SignedBuilderBid | deneb.SignedBuilderBid; @@ -87,9 +92,10 @@ export type LightClientOptimisticUpdate = | deneb.LightClientOptimisticUpdate; export type LightClientStore = altair.LightClientStore | capella.LightClientStore | deneb.LightClientStore; -export type SignedBeaconBlockAndBlobsSidecar = deneb.SignedBeaconBlockAndBlobsSidecar; - export type SSEPayloadAttributes = bellatrix.SSEPayloadAttributes | capella.SSEPayloadAttributes; + +export type SignedBeaconBlockAndBlobSidecars = deneb.SignedBeaconBlockAndBlobSidecars; + /** * Types known to change between forks */ @@ -104,7 +110,7 @@ export type AllForksTypes = { LightClientHeader: LightClientHeader; BuilderBid: BuilderBid; SignedBuilderBid: SignedBuilderBid; - SignedBeaconBlockAndBlobsSidecar: SignedBeaconBlockAndBlobsSidecar; + SignedBeaconBlockAndBlobSidecars: SignedBeaconBlockAndBlobSidecars; }; export type AllForksBlindedTypes = { @@ -277,5 +283,7 @@ export type AllForksLightClientSSZTypes = { }; export type AllForksBlobsSSZTypes = { - SignedBeaconBlockAndBlobsSidecar: AllForksTypeOf; + SignedBeaconBlockAndBlobSidecars: AllForksTypeOf; + BlobSidecar: AllForksTypeOf; + BlindedBlobSidecar: AllForksTypeOf; }; diff --git a/packages/types/src/bellatrix/types.ts b/packages/types/src/bellatrix/types.ts index f322e09920c3..dcd45d7c97ea 100644 --- a/packages/types/src/bellatrix/types.ts +++ b/packages/types/src/bellatrix/types.ts @@ -2,6 +2,7 @@ import {ValueOf} from "@chainsafe/ssz"; import * as ssz from "./sszTypes.js"; export type Transaction = ValueOf; +export type Transactions = ValueOf; export type ExecutionPayload = ValueOf; export type ExecutionPayloadHeader = ValueOf; export type BeaconBlockBody = ValueOf; diff --git a/packages/types/src/deneb/sszTypes.ts b/packages/types/src/deneb/sszTypes.ts index 9392ddac57e5..36babbc1fa73 100644 --- a/packages/types/src/deneb/sszTypes.ts +++ b/packages/types/src/deneb/sszTypes.ts @@ -3,7 +3,7 @@ import { HISTORICAL_ROOTS_LIMIT, FIELD_ELEMENTS_PER_BLOB, MAX_BLOBS_PER_BLOCK, - MAX_REQUEST_BLOCKS, + MAX_REQUEST_BLOB_SIDECARS, BYTES_PER_FIELD_ELEMENT, BLOCK_BODY_EXECUTION_PAYLOAD_DEPTH as EXECUTION_PAYLOAD_DEPTH, EPOCHS_PER_SYNC_COMMITTEE_PERIOD, @@ -14,7 +14,19 @@ import {ssz as phase0Ssz} from "../phase0/index.js"; import {ssz as altairSsz} from "../altair/index.js"; import {ssz as capellaSsz} from "../capella/index.js"; -const {UintNum64, Slot, Root, BLSSignature, UintBn256, Bytes32, Bytes48, Bytes96, BLSPubkey} = primitiveSsz; +const { + UintNum64, + Slot, + Root, + BLSSignature, + UintBn256, + Bytes32, + Bytes48, + Bytes96, + BLSPubkey, + BlobIndex, + ValidatorIndex, +} = primitiveSsz; // Polynomial commitments // https://github.com/ethereum/consensus-specs/blob/dev/specs/eip4844/polynomial-commitments.md @@ -34,8 +46,12 @@ export const KZGProof = Bytes48; export const Blob = new ByteVectorType(BYTES_PER_FIELD_ELEMENT * FIELD_ELEMENTS_PER_BLOB); export const Blobs = new ListCompositeType(Blob, MAX_BLOBS_PER_BLOCK); +export const BlindedBlob = Bytes32; +export const BlindedBlobs = new ListCompositeType(BlindedBlob, MAX_BLOBS_PER_BLOCK); + export const VersionedHash = Bytes32; export const BlobKzgCommitments = new ListCompositeType(KZGCommitment, MAX_BLOBS_PER_BLOCK); +export const KZGProofs = new ListCompositeType(KZGProof, MAX_BLOBS_PER_BLOCK); // Constants @@ -70,15 +86,26 @@ export const PolynomialAndCommitment = new ContainerType( // ReqResp types // ============= -export const BlobsSidecarsByRangeRequest = new ContainerType( +export const BlobSidecarsByRangeRequest = new ContainerType( { startSlot: Slot, count: UintNum64, }, - {typeName: "BlobsSidecarsByRangeRequest", jsonCase: "eth2"} + {typeName: "BlobSidecarsByRangeRequest", jsonCase: "eth2"} +); + +export const BlobIdentifier = new ContainerType( + { + blockRoot: Root, + index: BlobIndex, + }, + {typeName: "BlobIdentifier", jsonCase: "eth2"} ); -export const BeaconBlockAndBlobsSidecarByRootRequest = new ListCompositeType(Root, MAX_REQUEST_BLOCKS); +export const BlobSidecarsByRootRequest = new ListCompositeType( + BlobIdentifier, + MAX_REQUEST_BLOB_SIDECARS * MAX_BLOBS_PER_BLOCK +); // Beacon Chain types // https://github.com/ethereum/consensus-specs/blob/dev/specs/eip4844/beacon-chain.md#containers @@ -126,22 +153,71 @@ export const SignedBeaconBlock = new ContainerType( {typeName: "SignedBeaconBlock", jsonCase: "eth2"} ); -export const BlobsSidecar = new ContainerType( +export const BlobSidecar = new ContainerType( { - beaconBlockRoot: Root, - beaconBlockSlot: Slot, - blobs: Blobs, - kzgAggregatedProof: KZGProof, + blockRoot: Root, + index: BlobIndex, + slot: Slot, + blockParentRoot: Root, + proposerIndex: ValidatorIndex, + blob: Blob, + kzgCommitment: KZGCommitment, + kzgProof: KZGProof, + }, + {typeName: "BlobSidecar", jsonCase: "eth2"} +); + +export const BlobSidecars = new ListCompositeType(BlobSidecar, MAX_BLOBS_PER_BLOCK); + +export const BlobSidecarsWrapper = new ContainerType( + { + blockRoot: Root, + slot: Slot, + blobSidecars: BlobSidecars, + }, + {typeName: "BlobSidecarsWrapper", jsonCase: "eth2"} +); + +export const SignedBlobSidecar = new ContainerType( + { + message: BlobSidecar, + signature: BLSSignature, + }, + {typeName: "SignedBlobSidecar", jsonCase: "eth2"} +); + +export const BlindedBlobSidecar = new ContainerType( + { + blockRoot: Root, + index: BlobIndex, + slot: Slot, + blockParentRoot: Root, + proposerIndex: ValidatorIndex, + blobRoot: BlindedBlob, + kzgCommitment: KZGCommitment, + kzgProof: KZGProof, + }, + {typeName: "BlindedBlobSidecar", jsonCase: "eth2"} +); + +export const BlindedBlobSidecars = new ListCompositeType(BlindedBlobSidecar, MAX_BLOBS_PER_BLOCK); + +export const SignedBlindedBlobSidecar = new ContainerType( + { + message: BlindedBlobSidecar, + signature: BLSSignature, }, - {typeName: "BlobsSidecar", jsonCase: "eth2"} + {typeName: "SignedBlindedBlobSidecar", jsonCase: "eth2"} ); -export const SignedBeaconBlockAndBlobsSidecar = new ContainerType( +// TODO: deneb cleanup once the builder-api gets rectified for deneb +// as the type might be used in builder getHeader responses +export const SignedBeaconBlockAndBlobSidecars = new ContainerType( { beaconBlock: SignedBeaconBlock, - blobsSidecar: BlobsSidecar, + blobSidecars: BlobSidecars, }, - {typeName: "SignedBeaconBlockAndBlobsSidecar", jsonCase: "eth2"} + {typeName: "SignedBeaconBlockAndBlobSidecars", jsonCase: "eth2"} ); export const BlindedBeaconBlockBody = new ContainerType( diff --git a/packages/types/src/deneb/types.ts b/packages/types/src/deneb/types.ts index bb419bd3d00a..77ba84106a02 100644 --- a/packages/types/src/deneb/types.ts +++ b/packages/types/src/deneb/types.ts @@ -3,16 +3,28 @@ import * as ssz from "./sszTypes.js"; export type KZGProof = ValueOf; export type KZGCommitment = ValueOf; + export type Blob = ValueOf; export type Blobs = ValueOf; -export type BlobsSidecar = ValueOf; +export type BlindedBlob = ValueOf; +export type BlindedBlobs = ValueOf; +export type BlobSidecar = ValueOf; +export type BlobSidecars = ValueOf; +export type BlobSidecarsWrapper = ValueOf; +export type BlindedBlobSidecar = ValueOf; +export type BlindedBlobSidecars = ValueOf; +export type SignedBlobSidecar = ValueOf; +export type SignedBlindedBlobSidecar = ValueOf; + +export type BlobIdentifier = ValueOf; export type BlobKzgCommitments = ValueOf; +export type KZGProofs = ValueOf; export type Polynomial = ValueOf; export type PolynomialAndCommitment = ValueOf; export type BLSFieldElement = ValueOf; -export type BlobsSidecarsByRangeRequest = ValueOf; -export type BeaconBlockAndBlobsSidecarByRootRequest = ValueOf; +export type BlobSidecarsByRangeRequest = ValueOf; +export type BlobSidecarsByRootRequest = ValueOf; export type ExecutionPayload = ValueOf; export type ExecutionPayloadHeader = ValueOf; @@ -20,7 +32,7 @@ export type ExecutionPayloadHeader = ValueOf; export type BeaconBlockBody = ValueOf; export type BeaconBlock = ValueOf; export type SignedBeaconBlock = ValueOf; -export type SignedBeaconBlockAndBlobsSidecar = ValueOf; +export type SignedBeaconBlockAndBlobSidecars = ValueOf; export type BeaconState = ValueOf; diff --git a/packages/types/src/primitive/sszTypes.ts b/packages/types/src/primitive/sszTypes.ts index 39a62c2c9b33..65c81d1247b9 100644 --- a/packages/types/src/primitive/sszTypes.ts +++ b/packages/types/src/primitive/sszTypes.ts @@ -52,6 +52,7 @@ export const WithdrawalIndex = UintNum64; export const Gwei = UintBn64; export const Wei = UintBn256; export const Root = new ByteVectorType(32); +export const BlobIndex = UintNum64; export const Version = Bytes4; export const DomainType = Bytes4; diff --git a/packages/types/src/primitive/types.ts b/packages/types/src/primitive/types.ts index 7d4349d75e49..53422cc9b995 100644 --- a/packages/types/src/primitive/types.ts +++ b/packages/types/src/primitive/types.ts @@ -29,6 +29,7 @@ export type CommitteeIndex = UintNum64; export type SubcommitteeIndex = UintNum64; export type ValidatorIndex = UintNum64; export type WithdrawalIndex = UintNum64; +export type BlobIndex = UintNum64; export type Gwei = UintBn64; export type Wei = UintBn256; export type Root = Bytes32; diff --git a/packages/types/src/utils/typeguards.ts b/packages/types/src/utils/typeguards.ts index c918915a47d7..9e1cdcbceb81 100644 --- a/packages/types/src/utils/typeguards.ts +++ b/packages/types/src/utils/typeguards.ts @@ -3,8 +3,11 @@ import { FullOrBlindedSignedBeaconBlock, FullOrBlindedExecutionPayload, ExecutionPayloadHeader, + FullOrBlindedBlobSidecar, + FullOrBlindedSignedBlobSidecar, } from "../allForks/types.js"; import {ts as bellatrix} from "../bellatrix/index.js"; +import {ts as deneb} from "../deneb/index.js"; export function isBlindedExecution(payload: FullOrBlindedExecutionPayload): payload is ExecutionPayloadHeader { // we just check transactionsRoot for determinging as it the base field @@ -21,3 +24,13 @@ export function isBlindedSignedBeaconBlock( ): signedBlock is bellatrix.SignedBlindedBeaconBlock { return (signedBlock as bellatrix.SignedBlindedBeaconBlock).message.body.executionPayloadHeader !== undefined; } + +export function isBlindedBlobSidecar(blob: FullOrBlindedBlobSidecar): blob is deneb.BlindedBlobSidecar { + return (blob as deneb.BlindedBlobSidecar).blobRoot !== undefined; +} + +export function isBlindedSignedBlobSidecar( + blob: FullOrBlindedSignedBlobSidecar +): blob is deneb.SignedBlindedBlobSidecar { + return (blob as deneb.SignedBlindedBlobSidecar).message.blobRoot !== undefined; +} diff --git a/packages/validator/src/services/block.ts b/packages/validator/src/services/block.ts index a9663c69aa6c..67e51ea5be12 100644 --- a/packages/validator/src/services/block.ts +++ b/packages/validator/src/services/block.ts @@ -8,6 +8,8 @@ import { isBlindedBeaconBlock, Wei, BlockSource, + isBlindedBlobSidecar, + deneb, } from "@lodestar/types"; import {ChainForkConfig} from "@lodestar/config"; import {ForkName} from "@lodestar/params"; @@ -116,11 +118,22 @@ export class BlockProposingService { this.logger.debug("Produced block", {...debugLogCtx, ...block.debugLogCtx}); this.metrics?.blocksProduced.inc(); - const signedBlock = await this.validatorStore.signBlock(pubkey, block.data, slot); - - this.metrics?.proposerStepCallPublishBlock.observe(this.clock.secFromSlot(slot)); + const signedPromises: Promise[] = []; + signedPromises.push( + this.validatorStore.signBlock(pubkey, block.data, slot).then((signedBlock) => { + this.metrics?.proposerStepCallPublishBlock.observe(this.clock.secFromSlot(slot)); + return this.publishBlockWrapper(signedBlock); + }) + ); + for (const blob of block.blobs ?? []) { + signedPromises.push( + this.validatorStore.signBlob(pubkey, blob, slot).then((signedBlob) => { + return this.publishBlobWrapper(signedBlob); + }) + ); + } - await this.publishBlockWrapper(signedBlock).catch((e: Error) => { + await Promise.all(signedPromises).catch((e: Error) => { this.metrics?.blockProposingErrors.inc({error: "publish"}); throw extendError(e, "Failed to publish block"); }); @@ -139,12 +152,24 @@ export class BlockProposingService { ); }; + private publishBlobWrapper = async (signedBlob: allForks.FullOrBlindedSignedBlobSidecar): Promise => { + ApiError.assert( + isBlindedBlobSidecar(signedBlob.message) + ? await this.api.beacon.publishBlindedBlob(signedBlob as deneb.SignedBlindedBlobSidecar) + : await this.api.beacon.publishBlob(signedBlob as deneb.SignedBlobSidecar) + ); + }; + private produceBlockWrapper = async ( slot: Slot, randaoReveal: BLSSignature, graffiti: string, {expectedFeeRecipient, strictFeeRecipientCheck, isBuilderEnabled, builderSelection}: ProduceBlockOpts - ): Promise<{data: allForks.FullOrBlindedBeaconBlock} & {debugLogCtx: Record}> => { + ): Promise< + {data: allForks.FullOrBlindedBeaconBlock; blobs?: allForks.FullOrBlindedBlobSidecars} & { + debugLogCtx: Record; + } + > => { // Start calls for building execution and builder blocks const blindedBlockPromise = isBuilderEnabled ? this.produceBlindedBlock(slot, randaoReveal, graffiti) : null; const fullBlockPromise = this.produceBlock(slot, randaoReveal, graffiti); @@ -236,10 +261,16 @@ export class BlockProposingService { }; private getBlockWithDebugLog( - fullOrBlindedBlock: {data: allForks.FullOrBlindedBeaconBlock; blockValue: Wei}, + fullOrBlindedBlock: { + data: allForks.FullOrBlindedBeaconBlock; + blockValue: Wei; + blobs?: allForks.FullOrBlindedBlobSidecars; + }, source: BlockSource, {expectedFeeRecipient, strictFeeRecipientCheck}: {expectedFeeRecipient: string; strictFeeRecipientCheck: boolean} - ): {data: allForks.FullOrBlindedBeaconBlock} & {debugLogCtx: Record} { + ): {data: allForks.FullOrBlindedBeaconBlock; blobs?: allForks.FullOrBlindedBlobSidecars} & { + debugLogCtx: Record; + } { const debugLogCtx = { source: source, // winston logger doesn't like bigint @@ -266,12 +297,17 @@ export class BlockProposingService { transactions !== undefined ? {transactions} : {}, withdrawals !== undefined ? {withdrawals} : {} ); + Object.assign(debugLogCtx, fullOrBlindedBlock.blobs !== undefined ? {blobs: fullOrBlindedBlock.blobs.length} : {}); - return {...fullOrBlindedBlock, debugLogCtx}; + return {...fullOrBlindedBlock, blobs: fullOrBlindedBlock.blobs, debugLogCtx}; } /** Wrapper around the API's different methods for producing blocks across forks */ - private produceBlock: ServerApi["produceBlock"] = async (slot, randaoReveal, graffiti) => { + private produceBlock: ServerApi["produceBlock"] & {blobs?: deneb.BlobSidecars} = async ( + slot, + randaoReveal, + graffiti + ) => { switch (this.config.getForkName(slot)) { case ForkName.phase0: { const res = await this.api.validator.produceBlock(slot, randaoReveal, graffiti); @@ -283,18 +319,58 @@ export class BlockProposingService { default: { const res = await this.api.validator.produceBlockV2(slot, randaoReveal, graffiti); ApiError.assert(res, "Failed to produce block: validator.produceBlockV2"); - return res.response; + + const {response} = res; + const blobKzgCommitmentsLen = (response.data.body as deneb.BeaconBlockBody).blobKzgCommitments?.length ?? 0; + let blobs: deneb.BlobSidecars | undefined; + + if (blobKzgCommitmentsLen > 0) { + const blockRoot = this.config.getForkTypes(slot).BeaconBlock.hashTreeRoot(response.data); + blobs = await Promise.all( + Array.from({length: blobKzgCommitmentsLen}, async (_v, i) => { + const blobRes = await this.api.validator.getBlob(blockRoot, i); + ApiError.assert(blobRes, `Failed to fetch blob=${i} validator.produceBlockV2`); + const { + response: {data: blob}, + } = blobRes; + return blob; + }) + ); + } else { + blobs = undefined; + } + + return {...response, blobs}; } } }; - private produceBlindedBlock: ServerApi["produceBlindedBlock"] = async ( - slot, - randaoReveal, - graffiti - ) => { + private produceBlindedBlock: ServerApi["produceBlindedBlock"] & { + blobs?: deneb.BlindedBlobSidecars; + } = async (slot, randaoReveal, graffiti) => { const res = await this.api.validator.produceBlindedBlock(slot, randaoReveal, graffiti); ApiError.assert(res, "Failed to produce block: validator.produceBlindedBlock"); - return res.response; + + const {response} = res; + const blobKzgCommitmentsLen = (response.data.body as deneb.BeaconBlockBody).blobKzgCommitments?.length ?? 0; + let blobs: deneb.BlindedBlobSidecars | undefined; + + if (blobKzgCommitmentsLen > 0) { + const blockRoot = this.config.getBlindedForkTypes(slot).BeaconBlock.hashTreeRoot(response.data); + blobs = await Promise.all( + Array.from({length: blobKzgCommitmentsLen}, async (_v, i) => { + const blobRes = await this.api.validator.getBlindedBlob(blockRoot, i); + ApiError.assert(blobRes, `Failed to fetch blindedBlob=${i} validator.produceBlindedBlock`); + const { + response: {data: blob}, + } = blobRes; + return blob; + }) + ); + } else { + blobs = undefined; + } + + return {...response, blobs}; }; } diff --git a/packages/validator/src/services/validatorStore.ts b/packages/validator/src/services/validatorStore.ts index 61432ce892a9..0dbd9077257b 100644 --- a/packages/validator/src/services/validatorStore.ts +++ b/packages/validator/src/services/validatorStore.ts @@ -5,6 +5,7 @@ import { computeDomain, ZERO_HASH, blindedOrFullBlockHashTreeRoot, + blindedOrFullBlobSidecarHashTreeRoot, } from "@lodestar/state-transition"; import {BeaconConfig} from "@lodestar/config"; import { @@ -18,6 +19,7 @@ import { DOMAIN_SYNC_COMMITTEE_SELECTION_PROOF, DOMAIN_VOLUNTARY_EXIT, DOMAIN_APPLICATION_BUILDER, + DOMAIN_BLOB_SIDECAR, } from "@lodestar/params"; import type {SecretKey} from "@chainsafe/bls/types"; import { @@ -354,6 +356,37 @@ export class ValidatorStore { } as allForks.FullOrBlindedSignedBeaconBlock; } + async signBlob( + pubkey: BLSPubkey, + blindedOrFull: allForks.FullOrBlindedBlobSidecar, + currentSlot: Slot + ): Promise { + // Make sure the block slot is not higher than the current slot to avoid potential attacks. + if (blindedOrFull.slot > currentSlot) { + throw Error(`Not signing block with slot ${blindedOrFull.slot} greater than current slot ${currentSlot}`); + } + + // Duties are filtered before-hard by doppelganger-safe, this assert should never throw + this.assertDoppelgangerSafe(pubkey); + + const signingSlot = blindedOrFull.slot; + const domain = this.config.getDomain(signingSlot, DOMAIN_BLOB_SIDECAR); + const blobRoot = blindedOrFullBlobSidecarHashTreeRoot(this.config, blindedOrFull); + // Don't use `computeSigningRoot()` here to compute the objectRoot in typesafe function blindedOrFullBlockHashTreeRoot() + const signingRoot = ssz.phase0.SigningData.hashTreeRoot({objectRoot: blobRoot, domain}); + + // Slashing protection is not required as blobs are binded to blocks which are already protected + const signableMessage: SignableMessage = { + type: SignableMessageType.BLOB, + data: blindedOrFull, + }; + + return { + message: blindedOrFull, + signature: await this.getSignature(pubkey, signingRoot, signingSlot, signableMessage), + } as allForks.FullOrBlindedSignedBlobSidecar; + } + async signRandao(pubkey: BLSPubkey, slot: Slot): Promise { const signingSlot = slot; const domain = this.config.getDomain(slot, DOMAIN_RANDAO); diff --git a/packages/validator/src/util/externalSignerClient.ts b/packages/validator/src/util/externalSignerClient.ts index 43db6c33cc1e..25dcfe3cd256 100644 --- a/packages/validator/src/util/externalSignerClient.ts +++ b/packages/validator/src/util/externalSignerClient.ts @@ -15,6 +15,7 @@ export enum SignableMessageType { AGGREGATE_AND_PROOF = "AGGREGATE_AND_PROOF", ATTESTATION = "ATTESTATION", BLOCK_V2 = "BLOCK_V2", + BLOB = "BLOB", DEPOSIT = "DEPOSIT", RANDAO_REVEAL = "RANDAO_REVEAL", VOLUNTARY_EXIT = "VOLUNTARY_EXIT", @@ -64,6 +65,7 @@ export type SignableMessage = | {type: SignableMessageType.AGGREGATE_AND_PROOF; data: phase0.AggregateAndProof} | {type: SignableMessageType.ATTESTATION; data: phase0.AttestationData} | {type: SignableMessageType.BLOCK_V2; data: allForks.FullOrBlindedBeaconBlock} + | {type: SignableMessageType.BLOB; data: allForks.FullOrBlindedBlobSidecar} | {type: SignableMessageType.DEPOSIT; data: ValueOf} | {type: SignableMessageType.RANDAO_REVEAL; data: {epoch: Epoch}} | {type: SignableMessageType.VOLUNTARY_EXIT; data: phase0.VoluntaryExit} @@ -86,6 +88,7 @@ const requiresForkInfo: Record = { [SignableMessageType.SYNC_COMMITTEE_CONTRIBUTION_AND_PROOF]: true, [SignableMessageType.VALIDATOR_REGISTRATION]: false, [SignableMessageType.BLS_TO_EXECUTION_CHANGE]: true, + [SignableMessageType.BLOB]: true, }; type Web3SignerSerializedRequest = { @@ -229,5 +232,9 @@ function serializerSignableMessagePayload(config: BeaconConfig, payload: Signabl case SignableMessageType.BLS_TO_EXECUTION_CHANGE: return {BLS_TO_EXECUTION_CHANGE: ssz.capella.BLSToExecutionChange.toJson(payload.data)}; + + case SignableMessageType.BLOB: + // TODO: freetheblobs + throw Error("web3signer for blob signing not yet implemented"); } } diff --git a/packages/validator/src/util/params.ts b/packages/validator/src/util/params.ts index 511e16815f2b..8288e6f9f92d 100644 --- a/packages/validator/src/util/params.ts +++ b/packages/validator/src/util/params.ts @@ -129,8 +129,7 @@ function getSpecCriticalParams(localConfig: ChainConfig): Record