Free the blobs

fix the types

rejig the new constants in params

add comment for cleanup

update reqresp

fix api package

rename blobs repo

commit the wip modifications

further appropriate renaming

further references update

further reference updates

continue refac

fix reqresp build

further refac

further refac

fix api

fix db interface

fix beacondb alloc

build

fix api

improve blob verificaion

correct validation call

fixes

fix the produce block/blobs flow

reduce diff

blob gossip validation

update validations

cleanup block vali

reduce diff

handle gossip of block and blob

fix test for timebeing

modify publishing flow

fix import flow

onsidecarbyrange fix and some type fixes

fix sidecars by root

prune blockinput cache

fix kzg interface

small renaming

interface rename

fix fetch blockmaybeblobs by range test

fix build lint issues for now

c-kzg version fix

FullOrBlindedBlobSidecar changes

fix tests

complete the blob publishing flow

fix test

get the single node run functional

get the gossip blob flow working

fix peer syncing using req/resp

fix sidecar by root check

refactor blobsidecars hotdb and remove archive

add blob gossip validation flow

fix topic

fix the validation condition

add blob validation and test various sync modes

fix tests

rebase fixes

enable deneb spec tests

make blobsbyroot multi block

fixes

cleanup defunt builder endpoint

archive blobs post finalization uptill the blob window

serve finalized blobs within the blob prune window

fix test

fix test

lookup in archive as well

cleanup and improvements

packages/api/src/beacon/routes/beacon/block.ts

@@ -187,18 +187,43 @@ export type Api = {
     >
   >;
   /**
-   * Get block BlobsSidecar
-   * Retrieves BlobsSidecar included in requested block.
+   * Get block BlobSidecar
+   * Retrieves BlobSidecar included in requested block.
    * @param blockId Block identifier.
    * Can be one of: "head" (canonical head in node's view), "genesis", "finalized", \<slot\>, \<hex encoded blockRoot with 0x prefix\>.
    */
-  getBlobsSidecar(
+  getBlobSidecars(
     blockId: BlockId
   ): Promise<
     ApiClientResponse<{
-      [HttpStatusCode.OK]: {executionOptimistic: ExecutionOptimistic; data: deneb.BlobsSidecar};
+      [HttpStatusCode.OK]: {executionOptimistic: ExecutionOptimistic; data: deneb.BlobSidecars};
     }>
   >;
+  /**
+   * Publish a signed blob.
+   */
+  publishBlob(
+    blob: deneb.SignedBlobSidecar
+  ): Promise<
+    ApiClientResponse<
+      {
+        [HttpStatusCode.OK]: void;
+        [HttpStatusCode.ACCEPTED]: void;
+      },
+      HttpStatusCode.BAD_REQUEST | HttpStatusCode.SERVICE_UNAVAILABLE
+    >
+  >;
+  publishBlindedBlob(
+    blob: deneb.SignedBlindedBlobSidecar
+  ): Promise<
+    ApiClientResponse<
+      {
+        [HttpStatusCode.OK]: void;
+        [HttpStatusCode.ACCEPTED]: void;
+      },
+      HttpStatusCode.BAD_REQUEST | HttpStatusCode.SERVICE_UNAVAILABLE
+    >
+  >;
 };
 
 /**
@@ -213,7 +238,9 @@ export const routesData: RoutesData<Api> = {
   getBlockRoot: {url: "/eth/v1/beacon/blocks/{block_id}/root", method: "GET"},
   publishBlock: {url: "/eth/v1/beacon/blocks", method: "POST"},
   publishBlindedBlock: {url: "/eth/v1/beacon/blinded_blocks", method: "POST"},
-  getBlobsSidecar: {url: "/eth/v1/beacon/blobs_sidecars/{block_id}", method: "GET"},
+  getBlobSidecars: {url: "/eth/v1/beacon/blob_sidecars/{block_id}", method: "GET"},
+  publishBlob: {url: "/eth/v1/beacon/blob_sidecars", method: "POST"},
+  publishBlindedBlob: {url: "/eth/v1/beacon/blinded_blob_sidecars", method: "POST"},
 };
 
 /* eslint-disable @typescript-eslint/naming-convention */
@@ -229,7 +256,9 @@ export type ReqTypes = {
   getBlockRoot: BlockIdOnlyReq;
   publishBlock: {body: unknown};
   publishBlindedBlock: {body: unknown};
-  getBlobsSidecar: BlockIdOnlyReq;
+  getBlobSidecars: BlockIdOnlyReq;
+  publishBlob: {body: unknown};
+  publishBlindedBlob: {body: unknown};
 };
 
 export function getReqSerializers(config: ChainForkConfig): ReqSerializers<Api, ReqTypes> {
@@ -272,7 +301,9 @@ export function getReqSerializers(config: ChainForkConfig): ReqSerializers<Api,
     getBlockRoot: blockIdOnlyReq,
     publishBlock: reqOnlyBody(AllForksSignedBeaconBlock, Schema.Object),
     publishBlindedBlock: reqOnlyBody(AllForksSignedBlindedBeaconBlock, Schema.Object),
-    getBlobsSidecar: blockIdOnlyReq,
+    getBlobSidecars: blockIdOnlyReq,
+    publishBlob: reqOnlyBody(ssz.deneb.SignedBlobSidecar, Schema.Object),
+    publishBlindedBlob: reqOnlyBody(ssz.deneb.SignedBlindedBlobSidecar, Schema.Object),
   };
 }
 
@@ -294,6 +325,6 @@ export function getReturnTypes(): ReturnTypes<Api> {
     getBlockHeader: ContainerDataExecutionOptimistic(BeaconHeaderResType),
     getBlockHeaders: ContainerDataExecutionOptimistic(ArrayOf(BeaconHeaderResType)),
     getBlockRoot: ContainerDataExecutionOptimistic(RootContainer),
-    getBlobsSidecar: ContainerDataExecutionOptimistic(ssz.deneb.BlobsSidecar),
+    getBlobSidecars: ContainerDataExecutionOptimistic(ssz.deneb.BlobSidecars),
   };
 }

packages/api/src/beacon/routes/validator.ts

@@ -17,6 +17,7 @@ import {
   RootHex,
   StringType,
   Wei,
+  deneb,
 } from "@lodestar/types";
 import {ApiClientResponse} from "../../interfaces.js";
 import {HttpStatusCode} from "../../utils/client/httpStatusCode.js";
@@ -209,6 +210,26 @@ export type Api = {
     >
   >;
 
+  getBlob(
+    blockRoot: Root,
+    index: number
+  ): Promise<
+    ApiClientResponse<
+      {[HttpStatusCode.OK]: {data: deneb.BlobSidecar}},
+      HttpStatusCode.BAD_REQUEST | HttpStatusCode.SERVICE_UNAVAILABLE
+    >
+  >;
+
+  getBlindedBlob(
+    blockRoot: Root,
+    index: number
+  ): Promise<
+    ApiClientResponse<
+      {[HttpStatusCode.OK]: {data: deneb.BlindedBlobSidecar}},
+      HttpStatusCode.BAD_REQUEST | HttpStatusCode.SERVICE_UNAVAILABLE
+    >
+  >;
+
   /**
    * Produce an attestation data
    * Requests that the beacon node produce an AttestationData.
@@ -323,6 +344,8 @@ export const routesData: RoutesData<Api> = {
   getSyncCommitteeDuties: {url: "/eth/v1/validator/duties/sync/{epoch}", method: "POST"},
   produceBlock: {url: "/eth/v1/validator/blocks/{slot}", method: "GET"},
   produceBlockV2: {url: "/eth/v2/validator/blocks/{slot}", method: "GET"},
+  getBlob: {url: "/eth/v1/validator/blob/{root}/${index}", method: "GET"},
+  getBlindedBlob: {url: "/eth/v1/validator/blinded_blob/{root}/${index}", method: "GET"},
   produceBlindedBlock: {url: "/eth/v1/validator/blinded_blocks/{slot}", method: "GET"},
   produceAttestationData: {url: "/eth/v1/validator/attestation_data", method: "GET"},
   produceSyncCommitteeContribution: {url: "/eth/v1/validator/sync_committee_contribution", method: "GET"},
@@ -344,6 +367,8 @@ export type ReqTypes = {
   produceBlock: {params: {slot: number}; query: {randao_reveal: string; graffiti: string}};
   produceBlockV2: {params: {slot: number}; query: {randao_reveal: string; graffiti: string}};
   produceBlindedBlock: {params: {slot: number}; query: {randao_reveal: string; graffiti: string}};
+  getBlob: {params: {root: string; index: number}};
+  getBlindedBlob: {params: {root: string; index: number}};
   produceAttestationData: {query: {slot: number; committee_index: number}};
   produceSyncCommitteeContribution: {query: {slot: number; subcommittee_index: number; beacon_block_root: string}};
   getAggregatedAttestation: {query: {attestation_data_root: string; slot: number}};
@@ -389,6 +414,14 @@ export function getReqSerializers(): ReqSerializers<Api, ReqTypes> {
     },
   };
 
+  const getBlobReqSerializer: ReqSerializers<Api, ReqTypes>["getBlob"] = {
+    writeReq: (root, index) => ({
+      params: {root: toHexString(root), index},
+    }),
+    parseReq: ({params}) => [fromHexString(params.root), params.index],
+    schema: {params: {root: Schema.StringRequired, index: Schema.UintRequired}},
+  };
+
   return {
     getAttesterDuties: {
       writeReq: (epoch, indexes) => ({params: {epoch}, body: indexes.map((i) => toU64Str(i))}),
@@ -420,6 +453,9 @@ export function getReqSerializers(): ReqSerializers<Api, ReqTypes> {
     produceBlockV2: produceBlock,
     produceBlindedBlock: produceBlock,
 
+    getBlob: getBlobReqSerializer,
+    getBlindedBlob: getBlobReqSerializer,
+
     produceAttestationData: {
       writeReq: (index, slot) => ({query: {slot, committee_index: index}}),
       parseReq: ({query}) => [query.committee_index, query.slot],
@@ -529,6 +565,8 @@ export function getReturnTypes(): ReturnTypes<Api> {
         return ssz[fork].BlindedBeaconBlock;
       })
     ),
+    getBlob: ContainerData(ssz.deneb.BlobSidecar),
+    getBlindedBlob: ContainerData(ssz.deneb.BlindedBlobSidecar),
     produceAttestationData: ContainerData(ssz.phase0.AttestationData),
     produceSyncCommitteeContribution: ContainerData(ssz.altair.SyncCommitteeContribution),
     getAggregatedAttestation: ContainerData(ssz.phase0.Attestation),

packages/api/src/builder/routes.ts

@@ -1,6 +1,6 @@
 import {ssz, allForks, bellatrix, Slot, Root, BLSPubkey} from "@lodestar/types";
 import {fromHexString, toHexString} from "@chainsafe/ssz";
-import {ForkName, isForkExecution, isForkBlobs} from "@lodestar/params";
+import {ForkName, isForkExecution} from "@lodestar/params";
 import {ChainForkConfig} from "@lodestar/config";
 
 import {
@@ -42,14 +42,6 @@ export type Api = {
       HttpStatusCode.SERVICE_UNAVAILABLE
     >
   >;
-  submitBlindedBlockV2(
-    signedBlock: allForks.SignedBlindedBeaconBlock
-  ): Promise<
-    ApiClientResponse<
-      {[HttpStatusCode.OK]: {data: allForks.SignedBeaconBlockAndBlobsSidecar; version: ForkName}},
-      HttpStatusCode.SERVICE_UNAVAILABLE
-    >
-  >;
 };
 
 /**
@@ -60,7 +52,6 @@ export const routesData: RoutesData<Api> = {
   registerValidator: {url: "/eth/v1/builder/validators", method: "POST"},
   getHeader: {url: "/eth/v1/builder/header/{slot}/{parent_hash}/{pubkey}", method: "GET"},
   submitBlindedBlock: {url: "/eth/v1/builder/blinded_blocks", method: "POST"},
-  submitBlindedBlockV2: {url: "/eth/v2/builder/blinded_blocks", method: "POST"},
 };
 
 /* eslint-disable @typescript-eslint/naming-convention */
@@ -69,7 +60,6 @@ export type ReqTypes = {
   registerValidator: {body: unknown};
   getHeader: {params: {slot: Slot; parent_hash: string; pubkey: string}};
   submitBlindedBlock: {body: unknown};
-  submitBlindedBlockV2: {body: unknown};
 };
 
 export function getReqSerializers(config: ChainForkConfig): ReqSerializers<Api, ReqTypes> {
@@ -86,7 +76,6 @@ export function getReqSerializers(config: ChainForkConfig): ReqSerializers<Api,
       },
     },
     submitBlindedBlock: getBeaconReqSerializers(config)["publishBlindedBlock"],
-    submitBlindedBlockV2: getBeaconReqSerializers(config)["publishBlindedBlock"],
   };
 }
 
@@ -98,10 +87,5 @@ export function getReturnTypes(): ReturnTypes<Api> {
     submitBlindedBlock: WithVersion((fork: ForkName) =>
       isForkExecution(fork) ? ssz.allForksExecution[fork].ExecutionPayload : ssz.bellatrix.ExecutionPayload
     ),
-    submitBlindedBlockV2: WithVersion((fork: ForkName) =>
-      isForkBlobs(fork)
-        ? ssz.allForksBlobs[fork].SignedBeaconBlockAndBlobsSidecar
-        : ssz.deneb.SignedBeaconBlockAndBlobsSidecar
-    ),
   };
 }

packages/api/test/unit/beacon/testData/beacon.ts

@@ -56,9 +56,17 @@ export const testData: GenericServerTestCases<Api> = {
     args: [getDefaultBlindedBlock(64)],
     res: undefined,
   },
-  getBlobsSidecar: {
+  getBlobSidecars: {
     args: ["head"],
-    res: {executionOptimistic: true, data: ssz.deneb.BlobsSidecar.defaultValue()},
+    res: {executionOptimistic: true, data: ssz.deneb.BlobSidecars.defaultValue()},
+  },
+  publishBlob: {
+    args: [ssz.deneb.SignedBlobSidecar.defaultValue()],
+    res: undefined,
+  },
+  publishBlindedBlob: {
+    args: [ssz.deneb.SignedBlindedBlobSidecar.defaultValue()],
+    res: undefined,
   },
 
   // pool

packages/api/test/unit/beacon/testData/validator.ts

@@ -98,4 +98,12 @@ export const testData: GenericServerTestCases<Api> = {
     args: [[ssz.bellatrix.SignedValidatorRegistrationV1.defaultValue()]],
     res: undefined,
   },
+  getBlob: {
+    args: [ZERO_HASH, 0],
+    res: {data: ssz.deneb.BlobSidecar.defaultValue()},
+  },
+  getBlindedBlob: {
+    args: [ZERO_HASH, 0],
+    res: {data: ssz.deneb.BlindedBlobSidecar.defaultValue()},
+  },
 };

packages/api/test/unit/builder/testData.ts

@@ -26,8 +26,4 @@ export const testData: GenericServerTestCases<Api> = {
     args: [ssz.deneb.SignedBlindedBeaconBlock.defaultValue()],
     res: {version: ForkName.bellatrix, data: ssz.bellatrix.ExecutionPayload.defaultValue()},
   },
-  submitBlindedBlockV2: {
-    args: [ssz.deneb.SignedBlindedBeaconBlock.defaultValue()],
-    res: {version: ForkName.deneb, data: ssz.deneb.SignedBeaconBlockAndBlobsSidecar.defaultValue()},
-  },
 };

packages/beacon-node/package.json

@@ -131,7 +131,7 @@
     "@multiformats/multiaddr": "^11.0.0",
     "@types/datastore-level": "^3.0.0",
     "buffer-xor": "^2.0.2",
-    "c-kzg": "^1.0.9",
+    "c-kzg": "^2.0.0",
     "cross-fetch": "^3.1.4",
     "datastore-core": "^8.0.1",
     "datastore-level": "^9.0.1",
@@ -157,9 +157,6 @@
     "varint": "^6.0.0",
     "xxhash-wasm": "1.0.1"
   },
-  "peerDependencies": {
-    "c-kzg": "^1.0.7"
-  },
   "devDependencies": {
     "@types/bl": "^5.0.1",
     "@types/eventsource": "^1.1.5",