Update web3.storage API

README.md

@@ -7,7 +7,7 @@ Evaluate service
 ## Development
 
 ```bash
-$ WALLET_SEED=$(cat secrets/mnemonic) WEB3_STORAGE_API_TOKEN=$(cat secrets/web3storage) npm start
+$ WALLET_SEED=$(cat secrets/mnemonic) npm start
 $ npm test
 ```
 

bin/dry-run.js

@@ -1,6 +1,6 @@
 import { IE_CONTRACT_ADDRESS, RPC_URL } from '../lib/config.js'
 import { evaluate } from '../lib/evaluate.js'
-import { preprocess, fetchMeasurementsViaGateway } from '../lib/preprocess.js'
+import { preprocess, fetchMeasurements } from '../lib/preprocess.js'
 import { fetchRoundDetails } from '../lib/spark-api.js'
 import { Point } from '../lib/telemetry.js'
 import { readFile, writeFile, mkdir } from 'node:fs/promises'
@@ -50,15 +50,15 @@ const recordTelemetry = (measurementName, fn) => {
   console.log('TELEMETRY %s %o', measurementName, point.fields)
 }
 
-const fetchMeasurements = async (cid) => {
+const fetchMeasurementsWithCache = async (cid) => {
   const pathOfCachedResponse = path.join(cacheDir, cid + '.json')
   try {
     return JSON.parse(await readFile(pathOfCachedResponse, 'utf-8'))
   } catch (err) {
     if (err.code !== 'ENOENT') console.warn('Cannot read cached measurements:', err)
   }
 
-  const measurements = await fetchMeasurementsViaGateway(cid)
+  const measurements = await fetchMeasurements(cid)
   await writeFile(pathOfCachedResponse, JSON.stringify(measurements))
   return measurements
 }
@@ -73,7 +73,7 @@ for (const cid of measurementCids) {
     roundIndex,
     rounds,
     cid,
-    fetchMeasurements,
+    fetchMeasurements: fetchMeasurementsWithCache,
     recordTelemetry,
     logger: console
   })

bin/spark-evaluate.js

@@ -6,15 +6,13 @@ import assert from 'node:assert'
 import { ethers } from 'ethers'
 import { fileURLToPath } from 'node:url'
 import { newDelegatedEthAddress } from '@glif/filecoin-address'
-import { Web3Storage } from 'web3.storage'
 import { recordTelemetry } from '../lib/telemetry.js'
 import fs from 'node:fs/promises'
-import { fetchMeasurementsViaClient } from '../lib/preprocess.js'
+import { fetchMeasurements } from '../lib/preprocess.js'
 
 const {
   SENTRY_ENVIRONMENT = 'development',
-  WALLET_SEED,
-  WEB3_STORAGE_API_TOKEN
+  WALLET_SEED
 } = process.env
 
 Sentry.init({
@@ -25,7 +23,6 @@ Sentry.init({
 })
 
 assert(WALLET_SEED, 'WALLET_SEED required')
-assert(WEB3_STORAGE_API_TOKEN, 'WEB3_STORAGE_API_TOKEN required')
 
 const provider = new ethers.providers.JsonRpcProvider(RPC_URL)
 const signer = ethers.Wallet.fromMnemonic(WALLET_SEED).connect(provider)
@@ -45,8 +42,6 @@ const ieContract = new ethers.Contract(
   provider
 )
 const ieContractWithSigner = ieContract.connect(signer)
-const web3Storage = new Web3Storage({ token: WEB3_STORAGE_API_TOKEN })
-const fetchMeasurements = (cid) => fetchMeasurementsViaClient(web3Storage, cid)
 
 startEvaluate({
   ieContract,

lib/preprocess.js

@@ -1,6 +1,9 @@
 import assert from 'node:assert'
 import { ethers } from 'ethers'
 import { ethAddressFromDelegated } from '@glif/filecoin-address'
+import { CarReader } from '@ipld/car'
+import { validateBlock } from '@web3-storage/car-block-validator'
+import { recursive as exporter } from 'ipfs-unixfs-exporter'
 import createDebug from 'debug'
 
 const debug = createDebug('spark:preprocess')
@@ -95,21 +98,35 @@ export const preprocess = async ({
   return validMeasurements
 }
 
-export const fetchMeasurementsViaClient = async (web3Storage, cid) => {
-  const res = await web3Storage.get(cid)
-  const files = await res.files()
-  const measurements = JSON.parse(await files[0].text())
-  return measurements
-}
-
-export const fetchMeasurementsViaGateway = async (cid) => {
-  const res = await fetch(`https://${encodeURIComponent(cid)}.ipfs.w3s.link/measurements.json`)
+export const fetchMeasurements = async cid => {
+  const res = await fetch(
+    `https://${encodeURIComponent(cid)}.ipfs.w3s.link?format=car`
+  )
   if (!res.ok) {
     const msg = `Cannot fetch measurements ${cid}: ${res.status}\n${await res.text()}`
     throw new Error(msg)
   }
-  const measurements = await res.json()
-  return measurements
+  const reader = await CarReader.fromIterable(res.body)
+  const entries = exporter(cid, {
+    async get (cid) {
+      const block = await reader.get(cid)
+      await validateBlock(block)
+      return block.bytes
+    }
+  })
+  for await (const entry of entries) {
+    // Depending on size, entries might be packaged as `file` or `raw`
+    // https://github.com/web3-storage/w3up/blob/e8bffe2ee0d3a59a977d2c4b7efe425699424e19/packages/upload-client/src/unixfs.js#L11
+    if (entry.type === 'file' || entry.type === 'raw') {
+      const bufs = []
+      for await (const buf of entry.content()) {
+        bufs.push(buf)
+      }
+      const json = Buffer.concat(bufs).toString()
+      return JSON.parse(json)
+    }
+  }
+  throw new Error('No measurements found')
 }
 
 export const parseParticipantAddress = filWalletAddress => {