feat: write invocations and receipts into ucan log (#592)

Fixes #581 

## Overview

- Integrates changes from #579
  - I have added `undici` to [pnpm overrides](https://pnpm.io/package_json)
- Gets rid of `raw` route in favor of `root` which now uses decoder/encoder based of content type
- Defines `UCANLog` interface for sending invocations & receipts to kenisis
   - As per #579 we probably want to use CF Queue for failure tolerance which can be a followup improvement
- Type fixes to unblock some changes
   - You can't do `@implements {import('./bindings').UCANLog}` nor you can type alias and do implements because it no longer an interface. Solution is to do `import * as API from "./bindings.js"` in which case you can `@implements {API.UCANLog}`, which is why end up changing `bindings.d.ts` to `bindings.ts` + `bindings.js`.
   - Turns out `.d.ts` files aren't type checked so ☝️ uncovered bunch of errors I had to fix

## Notes

- I'm not happy how this reaches deep into ucanto & I'd like to uplift changes around encode / decode / execute into ucanto. Once that is done I'll make a followup change here to replace some of these with ucanto exposed functionality
   - It also would make it easier to integrate things in the upload-api
- I do not seem to have privileges to add vars and secrets in this, so someone with enough privileges should add `UCAN_LOG_URL` or` UCAN_LOG_BASIC_AUTH` so someone needs to do it to make this work.
- I end up sending invocations and receipts to the same endpoint but different content-type header which I think makes sense.
   - Also unlike #579 I've decided to decouple endpoint from upload endpoint as they're logically unrelated.

.github/workflows/reusable-deploy-api.yml

@@ -24,6 +24,8 @@ on:
         required: true
       LOGTAIL_TOKEN:
         required: true
+      UCAN_LOG_BASIC_AUTH:
+        required: true
 
 jobs:
   deploy-api:
@@ -58,10 +60,12 @@ jobs:
             PRIVATE_KEY
             SENTRY_DSN
             LOGTAIL_TOKEN
+            UCAN_LOG_BASIC_AUTH
         env:
           POSTMARK_TOKEN: ${{ secrets.POSTMARK_TOKEN }}
           PRIVATE_KEY: ${{ secrets.PRIVATE_KEY }}
           SENTRY_DSN: ${{ secrets.SENTRY_DSN }}
           SENTRY_UPLOAD: ${{ secrets.SENTRY_UPLOAD }}
           SENTRY_TOKEN: ${{ secrets.SENTRY_TOKEN }}
           LOGTAIL_TOKEN: ${{ secrets.LOGTAIL_TOKEN }}
+          UCAN_LOG_BASIC_AUTH: ${{ secrets.UCAN_LOG_BASIC_AUTH }}

packages/access-api/package.json

@@ -8,6 +8,7 @@
   "private": true,
   "scripts": {
     "lint": "tsc --build && eslint '**/*.{js,ts}' && prettier --check '**/*.{js,ts,yml,json}' --ignore-path ../../.gitignore",
+    "lint:fix": "prettier --check '**/*.{js,ts,yml,json}' --ignore-path ../../.gitignore --write",
     "dev": "scripts/cli.js dev",
     "build": "scripts/cli.js build",
     "check": "tsc --build",
@@ -41,6 +42,7 @@
     "@cloudflare/workers-types": "^3.19.0",
     "@databases/split-sql-query": "^1.0.3",
     "@databases/sql": "^3.2.0",
+    "@miniflare/core": "^2.11.0",
     "@miniflare/r2": "^2.12.1",
     "@sentry/cli": "2.7.0",
     "@types/assert": "^1.5.6",
@@ -144,5 +146,10 @@
       "hd-scripts",
       "better-sqlite3"
     ]
+  },
+  "pnpm": {
+    "overrides": {
+      "undici": "^5.20.0"
+    }
   }
 }

packages/access-api/src/bindings.js

@@ -0,0 +1 @@
+export {}

packages/access-api/src/bindings.ts

@@ -0,0 +1,219 @@
+import type { Logging } from '@web3-storage/worker-utils/logging'
+import type { Handler as _Handler } from '@web3-storage/worker-utils/router'
+import { Spaces } from './models/spaces.js'
+import { Validations } from './models/validations.js'
+import { loadConfig } from './config.js'
+import { ConnectionView, Signer as EdSigner } from '@ucanto/principal/ed25519'
+import { Accounts } from './models/accounts.js'
+import { DelegationsStorage as Delegations } from './types/delegations.js'
+import { ProvisionsStorage } from './types/provisions.js'
+import { R2Bucket } from '@miniflare/r2'
+import { DID, Link, Delegation, Signature, Block } from '@ucanto/interface'
+export * from '@ucanto/interface'
+
+export {}
+
+// CF Analytics Engine types not available yet
+export interface AnalyticsEngine {
+  writeDataPoint: (event: AnalyticsEngineEvent) => void
+}
+
+export interface AnalyticsEngineEvent {
+  readonly doubles?: number[]
+  readonly blobs?: Array<ArrayBuffer | string | null>
+}
+
+export interface Email {
+  sendValidation: (input: { to: string; url: string }) => Promise<void>
+  send: (input: {
+    to: string
+    textBody: string
+    subject: string
+  }) => Promise<void>
+}
+
+// We can't use interface here or it will not extend Record and cause type error
+// eslint-disable-next-line @typescript-eslint/consistent-type-definitions
+export type Env = {
+  // vars
+  ENV: string
+  DEBUG: string
+  /**
+   * publicly advertised decentralized identifier of the running api service
+   * * this may be used to filter incoming ucanto invocations
+   */
+  DID: `did:web:${string}`
+  // URLs to upload-api so we proxy invocations to it
+  UPLOAD_API_URL: string
+  // secrets
+  PRIVATE_KEY: string
+  SENTRY_DSN: string
+  POSTMARK_TOKEN: string
+  POSTMARK_SENDER?: string
+  UCAN_LOG_URL?: string
+  UCAN_LOG_BASIC_AUTH?: string
+
+  /** CSV DIDs of services that can be used to provision spaces. */
+  PROVIDERS?: string
+  DEBUG_EMAIL?: string
+  LOGTAIL_TOKEN: string
+  // bindings
+  SPACES: KVNamespace
+  VALIDATIONS: KVNamespace
+  W3ACCESS_METRICS: AnalyticsEngine
+  /**
+   * will be used for storing env.models.delegations CARs
+   */
+  DELEGATIONS_BUCKET: R2Bucket
+  // eslint-disable-next-line @typescript-eslint/naming-convention
+  __D1_BETA__: D1Database
+}
+
+export interface HandlerContext {
+  waitUntil: (promise: Promise<any>) => void
+}
+
+export interface RouteContext {
+  log: Logging
+  signer: EdSigner.Signer
+  config: ReturnType<typeof loadConfig>
+  url: URL
+  email: Email
+  ucanLog: UCANLog
+  models: {
+    accounts: Accounts
+    delegations: Delegations
+    spaces: Spaces
+    provisions: ProvisionsStorage
+    validations: Validations
+  }
+  uploadApi: ConnectionView<any>
+}
+
+export interface UCANLog {
+  /**
+   * This can fail if it is unable to write to the underlying store. Handling
+   * invocations will be blocked until write is complete. Implementation may
+   * choose to do several retries before failing.
+   *
+   * @param car - UCAN invocations in CAR. Each invocation is a root in the CAR.
+   */
+  logInvocations: (car: Uint8Array) => Promise<void>
+  /**
+   * Takes DAG-CBOR encoded invocation receipts. It is not allowed to fail and
+   * promise is only going to be used to keep worker waiting. It is not allowed
+   * to fail because by the time it is called invocation handler has already
+   * ran and did some IO which can't be rolled back. So it's up to implementation
+   * to either keep retrying or to store receipts in some queue and retry later.
+   *
+   * @see https://github.com/ucan-wg/invocation/#8-receipt
+   *
+   * @param receipt - DAG-CBOR encoded invocation receipt
+   */
+  logReceipt: (block: ReceiptBlock) => Promise<void>
+}
+
+export interface Receipt {
+  ran: Link
+  out: ReceiptResult
+  meta: Record<string, unknown>
+  iss?: DID
+  prf?: Array<Link<Delegation>>
+
+  s: Signature
+}
+
+export interface ReceiptBlock extends Block<Receipt> {
+  data: Receipt
+}
+
+/**
+ * Defines result type as per invocation spec
+ *
+ * @see https://github.com/ucan-wg/invocation/#6-result
+ */
+
+export type ReceiptResult<T = unknown, X extends {} = {}> = Variant<{
+  ok: T
+  error: X
+}>
+
+/**
+ * Utility type for defining a [keyed union] type as in IPLD Schema. In practice
+ * this just works around typescript limitation that requires discriminant field
+ * on all variants.
+ *
+ * ```ts
+ * type Result<T, X> =
+ *   | { ok: T }
+ *   | { error: X }
+ *
+ * const demo = (result: Result<string, Error>) => {
+ *   if (result.ok) {
+ *   //  ^^^^^^^^^ Property 'ok' does not exist on type '{ error: Error; }`
+ *   }
+ * }
+ * ```
+ *
+ * Using `Variant` type we can define same union type that works as expected:
+ *
+ * ```ts
+ * type Result<T, X> = Variant<{
+ *   ok: T
+ *   error: X
+ * }>
+ *
+ * const demo = (result: Result<string, Error>) => {
+ *   if (result.ok) {
+ *     result.ok.toUpperCase()
+ *   }
+ * }
+ * ```
+ *
+ * [keyed union]:https://ipld.io/docs/schemas/features/representation-strategies/#union-keyed-representation
+ */
+export type Variant<U extends Record<string, unknown>> = {
+  [Key in keyof U]: { [K in Exclude<keyof U, Key>]?: never } & {
+    [K in Key]: U[Key]
+  }
+}[keyof U]
+
+export type Handler = _Handler<RouteContext>
+
+export type Bindings = Record<
+  string,
+  | string
+  | undefined
+  | KVNamespace
+  | DurableObjectNamespace
+  | CryptoKey
+  | D1Database
+  | AnalyticsEngine
+  | R2Bucket
+>
+
+// eslint-disable-next-line @typescript-eslint/no-namespace
+declare namespace ModuleWorker {
+  type FetchHandler<Environment extends Bindings = Bindings> = (
+    request: Request,
+    env: Environment,
+    ctx: Pick<FetchEvent, 'waitUntil' | 'passThroughOnException'>
+  ) => Promise<Response> | Response
+
+  type CronHandler<Environment extends Bindings = Bindings> = (
+    event: Omit<ScheduledEvent, 'waitUntil'>,
+    env: Environment,
+    ctx: Pick<ScheduledEvent, 'waitUntil'>
+  ) => Promise<void> | void
+}
+
+export interface ModuleWorker {
+  fetch?: ModuleWorker.FetchHandler<Env>
+  scheduled?: ModuleWorker.CronHandler<Env>
+}
+
+// D1 types
+
+export interface D1ErrorRaw extends Error {
+  cause: Error & { code: string }
+}