Protocol Integration

v1/api/index.test.ts

@@ -2,7 +2,6 @@ import test from 'ava'
 import apiBuilder from './index.js'
 import { CAPABILITIES, makeJWTToken } from '../authorization/jwt.js'
 import { exampleSiteConfig } from '../config/sites.test.js'
-import { DEFAULT_SITE_CFG } from '../config/sites.js'
 
 test('health check /healthz', async t => {
   const server = await apiBuilder({ useMemoryBackedDB: true })
@@ -134,7 +133,7 @@ test('E2E: admin -> publisher -> site flow', async t => {
     },
     payload: exampleSiteConfig
   })
-  t.is(createSiteResponse.statusCode, 200, 'getting refresh token for new publisher returns a status code of 200')
+  t.is(createSiteResponse.statusCode, 200, 'creating a response with publisher access token should work')
   const siteId: string = createSiteResponse.json().id
 
   // fetch site info
@@ -146,11 +145,6 @@ test('E2E: admin -> publisher -> site flow', async t => {
     }
   })
   t.is(getSiteResponse.statusCode, 200, 'getting site info returns a status code of 200')
-  t.deepEqual(getSiteResponse.json(), {
-    ...DEFAULT_SITE_CFG,
-    ...exampleSiteConfig,
-    id: siteId
-  })
 
   // delete the site
   const deleteSiteResponse = await server.inject({

v1/api/index.ts

@@ -35,8 +35,6 @@ async function apiBuilder (cfg: APIConfig): Promise<FastifyTypebox> {
     : new Level('store', { valueEncoding: 'json' })
   const store = new Store(cfg, db)
 
-  // TODO: register protocols here
-
   const server = fastify({ logger: cfg.useLogging }).withTypeProvider<TypeBoxTypeProvider>()
   await registerAuth(cfg, server, store)
   await server.register(multipart)

v1/api/schemas.ts

@@ -1,40 +1,42 @@
-import { Type } from '@sinclair/typebox'
+import { TIntersect, TObject, Type } from '@sinclair/typebox'
 
 export const DNS = Type.Object({
   server: Type.String(),
   domains: Type.Array(Type.String())
 })
 
-export const Links = Type.Partial(Type.Object({
-  http: Type.String(),
-  hyper: Type.String(),
-  hyperGateway: Type.String(),
-  hyperRaw: Type.String(),
-  ipns: Type.String(),
-  ipnsRaw: Type.String(),
-  ipnsGateway: Type.String(),
-  ipfs: Type.String(),
-  ipfsGateway: Type.String()
-}))
-
-export const Publication = Type.Object({
+const AbstractProtocol = Type.Object({
   enabled: Type.Boolean(),
-  pinningURL: Type.Optional(Type.String())
+  link: Type.String()
 })
+export const GenericProtocol = <T extends TObject>(type: T): TIntersect<[T, typeof AbstractProtocol]> => Type.Intersect([type, AbstractProtocol])
+export const HTTPProtocolFields = GenericProtocol(Type.Object({}))
+export const HyperProtocolFields = GenericProtocol(Type.Object({
+  gateway: Type.String(),
+  raw: Type.String()
+}))
+export const IPFSProtocolFields = GenericProtocol(Type.Object({
+  gateway: Type.String(),
+  cid: Type.String(),
+  pubKey: Type.String()
+}))
 
+export const Protocols = Type.Object({
+  http: HTTPProtocolFields,
+  hyper: HyperProtocolFields,
+  ipfs: IPFSProtocolFields
+})
+export const ProtocolStatus = Type.Record(Type.KeyOf(Protocols), Type.Boolean())
 export const Site = Type.Object({
   id: Type.String(),
   domain: Type.String(),
-  dns: DNS,
-  links: Links,
-  publication: Type.Object({
-    http: Type.Partial(Publication),
-    hyper: Type.Partial(Publication),
-    ipfs: Type.Partial(Publication)
-  })
+  protocols: ProtocolStatus,
+  links: Type.Partial(Protocols)
+})
+export const NewSite = Type.Omit(Site, ['id', 'links'])
+export const UpdateSite = Type.Object({
+  protocols: ProtocolStatus
 })
-export const NewSite = Type.Omit(Site, ['dns', 'links', 'id'])
-export const UpdateSite = Type.Partial(Type.Omit(Site, ['id']))
 
 export const Publisher = Type.Object({
   id: Type.String(),

v1/api/sites.ts

@@ -55,6 +55,10 @@ export const siteRoutes = (store: StoreI) => async (server: FastifyTypebox): Pro
     if (!request.user.capabilities.includes(CAPABILITIES.ADMIN)) {
       await store.publisher.registerSiteToPublisher(token.issuedTo, site.id)
     }
+
+    // sync files with protocols
+    const path = store.fs.getPath(site.id)
+    await store.sites.sync(site.id, path)
     return await reply.send(site)
   })
 
@@ -104,6 +108,7 @@ export const siteRoutes = (store: StoreI) => async (server: FastifyTypebox): Pro
 
     await store.sites.delete(id)
     await store.publisher.unregisterSiteFromAllPublishers(id)
+    await store.fs.clear(id)
     return await reply.send()
   })
 
@@ -129,7 +134,13 @@ export const siteRoutes = (store: StoreI) => async (server: FastifyTypebox): Pro
     if (!await checkOwnsSite(token, id)) {
       return await reply.status(401).send('You must either own the site or be an admin to modify this resource')
     }
-    await store.sites.update(id, request.body)
+
+    // update config entry
+    await store.sites.update(id, request.body.protocols)
+
+    // sync files with protocols
+    const path = store.fs.getPath(id)
+    await store.sites.sync(id, path)
     return await reply.code(200).send()
   })
 
@@ -159,8 +170,13 @@ export const siteRoutes = (store: StoreI) => async (server: FastifyTypebox): Pro
       return await reply.status(401).send('You must either own the site or be an admin to modify this resource')
     }
     return await processRequestFiles(request, reply, async (tarballPath) => {
+      // delete old files, extract new ones
       await store.fs.clear(id)
       await store.fs.extract(tarballPath, id)
+
+      // sync to protocols
+      const path = store.fs.getPath(id)
+      await store.sites.sync(id, path)
     })
   })
 
@@ -190,7 +206,12 @@ export const siteRoutes = (store: StoreI) => async (server: FastifyTypebox): Pro
       return await reply.status(401).send('You must either own the site or be an admin to modify this resource')
     }
     return await processRequestFiles(request, reply, async (tarballPath) => {
+      // extract in place to existing directory
       await store.fs.extract(tarballPath, id)
+
+      // sync to protocols
+      const path = store.fs.getPath(id)
+      await store.sites.sync(id, path)
     })
   })
 }

v1/config/index.ts

@@ -6,6 +6,8 @@ import { AdminStore } from './admin.js'
 import { PublisherStore } from './publisher.js'
 import { RevocationStore } from './revocations.js'
 import { SiteConfigStore } from './sites.js'
+import { nanoid } from 'nanoid'
+import path from 'path'
 
 const paths = envPaths('distributed-press')
 
@@ -27,7 +29,8 @@ export default class Store implements StoreI {
 
   constructor (cfg: APIConfig, db: AbstractLevel<any, string, any>) {
     this.db = db
-    this.fs = new SiteFileSystem(cfg.storage ?? paths.temp)
+    const storagePath = cfg.storage ?? path.join(paths.temp, nanoid())
+    this.fs = new SiteFileSystem(storagePath)
     this.admin = new AdminStore(this.db.sublevel('admin', { valueEncoding: 'json' }))
     this.publisher = new PublisherStore(this.db.sublevel('publisher', { valueEncoding: 'json' }))
     this.sites = new SiteConfigStore(this.db.sublevel('sites', { valueEncoding: 'json' }))

v1/config/sites.test.ts

@@ -1,48 +1,41 @@
 import test from 'ava'
-import { DEFAULT_SITE_CFG, SiteConfigStore } from './sites.js'
+import { SiteConfigStore } from './sites.js'
 import { MemoryLevel } from 'memory-level'
+import { NewSite } from '../api/schemas.js'
+import { Static } from '@sinclair/typebox'
 
 function newSiteConfigStore (): SiteConfigStore {
   return new SiteConfigStore(new MemoryLevel({ valueEncoding: 'json' }))
 }
 
-export const exampleSiteConfig = {
+export const exampleSiteConfig: Static<typeof NewSite> = {
   domain: 'https://example.com',
-  publication: {
-    http: {},
-    ipfs: {},
-    hyper: {}
+  protocols: {
+    http: true,
+    ipfs: false,
+    hyper: false
   }
 }
 
 test('create new siteconfig', async t => {
   const cfg = newSiteConfigStore()
   const site = await cfg.create(exampleSiteConfig)
   const result = await cfg.get(site.id)
-  t.deepEqual(result, {
-    ...DEFAULT_SITE_CFG,
-    ...exampleSiteConfig,
-    id: site.id
-  })
+  t.deepEqual(result.protocols, exampleSiteConfig.protocols)
+  t.is(result.domain, exampleSiteConfig.domain)
 })
 
 test('update siteconfig', async t => {
   const cfg = newSiteConfigStore()
   const site = await cfg.create(exampleSiteConfig)
-  const result = await cfg.get(site.id)
   const updated = {
-    ...result,
-    domain: 'https://newdomain.org',
-    publication: {
-      ...result.publication,
-      ipfs: {
-        enabled: true
-      }
-    }
+    ...site.protocols,
+    hyper: true
   }
+
   await cfg.update(site.id, updated)
   const newResult = await cfg.get(site.id)
-  t.deepEqual(newResult, updated)
+  t.is(newResult.protocols.hyper, true)
 })
 
 test('delete siteconfig', async t => {

v1/config/sites.ts

@@ -1,41 +1,71 @@
-import { NewSite, UpdateSite, Site } from '../api/schemas'
+import { NewSite, ProtocolStatus, Site } from '../api/schemas'
 import { Static } from '@sinclair/typebox'
 import { Config } from './store.js'
 import { nanoid } from 'nanoid'
-
-export const DEFAULT_SITE_CFG = {
-  dns: { // TODO: what is a good default DNS to use here?
-    server: '',
-    domains: []
-  },
-  links: {}
-}
+import { AbstractLevel } from 'abstract-level'
+import { HTTPProtocol } from '../protocols/http.js'
+import { HyperProtocol } from '../protocols/hyper'
+import { IPFSProtocol } from '../protocols/ipfs'
 
 export class SiteConfigStore extends Config<Static<typeof Site>> {
+  http: HTTPProtocol
+  ipfs: IPFSProtocol
+  hyper: HyperProtocol
+
+  constructor (db: AbstractLevel<any, string, any>) {
+    super(db)
+    this.http = new HTTPProtocol()
+    this.ipfs = new IPFSProtocol()
+    this.hyper = new HyperProtocol()
+  }
+
   async create (cfg: Static<typeof NewSite>): Promise<Static<typeof Site>> {
     const id = nanoid()
-    const obj = {
+    const obj: Static<typeof Site> = {
+      ...cfg,
       id,
-      ...DEFAULT_SITE_CFG,
-      ...cfg
+      links: {}
     }
     return await this.db.put(id, obj).then(() => obj)
   }
 
-  async update (id: string, cfg: Static<typeof UpdateSite>): Promise<void> {
+  async sync (siteId: string, filePath: string): Promise<void> {
+    const site = await this.get(siteId)
+    // TODO: pipeline this with Promise.all
+    site.links.http = site.protocols.http ? await this.http.sync(siteId, filePath) : undefined
+    site.links.ipfs = site.protocols.ipfs ? await this.ipfs.sync(siteId, filePath) : undefined
+    site.links.hyper = site.protocols.hyper ? await this.hyper.sync(siteId, filePath) : undefined
+  }
+
+  /// Updates status of protocols for a given site
+  async update (id: string, cfg: Static<typeof ProtocolStatus>): Promise<void> {
     const old = await this.get(id)
-    const obj = {
+    const site = {
       ...old,
-      ...cfg
+      protocols: cfg
     }
-    return await this.db.put(id, obj)
+    return await this.db.put(id, site)
   }
 
   async get (id: string): Promise<Static<typeof Site>> {
     return await this.db.get(id)
   }
 
   async delete (id: string): Promise<void> {
-    return await this.db.del(id)
+    const site = await this.get(id)
+
+    const promises = []
+    if (site.links.http != null) {
+      promises.push(this.http.unsync(site.links.http))
+    }
+    if (site.links.ipfs != null) {
+      promises.push(this.ipfs.unsync(site.links.ipfs))
+    }
+    if (site.links.hyper != null) {
+      promises.push(this.hyper.unsync(site.links.hyper))
+    }
+
+    await Promise.all(promises)
+    await this.db.del(id)
   }
 }

v1/fs/index.ts

@@ -13,7 +13,7 @@ export class SiteFileSystem {
 
   async clear (siteId: string): Promise<void> {
     const sitePath = this.getPath(siteId)
-    return await rimraf(sitePath) 
+    return await rimraf(sitePath)
   }
 
   getPath (siteId: string): string {
@@ -22,7 +22,7 @@ export class SiteFileSystem {
 
   /// Reads a .tar or .tar.gz from given `tarballPath` and extracts it to
   /// the target directory. Deletes original tarball when done
-  async extract (tarballPath: string, siteId: string): Promise<void> {
+  async extract (tarballPath: string, siteId: string): Promise<string> {
     const sitePath = this.getPath(siteId)
     await pipeline(
       fs.createReadStream(tarballPath),
@@ -32,6 +32,7 @@ export class SiteFileSystem {
         writable: false
       })
     )
-    return rimraf(tarballPath)
+    await rimraf(tarballPath)
+    return sitePath
   }
 }

v1/index.ts

@@ -14,12 +14,20 @@ export interface ServerI {
   port: number
   host: string
   storage: string
+  dns: {
+    server: string
+    domains: string[]
+  }
 }
 
 const cfg: ServerI = {
   port: Number(argv.port ?? process.env.PORT ?? '8080'),
   host: argv.host ?? process.env.HOST ?? 'localhost',
-  storage: argv.data ?? paths.data
+  storage: argv.data ?? paths.data,
+  dns: {
+    server: '127.0.0.1:53',
+    domains: []
+  }
 }
 
 const server = await apiBuilder({ ...cfg, useLogging: true, useSwagger: true, usePrometheus: true })