chore: refactor to use postgres (behind a flag) (#1024)

This PR refactors the primary database from redis to postgres when using
the `TRY_POSTGRES=1` flag.

It also ensures that inserts happen in both databases concurrently so we
can keep data in both until cutting over.

https://vercel.com/changelog/vercel-postgres-is-now-available-for-pro-users

CONTRIBUTING.md

@@ -30,11 +30,10 @@ Create a `.env` file in the root directory.
 ```
 # required settings
 REDIS_URL="redis://127.0.0.1:6379"
-GA_ID=""
 
 # optional settings
-export PORT="3000"
-export NPM_REGISTRY_URL="https://registry.npmjs.com"
+PORT="3000"
+NPM_REGISTRY_URL="https://registry.npmjs.com"
 ```
 
 ## Running the code

api/initdb.ts

@@ -0,0 +1,89 @@
+import { sql } from '@vercel/postgres';
+import { findAll } from '../src/util/backend/db-redis';
+import type { IncomingMessage, ServerResponse } from 'http';
+
+export default async function handler(_req: IncomingMessage, res: ServerResponse) {
+    // This is a temporary function we can use to test
+    if (process.env.VERCEL_ENV !== 'development') {
+        res.statusCode = 403;
+        res.end('403 Forbidden');
+        return;
+    }
+
+    /*
+    console.log(await sql`
+        CREATE COLLATION semver (
+            LOCALE = 'en-US-u-kn-true',
+            PROVIDER = 'icu'
+        );
+    `);
+
+    console.log(
+        await sql`
+    drop table if exists "packages";
+`,
+    );
+    console.log(
+        await sql`
+    CREATE TABLE "packages" (
+        "name" VARCHAR(214),
+        "version" VARCHAR(255) COLLATE semver,
+        "publishSize" INTEGER,
+        "installSize" INTEGER,
+        "publishFiles" INTEGER,
+        "installFiles" INTEGER,
+        PRIMARY KEY ("name", "version")
+    );
+`,
+    );
+*/
+    const result = await findAll('next');
+    console.log(`inserting ${Object.keys(result).length} rows`);
+
+    for (let pkg of Object.values(result)) {
+        console.log(`inserting row ${pkg.name}@${pkg.version}`);
+        await sql`
+        INSERT INTO "packages" values (${pkg.name}, ${pkg.version}, ${pkg.publishSize}, ${pkg.installSize}, ${pkg.publishFiles}, ${pkg.installFiles});
+    `;
+    }
+    /*
+  console.log(await sql`
+    SELECT *
+    FROM "packages"
+    ORDER BY version desc;
+  `);
+  */
+
+    res.end('success');
+}
+
+/**
+ CREATE COLLATION en_natural (
+  LOCALE = 'en-US-u-kn-true',
+  PROVIDER = 'icu'
+);
+
+CREATE TABLE test (
+  version varchar(20) collate en_natural
+);
+
+INSERT INTO test values
+  ('14.1.0'),
+  ('14.20.0'),
+  ('14.11.0'),
+  ('14.10.0'),
+  ('14.2.0'),
+  ('14.0.2'),
+  ('14.0.3-canary.1'),
+  ('14.0.3-canary.0'),
+  ('14.0.3-canary.9'),
+  ('14.0.3-canary.10'),
+  ('14.0.3-canary.11'),
+  ('14.0.3-canary.12'),
+  ('14.0.3');
+
+SELECT split_part(version, '-', 1) as one, NULLIF(split_part(version, '-', 2), '') as two
+FROM test
+ORDER BY split_part(version, '-', 1) desc, NULLIF(split_part(version, '-', 2), '') desc;
+ 
+ */

package.json

@@ -14,6 +14,7 @@
   "author": "styfle",
   "license": "MIT",
   "dependencies": {
+    "@vercel/postgres": "^0.5.1",
     "badgen": "^3.2.2",
     "ioredis": "^5.1.0",
     "lru-cache": "^10.1.0",

src/page-props/common.ts

@@ -1,4 +1,6 @@
-import { findOne, insert } from '../util/backend/db';
+import { findOne } from '../util/backend/db';
+import { insert as insertPostgres } from '../util/backend/db-postgres';
+import { insert as insertRedis } from '../util/backend/db-redis';
 import { getAllDistTags } from '../util/npm-api';
 import { calculatePackageSize } from '../util/backend/npm-stats';
 import { versionUnknown } from '../util/constants';
@@ -43,7 +45,7 @@ export async function getPkgDetails(
         const end = new Date();
         const sec = (end.getTime() - start.getTime()) / 1000;
         console.log(`Calculated size of ${name}@${version} in ${sec}s`);
-        insert(pkgSize);
+        await Promise.all([insertRedis(pkgSize), insertPostgres(pkgSize)]);
     }
 
     const result = {

src/util/backend/db-postgres.ts

@@ -0,0 +1,43 @@
+import { sql } from '@vercel/postgres';
+import type { PkgSize } from '../../types';
+
+export async function findAll(name: string) {
+    console.time('findAll (postgres)');
+    const { rows } = await sql`
+        SELECT *
+        FROM "packages"
+        WHERE name = ${name};
+    `;
+    console.timeEnd('findAll (postgres)');
+    const obj: { [key: string]: PkgSize } = {};
+    for (let row of rows) {
+        const payload = row as PkgSize;
+        obj[payload.version] = payload;
+    }
+    return obj;
+}
+
+export async function findOne(name: string, version: string) {
+    console.time('findOne (postgres)');
+    const { rows } = await sql`
+        SELECT *
+        FROM "packages"
+        WHERE name = ${name}
+        AND version = ${version};
+    `;
+    console.timeEnd('findOne (postgres)');
+    const reply = rows[0];
+
+    if (!reply) {
+        return null;
+    }
+
+    return reply as PkgSize;
+}
+
+export async function insert(pkg: PkgSize) {
+    const reply = await sql`
+        INSERT INTO "packages" VALUES (${pkg.name}, ${pkg.version}, ${pkg.publishSize}, ${pkg.installSize}, ${pkg.publishFiles}, ${pkg.installFiles});
+    `;
+    return reply;
+}