Localize NYPD, Magic and Olympics examples

.gitignore

@@ -218,3 +218,5 @@ playwright/.cache/
 
 .pyodide-xbuildenv
 benchmark_venv
+
+venv/

README.md

@@ -35,7 +35,7 @@ and/or [Jupyterlab](https://jupyterlab.readthedocs.io/en/stable/).
 ### Examples
 
 <!-- Examples -->
-<table><tbody><tr><td>editable</td><td>file</td><td>fractal</td></tr><tr><td><a href="https://perspective.finos.org/block?example=editable"><img height="125" src="https://perspective.finos.org/blocks/editable/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=file"><img height="125" src="https://perspective.finos.org/blocks/file/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=fractal"><img height="125" src="https://perspective.finos.org/blocks/fractal/preview.png"></img></a></td></tr><tr><td>market</td><td>raycasting</td><td>evictions</td></tr><tr><td><a href="https://perspective.finos.org/block?example=market"><img height="125" src="https://perspective.finos.org/blocks/market/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=raycasting"><img height="125" src="https://perspective.finos.org/blocks/raycasting/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=evictions"><img height="125" src="https://perspective.finos.org/blocks/evictions/preview.png"></img></a></td></tr><tr><td>streaming</td><td>covid</td><td>movies</td></tr><tr><td><a href="https://perspective.finos.org/block?example=streaming"><img height="125" src="https://perspective.finos.org/blocks/streaming/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=covid"><img height="125" src="https://perspective.finos.org/blocks/covid/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=movies"><img height="125" src="https://perspective.finos.org/blocks/movies/preview.png"></img></a></td></tr><tr><td>superstore</td><td>citibike</td><td>olympics</td></tr><tr><td><a href="https://perspective.finos.org/block?example=superstore"><img height="125" src="https://perspective.finos.org/blocks/superstore/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=citibike"><img height="125" src="https://perspective.finos.org/blocks/citibike/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=olympics"><img height="125" src="https://perspective.finos.org/blocks/olympics/preview.png"></img></a></td></tr><tr><td>jupyterlab</td><td>magic</td><td>nft</td></tr><tr><td><a href="http://beta.mybinder.org/v2/gh/finos/perspective/master?urlpath=lab/tree/examples/jupyter-notebooks"><img height="125" src="https://perspective.finos.org/img/jupyterlab.png"></img></a></td><td><a href="https://texodus.github.io/mtg-perspective/?seasons-in-the-abyss-67"><img height="125" src="https://perspective.finos.org/img/mtg_preview.png"></img></a></td><td><a href="https://sc1f.github.io/pudgy-penguin-perspective/"><img height="125" src="https://raw.githubusercontent.com/sc1f/pudgy-penguin-perspective/pages/meta.png"></img></a></td></tr><tr><td>nypd ccrb</td></tr><tr><td><a href="https://texodus.github.io/nypd-ccrb/"><img height="125" src="https://texodus.github.io/nypd-ccrb/preview.png"></img></a></td></tr></tbody></table>
+<table><tbody><tr><td>editable</td><td>file</td><td>fractal</td></tr><tr><td><a href="https://perspective.finos.org/block?example=editable"><img height="125" src="https://perspective.finos.org/blocks/editable/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=file"><img height="125" src="https://perspective.finos.org/blocks/file/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=fractal"><img height="125" src="https://perspective.finos.org/blocks/fractal/preview.png"></img></a></td></tr><tr><td>market</td><td>raycasting</td><td>evictions</td></tr><tr><td><a href="https://perspective.finos.org/block?example=market"><img height="125" src="https://perspective.finos.org/blocks/market/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=raycasting"><img height="125" src="https://perspective.finos.org/blocks/raycasting/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=evictions"><img height="125" src="https://perspective.finos.org/blocks/evictions/preview.png"></img></a></td></tr><tr><td>nypd</td><td>magic</td><td>streaming</td></tr><tr><td><a href="https://perspective.finos.org/block?example=nypd"><img height="125" src="https://perspective.finos.org/blocks/nypd/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=magic"><img height="125" src="https://perspective.finos.org/blocks/magic/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=streaming"><img height="125" src="https://perspective.finos.org/blocks/streaming/preview.png"></img></a></td></tr><tr><td>covid</td><td>movies</td><td>superstore</td></tr><tr><td><a href="https://perspective.finos.org/block?example=covid"><img height="125" src="https://perspective.finos.org/blocks/covid/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=movies"><img height="125" src="https://perspective.finos.org/blocks/movies/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=superstore"><img height="125" src="https://perspective.finos.org/blocks/superstore/preview.png"></img></a></td></tr><tr><td>citibike</td><td>olympics</td><td>jupyterlab</td></tr><tr><td><a href="https://perspective.finos.org/block?example=citibike"><img height="125" src="https://perspective.finos.org/blocks/citibike/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=olympics"><img height="125" src="https://perspective.finos.org/blocks/olympics/preview.png"></img></a></td><td><a href="http://beta.mybinder.org/v2/gh/finos/perspective/master?urlpath=lab/tree/examples/jupyter-notebooks"><img height="125" src="https://perspective.finos.org/img/jupyterlab.png"></img></a></td></tr></tbody></table>
 <!-- Examples -->
 
 ### Documentation

docs/build.js

@@ -100,11 +100,21 @@ async function run_with_theme(page, is_dark = false) {
 }
 
 async function run() {
-    const browser = await puppeteer.launch({ headless: true });
-    const page = await browser.newPage();
-    await run_with_theme(page);
-    await run_with_theme(page, true);
-    await browser.close();
+    if (!fs.existsSync("static/features")) {
+        const browser = await puppeteer.launch({ headless: false });
+        const page = await browser.newPage();
+        await run_with_theme(page);
+        await run_with_theme(page, true);
+        await browser.close();
+    }
+
+    // TODO There is a typescript module annoyingly called `blocks`.
+    if (!fs.existsSync("static/blocks")) {
+        fs.mkdirSync("static/blocks");
+    }
+
+    const { dist_examples } = await import("../examples/blocks/index.mjs");
+    await dist_examples(`${__dirname}/static/blocks`);
 }
 
 function template(is_dark) {
@@ -137,7 +147,7 @@ function template(is_dark) {
         </perspective-viewer>
         <script type="module">
             import perspective from "/node_modules/@finos/perspective/dist/cdn/perspective.js";
-            const WORKER = window.perspective.worker();
+            const WORKER = perspective.worker();
             async function on_load() {
                 var el = document.getElementsByTagName('perspective-viewer')[0];
                 const plugin = await el.getPlugin("Heatmap");

docs/docusaurus.config.js

@@ -15,11 +15,6 @@ const darkCodeTheme = require("prism-react-renderer/themes/dracula");
 
 const fs = require("fs");
 
-// TODO There is a typescript module annoyinglt called `blocks`.
-const blocks = import("../examples/blocks/index.mjs");
-
-blocks.then(({ dist_examples }) => dist_examples(`${__dirname}/static/blocks`));
-
 const examples = fs.readdirSync("static/blocks").map((ex) => {
     const files = fs
         .readdirSync(`static/blocks/${ex}`)

docs/package.json

@@ -5,7 +5,7 @@
     "scripts": {
         "docusaurus": "docusaurus",
         "start": "docusaurus start",
-        "docs": "docusaurus build",
+        "docs": "node build.js && docusaurus build",
         "swizzle": "docusaurus swizzle",
         "deploy": "docusaurus deploy",
         "clean": "docusaurus clear",

examples/blocks/examples.js

@@ -17,6 +17,8 @@ const LOCAL_EXAMPLES = [
     "market",
     "raycasting",
     "evictions",
+    "nypd",
+    "magic",
     "streaming",
     "covid",
     "movies",
@@ -36,21 +38,6 @@ exports.get_examples = function get_examples(
             url: "http://beta.mybinder.org/v2/gh/finos/perspective/master?urlpath=lab/tree/examples/jupyter-notebooks",
             name: "jupyterlab",
         },
-        {
-            img: "https://perspective.finos.org/img/mtg_preview.png",
-            url: "https://texodus.github.io/mtg-perspective/?seasons-in-the-abyss-67",
-            name: "magic",
-        },
-        {
-            img: "https://raw.githubusercontent.com/sc1f/pudgy-penguin-perspective/pages/meta.png",
-            url: "https://sc1f.github.io/pudgy-penguin-perspective/",
-            name: "nft",
-        },
-        {
-            img: "https://texodus.github.io/nypd-ccrb/preview.png",
-            url: "https://texodus.github.io/nypd-ccrb/",
-            name: "nypd ccrb",
-        },
     ];
 
     const hashes = LOCAL_EXAMPLES.map((x) => ({

examples/blocks/index.mjs

@@ -30,7 +30,7 @@ const replacements = {
     "perspective-workspace/dist/cdn/perspective-workspace.js": `perspective-workspace@${version}/dist/cdn/perspective-workspace.js`,
 };
 
-export function dist_examples(
+export async function dist_examples(
     outpath = `${__dirname}/../../docs/static/blocks`
 ) {
     sh`mkdir -p ${outpath}`.runSync();
@@ -45,9 +45,14 @@ export function dist_examples(
     for (const name of LOCAL_EXAMPLES) {
         // Copy
         if (fs.existsSync(`${__dirname}/src/${name}`)) {
+            // Copy
             for (const filename of fs.readdirSync(`${__dirname}/src/${name}`)) {
                 sh`mkdir -p ${outpath}/${name}`.runSync();
-                if (filename.endsWith(".js") || filename.endsWith(".html")) {
+                if (
+                    filename.endsWith(".mjs") ||
+                    filename.endsWith(".js") ||
+                    filename.endsWith(".html")
+                ) {
                     let filecontents = fs
                         .readFileSync(`${__dirname}/src/${name}/${filename}`)
                         .toString();
@@ -61,6 +66,12 @@ export function dist_examples(
                         `${outpath}/${name}/${filename}`,
                         filecontents
                     );
+
+                    if (filename.endsWith("build.mjs")) {
+                        console.log("Building " + name);
+                        const script = `${outpath}/${name}/build.mjs`;
+                        sh`node ${script}`.runSync();
+                    }
                 } else if (filename !== ".git") {
                     sh`cp ${__dirname}/src/${name}/${filename} ${outpath}/${name}/${filename}`.runSync();
                 }

examples/blocks/server.mjs

@@ -19,7 +19,7 @@ import { dirname } from "path";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
-dist_examples(`${__dirname}/dist`);
+await dist_examples(`${__dirname}/dist`);
 
 const template = (body) => `
 <!DOCTYPE html>

examples/blocks/src/magic/.block

@@ -0,0 +1,2 @@
+license: apache-2.0
+height: 800

examples/blocks/src/magic/README.md

@@ -0,0 +1,7 @@
+A demo showing some visualization of decks of from the collectable trading card
+game ["Magic: The Gathering"](https://magic.wizards.com/en). The dataset is a
+single ~90,000 row database of every card ever published, which can be explored
+as before loading a deck. By clicking the "Load Deck" button, a user created
+deck provided by the [Tapped Out API](https://tappedout.net/) can be supplied
+to generate some simple analytics by projecting cards from this deck using
+Perspective's filtering.

examples/blocks/src/magic/build.mjs

@@ -0,0 +1,188 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+import perspective from "@finos/perspective";
+
+import * as fs from "fs";
+import * as https from "https";
+import * as url from "url";
+
+const __dirname = url.fileURLToPath(new URL(".", import.meta.url));
+
+const ARRAY_SIZE_HINTS = {
+    colors: 4,
+    keywords: 3,
+    types: 3,
+};
+
+function inferSchema(card) {
+    const schema = {
+        name: undefined,
+        manaCost: undefined,
+        convertedManaCost: undefined,
+        types: undefined,
+        colorIdentity: undefined,
+        keywords: undefined,
+    };
+    const accessors = {};
+    for (const field of Object.keys(card)) {
+        const value = card[field];
+        if (typeof value === "string") {
+            schema[field] = "string";
+            accessors[field] = (card) => card[field] || "-";
+        } else if (typeof value === "number") {
+            schema[field] = "integer";
+            accessors[field] = (card) => card[field] || null;
+        } else if (Array.isArray(value)) {
+            schema[field] = "string";
+            accessors[field] = (card) => (card[field] || ["-"]).join(",");
+            for (
+                let i = 0;
+                i < (ARRAY_SIZE_HINTS[field] || value.length);
+                i++
+            ) {
+                let frozen_i = i;
+                schema[`${field}_${i}`] = "string";
+                accessors[`${field}_${i}`] = (card) =>
+                    card[field] ? card[field][frozen_i] || "-" : "-";
+            }
+        } else {
+            console.warn(
+                `Dropping column "${field}" from ${JSON.stringify(card[field])}`
+            );
+        }
+    }
+    return { schema, accessors };
+}
+
+function download_json(url) {
+    return new Promise((resolve, reject) => {
+        https
+            .get(url, (resp) => {
+                let data = "";
+                resp.on("data", (chunk) => {
+                    data += chunk;
+                });
+                resp.on("end", () => {
+                    resolve(data);
+                });
+            })
+            .on("error", reject);
+    });
+}
+
+async function main() {
+    if (fs.existsSync(`${__dirname}/all_identifiers.arrow`)) {
+        return;
+    }
+
+    const buffer = JSON.parse(
+        await download_json(`https://mtgjson.com/api/v5/AllIdentifiers.json`)
+    );
+
+    let { _schema, accessors } = inferSchema(
+        buffer.data[Object.keys(buffer.data)[0]]
+    );
+
+    const schema = {
+        name: "string",
+        manaCost: "string",
+        convertedManaCost: "integer",
+        types: "string",
+        colorIdentity: "string",
+        text: "string",
+        keywords: "string",
+        artist: "string",
+        artistIds: "string",
+        artistIds_0: "string",
+        availability: "string",
+        availability_0: "string",
+        borderColor: "string",
+        colorIdentity_0: "string",
+        colors: "string",
+        colors_0: "string",
+        colors_1: "string",
+        colors_2: "string",
+        colors_3: "string",
+        edhrecRank: "integer",
+        edhrecSaltiness: "integer",
+        finishes: "string",
+        finishes_0: "string",
+        flavorText: "string",
+        // "foreignData": "string",
+        frameVersion: "string",
+        keywords_0: "string",
+        keywords_1: "string",
+        keywords_2: "string",
+        language: "string",
+        layout: "string",
+        manaValue: "integer",
+        number: "string",
+        power: "string",
+        // printings: "string",
+        // printings_0: "string",
+        // printings_1: "string",
+        promoTypes: "string",
+        promoTypes_0: "string",
+        promoTypes_1: "string",
+        promoTypes_2: "string",
+        rarity: "string",
+        // rulings: "string",
+        // rulings_0: "string",
+        securityStamp: "string",
+        setCode: "string",
+        subtypes: "string",
+        subtypes_0: "string",
+        supertypes: "string",
+        toughness: "string",
+        type: "string",
+        types_0: "string",
+        types_1: "string",
+        types_2: "string",
+        uuid: "string",
+        scryfallId: "string",
+    };
+
+    // schema["scryfallId"] = "string";
+    accessors["scryfallId"] = (card) => card.identifiers.scryfallId;
+
+    const table = await perspective.table(schema, { index: "uuid" });
+
+    let rows = [];
+    for (const uuid of Object.keys(buffer.data)) {
+        const card = buffer.data[uuid];
+        const row = {};
+        for (const field of Object.keys(schema)) {
+            try {
+                row[field] = accessors[field](card);
+            } catch (e) {
+                console.error(field);
+                throw e;
+            }
+        }
+        rows.push(row);
+        if (rows.length > 100) {
+            table.update(rows);
+            rows = [];
+        }
+    }
+
+    table.update(rows);
+    const arrow = await (await table.view()).to_arrow();
+    fs.writeFileSync(
+        `${__dirname}/all_identifiers.arrow`,
+        Buffer.from(arrow),
+        "binary"
+    );
+}
+
+await main();