Add number aggregates to date{time}s. Move examples to repo.

.gitignore

@@ -218,3 +218,5 @@ playwright/.cache/
 
 .pyodide-xbuildenv
 benchmark_venv
+
+venv/

README.md

@@ -35,7 +35,7 @@ and/or [Jupyterlab](https://jupyterlab.readthedocs.io/en/stable/).
 ### Examples
 
 <!-- Examples -->
-<table><tbody><tr><td>editable</td><td>file</td><td>fractal</td></tr><tr><td><a href="https://perspective.finos.org/block?example=editable"><img height="125" src="https://perspective.finos.org/blocks/editable/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=file"><img height="125" src="https://perspective.finos.org/blocks/file/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=fractal"><img height="125" src="https://perspective.finos.org/blocks/fractal/preview.png"></img></a></td></tr><tr><td>market</td><td>raycasting</td><td>evictions</td></tr><tr><td><a href="https://perspective.finos.org/block?example=market"><img height="125" src="https://perspective.finos.org/blocks/market/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=raycasting"><img height="125" src="https://perspective.finos.org/blocks/raycasting/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=evictions"><img height="125" src="https://perspective.finos.org/blocks/evictions/preview.png"></img></a></td></tr><tr><td>streaming</td><td>covid</td><td>movies</td></tr><tr><td><a href="https://perspective.finos.org/block?example=streaming"><img height="125" src="https://perspective.finos.org/blocks/streaming/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=covid"><img height="125" src="https://perspective.finos.org/blocks/covid/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=movies"><img height="125" src="https://perspective.finos.org/blocks/movies/preview.png"></img></a></td></tr><tr><td>superstore</td><td>citibike</td><td>olympics</td></tr><tr><td><a href="https://perspective.finos.org/block?example=superstore"><img height="125" src="https://perspective.finos.org/blocks/superstore/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=citibike"><img height="125" src="https://perspective.finos.org/blocks/citibike/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=olympics"><img height="125" src="https://perspective.finos.org/blocks/olympics/preview.png"></img></a></td></tr><tr><td>jupyterlab</td><td>magic</td><td>nft</td></tr><tr><td><a href="http://beta.mybinder.org/v2/gh/finos/perspective/master?urlpath=lab/tree/examples/jupyter-notebooks"><img height="125" src="https://perspective.finos.org/img/jupyterlab.png"></img></a></td><td><a href="https://texodus.github.io/mtg-perspective/?seasons-in-the-abyss-67"><img height="125" src="https://perspective.finos.org/img/mtg_preview.png"></img></a></td><td><a href="https://sc1f.github.io/pudgy-penguin-perspective/"><img height="125" src="https://raw.githubusercontent.com/sc1f/pudgy-penguin-perspective/pages/meta.png"></img></a></td></tr><tr><td>nypd ccrb</td></tr><tr><td><a href="https://texodus.github.io/nypd-ccrb/"><img height="125" src="https://texodus.github.io/nypd-ccrb/preview.png"></img></a></td></tr></tbody></table>
+<table><tbody><tr><td>editable</td><td>file</td><td>nypd</td></tr><tr><td><a href="https://perspective.finos.org/block?example=editable"><img height="125" src="https://perspective.finos.org/blocks/editable/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=file"><img height="125" src="https://perspective.finos.org/blocks/file/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=nypd"><img height="125" src="https://perspective.finos.org/blocks/nypd/preview.png"></img></a></td></tr><tr><td>magic</td><td>fractal</td><td>market</td></tr><tr><td><a href="https://perspective.finos.org/block?example=magic"><img height="125" src="https://perspective.finos.org/blocks/magic/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=fractal"><img height="125" src="https://perspective.finos.org/blocks/fractal/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=market"><img height="125" src="https://perspective.finos.org/blocks/market/preview.png"></img></a></td></tr><tr><td>raycasting</td><td>evictions</td><td>streaming</td></tr><tr><td><a href="https://perspective.finos.org/block?example=raycasting"><img height="125" src="https://perspective.finos.org/blocks/raycasting/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=evictions"><img height="125" src="https://perspective.finos.org/blocks/evictions/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=streaming"><img height="125" src="https://perspective.finos.org/blocks/streaming/preview.png"></img></a></td></tr><tr><td>covid</td><td>movies</td><td>superstore</td></tr><tr><td><a href="https://perspective.finos.org/block?example=covid"><img height="125" src="https://perspective.finos.org/blocks/covid/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=movies"><img height="125" src="https://perspective.finos.org/blocks/movies/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=superstore"><img height="125" src="https://perspective.finos.org/blocks/superstore/preview.png"></img></a></td></tr><tr><td>citibike</td><td>olympics</td><td>jupyterlab</td></tr><tr><td><a href="https://perspective.finos.org/block?example=citibike"><img height="125" src="https://perspective.finos.org/blocks/citibike/preview.png"></img></a></td><td><a href="https://perspective.finos.org/block?example=olympics"><img height="125" src="https://perspective.finos.org/blocks/olympics/preview.png"></img></a></td><td><a href="http://beta.mybinder.org/v2/gh/finos/perspective/master?urlpath=lab/tree/examples/jupyter-notebooks"><img height="125" src="https://perspective.finos.org/img/jupyterlab.png"></img></a></td></tr><tr><td>nft</td></tr><tr><td><a href="https://sc1f.github.io/pudgy-penguin-perspective/"><img height="125" src="https://raw.githubusercontent.com/sc1f/pudgy-penguin-perspective/pages/meta.png"></img></a></td></tr></tbody></table>
 <!-- Examples -->
 
 ### Documentation

examples/blocks/examples.js

@@ -13,6 +13,8 @@
 const LOCAL_EXAMPLES = [
     "editable",
     "file",
+    "nypd",
+    "magic",
     "fractal",
     "market",
     "raycasting",
@@ -36,21 +38,11 @@ exports.get_examples = function get_examples(
             url: "http://beta.mybinder.org/v2/gh/finos/perspective/master?urlpath=lab/tree/examples/jupyter-notebooks",
             name: "jupyterlab",
         },
-        {
-            img: "https://perspective.finos.org/img/mtg_preview.png",
-            url: "https://texodus.github.io/mtg-perspective/?seasons-in-the-abyss-67",
-            name: "magic",
-        },
         {
             img: "https://raw.githubusercontent.com/sc1f/pudgy-penguin-perspective/pages/meta.png",
             url: "https://sc1f.github.io/pudgy-penguin-perspective/",
             name: "nft",
         },
-        {
-            img: "https://texodus.github.io/nypd-ccrb/preview.png",
-            url: "https://texodus.github.io/nypd-ccrb/",
-            name: "nypd ccrb",
-        },
     ];
 
     const hashes = LOCAL_EXAMPLES.map((x) => ({

examples/blocks/index.mjs

@@ -30,7 +30,7 @@ const replacements = {
     "perspective-workspace/dist/cdn/perspective-workspace.js": `perspective-workspace@${version}/dist/cdn/perspective-workspace.js`,
 };
 
-export function dist_examples(
+export async function dist_examples(
     outpath = `${__dirname}/../../docs/static/blocks`
 ) {
     sh`mkdir -p ${outpath}`.runSync();
@@ -45,9 +45,14 @@ export function dist_examples(
     for (const name of LOCAL_EXAMPLES) {
         // Copy
         if (fs.existsSync(`${__dirname}/src/${name}`)) {
+            // Copy
             for (const filename of fs.readdirSync(`${__dirname}/src/${name}`)) {
                 sh`mkdir -p ${outpath}/${name}`.runSync();
-                if (filename.endsWith(".js") || filename.endsWith(".html")) {
+                if (
+                    filename.endsWith(".mjs") ||
+                    filename.endsWith(".js") ||
+                    filename.endsWith(".html")
+                ) {
                     let filecontents = fs
                         .readFileSync(`${__dirname}/src/${name}/${filename}`)
                         .toString();
@@ -61,6 +66,11 @@ export function dist_examples(
                         `${outpath}/${name}/${filename}`,
                         filecontents
                     );
+
+                    if (filename.endsWith("build.mjs")) {
+                        console.log("Building " + name);
+                        await import(`${__dirname}/dist/${name}/build.mjs`);
+                    }
                 } else if (filename !== ".git") {
                     sh`cp ${__dirname}/src/${name}/${filename} ${outpath}/${name}/${filename}`.runSync();
                 }

examples/blocks/server.mjs

@@ -19,7 +19,7 @@ import { dirname } from "path";
 
 const __dirname = dirname(fileURLToPath(import.meta.url));
 
-dist_examples(`${__dirname}/dist`);
+await dist_examples(`${__dirname}/dist`);
 
 const template = (body) => `
 <!DOCTYPE html>

examples/blocks/src/magic/.block

@@ -0,0 +1,2 @@
+license: apache-2.0
+height: 800

examples/blocks/src/magic/README.md

@@ -0,0 +1,3 @@
+A demo showing some visualization of decks of Magic: The Gathering decks.
+
+Deck lists were provided by Evan Wolfe, thanks!!

examples/blocks/src/magic/build.mjs

@@ -0,0 +1,158 @@
+// ┏━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
+// ┃ ██████ ██████ ██████       █      █      █      █      █ █▄  ▀███ █       ┃
+// ┃ ▄▄▄▄▄█ █▄▄▄▄▄ ▄▄▄▄▄█  ▀▀▀▀▀█▀▀▀▀▀ █ ▀▀▀▀▀█ ████████▌▐███ ███▄  ▀█ █ ▀▀▀▀▀ ┃
+// ┃ █▀▀▀▀▀ █▀▀▀▀▀ █▀██▀▀ ▄▄▄▄▄ █ ▄▄▄▄▄█ ▄▄▄▄▄█ ████████▌▐███ █████▄   █ ▄▄▄▄▄ ┃
+// ┃ █      ██████ █  ▀█▄       █ ██████      █      ███▌▐███ ███████▄ █       ┃
+// ┣━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┫
+// ┃ Copyright (c) 2017, the Perspective Authors.                              ┃
+// ┃ ╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌╌ ┃
+// ┃ This file is part of the Perspective library, distributed under the terms ┃
+// ┃ of the [Apache License 2.0](https://www.apache.org/licenses/LICENSE-2.0). ┃
+// ┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
+
+import perspective from "@finos/perspective";
+
+import * as fs from "fs";
+import * as https from "https";
+import * as url from "url";
+
+const __dirname = url.fileURLToPath(new URL(".", import.meta.url));
+
+const ARRAY_SIZE_HINTS = {
+    colors: 4,
+    keywords: 3,
+    types: 3,
+};
+
+function inferSchema(card) {
+    const schema = {
+        name: undefined,
+        manaCost: undefined,
+        convertedManaCost: undefined,
+        types: undefined,
+        colorIdentity: undefined,
+        keywords: undefined,
+    };
+    const accessors = {};
+    for (const field of Object.keys(card)) {
+        const value = card[field];
+        if (typeof value === "string") {
+            schema[field] = "string";
+            accessors[field] = (card) => card[field] || "-";
+        } else if (typeof value === "number") {
+            schema[field] = "integer";
+            accessors[field] = (card) => card[field] || null;
+        } else if (Array.isArray(value)) {
+            schema[field] = "string";
+            accessors[field] = (card) => (card[field] || ["-"]).join(",");
+            for (
+                let i = 0;
+                i < (ARRAY_SIZE_HINTS[field] || value.length);
+                i++
+            ) {
+                let frozen_i = i;
+                schema[`${field}_${i}`] = "string";
+                accessors[`${field}_${i}`] = (card) =>
+                    card[field] ? card[field][frozen_i] || "-" : "-";
+            }
+        } else {
+            console.warn(
+                `Dropping column "${field}" from ${JSON.stringify(card[field])}`
+            );
+        }
+    }
+    return { schema, accessors };
+}
+
+function download_json(url) {
+    return new Promise((resolve, reject) => {
+        https
+            .get(url, (resp) => {
+                let data = "";
+                resp.on("data", (chunk) => {
+                    data += chunk;
+                });
+                resp.on("end", () => {
+                    resolve(data);
+                });
+            })
+            .on("error", reject);
+    });
+}
+
+async function main() {
+    if (!fs.existsSync(`${__dirname}/AllIdentifiers.json`)) {
+        const json = await download_json(
+            `https://mtgjson.com/api/v5/AllIdentifiers.json`
+        );
+
+        fs.writeFileSync(`${__dirname}/AllIdentifiers.json`, json);
+    }
+
+    const buffer = JSON.parse(
+        fs.readFileSync(`${__dirname}/AllIdentifiers.json`)
+    );
+    let { schema, accessors } = inferSchema(
+        buffer.data[Object.keys(buffer.data)[0]]
+    );
+
+    schema["scryfallId"] = "string";
+    accessors["scryfallId"] = (card) => card.identifiers.scryfallId;
+
+    const table = await perspective.table(schema, { index: "uuid" });
+
+    let rows = [];
+    for (const uuid of Object.keys(buffer.data)) {
+        const card = buffer.data[uuid];
+        const row = {};
+        for (const field of Object.keys(schema)) {
+            try {
+                row[field] = accessors[field](card);
+            } catch (e) {
+                console.error(field);
+                throw e;
+            }
+        }
+        rows.push(row);
+        if (rows.length > 100) {
+            table.update(rows);
+            rows = [];
+        }
+    }
+
+    table.update(rows);
+    const arrow = await (await table.view()).to_arrow();
+    fs.writeFileSync(
+        `${__dirname}/all_identifiers.arrow`,
+        Buffer.from(arrow),
+        "binary"
+    );
+
+    // schema = {
+    //     name: "string",
+    //     count: "integer",
+    //     manaCost: "string",
+    //     convertedManaCost: "integer",
+    //     types: "string",
+    //     colorIdentity: "string",
+    //     keywords: "string",
+    //     group: "string",
+    //     ...schema,
+    // };
+
+    // const deck_table = await perspective.table(schema);
+    // const arrow2 = await (await deck_table.view()).to_arrow();
+    // fs.writeFileSync("./data/deck.arrow", Buffer.from(arrow2), "binary");
+
+    // if (!fs.existsSync("data/symbology.json")) {
+    //     const json = await download_json(`https://api.scryfall.com/symbology`);
+    //     fs.writeFileSync("data/symbology.json", json);
+    // }
+
+    // const json2 = require("../data/symbology.json");
+    // const symbology = await perspective.table(json2.data, { index: "symbol" });
+    // const arrow3 = await (await symbology.view()).to_arrow();
+    // fs.writeFileSync("./data/symbology.arrow", Buffer.from(arrow3), "binary");
+}
+
+await main();