Lazperf upgrades and performance optimizations for point coloring

 - WASM laz-perf and newer asm.js fallback
 - Move worker out of the main module
 - Performance fixes for point-recoloring

Procfile

@@ -2,4 +2,4 @@ lib: sh -c "cd lib && npm run dev"
 lts: sh -c "cd lib && npm run devtest"
 clj: sh -c "cd renderer && lein cljsbuild auto dev"
 cts: sh -c "cd renderer && lein doo node test auto"
-svr: env PORT=3000 http-server > /dev/null
+svr: env PORT=3000 http-server

dev.sh

@@ -6,7 +6,7 @@ PWD=`pwd`
 
 # move vendor stuff to dist
 mkdir -p lib/dist
-cp vendor/laz-perf.js lib/dist/
+cp lib/lib/vendor/* lib/dist/
 
 cd lib && npm install && cd ..
 

index.html

@@ -34,13 +34,13 @@
 
         var viewer = new Plasio.PointCloudViewer(document.getElementById("app"), {
             server: SERVER,
-            resource: "autzen", //["mn-z", RESOURCE],
+            resource: "dc", //["mn-z", RESOURCE],
             brushes: [
                     //'local://ramp?field=Intensity'
                 //'local://ramp?field=Intensity'
                 // 'local://ramp?field=Z&step=20'
-                'local://color'
-                //'remote://imagery?url=http%3A%2F%2Fapi.tiles.mapbox.com%2Fv4%2Fmapbox.satellite%2F%7B%7Bz%7D%7D%2F%7B%7Bx%7D%7D%2F%7B%7By%7D%7D.jpg70%3Faccess_token%3Dpk.eyJ1IjoiaG9idSIsImEiOiItRUhHLW9NIn0.RJvshvzdstRBtmuzSzmLZw',
+                //'local://color'
+                'remote://imagery?url=http%3A%2F%2Fapi.tiles.mapbox.com%2Fv4%2Fmapbox.satellite%2F%7B%7Bz%7D%7D%2F%7B%7Bx%7D%7D%2F%7B%7By%7D%7D.jpg70%3Faccess_token%3Dpk.eyJ1IjoiaG9idSIsImEiOiItRUhHLW9NIn0.RJvshvzdstRBtmuzSzmLZw',
                 //'remote://imagery?url=http%3A%2F%2Fapi.tiles.mapbox.com%2Fv4%2Fmapbox.satellite%2F%7B%7Bz%7D%7D%2F%7B%7Bx%7D%7D%2F%7B%7By%7D%7D.jpg70%3Faccess_token%3Dpk.eyJ1IjoiaG9idSIsImEiOiItRUhHLW9NIn0.RJvshvzdstRBtmuzSzmLZw'
             ],
             baseDepth: 9,

lib/index-webworker.js

@@ -0,0 +1,2 @@
+require("babel-polyfill");
+require("expose?PlasioWorker!./lib/loader-worker.js");

lib/lib/buffer-loaders.js

@@ -9,8 +9,6 @@ var _ = require("lodash");
 var LRU = require("lru-cache");
 var vec3 = require("gl-matrix").vec3;
 
-const LoaderWorker = require('worker-loader?inline!./loader-worker');
-
 import { Promise } from 'bluebird';
 
 import { TileLoader } from "./tile-loaders";
@@ -34,7 +32,8 @@ class BufferLoader {
     }
 
     _newWorker(index) {
-        const w = new LoaderWorker();
+        const webWorkerPath = window.PLASIO_WEB_WORKER_PATH || "lib/dist/plasio.webworker.js";
+        const w = new Worker(webWorkerPath);
 
         w.id = index;
         w.processQueue = {};
@@ -45,6 +44,7 @@ class BufferLoader {
 
         w.onerror = (e) => {
             console.log("WARNING: Worker crashed!", w);
+            console.error(e);
             this._handleError(w);
         };
 

lib/lib/frustum-lod.js

@@ -134,7 +134,6 @@ class TreeInfoCache {
             if (this._isPrefetchLevel(node) && node.depthEnd <= node.stopSplitDepth) {
                 // yes its a pre-fetch level, so query it and load stuff up
                 const nodeBounds = node.bounds();
-                console.log("TREE: PREFETCH! ", node.id, node.depthEnd, "->", node.depthEnd + stepSize);
 
                 const res = await this.loader.loadHierarchyInfo(
                     nodeBounds, this.geoTransform,
@@ -499,9 +498,11 @@ class TreeWalker {
 
         // is this node in view?  Make sure to adjust by the tree's render space offset
         const intersects = node.intersects(frustum, this.renderSpaceTreeOffset);
+        /*
         if (node.id.length == 2) {
             console.log("xx", node.id, intersects);
         }
+        */
 
         if (!intersects)
             return this._rejectNode(node, fRemove);
@@ -589,19 +590,23 @@ class TreeWalker {
 
         // Only the revisions that make it to the end are responsible for removal
         if (completedWithRevision === this.revision - 1) {
+            /*
             let s1 = '';
             for (let k in this.nodesInScene) {
                 const n = this.nodesInScene[k];
                 s1 = s1 + " [" + n.node.id + ":" + n.node.depthEnd + ":" + n.revision + "] "
             }
+            */
             this._doCleanup(revision, fRemove);
 
+            /*
             let s = '';
             for (let k in this.nodesInScene) {
                 const n = this.nodesInScene[k];
                 s = s + " [" + n.node.id + ":" + n.node.depthEnd + ":" + n.revision + "] "
             }
             console.log('nodes in scene:', s);
+            */
 
             // Only remove params if we are still responsible for it.
             if (revision === this.activeParams.revision) {
@@ -794,15 +799,14 @@ export class FrustumLODNodePolicy extends EventEmitter {
 
     async _loadRootNode() {
         const [x1, y1, z1, x2, y2, z2] = this.renderBounds;
-        console.log('render-bounds:', this.renderBounds);
+
         const baseBox = new Box(
             x1, y1, z1, x2 - x1, y2 - y1, z2 - z1,
             "R", 0, this.baseDepth, null,
             this.stopSplitDepth,
             this.hardStopDepth
         );
 
-        console.log("trying to load root node", baseBox);
         return await this._loadNode(baseBox);
     }
 

lib/lib/loader-worker.js

@@ -1,10 +1,6 @@
 // Loader worker
 // Loads and processes data buffers from greyhound
 
-// load laz-perf in global namespace
-
-require('script-loader!./vendor/laz-perf.js?compact=false');
-
 import 'babel-polyfill'
 
 import { Promise } from "bluebird";
@@ -24,6 +20,7 @@ class BufferDownloadPipeline {
        this.pending = [];
 
        this.activeCount = 0;
+       this.ready = false;
    }
 
     /**
@@ -45,6 +42,14 @@ class BufferDownloadPipeline {
        this._doNext();
    }
 
+    /**
+     * Mark the queue as ready, indicating that decompression runtime is ready.
+     */
+   markReady() {
+       this.ready = true;
+       this._doNext();
+   }
+
    _notifyParent(task, status, result) {
        const payload = {
            id: task.id,
@@ -61,7 +66,7 @@ class BufferDownloadPipeline {
    }
 
     _doNext() {
-       if (this.activeCount >= this.activeDownloads)
+       if (this.activeCount >= this.activeDownloads || !this.ready)
            return; // No more tasks can be run, so wait for things to complete
 
         const task = this.pending.shift();
@@ -405,9 +410,23 @@ function processTask(task) {
     });
 }
 
-
 const pipeline = new BufferDownloadPipeline();
 
+self.Module = {};
+self.Module['onRuntimeInitialized'] = function() {
+    console.log("Worker runtime is ready.");
+    pipeline.markReady();
+};
+
+if (WebAssembly) {
+    console.log ("Worker: using webassembly laz-perf module.");
+    importScripts("laz-perf.js");
+}
+else {
+    console.log ("Worker: using asm.js laz-perf module.");
+    importScripts("laz-perf.asm.js");
+}
+
 onmessage = (event) => {
     const data = event.data;
 

lib/lib/modes/base-mode.js

@@ -15,7 +15,6 @@ export class BaseMode {
 
     invokeHandler(event, params) {
         let handler = this.handlers[event];
-        console.log ("-- --", event, "on", this.name, "handler?", handler ? "YES" : "NO", "params:", params)
 
         if(handler) {
             return handler.call(this, params);

lib/lib/point-buffer-cache.js

@@ -147,7 +147,6 @@ export class PointBufferCache {
         };
 
         // Determine this buffers impact on already loaded buffers.
-        console.time('impacted-nodes-determination');
         const impactedNodes = [];
         cleanedUpBrushes.forEach((brush, index) => {
             const {strategy, params} = brush.nodeSelectionStrategy(bufferParams);
@@ -181,7 +180,6 @@ export class PointBufferCache {
                 }
             }
         });
-        console.timeEnd('impacted-nodes-determination');
 
         this._queueRecolorTasks(impactedNodes, bufferParams.geoTransform, bufferParams.pointCloudBufferStats);
         return outputBuffer;
@@ -236,47 +234,51 @@ export class PointBufferCache {
             totalPoints, data, schema
         } = inputBufferParams;
 
-        const point = {};
+        var point = {};
         const numItems = schema.length;
-        const pointOffsets = schema.map(({name}, index) => {
-            return [name.toLowerCase(), index];
+        const schemaRenamed = schema.slice(0).map(s => {
+            s.name = s.name.toLowerCase();
+            return s;
         });
 
         const decomposePoint = (index) => {
             const offset = index * numItems;
             for (let i = 0 ; i < numItems ; i ++) {
-                const [name, fieldOffset] = pointOffsets[i];
-                point[name] = data[offset + fieldOffset];
+                point[schemaRenamed[i].name] = data[offset + i];
             }
             return point;
         };
 
         const colorPoints = (start, end) => {
-            let writeOffset = start * outputPointSize;
-            let color = [0, 0, 0];
-
-            for (let i = start; i < end; i++) {
-                const p = decomposePoint(i);
-                outputBuffer[writeOffset + 0] = p.x;
-                outputBuffer[writeOffset + 1] = p.y;
-                outputBuffer[writeOffset + 2] = p.z;
-
-                for (let bi = 0, bl = brushes.length; bi < bl; bi++) {
-                    const brush = brushes[bi];
-                    if (brush) {
-                        brush.colorPoint(color, p);
-                        outputBuffer[writeOffset + 3 + bi] = compressColor(color);
+            return new Promise((accept, reject) => {
+                setTimeout(() => {
+                    let writeOffset = start * outputPointSize;
+                    let color = [0, 0, 0];
+
+                    for (let i = start; i < end; i++) {
+                        const p = decomposePoint(i);
+                        outputBuffer[writeOffset] = p.x;
+                        outputBuffer[writeOffset + 1] = p.y;
+                        outputBuffer[writeOffset + 2] = p.z;
+
+                        for (let bi = 0, bl = brushes.length; bi < bl; bi++) {
+                            const brush = brushes[bi];
+                            if (brush) {
+                                brush.colorPoint(color, p);
+                                outputBuffer[writeOffset + 3 + bi] = compressColor(color);
+                            }
+                        }
+
+                        writeOffset += outputPointSize;
                     }
-                }
-
-                writeOffset += outputPointSize;
-            }
+                    accept();
+                }, 0);
+            });
         };
 
         const coloringTasks = [];
         const POINTS_PER_TASK = 1000;
 
-        console.time('color');
         for (let i = 0 ; i < totalPoints ; i += POINTS_PER_TASK) {
             const start = i;
             const end = Math.min(start + POINTS_PER_TASK, totalPoints);
@@ -285,7 +287,6 @@ export class PointBufferCache {
         }
 
         await Promise.all(coloringTasks);
-        console.timeEnd('color');
     }
 
     async _recolorNode(node, brushes, geoTransform, pointCloudBufferStats) {
@@ -367,8 +368,6 @@ export class PointBufferCache {
             }
 
             // if this task is already queued, remove it, since it will be re-queued at the end.
-            console.log(node.treePath, foundIndex > -1 ? 'hit' : 'miss', this.recolorTasks.length, this._recolorRunning);
-
             if (foundIndex >= 0) {
                 const task = this.recolorTasks[foundIndex];
                 this.recolorTasks.splice(foundIndex, 1);
@@ -398,14 +397,12 @@ export class PointBufferCache {
         while(this.recolorTasks.length > 0) {
             await Promise.delay(100);
 
-            const task = this.recolorTasks.shift()
+            const task = this.recolorTasks.shift();
             if (task) {
                 const [
                     node, brushes, params
                 ] = task;
 
-                console.log('re-processing: ' + node.treePath + ' with ' + brushes.length + ' brushes.');
-                console.log('re-coloring', task);
                 await this._recolorNode(node, brushes, params.geoTransform, params.pointCloudBufferStats);
 
                 // Mark buffer updated since we want the GPU to re-load it.
@@ -417,7 +414,6 @@ export class PointBufferCache {
             }
         }
 
-        console.log('recolor queue exhausted');
         this._recolorRunning = false;
     }
 }

lib/webpack.config.js

@@ -1,6 +1,8 @@
 var webpack = require('webpack');
 
-module.exports = {
+
+
+var plasioLibConfig = {
     entry: [
         './index.js'
     ],
@@ -33,3 +35,39 @@ module.exports = {
         ]
     }
 };
+
+var plasioWebWorkerConfig = {
+    entry: [
+        './index-webworker.js'
+    ],
+
+    resolve: {
+        extensions: ['', '.js' ]
+    },
+
+    output: {
+        path: './dist',
+        filename: 'plasio.webworker.js'
+    },
+
+    module: {
+        loaders: [
+            {
+                test: /\.jsx?$/,
+                exclude: /node_modules/,
+                loader: 'babel',
+                query: {
+                    cacheDirectory: true,
+                    presets: ["es2015"]
+                }
+            },
+            {
+                test: /\.tsx?$/,
+                exclude: /node_modules/,
+                loader: 'ts-loader',
+            }
+        ]
+    }
+};
+
+module.exports = [plasioLibConfig, plasioWebWorkerConfig];

renderer/src/renderer/engine/render.cljs

@@ -534,6 +534,7 @@
                                             (get-in % [:point-buffer :key])
                                             true)))
                             (vals (:point-buffers state)))]
+      (println "-- -- picker, to draw:" (count buffers-to-draw))
       (draw/draw-all-buffers gl buffers-to-draw nil nil
                              shader-context
                              picker-uniform-map