Merge pull request #64 from pelias/wip_merge

wip merge

.travis.yml

@@ -9,13 +9,19 @@ node_js:
 matrix:
   fast_finish: true
 script: npm run travis
+env:
+  - BUCKET=http://pelias-data.s3.amazonaws.com/placeholder
 before_install:
   - npm i -g npm@^3.0.0
 before_script:
   - npm prune
   - mkdir data
-  - curl -s http://pelias-data.s3.amazonaws.com/placeholder/archive/$(date +%Y-%m-%d)/graph.json.gz | gunzip > data/graph.json;
-  - curl -s http://pelias-data.s3.amazonaws.com/placeholder/archive/$(date +%Y-%m-%d)/store.sqlite3.gz | gunzip > data/store.sqlite3;
+  - 'curl -sfo data/graph.json.gz ${BUCKET}/archive/$(date +%Y-%m-%d)/graph.json.gz || true'
+  - 'curl -sfo data/store.sqlite3.gz ${BUCKET}/archive/$(date +%Y-%m-%d)/store.sqlite3.gz || true'
+  - '[ -e data/graph.json.gz ] || curl -so data/graph.json.gz ${BUCKET}/graph.json.gz'
+  - '[ -e data/store.sqlite3.gz ] || curl -so data/store.sqlite3.gz ${BUCKET}/store.sqlite3.gz'
+  - gunzip data/graph.json.gz
+  - gunzip data/store.sqlite3.gz
 after_success:
   - npm install -g npx
   - npx -p node@8 npm run semantic-release

cmd/download_extract.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+set -euo pipefail
+
+# directory of this file
+DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+
+# placetypes to download and extract
+PLACETYPES=( 'ocean' 'continent' 'marinearea' 'empire' 'country' 'dependency' 'disputed' 'macroregion' 'region',
+             'macrocounty' 'county' 'localadmin' 'locality' 'borough' 'macrohood' 'neighbourhood' )
+
+# download and extract fields from contents of tar
+function extract {
+  curl -so "/tmp/wof-${1}-latest-bundle.tar.bz2" "https://whosonfirst.mapzen.com/bundles/wof-${1}-latest-bundle.tar.bz2"
+  tar --wildcards '*.geojson' -jx --to-command 'jq -cMf "${DIR}/jq.filter"' -f "/tmp/wof-${1}-latest-bundle.tar.bz2"
+  rc=$?; if [[ $rc != 0 ]]; then
+    >&2 echo "/tmp/wof-${1}-latest-bundle.tar.bz2"
+    >&2 echo "command exited with status: $rc"
+  fi
+}
+
+# export variables required by the 'extract' function
+export -f extract
+export DIR
+
+# run the import
+parallel \
+  --no-notice \
+  --line-buffer \
+  --jobs -1 \
+  extract \
+  ::: "${PLACETYPES[@]}"

package.json

@@ -7,6 +7,7 @@
   "scripts": {
     "test": "npm run units",
     "units": "node test/units.js | tap-spec",
+    "integration": "node test/integration.js | tap-spec",
     "funcs": "for case in test/cases/*.txt; do node test/case.js $case; done",
     "start": "node server/http.js",
     "extract": "bash ./cmd/extract.sh",
@@ -19,7 +20,7 @@
     "check-dependencies": "node_modules/.bin/npm-check --skip-unused --production",
     "coverage": "node_modules/.bin/istanbul cover test/unit/run.js",
     "validate": "npm ls",
-    "travis": "npm run check-dependencies && npm test"
+    "travis": "npm run check-dependencies && npm run units && npm run integration && npm run funcs"
   },
   "repository": {
     "type": "git",

server/demo/index.html

@@ -63,8 +63,7 @@
 
       function exec( text ){
 
-        $('#results li').remove();
-        $('#tokens').empty();
+        $('#results').empty();
         clearMap();
 
         search( text, function( results ){
@@ -131,12 +130,16 @@
 
       function render( results, groups ){
 
+        $('#results').empty();
+        $('#tokens').empty();
+        clearMap();
+
         // display token groups
         groups.forEach( function( win ){
           var buttons = win.map( function( token ){
             return '<li type="button" class="btn btn-default" style="margin-top: 5px;"><span>' + token + '</span></li>';
           });
-          $("#tokens").append('<li><ul style="margin: 0; padding: 5px; padding-top: 0; list-style: none; background-color: #efefef; margin-bottom: 5px;">' + buttons.join('\n') + '</ul></li>' );
+          $("#tokens").html('<li><ul style="margin: 0; padding: 5px; padding-top: 0; list-style: none; background-color: #efefef; margin-bottom: 5px;">' + buttons.join('\n') + '</ul></li>' );
         });
 
         // render the results
@@ -231,6 +234,11 @@
           return false;
         });
 
+        $('#search').on( 'keyup', _.debounce( function( e ){
+          exec( $('#text').val() );
+          return false;
+        }, 200));
+
         $('#go').on( 'click', function( e ){
           $('#search').submit();
           return false;

server/http.js

@@ -1,14 +1,38 @@
 
-var express = require('express'),
-    Placeholder = require('../Placeholder.js');
+/**
+  The http server improves performance on multicore machines by using the
+  node core 'cluster' module to fork worker processes.
 
-const morgan = require( 'morgan' );
-const logger = require('pelias-logger').get('interpolation');
-const through = require( 'through2' );
+  The default setting is to use all available CPUs, this will spawn 32 child
+  processes on a 32 core machine.
+
+  If you would like to disable this feature (maybe because you are running
+  inside a container) then you can do so by setting the env var CPUS=1
+
+  You may also specify exactly how many child processes you would like to
+  spawn by setting the env var to a numeric value >1, eg CPUS=4
+
+  If the CPUS env var is set less than 1 or greater than os.cpus().length
+  then the default setting will be used (using all available cores).
+**/
+
+const os = require('os');
+const morgan = require('morgan');
+const express = require('express');
+const cluster = require('cluster');
+const through = require('through2');
 const _ = require('lodash');
 
-// optionally override port using env var
+const Placeholder = require('../Placeholder.js');
+const logger = require('pelias-logger').get('placeholder');
+
+// select the amount of cpus we will use
+const envCpus = parseInt( process.env.CPUS, 10 );
+const cpus = Math.min( Math.max( envCpus, 1 ), os.cpus().length );
+
+// optionally override port/host using env var
 var PORT = process.env.PORT || 3000;
+var HOST = process.env.HOST || undefined;
 var app = express();
 
 function log() {
@@ -35,12 +59,19 @@ app.use(log());
 
 // init placeholder
 console.error( 'loading data' );
-var ph = new Placeholder();
+var ph = new Placeholder({ readonly: true });
 ph.load();
 
 // store $ph on app
 app.locals.ph = ph;
 
+// generic http headers
+app.use((req, res, next) => {
+  res.header('Charset','utf8');
+  res.header('Cache-Control','public, max-age=120');
+  next();
+});
+
 // routes
 app.get( '/parser/search', require( './routes/search' ) );
 app.get( '/parser/findbyid', require( './routes/findbyid' ) );
@@ -49,15 +80,46 @@ app.get( '/parser/tokenize', require( './routes/tokenize' ) );
 
 // demo page
 app.use('/demo', express.static( __dirname + '/demo' ));
-app.use('/', function( req, res ){ res.redirect('/demo#eng'); });
-
-// start server
-app.listen( PORT, function() {
-  console.log( 'server listening on port', PORT );
-});
+app.use('/', (req, res) => { res.redirect('/demo#eng'); });
 
 // handle SIGTERM (required for fast docker restarts)
-process.on('SIGTERM', function(){
+process.on('SIGTERM', () => {
   ph.close();
   app.close();
 });
+
+// start multi-threaded server
+if( cpus > 1 ){
+  if( cluster.isMaster ){
+    console.error('[master] using %d cpus', cpus);
+
+    // worker exit event
+    cluster.on('exit', (worker, code, signal) => {
+      console.error('[master] worker died', worker.process.pid);
+    });
+
+    // worker fork event
+    cluster.on('fork', (worker, code, signal) => {
+      console.error('[master] worker forked', worker.process.pid);
+    });
+
+    // fork workers
+    for( var c=0; c<cpus; c++ ){
+      cluster.fork();
+    }
+
+  } else {
+    app.listen( PORT, HOST, () => {
+      console.error('[worker %d] listening on %s:%s', process.pid, HOST||'0.0.0.0', PORT);
+    });
+  }
+}
+
+// start single-threaded server
+else {
+  console.error('[master] using %d cpus', cpus);
+
+  app.listen( PORT, HOST, () => {
+    console.log('[master] listening on %s:%s', HOST||'0.0.0.0', PORT);
+  });
+}

test/integration.js

@@ -0,0 +1,25 @@
+var tape = require('tape');
+var path = require('path');
+
+var tests = [
+  './prototype/tokenize',
+  './prototype/query',
+  './prototype/wof',
+  './functional',
+];
+
+// test runner
+tests.map( function( testpath ){
+
+  var file = require( testpath );
+
+  var test = function( name, func ) {
+    return tape( path.normalize( testpath ) + ': ' + name , func );
+  };
+
+  for( var testCase in file ){
+    if( 'function' === typeof file[testCase] ){
+      file[testCase]( test );
+    }
+  }
+});

test/units.js

@@ -6,10 +6,6 @@ var tests = [
   './lib/permutations',
   './lib/sorted',
   './lib/TokenGraph',
-  './prototype/tokenize',
-  './prototype/query',
-  './prototype/wof',
-  './functional',
 ];
 
 // test runner