Merge pull request #45 from mapbox/smelloscope

Expand GeoJSON filetype detection logic

.travis.yml

@@ -1,10 +1,6 @@
 language: node_js
+
 node_js:
 - '0.10'
 - '4'
-before_script:
-- echo "$TRAVIS_SECURE_ENV_VARS"
-- echo "$TRAVIS_PULL_REQUEST"
-- echo "$TRAVIS_BRANCH"
-- echo "$TRAVIS_COMMIT"
-- echo "$TRAVIS_REPO_SLUG"
+- '6'

index.js

@@ -3,6 +3,18 @@ var isgeocsv = require('detect-geocsv');
 var invalid = require('./lib/invalid');
 var fs = require('fs');
 var bf = require('buffer');
+var semver = require('semver');
+
+if (semver.major(process.version) > 0) {
+    function zlib_gunzip(buffer,zlib_opts,callback) {
+        zlib.gunzip(buffer, zlib_opts, callback);
+    }
+} else {
+    // node v0.10 does not support options passed to zlib.gunzip
+    function zlib_gunzip(buffer,zlib_opts,callback) {
+        zlib.gunzip(buffer, callback);
+    }
+}
 
 module.exports.sniff = sniff;
 module.exports.waft = waft;
@@ -23,6 +35,7 @@ function sniff(buffer, callback) {
 
         if (header.indexOf('\"tilejson\":') !== -1) return callback(null, 'tilejson');
         if ((header.indexOf('\"arcs\":') !== -1) || (header.indexOf('\"objects\":') !== -1)) return callback(null, 'topojson');
+        if ((header.indexOf('\"features\":') !== -1) || (header.indexOf('\"geometries\":') !== -1) || (header.indexOf('\"coordinates\":') !== -1)) return callback(null, 'geojson');
         if (header.indexOf('\"type\":') !== -1) {
             var m = /"type":\s?"(.+?)"/.exec(header);
             if (!m) {
@@ -39,7 +52,6 @@ function sniff(buffer, callback) {
                 m[1] === 'MultiPolygon' ||
                 m[1] === 'GeometryCollection') return callback(null, 'geojson');
         }
-
         return callback(invalid('Unknown filetype'));
     }
 
@@ -76,8 +88,7 @@ function sniff(buffer, callback) {
         return callback(null, 'csv');
     }
 
-    zlib.gunzip(buffer, function (err, output) {
-        if (err) return callback(invalid('Unknown filetype'));
+    function returnOutput(output,callback) {
         //check for tm2z
         if (output.toString('ascii', 257, 262) === 'ustar') return callback(null, 'tm2z');
         //check for serial tiles
@@ -92,6 +103,12 @@ function sniff(buffer, callback) {
 
         //default to unknown
         return callback(invalid('Unknown filetype'));
+    }
+
+    var zlib_opts = {finishFlush: zlib.Z_SYNC_FLUSH };
+    zlib_gunzip(buffer, zlib_opts, function (err, output) {
+        if (err) return callback(invalid('Unknown filetype'));
+        returnOutput(output,callback);
     });
 }
 

package.json

@@ -9,7 +9,8 @@
   },
   "dependencies": {
     "detect-geocsv": "0.1.0",
-    "buffer": "^3.2.2"
+    "buffer": "^3.2.2",
+    "semver": "~5.3.0"
   },
   "devDependencies": {
     "tape": "3.0.x",

test/data/crs-geojson.json

@@ -0,0 +1,21 @@
+{
+  "crs": {
+    "type": "name", 
+    "properties": {
+      "name": "urn:ogc:def:crs:OGC:1.3:CRS84"
+    }
+  }, 
+  "type": "FeatureCollection", 
+  "features": [
+    {
+      "type": "Feature",
+      "geometry": {
+        "type": "Point",
+        "coordinates": [125.6, 10.1]
+      },
+      "properties": {
+        "name": "Some Island name"
+      }
+    }
+  ]
+}

test/data/valid-coordinates.json

@@ -0,0 +1,6 @@
+{  "coordinates": [
+    [[[102.0, 2.0], [102.0, 3.0], [103.0, 3.0], [103.0, 2.0], [102.0, 2.0]]],
+    [[[100.0, 0.0], [100.0, 1.0], [101.0, 1.0], [101.0, 0.0], [100.0, 0.0]],
+     [[100.2, 0.2], [100.8, 0.2], [100.8, 0.8], [100.2, 0.8], [100.2, 0.2]]]
+    ], "type": "MultiPolygon"
+  }

test/data/valid-geometries.json

@@ -0,0 +1,10 @@
+{  "geometries": [
+    { "type": "Point",
+      "coordinates": [100.0, 0.0]
+      },
+    { "type": "LineString",
+      "coordinates": [ [101.0, 0.0], [102.0, 1.0] ]
+      }
+  ],
+  "type": "GeometryCollection"
+}

test/data/xtracharacters.json

@@ -0,0 +1,25 @@
+{
+  "features": [
+    {
+      "properties": {
+        "name": "Phasellus nec bibendum lacus. Nullam sit amet leo ipsum. Aliquam erat volutpat. Morbi sollicitudin risus at tristique gravida. Duis ornare ullamcorper arcu ut malesuada. Ut sapien massa, aliquet lobortis iaculis sed, venenatis eget ante. Sed aliquet nunc nunc, sit amet tincidunt libero aliquet nec. Curabitur nec dui ut ex blandit sagittis id at augue. Suspendisse et metus eget lorem lacinia congue. Aenean dictum finibus condimentum. Donec ultricies convallis orci, sit amet sodales orci gravida id. Nam sagittis neque vel congue maximus. Mauris porta quam non hendrerit consectetur. Aliquam congue nibh at nisl pulvinar varius. Nunc in ullamcorper tellus. Sed hendrerit metus metus, quis tincidunt felis consectetur eu."
+      },
+      "type": "Feature",
+      "geometry": {
+        "type": "Point",
+        "coordinates": [125.6, 10.1]
+      }
+    },
+    {
+      "type": "Feature",
+      "geometry": {
+        "type": "Point",
+        "coordinates": [125.6, 10.1]
+      },
+      "properties": {
+        "name": "Dinagat Islands"
+      }
+    }
+  ],
+  "type": "FeatureCollection"
+}

test/index.test.js

@@ -147,6 +147,118 @@ tape('[GeoJson] Sniffing file: should return geojson filetype and omnivore proto
         });
     });
 });
+tape('[GeoJson-CRS] Sniffing file: should return geojson file type', function(assert) {
+    var filepath = path.resolve('./test/data/crs-geojson.json');
+    var expectedFiletype = 'geojson';
+    var buffer;
+    try {
+        fs.statSync(filepath);
+        buffer = new Buffer(512);
+        var fd = fs.openSync(filepath, 'r');
+        fs.readSync(fd, buffer, 0, 512, 0);
+        fs.closeSync(fd);
+    } catch (err) {
+        return assert.end(err);
+    }
+    filesniffer.sniff(buffer, function(err, filetype) {
+        if (err) return assert.end(err);
+        assert.ok(err === null);
+        try {
+            assert.equal(filetype, expectedFiletype);
+        } catch (err) {
+            return assert.end(err);
+        }
+        filesniffer.waft(buffer, function(err, protocol) {
+            assert.ifError(err);
+            assert.equal(protocol, 'omnivore:');
+            assert.end();
+        });
+    });
+});
+tape('[GeoJson-geometries] Sniffing file: should return geojson file type', function(assert) {
+    var filepath = path.resolve('./test/data/valid-geometries.json');
+    var expectedFiletype = 'geojson';
+    var buffer;
+    try {
+        fs.statSync(filepath);
+        buffer = new Buffer(512);
+        var fd = fs.openSync(filepath, 'r');
+        fs.readSync(fd, buffer, 0, 512, 0);
+        fs.closeSync(fd);
+    } catch (err) {
+        return assert.end(err);
+    }
+    filesniffer.sniff(buffer, function(err, filetype) {
+        if (err) return assert.end(err);
+        assert.ok(err === null);
+        try {
+            assert.equal(filetype, expectedFiletype);
+        } catch (err) {
+            return assert.end(err);
+        }
+        filesniffer.waft(buffer, function(err, protocol) {
+            assert.ifError(err);
+            assert.equal(protocol, 'omnivore:');
+            assert.end();
+        });
+    });
+});
+tape('[GeoJson-coordinates] Sniffing file: should return geojson file type', function(assert) {
+    var filepath = path.resolve('./test/data/valid-coordinates.json');
+    var expectedFiletype = 'geojson';
+    var buffer;
+    try {
+        fs.statSync(filepath);
+        buffer = new Buffer(512);
+        var fd = fs.openSync(filepath, 'r');
+        fs.readSync(fd, buffer, 0, 512, 0);
+        fs.closeSync(fd);
+    } catch (err) {
+        return assert.end(err);
+    }
+    filesniffer.sniff(buffer, function(err, filetype) {
+        if (err) return assert.end(err);
+        assert.ok(err === null);
+        try {
+            assert.equal(filetype, expectedFiletype);
+        } catch (err) {
+            return assert.end(err);
+        }
+        filesniffer.waft(buffer, function(err, protocol) {
+            assert.ifError(err);
+            assert.equal(protocol, 'omnivore:');
+            assert.end();
+        });
+    });
+});
+tape('[GeoJson-Other] Sniffing file: should return geojson file type', function(assert) {
+    var filepath = path.resolve('./test/data/xtracharacters.json');
+    var expectedFiletype = 'geojson';
+    var buffer;
+    try {
+        fs.statSync(filepath);
+        buffer = new Buffer(512);
+        var fd = fs.openSync(filepath, 'r');
+        fs.readSync(fd, buffer, 0, 512, 0);
+        fs.closeSync(fd);
+    } catch (err) {
+        return assert.end(err);
+    }
+    filesniffer.sniff(buffer, function(err, filetype) {
+        if (err) return assert.end(err);
+        assert.ok(err === null);
+        try {
+            assert.equal(filetype, expectedFiletype);
+        } catch (err) {
+            return assert.end(err);
+        }
+        filesniffer.waft(buffer, function(err, protocol) {
+            assert.ifError(err);
+            assert.equal(protocol, 'omnivore:');
+            assert.end();
+        });
+    });
+});
 tape('[GeoJson] Sniffing file with nested type property: should return geojson filetype and omnivore protocol', function(assert) {
     var filepath = path.resolve('./test/data/valid-nested-type.json');
     var expectedFiletype = 'geojson';