Skip to content

Commit

Permalink
update README and tests
Browse files Browse the repository at this point in the history
sedenardi committed Feb 16, 2017
1 parent c942d14 commit d8140a5
Showing 4 changed files with 99 additions and 89 deletions.
37 changes: 18 additions & 19 deletions README.md
Original file line number Diff line number Diff line change
@@ -15,19 +15,18 @@ A node.js module for working with NYC's [BYTES of the BIG APPLE datasets](http:/

npm install nyc-bytes

Each dataset is exposed as a singleton object that must be initialized. This ensures that the underlying files are downloaded, extracted, and ready for use. Attach a `ready` listener to continue execution once the dataset has finished initializing.
Each dataset is exposed as a singleton object that must be initialized. This ensures that the underlying files are downloaded, extracted, and ready for use. Initializing the dataset returns a `Promise` that returns once the dataset has finished initializing.

var Bytes = require('nyc-bytes');
const Bytes = require('nyc-bytes');

var dataset = Bytes.Pluto;
dataset.on('ready', function() {
const dataset = Bytes.Pluto;
dataset.init().then(() => {
console.log('Dataset ready.');
//do something
const stream = dataset.stream();
// do something with stream
}).catch((err) => {
console.error(err);
});
dataset.on('error', function(err) {
console.log('Error: ' + err.message);
});
dataset.init();

#####Datasets
* Pluto - `var dataset = Bytes.Pluto;`
@@ -37,31 +36,31 @@ Each dataset is exposed as a singleton object that must be initialized. This ens

##dataset.stream([options])

The dataset's underlying data is accessable like any other standard node stream.
The dataset's underlying data is accessible like any other standard node stream.

var stream = dataset.stream();
stream.on('readable', function() {
var record = stream.read();
const stream = dataset.stream();
stream.on('readable', () => {
const record = stream.read();
// do something with the record
});
stream.on('end', function() {
stream.on('end', () => {
console.log('finished');
});

You can also use the stream in [flowing mode](http://nodejs.org/api/stream.html#stream_event_data) by attaching a `data` event listener.

var stream = dataset.stream();
stream.on('data', function(record) {
const stream = dataset.stream();
stream.on('data', (record) => {
// do something with the record
});
stream.on('end', function() {
stream.on('end', () => {
console.log('finished');
});

Lastly, you can also pipe the stream like you would any other readable stream.

var stream = dataset.stream();
var writableStream = somehowGetWritableStream();
const stream = dataset.stream();
const writableStream = somehowGetWritableStream();
stream.pipe(writableStream);

##options
66 changes: 37 additions & 29 deletions test/testFunctions.js
Original file line number Diff line number Diff line change
@@ -1,19 +1,25 @@
var assert = require('assert'),
TestStream = require('./testStream.js');
'use strict';

var boroughTest = function(dataset, streamOptions, count) {
var testData = {};
/* global before */
/* global it */
/* global describe */

const assert = require('assert');
const TestStream = require('./testStream.js');

const boroughTest = function(dataset, streamOptions, count) {
let testData = {};
before(function(done) {
this.timeout(0);
var stream = dataset.stream(streamOptions);
var testStream = new TestStream();
const stream = dataset.stream(streamOptions);
const testStream = new TestStream({ objectMode: true });
testStream.on('finish', function() {
testData = this.testData();
done();
});
stream.pipe(testStream);
});
it('Record count matches', function() {
it('Record count matches', () => {
assert.equal(count, testData.count);
});
};
@@ -22,48 +28,50 @@ module.exports.allTests = function(dataset, options, counts, allOnly, noInit) {
if (typeof noInit === 'undefined' || !noInit) {
before(function(done) {
this.timeout(0);
dataset.on('ready', done);
dataset.on('error', done);
dataset.init();
dataset.init().then(() => {
done();
}).catch((err) => {
done(err);
});
});
}
if (!allOnly) {
describe('Individual boroughs', function() {
describe('Manhattan', function() {
var opt = { boroughs: ['MN'] };
for (var k in options)
describe('Individual boroughs', () => {
describe('Manhattan', () => {
const opt = { boroughs: ['MN'] };
for (let k in options)
opt[k] = options[k];
boroughTest(dataset, opt, counts.MN);
});
describe('Bronx', function() {
var opt = { boroughs: ['BX'] };
for (var k in options)
describe('Bronx', () => {
const opt = { boroughs: ['BX'] };
for (let k in options)
opt[k] = options[k];
boroughTest(dataset, opt, counts.BX);
});
describe('Brooklyn', function() {
var opt = { boroughs: ['BK'] };
for (var k in options)
describe('Brooklyn', () => {
const opt = { boroughs: ['BK'] };
for (let k in options)
opt[k] = options[k];
boroughTest(dataset, opt, counts.BK);
});
describe('Queens', function() {
var opt = { boroughs: ['QN'] };
for (var k in options)
describe('Queens', () => {
const opt = { boroughs: ['QN'] };
for (let k in options)
opt[k] = options[k];
boroughTest(dataset, opt, counts.QN);
});
describe('Staten Island', function() {
var opt = { boroughs: ['SI'] };
for (var k in options)
describe('Staten Island', () => {
const opt = { boroughs: ['SI'] };
for (let k in options)
opt[k] = options[k];
boroughTest(dataset, opt, counts.SI);
});
});
} else {
describe('All boroughs', function() {
var opt = { };
for (var k in options)
describe('All boroughs', () => {
const opt = { };
for (let k in options)
opt[k] = options[k];
var totalCount = counts.Total;
boroughTest(dataset, opt, totalCount);
31 changes: 15 additions & 16 deletions test/testStream.js
Original file line number Diff line number Diff line change
@@ -1,20 +1,19 @@
var Writable = require('stream').Writable,
util = require('util');
'use strict';

var TestStream = function() {
Writable.call(this, {objectMode: true});
this.count = 0;
};
const Writable = require('stream').Writable;

util.inherits(TestStream, Writable);

TestStream.prototype._write = function(chunk, encoding, callback) {
this.count++;
callback();
};

TestStream.prototype.testData = function() {
return { count: this.count };
};
class TestStream extends Writable {
constructor(options) {
super(options);
this.count = 0;
}
_write(chunk, encoding, done) {
this.count++;
done();
}
testData() {
return { count: this.count };
}
}

module.exports = TestStream;
54 changes: 29 additions & 25 deletions test/tests.js
Original file line number Diff line number Diff line change
@@ -1,32 +1,36 @@
var Bytes = require('../lib/bytes.js'),
testFunctions = require('./testFunctions.js');
'use strict';

// describe('Pluto', function(){
// var dataset = Bytes.Pluto;
// var counts = { MN: 42958, BX: 89830, BK: 277131, QN: 324403, SI: 124048 };
// testFunctions.allTests(dataset, { }, counts);
// });
/* global describe */

const Bytes = require('../lib/bytes.js');
const testFunctions = require('./testFunctions.js');

describe('Pluto', () => {
var dataset = Bytes.Pluto;
var counts = { MN: 42958, BX: 89830, BK: 277131, QN: 324403, SI: 124048 };
testFunctions.allTests(dataset, { }, counts);
});

describe('MapPluto', function(){
var dataset = Bytes.MapPluto;
var counts = { MN: 42697, BX: 89687, BK: 276930, QN: 324181, SI: 123721 };
var counts = { MN: 42686, BX: 89685, BK: 276909, QN: 324168, SI: 123789 };
testFunctions.allTests(dataset, { }, counts);
});

// describe('ZoningTaxLot', function(){
// var dataset = Bytes.ZoningTaxLot;
// var counts = { MN: 42995, BX: 89798, BK: 277135, QN: 324346, SI: 124131 };
// testFunctions.allTests(dataset, { }, counts);
// });
//
// describe('PAD', function(){
// var dataset = Bytes.PAD;
// var bblCounts = { Total: 880814 };
// var adrCounts = { Total: 1292406 };
// describe('BBL', function(){
// testFunctions.allTests(dataset, { table: 'BBL' }, bblCounts, true);
// });
// describe('ADR', function(){
// testFunctions.allTests(dataset, { table: 'ADR' }, adrCounts, true, true);
// });
// });
describe('ZoningTaxLot', function(){
var dataset = Bytes.ZoningTaxLot;
var counts = { MN: 43017, BX: 89810, BK: 277218, QN: 324396, SI: 124169 };
testFunctions.allTests(dataset, { }, counts);
});

describe('PAD', function(){
var dataset = Bytes.PAD;
var bblCounts = { Total: 880969 };
var adrCounts = { Total: 1299356 };
describe('BBL', function(){
testFunctions.allTests(dataset, { table: 'BBL' }, bblCounts, true);
});
describe('ADR', function(){
testFunctions.allTests(dataset, { table: 'ADR' }, adrCounts, true, true);
});
});

0 comments on commit d8140a5

Please sign in to comment.