Skip to content

Commit

Permalink
Merge branch 'master' into greenkeeper/pelias-dbclient-2.2.2
Browse files Browse the repository at this point in the history
  • Loading branch information
Diana Shkolnikov authored Aug 23, 2017
2 parents b334235 + 1592705 commit 7ca0907
Show file tree
Hide file tree
Showing 3 changed files with 17 additions and 5 deletions.
4 changes: 2 additions & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@
"klaw-sync": "^2.1.0",
"lodash": "^4.5.1",
"node-version-checker": "^2.0.0",
"pelias-config": "2.10.0",
"pelias-config": "2.12.0",
"pelias-dbclient": "2.2.2",
"pelias-logger": "0.2.0",
"pelias-model": "5.0.0",
Expand All @@ -57,7 +57,7 @@
"pelias-mock-logger": "^1.1.0",
"precommit-hook": "^3.0.0",
"proxyquire": "^1.7.11",
"semantic-release": "^6.3.2",
"semantic-release": "^7.0.1",
"tap-dot": "^1.0.0",
"tape": "^4.2.2",
"temp": "^0.8.3"
Expand Down
11 changes: 11 additions & 0 deletions src/bundleList.js
Original file line number Diff line number Diff line change
Expand Up @@ -40,6 +40,10 @@ const venueRoles = [
'venue'
];

// the bundle index contains timestamped bundles and non tar.bz2 files.
// this regex is used to filter the bundle list, removing invalid entries.
// see: https://whosonfirst.mapzen.com/bundles/index.txt
const validBundleRegex = /-latest-bundle\.tar\.bz2$/;

function getPlacetypes() {
let roles = hierarchyRoles;
Expand Down Expand Up @@ -127,6 +131,13 @@ function initBundleBuckets(roles) {
function sortBundleByBuckets(roles, bundle, bundleBuckets) {
roles.forEach((role) => {
if (bundle.indexOf('-' + role + '-') !== -1) {

// skip invalid bundle names
if( !validBundleRegex.test( bundle ) ){
console.error( 'info: invalid bundle name skipped', bundle );
return;
}

bundleBuckets[role].push(bundle);
}
});
Expand Down
7 changes: 4 additions & 3 deletions utils/download_data_all.js
Original file line number Diff line number Diff line change
Expand Up @@ -27,10 +27,11 @@ function download(callback) {
// the README file is ignored (it just would get overridden by subsequent bundles)
// 3.) move the meta file to the meta files directory
function generateCommand(bundle, directory) {
const targetPath = bundle.replace('-bundle.tar.bz2', '.csv');
const csvFilename = bundle.replace(/-\d{8}T\d{6}-/, '-latest-') // support timestamped downloads
.replace('-bundle.tar.bz2', '.csv');

return 'curl https://whosonfirst.mapzen.com/bundles/' + bundle + ' | tar -xj --strip-components=1 --exclude=README.txt -C ' +
directory + ' && mv ' + path.join(directory, targetPath) + ' ' + path.join(directory, 'meta');
directory + ' && mv ' + path.join(directory, csvFilename) + ' ' + path.join(directory, 'meta');
}

bundles.generateBundleList((err, bundlesToDownload) => {
Expand All @@ -57,4 +58,4 @@ function download(callback) {
});
}

module.exports.download = download;
module.exports.download = download;

0 comments on commit 7ca0907

Please sign in to comment.