diff --git a/package.json b/package.json index 333afe4c..2816fd23 100644 --- a/package.json +++ b/package.json @@ -38,7 +38,7 @@ "klaw-sync": "^2.1.0", "lodash": "^4.5.1", "node-version-checker": "^2.0.0", - "pelias-config": "2.10.0", + "pelias-config": "2.12.0", "pelias-dbclient": "2.2.2", "pelias-logger": "0.2.0", "pelias-model": "5.0.0", @@ -57,7 +57,7 @@ "pelias-mock-logger": "^1.1.0", "precommit-hook": "^3.0.0", "proxyquire": "^1.7.11", - "semantic-release": "^6.3.2", + "semantic-release": "^7.0.1", "tap-dot": "^1.0.0", "tape": "^4.2.2", "temp": "^0.8.3" diff --git a/src/bundleList.js b/src/bundleList.js index 2ebbca31..ea265264 100644 --- a/src/bundleList.js +++ b/src/bundleList.js @@ -40,6 +40,10 @@ const venueRoles = [ 'venue' ]; +// the bundle index contains timestamped bundles and non tar.bz2 files. +// this regex is used to filter the bundle list, removing invalid entries. +// see: https://whosonfirst.mapzen.com/bundles/index.txt +const validBundleRegex = /-latest-bundle\.tar\.bz2$/; function getPlacetypes() { let roles = hierarchyRoles; @@ -127,6 +131,13 @@ function initBundleBuckets(roles) { function sortBundleByBuckets(roles, bundle, bundleBuckets) { roles.forEach((role) => { if (bundle.indexOf('-' + role + '-') !== -1) { + + // skip invalid bundle names + if( !validBundleRegex.test( bundle ) ){ + console.error( 'info: invalid bundle name skipped', bundle ); + return; + } + bundleBuckets[role].push(bundle); } }); diff --git a/utils/download_data_all.js b/utils/download_data_all.js index 5268e8fa..a33f673e 100644 --- a/utils/download_data_all.js +++ b/utils/download_data_all.js @@ -27,10 +27,11 @@ function download(callback) { // the README file is ignored (it just would get overridden by subsequent bundles) // 3.) move the meta file to the meta files directory function generateCommand(bundle, directory) { - const targetPath = bundle.replace('-bundle.tar.bz2', '.csv'); + const csvFilename = bundle.replace(/-\d{8}T\d{6}-/, '-latest-') // support timestamped downloads + .replace('-bundle.tar.bz2', '.csv'); return 'curl https://whosonfirst.mapzen.com/bundles/' + bundle + ' | tar -xj --strip-components=1 --exclude=README.txt -C ' + - directory + ' && mv ' + path.join(directory, targetPath) + ' ' + path.join(directory, 'meta'); + directory + ' && mv ' + path.join(directory, csvFilename) + ' ' + path.join(directory, 'meta'); } bundles.generateBundleList((err, bundlesToDownload) => { @@ -57,4 +58,4 @@ function download(callback) { }); } -module.exports.download = download; \ No newline at end of file +module.exports.download = download;