Merge pull request #1301 from jpwhite4/extendoverride

Add support for multiple groupby definition files in js etl.
ubccr · Jun 25, 2020 · 27a3b40 · 27a3b40
2 parents eaa64e1 + 9469799
commit 27a3b40
Show file tree

Hide file tree

Showing 2 changed files with 6 additions and 4 deletions.
diff --git a/etl/js/lib/etl_profile.js b/etl/js/lib/etl_profile.js
@@ -23,6 +23,7 @@ var events = require('events'),
 	config = require('../config.js');
 var etlv2 = require('./etlv2.js');
 var fs = require('fs');
+var glob = require('glob');
 var sorting = require('./sorting.js');
 
 var ETLProfile = module.exports = function (etlProfile) {
@@ -721,10 +722,10 @@ ETLProfile.prototype.integrateWithXDMoD = function () {
 
             writeRealmMetadata(realmName, this.version);
             var groupBys = {};
-            try {
-                groupBys = JSON.parse(fs.readFileSync(this.root + '/output_db/groupbys.json', 'utf8'));
-            } catch (err) {
-                // dont do anyting just be cool man
+            var includefiles = glob.sync(this.root + '/output_db/groupbys*.json');
+            includefiles.sort();
+            for (let i = 0; i < includefiles.length; i++) {
+                Object.assign(groupBys, JSON.parse(fs.readFileSync(includefiles[i], 'utf8')));
             }
             var xdmodInteg = new xdmodIntegrator(realmName, this.root);
 

diff --git a/etl/js/package.json b/etl/js/package.json
@@ -8,6 +8,7 @@
     "dependencies": {
         "cloneextend": "0.0.x",
         "ini": "1.2.x",
+        "glob": "7.1.x",
         "mongodb": "3.1.x",
         "mysql": "2.17.x",
         "tv4": "^1.0.13",