[BACK-43] Dedup hash val

lib/misc.js

@@ -1,15 +1,15 @@
 /*
  * == BSD2 LICENSE ==
  * Copyright (c) 2014, Tidepool Project
- * 
+ *
  * This program is free software; you can redistribute it and/or modify it under
  * the terms of the associated License, which is identical to the BSD 2-Clause
  * License as published by the Open Source Initiative at opensource.org.
- * 
+ *
  * This program is distributed in the hope that it will be useful, but WITHOUT
  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  * FOR A PARTICULAR PURPOSE. See the License for more details.
- * 
+ *
  * You should have received a copy of the License along with this program; if
  * not, you can obtain one from Tidepool Project at tidepool.org.
  * == BSD2 LICENSE ==
@@ -45,7 +45,7 @@ var except = amoeba.except;
  * @param fields an array of values to be concatenated together into a unique string
  * @returns {string} the base32 encoded hash of the delimited-concatenation of the provided fields (also known as a "unique" id)
  */
-exports.generateId = function(fields) {
+exports.generateId = function (fields) {
   var hasher = crypto.createHash('sha1');
 
   for (var i = 0; i < fields.length; ++i) {
@@ -65,3 +65,10 @@ exports.generateId = function(fields) {
   return base32hex.encodeBuffer(hasher.digest(), { paddingChar: '-' });
 };
 
+exports.generateHash = function (identityFields) {
+  const identityString = identityFields.join('|').trim();
+  return crypto
+    .createHash('sha256')
+    .update(identityString, 'binary')
+    .digest('base64');
+};

lib/schema/basal.js

@@ -17,14 +17,13 @@
 
 'use strict';
 
-var util = require('util');
-
 var _ = require('lodash');
 
 var schema = require('./schema.js');
 
 var idFields = ['type', 'deliveryType', 'deviceId', 'time'];
 schema.registerIdFields('basal', idFields);
+schema.registerFieldsForDuplicator('basal', ['deliveryType']);
 
 var mismatchedSeries = 'basal/mismatched-series';
 

lib/schema/bloodKetone.js

@@ -21,6 +21,7 @@ var schema = require('./schema.js');
 
 var idFields = ['type', 'deviceId', 'time'];
 schema.registerIdFields('bloodKetone', idFields);
+schema.registerFieldsForDuplicator('bloodKetone', ['units', 'value']);
 
 module.exports = schema.makeHandler('bloodKetone', {
   schema: {

lib/schema/bolus.js

@@ -17,14 +17,13 @@
 
 'use strict';
 
-var util = require('util');
-
 var _ = require('lodash');
 
 var schema = require('./schema.js');
 
 var idFields = ['type', 'subType', 'deviceId', 'time'];
 schema.registerIdFields('bolus', idFields);
+schema.registerFieldsForDuplicator('bolus', ['subType']);
 
 module.exports = function(streamDAO){
   return schema.makeSubHandler(

lib/schema/cbg.js

@@ -21,6 +21,7 @@ var schema = require('./schema.js');
 
 var idFields = ['type', 'deviceId', 'time'];
 schema.registerIdFields('cbg', idFields);
+schema.registerFieldsForDuplicator('cbg', ['units','value']);
 
 module.exports = schema.makeHandler('cbg', {
   schema: {

lib/schema/cgmSettings.js

@@ -18,9 +18,9 @@
 'use strict';
 
 var schema = require('./schema.js');
-
 var idFields = ['type', 'time', 'deviceId'];
 schema.registerIdFields('cgmSettings', idFields);
+schema.registerFieldsForDuplicator('cgmSettings');
 
 var lowHighAlertsSchema = {
   enabled: schema.isBoolean,

lib/schema/deviceEvent.js

@@ -29,6 +29,7 @@ var UNKNOWN_PREV = 'status/unknown-previous';
 
 var idFields = ['type', 'subType', 'time', 'deviceId'];
 schema.registerIdFields('deviceEvent', idFields);
+schema.registerFieldsForDuplicator('deviceEvent',['subType']);
 
 var statusReasons = ['manual', 'automatic'];
 

lib/schema/duplicate.js

@@ -0,0 +1,77 @@
+'use strict';
+
+var amoeba = require('amoeba');
+var except = amoeba.except;
+var misc = require('../misc.js');
+var schema = require('./schema.js');
+
+var idHashFieldMap = {};
+
+const getIdHashFields = function (type) {
+  var retVal = idHashFieldMap[type];
+  if (retVal == null) {
+    throw except.IAE('No known hashFields for type[%s]', type);
+  }
+  return retVal;
+};
+
+exports.registerFieldsForDuplicator = function (type, idHashFields = []) {
+  if (idHashFieldMap[type] == null) {
+    idHashFieldMap[type] = [
+      '_userId',
+      'deviceId',
+      'time',
+      'type',
+      ...idHashFields,
+    ];
+  } else {
+    throw except.IAE(
+      'Id hash fields for type[%s] already defined[%j], cannot set[%j]',
+      type,
+      idHashFieldMap[type],
+      idHashFields
+    );
+  }
+};
+
+exports.generateHash = function (datum) {
+  if (typeof datum === 'string') {
+    return datum;
+  }
+  var idHashFields = idHashFieldMap[datum.type];
+  if (!idHashFields) {
+    return '';
+  }
+  var vals = new Array(idHashFields.length);
+  for (var i = 0; i < idHashFields.length; ++i) {
+    let val = datum[idHashFields[i]];
+    if (val == null) {
+      throw except.IAE(
+        "Can't generate hash, field[%s] didn't exist on datum of type[%s]",
+        idHashFields[i],
+        datum.type
+      );
+    }
+    if (idHashFields[i] === 'time') {
+      const dateTime = new Date(val);
+      // NOTE: platform `time` is being returned minus millis so we need to do the same here
+      val = dateTime.toISOString().split('.')[0] + 'Z';
+    }
+    if (idHashFields[i] === 'value') {
+      if (
+        datum.type === 'smbg' ||
+        datum.type === 'bloodKetone' ||
+        datum.type === 'cbg'
+      ) {
+        // NOTE: platform `value` precision is being used so that the hash will be the same
+        if (val.toString().length > 7) {
+          val = schema.convertMgToMmolPrecision(val);
+        }
+      }
+    }
+    vals[i] = String(val);
+  }
+  return misc.generateHash(vals);
+};
+
+exports.getIdHashFields = getIdHashFields;

lib/schema/food.js

@@ -21,6 +21,7 @@ var schema = require('./schema.js');
 
 var idFields = ['type', 'deviceId', 'time'];
 schema.registerIdFields('food', idFields);
+schema.registerFieldsForDuplicator('food');
 
 module.exports = schema.makeHandler('food', {
   schema: {

lib/schema/insulin.js

@@ -21,6 +21,7 @@ var schema = require('./schema.js');
 
 var idFields = ['type', 'deviceId', 'time'];
 schema.registerIdFields('insulin', idFields);
+schema.registerFieldsForDuplicator('insulin');
 
 module.exports = schema.makeHandler('insulin', {
   schema: {

lib/schema/note.js

@@ -21,6 +21,7 @@ var schema = require('./schema.js');
 
 var idFields = ['type', 'time', 'creatorId', 'text', 'deviceId'];
 schema.registerIdFields('note', idFields);
+schema.registerFieldsForDuplicator('note');
 
 module.exports = function(streamDAO){
   return schema.makeHandler('note', {

lib/schema/physicalActivity.js

@@ -21,6 +21,7 @@ var schema = require('./schema.js');
 
 var idFields = ['type', 'deviceId', 'time'];
 schema.registerIdFields('physicalActivity', idFields);
+schema.registerFieldsForDuplicator('physicalActivity');
 
 var durationHoursSchema = {
   value: schema.and(

lib/schema/pumpSettings.js

@@ -23,6 +23,7 @@ var schema = require('./schema.js');
 
 var idFields = ['type', 'deviceId', 'time'];
 schema.registerIdFields('pumpSettings', idFields);
+schema.registerFieldsForDuplicator('pumpSettings');
 
 function forEachItem(obj, fn) {
   if (Array.isArray(obj)) {

lib/schema/reportedState.js

@@ -21,6 +21,7 @@ var schema = require('./schema.js');
 
 var idFields = ['type', 'deviceId', 'time'];
 schema.registerIdFields('reportedState', idFields);
+schema.registerFieldsForDuplicator('reportedState');
 
 var stateArraySchema = schema.isArrayWithValueSchema({
   state: schema.in('alcohol', 'cycle', 'hyperglycemiaSymptoms', 'hypoglycemiaSymptoms', 'illness', 'stress'),

lib/schema/schema.js

@@ -24,6 +24,7 @@ var except = amoeba.except;
 var util = require('util');
 
 var misc = require('../misc.js');
+var duplicate = require('./duplicate.js');
 
 exports.validDeviceTime = function(val) {
   if (!/^(\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d)$/.test(val)) {
@@ -252,6 +253,13 @@ exports.convertMgToMmol = function(mg) {
   return mg / 18.01559;
 };
 
+exports.convertMgToMmolPrecision = function(mgValue) {
+  const mmolLToMgdLConversionFactor = 18.01559;
+  const mmolLToMgdLPrecisionFactor = 100000.0;
+  let mmolVal = parseInt(mgValue / mmolLToMgdLConversionFactor * mmolLToMgdLPrecisionFactor + 0.5);
+  return mmolVal / mmolLToMgdLPrecisionFactor;
+};
+
 exports.convertUnits = function(datum) {
 var fields = Array.prototype.slice.call(arguments, 1);
     var normalUnits = exports.normalizeUnitName(datum.units);
@@ -513,3 +521,7 @@ exports.makeId = function(datum) {
 
   return exports.generateId(datum, idFields);
 };
+
+exports.generateHash = duplicate.generateHash;
+exports.registerFieldsForDuplicator = duplicate.registerFieldsForDuplicator;
+exports.getIdHashFields = duplicate.getIdHashFields;

lib/schema/smbg.js

@@ -1,15 +1,15 @@
 /*
  * == BSD2 LICENSE ==
  * Copyright (c) 2014, Tidepool Project
- * 
+ *
  * This program is free software; you can redistribute it and/or modify it under
  * the terms of the associated License, which is identical to the BSD 2-Clause
  * License as published by the Open Source Initiative at opensource.org.
- * 
+ *
  * This program is distributed in the hope that it will be useful, but WITHOUT
  * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
  * FOR A PARTICULAR PURPOSE. See the License for more details.
- * 
+ *
  * You should have received a copy of the License along with this program; if
  * not, you can obtain one from Tidepool Project at tidepool.org.
  * == BSD2 LICENSE ==
@@ -21,15 +21,16 @@ var schema = require('./schema.js');
 
 var idFields = ['type', 'deviceId', 'time', 'value'];
 schema.registerIdFields('smbg', idFields);
+schema.registerFieldsForDuplicator('smbg', ['units', 'value']);
 
 module.exports = schema.makeHandler('smbg', {
   schema: {
     deviceTime: schema.validDeviceTime,
     value: schema.isNumber,
     units: schema.in('mmol/L', 'mmol/l', 'mg/dL', 'mg/dl'),
-    subType: schema.ifExists(schema.in('manual', 'linked', ''))
+    subType: schema.ifExists(schema.in('manual', 'linked', '')),
   },
-  transform: function(datum, cb) {
+  transform: function (datum, cb) {
     return cb(null, schema.convertUnits(datum, 'value'));
-  }
+  },
 });

lib/schema/upload.js

@@ -21,6 +21,7 @@ var schema = require('./schema.js');
 
 var idFields = ['type', 'deviceId', 'time'];
 schema.registerIdFields('upload', idFields);
+schema.registerFieldsForDuplicator('upload');
 
 module.exports = schema.makeHandler('upload', {
   schema: {

lib/schema/urineKetone.js

@@ -21,6 +21,7 @@ var schema = require('./schema.js');
 
 var idFields = ['type', 'deviceId', 'time'];
 schema.registerIdFields('urineKetone', idFields);
+schema.registerFieldsForDuplicator('urineKetone');
 
 module.exports = schema.makeHandler('urineKetone', {
   schema: {

lib/schema/wizard.js

@@ -22,6 +22,7 @@ var settings = require('./pumpSettings.js');
 
 var idFields = ['type', 'deviceId', 'time'];
 schema.registerIdFields('wizard', idFields);
+schema.registerFieldsForDuplicator('wizard');
 
 var recommendedSchema = schema.and(
   schema.isObject,