Skip to content

Commit

Permalink
Merge pull request #847 from Nozbe/column-comparisons
Browse files Browse the repository at this point in the history 
Column comparisons & Fix unique index bug
  • Loading branch information
@techfort
techfort authored Jul 30, 2020
2 parents 8791bd9 + fabe503 commit 1c3114e
Show file tree
Hide file tree
Showing 4 changed files with 145 additions and 40 deletions.
67 changes: 67 additions & 0 deletions spec/generic/ops.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -428,4 +428,71 @@ describe("Individual operator tests", function() {
}
}).length).toBe(1)
});

it('$$op column comparisons work', function () {
var db = new loki('db');
var coll = db.addCollection('coll');

coll.insert({ a: null, b: 5 });
coll.insert({ a: '5', b: 5 });
coll.insert({ a: 5, b: 5 });
coll.insert({ a: 6, b: 5 });
coll.insert({ a: 3, b: 5 });
coll.insert({ a: 3, b: 'number' });

// standard case
expect(coll.find({ a: { $$eq: 'b' } }).length).toEqual(1);
expect(coll.find({ a: { $$aeq: 'b' } }).length).toEqual(2);
expect(coll.find({ a: { $$ne: 'b' } }).length).toEqual(5);
expect(coll.find({ a: { $$gt: 'b' } }).length).toEqual(1);
expect(coll.find({ a: { $$gte: 'b' } }).length).toEqual(3);

// function variant
expect(coll.find({ a: { $$gt: function (record) { return record.b - 1 } } }).length).toEqual(4);

// comparison on filtered rows
expect(coll.find({ b: { $gt: 0 }, a: { $$aeq: 'b' } }).length).toEqual(2);

// type
expect(coll.find({ a: { $$type: 'b' } }).length).toEqual(1);
expect(coll.find({ a: { $type: { $$eq: 'b' } } }).length).toEqual(1);

// $not, $and, $or
expect(coll.find({ a: { $not: { $$type: 'b' } } }).length).toEqual(5);
expect(coll.find({ a: { $and: [{ $type: 'number' }, { $$gte: 'b' }] } }).length).toEqual(2);
expect(coll.find({ a: { $or: [{ $eq: null }, { $$gt: 'b' }] } }).length).toEqual(2);

// $len
coll.insert({ text1: 'blablabla', len: 10 })
coll.insert({ text1: 'abcdef', len: 6 })
coll.insert({ text1: 'abcdef', len: 3 })
expect(coll.find({ text1: { $len: { $$eq: 'len' } } }).length).toEqual(1);

// $size
coll.insert({ array1: [1, 2, 3], size: 2 })
coll.insert({ array1: [1, 2], size: 1 })
coll.insert({ array1: [1, 2], size: 3 })
coll.insert({ array1: [1, 2, 3, 4], size: 5 })
expect(coll.find({ array1: { $size: { $$eq: 'size' } } }).length).toEqual(0);
expect(coll.find({ array1: { $size: { $$lt: 'size' } } }).length).toEqual(2);

// $elemMatch
coll.insert({ els: [{ a: 1, b: 2 }] })
coll.insert({ els: [{ a: 1, b: 2 }, { a: 2, b: 2 }] })
expect(coll.find({ els: { $elemMatch: { a: { $$eq: 'b' } } } }).length).toEqual(1);

// $elemMatch - dot scan
coll.insert({ els2: [{ a: { val: 1 }, b: 2 }] })
coll.insert({ els2: [{ a: { val: 1 }, b: 2 }, { a: { val: 2 }, b: 2 }] })
expect(coll.find({ els2: { $elemMatch: { 'a.val': { $$eq: 'b' } } } }).length).toEqual(1);

// dot notation
coll.insert({ c: { val: 5 }, b: 5 });
coll.insert({ c: { val: 6 }, b: 5 });
coll.insert({ c: { val: 7 }, b: 6 });
expect(coll.find({ 'c.val': { $$gt: 'b' } }).length).toEqual(2);

// dot notation - on filtered rows
expect(coll.find({ b: { $gt: 0 }, 'c.val': { $$gt: 'b' } }).length).toEqual(2);
});
});
25 changes: 17 additions & 8 deletions spec/generic/unique.spec.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ describe('Constraints', function () {
it('coll.clear should affect unique indices correctly', function() {
var db = new loki();
var coll = db.addCollection('users', { unique: ['username'] });

coll.insert({ username: 'joe', name: 'Joe' });
coll.insert({ username: 'jack', name: 'Jack' });
coll.insert({ username: 'jake', name: 'Jake' });
Expand All @@ -100,7 +100,7 @@ describe('Constraints', function () {

var db = new loki();
var coll = db.addCollection('users', { unique: ['username'] });

coll.insert({ username: 'joe', name: 'Joe' });
coll.insert({ username: 'jack', name: 'Jack' });
coll.insert({ username: 'jake', name: 'Jake' });
Expand All @@ -122,18 +122,18 @@ describe('Constraints', function () {
{name:'Jormungandr', legs: 0},
{name:'Hel', legs: 2}
];

var db = new loki('test.db');
var collection = db.addCollection("children", {
unique: ["name"]
});

data.forEach(function(c) {
collection.insert(JSON.parse(JSON.stringify(c)));
});

collection.findAndRemove();

// reinsert 2 of the 3 original docs
// implicitly 'expecting' that this will not throw exception on Duplicate key for property name(s)
collection.insert(JSON.parse(JSON.stringify(data[0])));
Expand All @@ -158,20 +158,20 @@ describe('Constraints', function () {
{name:'Jormungandr', legs: 0},
{name:'Hel', legs: 2}
];

var db = new loki('test.db');
var collection = db.addCollection("children", {
unique: ["name"]
});

data.forEach(function(c) {
collection.insert(JSON.parse(JSON.stringify(c)));
});

collection.chain().update(function(obj) {
obj.name = obj.name + '2';
});

// implicitly 'expecting' that this will not throw exception on Duplicate key for property name: Sleipnir
data.forEach(function(c) {
collection.insert(JSON.parse(JSON.stringify(c)));
Expand Down Expand Up @@ -222,4 +222,13 @@ describe('Constraints', function () {
expect(keys[4]).toEqual('Sleipnir');
expect(keys[5]).toEqual('Sleipnir2');
});
it('should not crash on unsafe strings', function () {
var db = new loki();
var coll = db.addCollection('local_storage', {
unique: ['key']
});
expect(coll.by('key', 'hasOwnProperty')).toBe(undefined);
coll.insert({ key: 'hasOwnProperty', name: 'hey' });
expect(coll.by('key', 'hasOwnProperty').name).toBe('hey');
});
});
84 changes: 52 additions & 32 deletions src/lokijs.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -406,9 +406,10 @@
* @param {array} paths - array of properties to drill into
* @param {function} fun - evaluation function to test with
* @param {any} value - comparative value to also pass to (compare) fun
* @param {any} extra - extra arg to also pass to compare fun
* @param {number} poffset - index of the item in 'paths' to start the sub-scan from
*/
function dotSubScan(root, paths, fun, value, poffset) {
function dotSubScan(root, paths, fun, value, extra, poffset) {
var pathOffset = poffset || 0;
var path = paths[pathOffset];

Expand All @@ -420,16 +421,16 @@
if (pathOffset + 1 >= paths.length) {
// if we have already expanded out the dot notation,
// then just evaluate the test function and value on the element
valueFound = fun(element, value);
valueFound = fun(element, value, extra);
} else if (Array.isArray(element)) {
for (var index = 0, len = element.length; index < len; index += 1) {
valueFound = dotSubScan(element[index], paths, fun, value, pathOffset + 1);
valueFound = dotSubScan(element[index], paths, fun, value, extra, pathOffset + 1);
if (valueFound === true) {
break;
}
}
} else {
valueFound = dotSubScan(element, paths, fun, value, pathOffset + 1);
valueFound = dotSubScan(element, paths, fun, value, extra, pathOffset + 1);
}

return valueFound;
Expand All @@ -448,10 +449,10 @@
return null;
}

function doQueryOp(val, op) {
function doQueryOp(val, op, record) {
for (var p in op) {
if (hasOwnProperty.call(op, p)) {
return LokiOps[p](val, op[p]);
return LokiOps[p](val, op[p], record);
}
}
return false;
Expand Down Expand Up @@ -591,16 +592,16 @@
}

if (property.indexOf('.') !== -1) {
return dotSubScan(item, property.split('.'), doQueryOp, b[property]);
return dotSubScan(item, property.split('.'), doQueryOp, b[property], item);
}
return doQueryOp(item[property], filter);
return doQueryOp(item[property], filter, item);
});
});
}
return false;
},

$type: function (a, b) {
$type: function (a, b, record) {
var type = typeof a;
if (type === 'object') {
if (Array.isArray(a)) {
Expand All @@ -609,23 +610,23 @@
type = 'date';
}
}
return (typeof b !== 'object') ? (type === b) : doQueryOp(type, b);
return (typeof b !== 'object') ? (type === b) : doQueryOp(type, b, record);
},

$finite: function (a, b) {
return (b === isFinite(a));
},

$size: function (a, b) {
$size: function (a, b, record) {
if (Array.isArray(a)) {
return (typeof b !== 'object') ? (a.length === b) : doQueryOp(a.length, b);
return (typeof b !== 'object') ? (a.length === b) : doQueryOp(a.length, b, record);
}
return false;
},

$len: function (a, b) {
$len: function (a, b, record) {
if (typeof a === 'string') {
return (typeof b !== 'object') ? (a.length === b) : doQueryOp(a.length, b);
return (typeof b !== 'object') ? (a.length === b) : doQueryOp(a.length, b, record);
}
return false;
},
Expand All @@ -638,22 +639,22 @@
// a is the value in the collection
// b is the nested query operation (for '$not')
// or an array of nested query operations (for '$and' and '$or')
$not: function (a, b) {
return !doQueryOp(a, b);
$not: function (a, b, record) {
return !doQueryOp(a, b, record);
},

$and: function (a, b) {
$and: function (a, b, record) {
for (var idx = 0, len = b.length; idx < len; idx += 1) {
if (!doQueryOp(a, b[idx])) {
if (!doQueryOp(a, b[idx], record)) {
return false;
}
}
return true;
},

$or: function (a, b) {
$or: function (a, b, record) {
for (var idx = 0, len = b.length; idx < len; idx += 1) {
if (doQueryOp(a, b[idx])) {
if (doQueryOp(a, b[idx], record)) {
return true;
}
}
Expand All @@ -669,6 +670,21 @@
}
};

// ops that can be used with { $$op: 'column-name' } syntax
var valueLevelOps = ['$eq', '$aeq', '$ne', '$dteq', '$gt', '$gte', '$lt', '$lte', '$jgt', '$jgte', '$jlt', '$jlte', '$type'];
valueLevelOps.forEach(function (op) {
var fun = LokiOps[op];
LokiOps['$' + op] = function (a, spec, record) {
if (typeof spec === 'string') {
return fun(a, record[spec]);
} else if (typeof spec === 'function') {
return fun(a, spec(record));
} else {
throw new Error('Invalid argument to $$ matcher');
}
};
});

// if an op is registered in this object, our 'calculateRange' can use it with our binary indices.
// if the op is registered to a function, we will run that function/op as a 2nd pass filter on results.
// those 2nd pass filter functions should be similar to LokiOps functions, accepting 2 vals to compare.
Expand Down Expand Up @@ -3547,7 +3563,7 @@
//
// For performance reasons, each case has its own if block to minimize in-loop calculations

var filter, rowIdx = 0;
var filter, rowIdx = 0, record;

// If the filteredrows[] is already initialized, use it
if (this.filterInitialized) {
Expand All @@ -3559,7 +3575,8 @@
property = property.split('.');
for (i = 0; i < len; i++) {
rowIdx = filter[i];
if (dotSubScan(t[rowIdx], property, fun, value)) {
record = t[rowIdx];
if (dotSubScan(record, property, fun, value, record)) {
result.push(rowIdx);
if (firstOnly) {
this.filteredrows = result;
Expand All @@ -3570,7 +3587,8 @@
} else {
for (i = 0; i < len; i++) {
rowIdx = filter[i];
if (fun(t[rowIdx][property], value)) {
record = t[rowIdx];
if (fun(record[property], value, record)) {
result.push(rowIdx);
if (firstOnly) {
this.filteredrows = result;
Expand All @@ -3589,7 +3607,8 @@
if (usingDotNotation) {
property = property.split('.');
for (i = 0; i < len; i++) {
if (dotSubScan(t[i], property, fun, value)) {
record = t[i];
if (dotSubScan(record, property, fun, value, record)) {
result.push(i);
if (firstOnly) {
this.filteredrows = result;
Expand All @@ -3600,7 +3619,8 @@
}
} else {
for (i = 0; i < len; i++) {
if (fun(t[i][property], value)) {
record = t[i];
if (fun(record[property], value, record)) {
result.push(i);
if (firstOnly) {
this.filteredrows = result;
Expand Down Expand Up @@ -4115,8 +4135,8 @@
DynamicView.prototype.constructor = DynamicView;

/**
* getSort() - used to get the current sort
*
* getSort() - used to get the current sort
*
* @returns function (sortFunction) or array (sortCriteria) or object (sortCriteriaSimple)
*/
DynamicView.prototype.getSort = function () {
Expand Down Expand Up @@ -7489,8 +7509,8 @@

function UniqueIndex(uniqueField) {
this.field = uniqueField;
this.keyMap = {};
this.lokiMap = {};
this.keyMap = Object.create(null);
this.lokiMap = Object.create(null);
}
UniqueIndex.prototype.keyMap = {};
UniqueIndex.prototype.lokiMap = {};
Expand Down Expand Up @@ -7537,12 +7557,12 @@
}
};
UniqueIndex.prototype.clear = function () {
this.keyMap = {};
this.lokiMap = {};
this.keyMap = Object.create(null);
this.lokiMap = Object.create(null);
};

function ExactIndex(exactField) {
this.index = {};
this.index = Object.create(null);
this.field = exactField;
}

Expand Down
9 changes: 9 additions & 0 deletions tutorials/Query Examples.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -319,6 +319,15 @@ var results = coll.find({
});
```

**$$eq, $$eq, etc.** - many filters support column comparisons - comparing one column in a document
with another (instead of between column and value):
```javascript
// fetch documents where foo > bar
var results = coll.find({{ foo: { $$gt: 'bar' } }})

// instead of passing second column name, you can pass a function that computes value to compare against
var results = coll.find({{ foo: { $$lt: doc => doc.bar + 1 } }})
```
### Features which support 'find' queries
These operators can be used to compose find filter objects which can be used within :
* Collection find()
Expand Down

0 comments on commit 1c3114e

Please sign in to comment.