Skip to content

Commit

Permalink
Activate DupSort for IDX
Browse files Browse the repository at this point in the history
  • Loading branch information
@stpietsch
stpietsch committed Mar 16, 2022
1 parent d8de362 commit f21bfb8
Show file tree
Hide file tree
Showing 10 changed files with 113 additions and 69 deletions.
7 changes: 2 additions & 5 deletions index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,6 @@ let id2 = await db.table(tableName).save({ title: "" });

await db.table(tableName).ensureIndex("title");

let data = await db.table(tableName).filter((row) => {
return row.title === "Mr.";
});
let r = await db.table(tableName).find((row) => row.title == "Mr.");

await db.delete();
await db.close();
console.log(r);
42 changes: 20 additions & 22 deletions lib/CodeInterpreter.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ export default class CodeInterpreter {

ret = {
*[Symbol.iterator]() {
let range = db.getRange({
let range = db.getRangeData({
start: dbKey(dbValues.LO)
,end: dbKey(dbValues.HI)
})
Expand Down Expand Up @@ -98,7 +98,7 @@ export default class CodeInterpreter {
*[Symbol.iterator]() {
try {
for(let id of result.recordset) {
let val = db.get(dbKey(id));
let val = db.getData(dbKey(id));
if(result.recordset.count) {
this.query.indexes[key] = result.recordset.count;
}
Expand Down Expand Up @@ -205,32 +205,32 @@ export default class CodeInterpreter {
queryIndex(key, value) {
if (this.#meta.indexes[key] && this.#meta.indexes[key].build) {
let db = this.#db
let idxKey = (indexName, value, id) => indexKey(this.#meta.name, indexName, value, id)
let idxKey = (indexName, value, ext) => indexKey(this.#meta.name, indexName, value, ext)

if(value === Object(value)) {
let conditionKey = Object.keys(value)[0];
let conditionValue = value[conditionKey];
if(conditionKey === "$not") {
return {
*[Symbol.iterator]() {
let range1 = db.getKeys({
start: idxKey(key, dbValues.LO, dbValues.LO)
,end: idxKey(key, conditionValue, dbValues.LO)
let range1 = db.getRangeIdx({
start: idxKey(key, dbValues.LO)
,end: idxKey(key, conditionValue)
})

for(let row of range1) {
this.count++
yield row.pop()
yield row.value
}

let range2 = db.getKeys({
let range2 = db.getRangeIdx({
start: idxKey(key, conditionValue, dbValues.HI)
,end: idxKey(key, dbValues.HI, dbValues.HI)
,end: idxKey(key, dbValues.HI)
})

for(let row of range2) {
this.count++
yield row.pop()
yield row.value
}
},
count: 0
Expand All @@ -239,14 +239,15 @@ export default class CodeInterpreter {
if(conditionKey === "$gte" || conditionKey === "$gt") {
return {
*[Symbol.iterator]() {
let range = db.getKeys({

let range = db.getRangeIdx({
start: idxKey(key, conditionValue, conditionKey === "$gte"? dbValues.LO : dbValues.HI)
,end: idxKey(key, dbValues.HI, dbValues.HI)
,end: idxKey(key, dbValues.HI)
})

for(let row of range) {
this.count++
yield row.pop()
yield row.value
}
},
count: 0
Expand All @@ -255,14 +256,14 @@ export default class CodeInterpreter {
if(conditionKey === "$lte" || conditionKey === "$lt") {
return {
*[Symbol.iterator]() {
let range = db.getKeys({
start: idxKey(key, dbValues.LO, dbValues.LO)
let range = db.getRangeIdx({
start: idxKey(key, dbValues.LO)
,end: idxKey(key, conditionValue, conditionKey === "$lte"? dbValues.HI : dbValues.LO)
})

for(let row of range) {
this.count++
yield row.pop()
yield row.value
}
},
count: 0
Expand All @@ -272,14 +273,11 @@ export default class CodeInterpreter {

return {
*[Symbol.iterator]() {
let range = db.getKeys({
start: idxKey(key, value, dbValues.LO)
,end: idxKey(key, value, dbValues.HI)
})
let range = db.getValuesIdx(idxKey(key, value))

for(let row of range) {
this.count++
yield row.pop()
yield row
}
},
count: 0
Expand All @@ -293,7 +291,7 @@ function interpreteStart(code, context) {
let expression = code.body[0].declarations[0].init;
if (["ArrowFunctionExpression", "FunctionExpression"].includes(expression.type)) {
let name = expression.params[0]?.name;
console.log(expression.body)
//console.log(expression.body)
let r = interpreteNode(expression.body, name, context);
return r
}
Expand Down
21 changes: 12 additions & 9 deletions lib/Database.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,10 @@ import ThreadPool from "./ThreadPool.js";
import { authenticate, createPasswordHash, verifyToken, createToken } from "./Authorization.js";
import { nanoid } from "nanoid/async";

let __dirname = path.dirname(fileURLToPath(import.meta.url));
const __dirname = path.dirname(fileURLToPath(import.meta.url));

const META = Symbol.for("meta");
const SECRET = Symbol.for("secret");

export default class Database {
#name;
Expand All @@ -19,14 +22,14 @@ export default class Database {
#closed = true;
#systemToken;
#meta() {
return this._db.get("meta");
return this._db.get(META);
}

#secret() {
if (this.#deleted === true) {
throw new Error("Database is deleted");
}
return this._db.get("secret");
return this._db.get(SECRET);
}

getUser(token) {
Expand Down Expand Up @@ -174,14 +177,14 @@ export default class Database {
},
};
meta.tables[name] = tableMeta;
this._db.putSync("meta", meta);
this._db.putSync(META, meta);
}
this.#loadedTables[name] = new Table(
tableMeta,
(data) => {
let meta = this.#meta();
meta.tables[name] = data;
this._db.putSync("meta", meta);
this._db.putSync(META, meta);
},
this._db,
this.#threadPool,
Expand Down Expand Up @@ -231,7 +234,7 @@ export default class Database {
this.#closed = false;

if (!this.#meta()) {
await this._db.put("meta", {
await this._db.put(META, {
name: this.#name,
version: pack.version,
manageGroups: {
Expand All @@ -242,21 +245,21 @@ export default class Database {
}

if (!this.#secret()) {
await this._db.put("secret", `${await nanoid()}${await nanoid()}${await nanoid()}`);
await this._db.put(SECRET, `${await nanoid()}${await nanoid()}${await nanoid()}`);
}

if (this.#meta().name !== this.#name) {
let meta = this.#meta();
meta.name = this.#name;
await this._db.put("meta", meta);
this._db.putSync(META, meta);
}

if (this.#meta().version !== pack.version) {
let upgrade = await import("./Upgrade.js");
await upgrade.default(this.#meta().version, pack.version);
let meta = this.#meta();
meta.version = pack.version;
await this._db.put("meta", meta);
this._db.putSync(META, meta);
}
await this.#threadPool.start();

Expand Down
51 changes: 47 additions & 4 deletions lib/Storage.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,11 +2,38 @@ import { open } from "lmdb";

export default class Storage {
#db;
#data;
#idx;
constructor(path) {
this.#db = open({
path: path,
sharedStructuresKey: Symbol.for("structures"),
noMemInit: true,
});
this.#data = this.#db.openDB("tableData");
this.#idx = this.#db.openDB("idx", {
dupSort: true,
});
}

getRange(options) {
return this.#db.getRange(options);
}

getRangeData(options) {
return this.#data.getRange(options);
}

getRangeIdx(options) {
return this.#idx.getRange(options);
}

getValuesIdx(key, options) {
return this.#idx.getValues(key, options);
}

getKeys(options) {
return this.#db.getKeys(options);
}

putSync(key, value) {
Expand All @@ -17,20 +44,36 @@ export default class Storage {
return this.#db.put(key, value);
}

putData(key, value) {
return this.#data.put(key, value);
}

putIdx(key, value) {
return this.#idx.put(key, value);
}

remove(key) {
return this.#db.remove(key);
}

removeData(key) {
return this.#data.remove(key);
}

removeIdx(key, value) {
return this.#idx.put(key, value);
}

get(key) {
return this.#db.get(key);
}

getRange(options) {
return this.#db.getRange(options);
getData(key) {
return this.#data.get(key);
}

getKeys(options) {
return this.#db.getKeys(options);
getIdx(key) {
return this.#idx.get(key);
}

close() {
Expand Down
20 changes: 11 additions & 9 deletions lib/Table.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -48,7 +48,7 @@ export default class Table {

async get(id, token = this.#token) {
let user = this.getUser(token);
let data = await this.#db.get(this.#tableKey(id));
let data = await this.#db.getData(this.#tableKey(id));
return data;
}

Expand Down Expand Up @@ -97,7 +97,9 @@ export default class Table {
for await (let row of query) {
query.destroy();
result = row;
result.getQuery = () => query.getQuery();
if (result !== undefined) {
result.getQuery = () => query.getQuery();
}
break;
}
return result;
Expand Down Expand Up @@ -125,7 +127,7 @@ export default class Table {

if (data._id) {
hadOld = true;
old = this.#db.get(this.#tableKey(data._id));
old = this.#db.getData(this.#tableKey(data._id));
}

for (let prop in this.#meta.indexes) {
Expand All @@ -134,14 +136,14 @@ export default class Table {
if (val === data[prop]) {
continue;
}
this.#db.remove(this.#indexKey(prop, val, id));
this.#db.removeIdx(this.#indexKey(prop, val), id);
}

let val = data[prop];
this.#db.put(this.#indexKey(prop, val, id), null);
this.#db.putIdx(this.#indexKey(prop, val), id);
}

this.#db.put(this.#tableKey(id), data);
this.#db.putData(this.#tableKey(id), data);
return id;
});
return id;
Expand All @@ -150,14 +152,14 @@ export default class Table {
async remove(id, token = this.#token) {
let old;
await this.#db.transaction(() => {
old = this.#db.get(this.#tableKey(id));
old = this.#db.getData(this.#tableKey(id));
if (old) {
for (let prop in this.#meta.indexes) {
let val = old[prop];
this.#db.remove(this.#indexKey(prop, val, id));
this.#db.removeIdx(this.#indexKey(prop, val), id);
}
}
this.#db.remove(this.#tableKey(id));
this.#db.removeData(this.#tableKey(id));
});
}
}
12 changes: 6 additions & 6 deletions lib/Thread.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -110,15 +110,15 @@ const fns = {
},
async createIndex({ name, meta }) {
await db.transaction(() => {
let table = db.getRange({
let table = db.getRangeData({
start: tableKey(meta.name, dbValues.LO),
end: tableKey(meta.name, dbValues.HI),
});

for (let row of table) {
let val = row.value[name];
let id = row.key.pop();
db.put(indexKey(meta.name, name, val, id), null);
db.putIdx(indexKey(meta.name, name, val), id);
}
});

Expand All @@ -128,12 +128,12 @@ const fns = {
},
async removeIndex({ name, meta }) {
await db.transaction(() => {
let table = db.getRange({
start: indexKey(meta.name, name, dbValues.LO, dbValues.LO),
end: indexKey(meta.name, name, dbValues.HI, dbValues.HI),
let table = db.getRangeIdx({
start: indexKey(meta.name, name, dbValues.LO),
end: indexKey(meta.name, name, dbValues.HI),
});
for (let { key } of table) {
db.remove(key);
db.removeIdx(key);
}
});

Expand Down
Loading

0 comments on commit f21bfb8

Please sign in to comment.