Skip to content

Commit

Permalink
feat(minato): add primary type for auto generated primary key (#36)
Browse files Browse the repository at this point in the history 
Co-authored-by: Shigma <[email protected]>
  • Loading branch information
@Hieuzest @shigma
Hieuzest and shigma authored Aug 25, 2023
1 parent cf2e8db commit 2834186
Show file tree
Hide file tree
Showing 5 changed files with 102 additions and 39 deletions.
10 changes: 9 additions & 1 deletion packages/core/src/model.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,9 @@ import { Eval, isEvalExpr } from './eval'
import { Selection } from './selection'
import { Flatten, Keys } from './utils'

export const Primary = Symbol('Primary')
export type Primary = (string | number) & { [Primary]: true }

export interface Field<T = any> {
type: Field.Type<T>
length?: number
Expand All @@ -24,7 +27,8 @@ export namespace Field {
export const object: Type[] = ['list', 'json']

export type Type<T = any> =
| T extends number ? 'integer' | 'unsigned' | 'float' | 'double' | 'decimal'
| T extends Primary ? 'primary'
: T extends number ? 'integer' | 'unsigned' | 'float' | 'double' | 'decimal'
: T extends string ? 'char' | 'string' | 'text'
: T extends boolean ? 'boolean'
: T extends Date ? 'timestamp' | 'date' | 'time'
Expand Down Expand Up @@ -121,6 +125,10 @@ export class Model<S = any> {
this.fields[key].deprecated = !!callback
}

if (typeof this.primary === 'string' && this.fields[this.primary]?.type === 'primary') {
this.autoInc = true
}

// check index
this.checkIndex(this.primary)
this.unique.forEach(index => this.checkIndex(index))
Expand Down
48 changes: 28 additions & 20 deletions packages/mongo/src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ export class MongoDriver extends Driver {

;[primary, ...unique].forEach((keys, index) => {
// use internal `_id` for single primary fields
if (this.config.optimizeIndex && !index && typeof keys === 'string') return
if (primary === keys && !index && this.getVirtualKey(table)) return

// if the index is already created, skip it
keys = makeArray(keys)
Expand Down Expand Up @@ -153,20 +153,20 @@ export class MongoDriver extends Driver {
const meta: Dict = { table, field: primary }
const found = await fields.findOne(meta)
let virtual = !!found?.virtual
const useVirtualKey = !!this.getVirtualKey(table)
// If _fields table was missing for any reason
// Test the type of _id to get its possible preference
if (!found) {
const doc = await this.db.collection(table).findOne()
if (doc) virtual = typeof doc._id !== 'object'
else {
// Empty collection, just set meta and return
fields.updateOne(meta, { $set: { virtual: this.config.optimizeIndex } }, { upsert: true })
fields.updateOne(meta, { $set: { virtual: useVirtualKey } }, { upsert: true })
logger.info('Successfully reconfigured table %s', table)
return
}
}

if (virtual === !!this.config.optimizeIndex) return
if (virtual === useVirtualKey) return
logger.info('Start migrating table %s', table)

if (found?.migrate && await this.db.listCollections({ name: '_migrate_' + table }).hasNext()) {
Expand All @@ -176,16 +176,16 @@ export class MongoDriver extends Driver {
await this.db.collection(table).aggregate([
{ $addFields: { _temp_id: '$_id' } },
{ $unset: ['_id'] },
{ $addFields: this.config.optimizeIndex ? { _id: '$' + primary } : { [primary]: '$_temp_id' } },
{ $unset: ['_temp_id', ...this.config.optimizeIndex ? [primary] : []] },
{ $addFields: useVirtualKey ? { _id: '$' + primary } : { [primary]: '$_temp_id' } },
{ $unset: ['_temp_id', ...useVirtualKey ? [primary] : []] },
{ $out: '_migrate_' + table },
]).toArray()
await fields.updateOne(meta, { $set: { migrate: true } }, { upsert: true })
}
await this.db.dropCollection(table).catch(noop)
await this.db.renameCollection('_migrate_' + table, table)
await fields.updateOne(meta,
{ $set: { virtual: this.config.optimizeIndex, migrate: false } },
{ $set: { virtual: useVirtualKey, migrate: false } },
{ upsert: true },
)
logger.info('Successfully migrated table %s', table)
Expand All @@ -201,7 +201,7 @@ export class MongoDriver extends Driver {

const coll = this.db.collection(table)
// Primary _id cannot be modified thus should always meet the requirements
if (!this.config.optimizeIndex) {
if (!this.getVirtualKey(table)) {
const bulk = coll.initializeOrderedBulkOp()
await coll.find().forEach((data) => {
bulk
Expand All @@ -211,9 +211,9 @@ export class MongoDriver extends Driver {
if (bulk.batches.length) await bulk.execute()
}

const [latest] = await coll.find().sort(this.config.optimizeIndex ? '_id' : primary, -1).limit(1).toArray()
const [latest] = await coll.find().sort(this.getVirtualKey(table) ? '_id' : primary, -1).limit(1).toArray()
await fields.updateOne(meta, {
$set: { autoInc: latest ? +latest[this.config.optimizeIndex ? '_id' : primary] : 0 },
$set: { autoInc: latest ? +latest[this.getVirtualKey(table) ? '_id' : primary] : 0 },
}, { upsert: true })
}

Expand Down Expand Up @@ -285,24 +285,24 @@ export class MongoDriver extends Driver {
}

private getVirtualKey(table: string) {
const { primary } = this.model(table)
if (typeof primary === 'string' && this.config.optimizeIndex) {
const { primary, fields } = this.model(table)
if (typeof primary === 'string' && (this.config.optimizeIndex || fields[primary]?.type === 'primary')) {
return primary
}
}

private patchVirtual(table: string, row: any) {
const { primary } = this.model(table)
if (typeof primary === 'string' && this.config.optimizeIndex) {
const { primary, fields } = this.model(table)
if (typeof primary === 'string' && (this.config.optimizeIndex || fields[primary]?.type === 'primary')) {
row[primary] = row['_id']
delete row['_id']
}
return row
}

private unpatchVirtual(table: string, row: any) {
const { primary } = this.model(table)
if (typeof primary === 'string' && this.config.optimizeIndex) {
const { primary, fields } = this.model(table)
if (typeof primary === 'string' && (this.config.optimizeIndex || fields[primary]?.type === 'primary')) {
row['_id'] = row[primary]
delete row[primary]
}
Expand Down Expand Up @@ -464,13 +464,19 @@ export class MongoDriver extends Driver {
private shouldEnsurePrimary(table: string) {
const model = this.model(table)
const { primary, autoInc } = model
return typeof primary === 'string' && autoInc
return typeof primary === 'string' && autoInc && model.fields[primary]?.type !== 'primary'
}

private shouldFillPrimary(table: string) {
const model = this.model(table)
const { primary, autoInc } = model
return typeof primary === 'string' && autoInc && model.fields[primary]?.type === 'primary'
}

private async ensurePrimary(table: string, data: any[]) {
const model = this.model(table)
const { primary, autoInc } = model
if (typeof primary === 'string' && autoInc) {
if (typeof primary === 'string' && autoInc && model.fields[primary]?.type !== 'primary') {
const missing = data.filter(item => !(primary in item))
if (!missing.length) return
const { value } = await this.db.collection('_fields').findOneAndUpdate(
Expand All @@ -495,8 +501,10 @@ export class MongoDriver extends Driver {
try {
data = model.create(data)
const copy = this.unpatchVirtual(table, { ...data })
await coll.insertOne(copy)
return data
const insertedId = (await coll.insertOne(copy)).insertedId
if (this.shouldFillPrimary(table)) {
return { ...data, [model.primary as string]: insertedId }
} else return data
} catch (err) {
if (err instanceof MongoError && err.code === 11000) {
throw new RuntimeError('duplicate-entry', err.message)
Expand Down
81 changes: 63 additions & 18 deletions packages/mongo/tests/migration.spec.ts
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { Database } from 'minato'
import { Database, Primary } from 'minato'
import Logger from 'reggol'
import { expect } from 'chai'
import { } from 'chai-shape'
Expand All @@ -18,15 +18,28 @@ interface Foo {
regex?: string
}

interface Bar {
id?: Primary
text?: string
value?: number
bool?: boolean
list?: number[]
timestamp?: Date
date?: Date
time?: Date
regex?: string
}

interface Tables {
temp1: Foo
temp2: Bar
}

describe('@minatojs/driver-mongo/migrate-virtualKey', () => {
const database: Database<Tables> = new Database()

const initialize = async (optimizeIndex: boolean) => {
logger.level = 3
const resetConfig = async (optimizeIndex: boolean) => {
await database.stopAll()
await database.connect('mongo', {
host: 'localhost',
port: 27017,
Expand All @@ -35,20 +48,23 @@ describe('@minatojs/driver-mongo/migrate-virtualKey', () => {
})
}

const finalize = async () => {
await database.stopAll()
logger.level = 2
}
beforeEach(async () => {
logger.level = 3
await database.connect('mongo', {
host: 'localhost',
port: 27017,
database: 'test',
optimizeIndex: false,
})
})

after(async () => {
afterEach(async () => {
await database.dropAll()
await database.stopAll()
logger.level = 2
})

it('reset optimizeIndex', async () => {
await initialize(false)

database.extend('temp1', {
id: 'unsigned',
text: 'string',
Expand All @@ -74,22 +90,51 @@ describe('@minatojs/driver-mongo/migrate-virtualKey', () => {
table.push(await database.create('temp1', { text: 'awesome baz' }))
await expect(database.get('temp1', {})).to.eventually.have.shape(table)

await finalize()
await initialize(true)
await resetConfig(true)
await expect(database.get('temp1', {})).to.eventually.have.shape(table)

await finalize()
await initialize(false)
await resetConfig(false)
await expect(database.get('temp1', {})).to.eventually.have.shape(table)

await (Object.values(database.drivers)[0] as MongoDriver).drop('_fields')
await finalize()
await initialize(true)
await resetConfig(true)
await expect(database.get('temp1', {})).to.eventually.have.shape(table)

await (Object.values(database.drivers)[0] as MongoDriver).drop('_fields')
await finalize()
await initialize(false)
await resetConfig(false)
await expect(database.get('temp1', {})).to.eventually.have.shape(table)
})

it('using primary', async () => {
database.extend('temp2', {
id: 'primary',
text: 'string',
value: 'integer',
bool: 'boolean',
list: 'list',
timestamp: 'timestamp',
date: 'date',
time: 'time',
regex: 'string',
})

const table: Bar[] = []
table.push(await database.create('temp2', {
text: 'awesome foo',
timestamp: new Date('2000-01-01'),
date: new Date('2020-01-01'),
time: new Date('2020-01-01 12:00:00'),
}))
table.push(await database.create('temp2', { text: 'awesome bar' }))
table.push(await database.create('temp2', { text: 'awesome baz' }))
await expect(database.get('temp2', {})).to.eventually.have.shape(table)

await (Object.values(database.drivers)[0] as MongoDriver).drop('_fields')
await resetConfig(true)
await expect(database.get('temp2', {})).to.eventually.have.shape(table)

await (Object.values(database.drivers)[0] as MongoDriver).drop('_fields')
await resetConfig(false)
await expect(database.get('temp2', {})).to.eventually.have.shape(table)
})
})
1 change: 1 addition & 0 deletions packages/mysql/src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ function getTypeDef({ type, length, precision, scale }: Field) {
case 'timestamp': return 'datetime(3)'
case 'boolean': return 'bit'
case 'integer': return getIntegerType(length)
case 'primary':
case 'unsigned': return `${getIntegerType(length)} unsigned`
case 'decimal': return `decimal(${precision}, ${scale}) unsigned`
case 'char': return `char(${length || 255})`
Expand Down
1 change: 1 addition & 0 deletions packages/sqlite/src/index.ts
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ const logger = new Logger('sqlite')

function getTypeDef({ type }: Field) {
switch (type) {
case 'primary':
case 'boolean':
case 'integer':
case 'unsigned':
Expand Down

0 comments on commit 2834186

Please sign in to comment.