Skip to content

Commit

Permalink
feat: one to one, many to many relationship in link options (#253)
Browse files Browse the repository at this point in the history 
* feat: complete link options and unified relationship lookup and calculation process

* fix: sqlite capability
  • Loading branch information
@tea-artist
tea-artist authored Nov 23, 2023
1 parent 3f34120 commit afde2b2
Show file tree
Hide file tree
Showing 66 changed files with 3,093 additions and 2,645 deletions.
14 changes: 3 additions & 11 deletions apps/nestjs-backend/src/db-provider/db.provider.interface.ts
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,5 @@
import type { DriverClient, IFilter } from '@teable-group/core';
import type { Knex } from 'knex';
import type { IOpsData } from '../features/calculation/batch.service';
import type { ITopoLinkOrder } from '../features/calculation/reference.service';
import type { IFieldInstance } from '../features/field/model/factory';
import type { IAggregationFunctionInterface } from './aggregation/aggregation-function.interface';
import type { IFilterQueryInterface } from './filter-query/filter-query.interface';
Expand All @@ -13,18 +11,12 @@ export interface IDbProvider {

batchInsertSql(tableName: string, insertData: ReadonlyArray<unknown>): string;

affectedRecordItemsQuerySql(
topoOrder: ITopoLinkOrder[],
originRecordIdItems: { dbTableName: string; id: string }[]
): string;

executeUpdateRecordsSqlList(params: {
dbTableName: string;
fieldMap: { [fieldId: string]: IFieldInstance };
opsData: IOpsData[];
tempTableName: string;
columnNames: string[];
userId: string;
idFieldName: string;
dbFieldNames: string[];
data: { id: string; values: { [key: string]: unknown } }[];
}): { insertTempTableSql: string; updateRecordSql: string };

aggregationFunction(dbTableName: string, field: IFieldInstance): IAggregationFunctionInterface;
Expand Down
124 changes: 15 additions & 109 deletions apps/nestjs-backend/src/db-provider/postgres.provider.ts
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
import { Logger } from '@nestjs/common';
import type { IFilter } from '@teable-group/core';
import { DriverClient, Relationship } from '@teable-group/core';
import { DriverClient } from '@teable-group/core';
import type { Knex } from 'knex';
import { map } from 'lodash';
import type { IOpsData } from '../features/calculation/batch.service';
import type { ITopoLinkOrder } from '../features/calculation/reference.service';
import type { IFieldInstance } from '../features/field/model/factory';
import type { IAggregationFunctionInterface } from './aggregation/aggregation-function.interface';
import { AggregationFunctionPostgres } from './aggregation/aggregation-function.postgres';
Expand All @@ -29,125 +26,34 @@ export class PostgresProvider implements IDbProvider {
return this.knex.insert(insertData).into(tableName).toQuery();
}

affectedRecordItemsQuerySql(
topoOrder: ITopoLinkOrder[],
originRecordIdItems: { dbTableName: string; id: string }[]
): string {
// Initialize the base case for the recursive CTE
const initTableName = topoOrder[0].linkedTable;
const initQuery = this.knex
.select({
__id: '__id',
dbTableName: this.knex.raw('?', initTableName),
selectIn: this.knex.raw('?', null),
relationTo: this.knex.raw('?::varchar', null),
fieldId: this.knex.raw('?', null),
})
.from(initTableName)
.whereIn('__id', map(originRecordIdItems, 'id'));

let finalQuery = this.knex.queryBuilder();
// Iterate over the nodes in topological order
for (let i = 0; i < topoOrder.length; i++) {
const currentOrder = topoOrder[i];
const { fieldId, foreignKeyField, dbTableName, linkedTable } = currentOrder;
const affectedRecordsTable = `affected_records_${i}`;

// Append the current node to the recursive CTE
if (currentOrder.relationship === Relationship.OneMany) {
const oneManyQuery = this.knex
.select({
__id: this.knex.ref(`${linkedTable}.${foreignKeyField}`),
dbTableName: this.knex.raw('?', dbTableName),
selectIn: this.knex.raw('?', `${linkedTable}#${foreignKeyField}`),
relationTo: this.knex.raw('?', null),
fieldId: this.knex.raw('?', fieldId),
})
.from(linkedTable)
.join(affectedRecordsTable, `${linkedTable}.__id`, '=', `${affectedRecordsTable}.__id`)
.where(`${affectedRecordsTable}.dbTableName`, linkedTable);

const recursiveQuery =
i < 1 ? initQuery : this.knex.select('*').from(`affected_records_${i - 1}`);

finalQuery = finalQuery.withRecursive(
affectedRecordsTable,
recursiveQuery.union(oneManyQuery)
);
} else {
const manyOneQuery = this.knex
.select({
__id: this.knex.ref(`${dbTableName}.__id`),
dbTableName: this.knex.raw('?', dbTableName),
selectIn: this.knex.raw('?', null),
relationTo: this.knex.ref(`${affectedRecordsTable}.__id`),
fieldId: this.knex.raw('?', fieldId),
})
.from(dbTableName)
.join(
affectedRecordsTable,
`${dbTableName}.${foreignKeyField}`,
'=',
`${affectedRecordsTable}.__id`
)
.where(`${affectedRecordsTable}.dbTableName`, linkedTable);

const recursiveQuery =
i < 1 ? initQuery : this.knex.select('*').from(`affected_records_${i - 1}`);

finalQuery = finalQuery.withRecursive(
affectedRecordsTable,
recursiveQuery.union(manyOneQuery)
);
}
}

// Construct the final query using the recursive CTE
finalQuery = finalQuery.select('*').from(`affected_records_${topoOrder.length - 1}`);

// this.logger.log('affectedRecordItemsSql：%s', finalQuery.toQuery());
return finalQuery.toQuery();
}

executeUpdateRecordsSqlList(params: {
dbTableName: string;
fieldMap: { [fieldId: string]: IFieldInstance };
opsData: IOpsData[];
tempTableName: string;
columnNames: string[];
userId: string;
}): { insertTempTableSql: string; updateRecordSql: string } {
const { dbTableName, fieldMap, opsData, tempTableName, columnNames, userId } = params;

// 2.initialize temporary table data
const insertRowsData = opsData.map((data) => {
idFieldName: string;
dbFieldNames: string[];
data: { id: string; values: { [key: string]: unknown } }[];
}) {
const { dbTableName, tempTableName, idFieldName, dbFieldNames, data } = params;
const insertRowsData = data.map((item) => {
return {
__id: data.recordId,
__version: data.version + 1,
__last_modified_time: new Date().toISOString(),
__last_modified_by: userId,
...Object.entries(data.updateParam).reduce<{ [dbFieldName: string]: unknown }>(
(pre, [fieldId, value]) => {
const field = fieldMap[fieldId];
const { dbFieldName } = field;
pre[dbFieldName] = field.convertCellValue2DBValue(value);
return pre;
},
{}
),
[idFieldName]: item.id,
...item.values,
};
});

// initialize temporary table data
const insertTempTableSql = this.knex.insert(insertRowsData).into(tempTableName).toQuery();

// 3.update data
const updateColumns = columnNames.reduce<{ [key: string]: unknown }>((pre, columnName) => {
// update data
const updateColumns = dbFieldNames.reduce<{ [key: string]: unknown }>((pre, columnName) => {
pre[columnName] = this.knex.ref(`${tempTableName}.${columnName}`);
return pre;
}, {});

const updateRecordSql = this.knex(dbTableName)
.update(updateColumns)
.updateFrom(tempTableName)
.where(`${dbTableName}.__id`, this.knex.ref(`${tempTableName}.__id`))
.where(`${dbTableName}.${idFieldName}`, this.knex.ref(`${tempTableName}.${idFieldName}`))
.toQuery();

return { insertTempTableSql, updateRecordSql };
Expand Down
116 changes: 14 additions & 102 deletions apps/nestjs-backend/src/db-provider/sqlite.provider.ts
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,7 @@
import { Logger } from '@nestjs/common';
import type { IFilter } from '@teable-group/core';
import { DriverClient, Relationship } from '@teable-group/core';
import { DriverClient } from '@teable-group/core';
import type { Knex } from 'knex';
import { map } from 'lodash';
import type { IOpsData } from '../features/calculation/batch.service';
import type { ITopoLinkOrder } from '../features/calculation/reference.service';
import type { IFieldInstance } from '../features/field/model/factory';
import type { IAggregationFunctionInterface } from './aggregation/aggregation-function.interface';
import { AggregationFunctionSqlite } from './aggregation/aggregation-function.sqlite';
Expand Down Expand Up @@ -39,116 +36,31 @@ export class SqliteProvider implements IDbProvider {
return this.knex.raw(sql + body, bindings).toQuery();
}

affectedRecordItemsQuerySql(
topoOrder: ITopoLinkOrder[],
originRecordIdItems: { dbTableName: string; id: string }[]
): string {
// Initialize the base case for the recursive CTE
const initTableName = topoOrder[0].linkedTable;
const cteQuery = this.knex
.select({
__id: '__id',
dbTableName: this.knex.raw('?', initTableName),
selectIn: this.knex.raw('?', null),
relationTo: this.knex.raw('?', null),
fieldId: this.knex.raw('?', null),
})
.from(initTableName)
.whereIn('__id', map(originRecordIdItems, 'id'));

// Iterate over the nodes in topological order
for (let i = 0; i < topoOrder.length; i++) {
const currentOrder = topoOrder[i];
const { fieldId, foreignKeyField, dbTableName, linkedTable } = currentOrder;
const affectedRecordsTable = `affected_records`;

// Append the current node to the recursive CTE
if (currentOrder.relationship === Relationship.OneMany) {
const oneManyQuery = this.knex
.select({
__id: this.knex.ref(`${linkedTable}.${foreignKeyField}`),
dbTableName: this.knex.raw('?', dbTableName),
selectIn: this.knex.raw('?', `${linkedTable}#${foreignKeyField}`),
relationTo: this.knex.raw('?', null),
fieldId: this.knex.raw('?', fieldId),
})
.from(linkedTable)
.join(affectedRecordsTable, `${linkedTable}.__id`, '=', `${affectedRecordsTable}.__id`)
.where(`${affectedRecordsTable}.dbTableName`, linkedTable);

cteQuery.union(oneManyQuery);
} else {
const manyOneQuery = this.knex
.select({
__id: this.knex.ref(`${dbTableName}.__id`),
dbTableName: this.knex.raw('?', dbTableName),
selectIn: this.knex.raw('?', null),
relationTo: this.knex.ref(`${affectedRecordsTable}.__id`),
fieldId: this.knex.raw('?', fieldId),
})
.from(dbTableName)
.join(
affectedRecordsTable,
`${dbTableName}.${foreignKeyField}`,
'=',
`${affectedRecordsTable}.__id`
)
.where(`${affectedRecordsTable}.dbTableName`, linkedTable);

cteQuery.union(manyOneQuery);
}
}

// Construct the final query using the recursive CTE
const finalQuery = this.knex
.withRecursive('affected_records', cteQuery)
.select('*')
.from(`affected_records`);

// this.logger.log('affectedRecordItemsSql：%s', finalQuery.toQuery());
return finalQuery.toQuery();
}

executeUpdateRecordsSqlList(params: {
dbTableName: string;
fieldMap: { [fieldId: string]: IFieldInstance };
opsData: IOpsData[];
tempTableName: string;
columnNames: string[];
userId: string;
}): {
insertTempTableSql: string;
updateRecordSql: string;
} {
const { dbTableName, fieldMap, opsData, tempTableName, columnNames, userId } = params;

// 2.initialize temporary table data
const insertRowsData = opsData.map((data) => {
idFieldName: string;
dbFieldNames: string[];
data: { id: string; values: { [key: string]: unknown } }[];
}) {
const { dbTableName, tempTableName, idFieldName, dbFieldNames, data } = params;
const insertRowsData = data.map((item) => {
return {
__id: data.recordId,
__version: data.version + 1,
__last_modified_time: new Date().toISOString(),
__last_modified_by: userId,
...Object.entries(data.updateParam).reduce<{ [dbFieldName: string]: unknown }>(
(pre, [fieldId, value]) => {
const field = fieldMap[fieldId];
const { dbFieldName } = field;
pre[dbFieldName] = field.convertCellValue2DBValue(value);
return pre;
},
{}
),
[idFieldName]: item.id,
...item.values,
};
});

// initialize temporary table data
const insertTempTableSql = this.batchInsertSql(tempTableName, insertRowsData);

// 3.update data
const updateColumns = columnNames.reduce<{ [key: string]: unknown }>((pre, columnName) => {
// update data
const updateColumns = dbFieldNames.reduce<{ [key: string]: unknown }>((pre, columnName) => {
pre[columnName] = this.knex.ref(`${tempTableName}.${columnName}`);
return pre;
}, {});
let updateRecordSql = this.knex(dbTableName).update(updateColumns).toQuery();
updateRecordSql += ` FROM \`${tempTableName}\` WHERE ${dbTableName}.__id = ${tempTableName}.__id`;
updateRecordSql += ` FROM \`${tempTableName}\` WHERE ${dbTableName}.${idFieldName} = ${tempTableName}.${idFieldName}`;

return { insertTempTableSql, updateRecordSql };
}
Expand Down
Loading

0 comments on commit afde2b2

Please sign in to comment.