diff --git a/packages/cli/src/config.ts b/packages/cli/src/config.ts index fd753139..80bee2d2 100644 --- a/packages/cli/src/config.ts +++ b/packages/cli/src/config.ts @@ -1,14 +1,14 @@ /** @fileoverview Config file parser */ -import { createRequire } from 'module'; +import { Type } from '@pgtyped/query'; import * as Either from 'fp-ts/lib/Either.js'; -import { join, isAbsolute } from 'path'; import * as t from 'io-ts'; import { reporter } from 'io-ts-reporters'; +import { createRequire } from 'module'; +import { isAbsolute, join } from 'path'; import tls from 'tls'; -import { default as dbUrlModule, DatabaseConfig } from 'ts-parse-database-url'; +import { DatabaseConfig, default as dbUrlModule } from 'ts-parse-database-url'; import { TypeDefinition } from './types.js'; -import { Type } from '@pgtyped/query'; // module import hack const { default: parseDatabaseUri } = dbUrlModule as any; @@ -25,12 +25,27 @@ const TSTransformCodec = t.type({ ...transformCodecProps, }); +const TSTypedSQLTagTransformCodec = t.type({ + mode: t.literal('ts-implicit'), + include: t.string, + functionName: t.string, + emitFileName: t.string, +}); + +export type TSTypedSQLTagTransformConfig = t.TypeOf< + typeof TSTypedSQLTagTransformCodec +>; + const SQLTransformCodec = t.type({ mode: t.literal('sql'), ...transformCodecProps, }); -const TransformCodec = t.union([TSTransformCodec, SQLTransformCodec]); +const TransformCodec = t.union([ + TSTransformCodec, + SQLTransformCodec, + TSTypedSQLTagTransformCodec, +]); export type TransformConfig = t.TypeOf; @@ -193,7 +208,7 @@ export function parseConfig( ? convertParsedURLToDBConfig(parseDatabaseUri(dbUri)) : {}; - if (transforms.some((tr) => !!tr.emitFileName)) { + if (transforms.some((tr) => tr.mode !== 'ts-implicit' && !!tr.emitFileName)) { // tslint:disable:no-console console.log( 'Warning: Setting "emitFileName" is deprecated. Consider using "emitTemplate" instead.', diff --git a/packages/cli/src/generator.test.ts b/packages/cli/src/generator.test.ts index d4de45e0..5e8e6741 100644 --- a/packages/cli/src/generator.test.ts +++ b/packages/cli/src/generator.test.ts @@ -1,11 +1,13 @@ -import { ParameterTransform } from '@pgtyped/runtime'; - -import { parseSQLFile } from '@pgtyped/parser'; +import { parseSQLFile, TSQueryAST } from '@pgtyped/parser'; import { IQueryTypes } from '@pgtyped/query/lib/actions.js'; +import { ParameterTransform } from '@pgtyped/runtime'; +import { pascalCase } from 'pascal-case'; import { ParsedConfig } from './config.js'; import { escapeComment, generateInterface, + genTypedSQLOverloadFunctions, + TSTypedQuery, ProcessingMode, queryToTypeDeclarations, } from './generator.js'; @@ -657,3 +659,23 @@ export type IGetNotificationsParams = never; `; expect(result).toEqual(expected); }); + +test('should generate the correct SQL overload functions', async () => { + const queryStringTS = ` + const getUsers = sql\`SELECT id from users\`; + `; + const query = parsedQuery(ProcessingMode.TS, queryStringTS); + const typedQuery: TSTypedQuery = { + mode: 'ts' as const, + fileName: 'test.ts', + query: { + name: query.ast.name, + ast: query.ast as TSQueryAST, + queryTypeAlias: `I${pascalCase(query.ast.name)}Query`, + }, + typeDeclaration: '', + }; + const result = genTypedSQLOverloadFunctions('sqlFunc', [typedQuery]); + const expected = `export function sqlFunc(s: \`SELECT id from users\`): ReturnType>;`; + expect(result).toEqual(expected); +}); diff --git a/packages/cli/src/generator.ts b/packages/cli/src/generator.ts index 4e5f5e44..45a3efb5 100644 --- a/packages/cli/src/generator.ts +++ b/packages/cli/src/generator.ts @@ -1,9 +1,3 @@ -import { - ParameterTransform, - processSQLQueryIR, - processTSQueryAST, -} from '@pgtyped/runtime'; - import { parseSQLFile, prettyPrintEvents, @@ -14,12 +8,17 @@ import { } from '@pgtyped/parser'; import { getTypes, TypeSource } from '@pgtyped/query'; +import { + ParameterTransform, + processSQLQueryIR, + processTSQueryAST, +} from '@pgtyped/runtime'; import { camelCase } from 'camel-case'; import { pascalCase } from 'pascal-case'; import path from 'path'; -import { ParsedConfig } from './config.js'; -import { TypeAllocator, TypeMapping, TypeScope } from './types.js'; +import { ParsedConfig, TransformConfig } from './config.js'; import { parseCode as parseTypescriptFile } from './parseTypescript.js'; +import { TypeAllocator, TypeDefinitions, TypeScope } from './types.js'; import { IQueryTypes } from '@pgtyped/query/lib/actions.js'; export enum ProcessingMode { @@ -262,54 +261,67 @@ export async function queryToTypeDeclarations( ); } -type ITypedQuery = - | { - mode: 'ts'; - fileName: string; - query: { - name: string; - ast: TSQueryAST; - }; - typeDeclaration: string; - } - | { - mode: 'sql'; - fileName: string; - query: { - name: string; - ast: SQLQueryAST; - ir: SQLQueryIR; - paramTypeAlias: string; - returnTypeAlias: string; - }; - typeDeclaration: string; - }; +export type TSTypedQuery = { + mode: 'ts'; + fileName: string; + query: { + name: string; + ast: TSQueryAST; + queryTypeAlias: string; + }; + typeDeclaration: string; +}; + +type SQLTypedQuery = { + mode: 'sql'; + fileName: string; + query: { + name: string; + ast: SQLQueryAST; + ir: SQLQueryIR; + paramTypeAlias: string; + returnTypeAlias: string; + }; + typeDeclaration: string; +}; -async function generateTypedecsFromFile( +export type TypedQuery = TSTypedQuery | SQLTypedQuery; +export type TypeDeclarationSet = { + typedQueries: TypedQuery[]; + typeDefinitions: TypeDefinitions; + fileName: string; +}; +export async function generateTypedecsFromFile( contents: string, fileName: string, connection: any, - mode: 'ts' | 'sql', + transform: TransformConfig, types: TypeAllocator, config: ParsedConfig, -): Promise { - const results: ITypedQuery[] = []; +): Promise { + const typedQueries: TypedQuery[] = []; const interfacePrefix = config.hungarianNotation ? 'I' : ''; const typeSource: TypeSource = (query) => getTypes(query, connection); const { queries, events } = - mode === 'ts' - ? parseTypescriptFile(contents, fileName) - : parseSQLFile(contents); + transform.mode === 'sql' + ? parseSQLFile(contents) + : parseTypescriptFile(contents, fileName, transform); + if (events.length > 0) { prettyPrintEvents(contents, events); if (events.find((e) => 'critical' in e)) { - return results; + return { + typedQueries, + typeDefinitions: types.toTypeDefinitions(), + fileName, + }; } } + for (const queryAST of queries) { - let typedQuery: ITypedQuery; - if (mode === 'sql') { + let typedQuery: TypedQuery; + if (transform.mode === 'sql') { const sqlQueryAST = queryAST as SQLQueryAST; const result = await queryToTypeDeclarations( { ast: sqlQueryAST, mode: ProcessingMode.SQL }, @@ -350,55 +362,22 @@ async function generateTypedecsFromFile( query: { name: tsQueryAST.name, ast: tsQueryAST, + queryTypeAlias: `${interfacePrefix}${pascalCase( + tsQueryAST.name, + )}Query`, }, typeDeclaration: result, }; } - results.push(typedQuery); + typedQueries.push(typedQuery); } - return results; + return { typedQueries, typeDefinitions: types.toTypeDefinitions(), fileName }; } -export async function generateDeclarationFile( - contents: string, - fileName: string, - connection: any, - mode: 'ts' | 'sql', - config: ParsedConfig, - decsFileName: string, -): Promise<{ typeDecs: ITypedQuery[]; declarationFileContents: string }> { - const types = new TypeAllocator(TypeMapping(config.typesOverrides)); - - if (mode === 'sql') { - // Second parameter has no effect here, we could have used any value - types.use( - { name: 'PreparedQuery', from: '@pgtyped/runtime' }, - TypeScope.Return, - ); - } - const typeDecs = await generateTypedecsFromFile( - contents, - fileName, - connection, - mode, - types, - config, - ); - - // file paths in generated files must be stable across platforms - // https://github.com/adelsz/pgtyped/issues/230 - const isWindowsPath = path.sep === '\\'; - // always emit POSIX paths - const stableFilePath = isWindowsPath - ? fileName.replace(/\\/g, '/') - : fileName; - - let declarationFileContents = ''; - declarationFileContents += `/** Types generated for queries found in "${stableFilePath}" */\n`; - declarationFileContents += types.declaration(decsFileName); - declarationFileContents += '\n'; +export function generateDeclarations(typeDecs: TypedQuery[]): string { + let typeDeclarations = ''; for (const typeDec of typeDecs) { - declarationFileContents += typeDec.typeDeclaration; + typeDeclarations += typeDec.typeDeclaration; if (typeDec.mode === 'ts') { continue; } @@ -406,20 +385,52 @@ export async function generateDeclarationFile( .split('\n') .map((s: string) => ' * ' + s) .join('\n'); - declarationFileContents += `const ${ - typeDec.query.name - }IR: any = ${JSON.stringify(typeDec.query.ir)};\n\n`; - declarationFileContents += + typeDeclarations += `const ${typeDec.query.name}IR: any = ${JSON.stringify( + typeDec.query.ir, + )};\n\n`; + typeDeclarations += `/**\n` + ` * Query generated from SQL:\n` + ` * \`\`\`\n` + `${queryPP}\n` + ` * \`\`\`\n` + ` */\n`; - declarationFileContents += + typeDeclarations += `export const ${typeDec.query.name} = ` + `new PreparedQuery<${typeDec.query.paramTypeAlias},${typeDec.query.returnTypeAlias}>` + `(${typeDec.query.name}IR);\n\n\n`; } - return { declarationFileContents, typeDecs }; + return typeDeclarations; +} + +export function generateDeclarationFile(typeDecSet: TypeDeclarationSet) { + // file paths in generated files must be stable across platforms + // https://github.com/adelsz/pgtyped/issues/230 + const isWindowsPath = path.sep === '\\'; + // always emit POSIX paths + const stableFilePath = isWindowsPath + ? typeDecSet.fileName.replace(/\\/g, '/') + : typeDecSet.fileName; + + let content = `/** Types generated for queries found in "${stableFilePath}" */\n`; + content += TypeAllocator.typeDefinitionDeclarations( + typeDecSet.fileName, + typeDecSet.typeDefinitions, + ); + content += '\n'; + content += generateDeclarations(typeDecSet.typedQueries); + return content; +} + +export function genTypedSQLOverloadFunctions( + functionName: string, + typedQueries: TSTypedQuery[], +) { + return typedQueries + .map( + (typeDec) => + `export function ${functionName}(s: \`${typeDec.query.ast.text}\`): ReturnType>;`, + ) + .filter((s) => s) + .join('\n'); } diff --git a/packages/cli/src/index.ts b/packages/cli/src/index.ts index 76e22bb2..a0e5cd99 100644 --- a/packages/cli/src/index.ts +++ b/packages/cli/src/index.ts @@ -3,31 +3,29 @@ import { startup } from '@pgtyped/query'; import { AsyncQueue } from '@pgtyped/wire'; import chokidar from 'chokidar'; -import { globSync } from 'glob'; import nun from 'nunjucks'; + +import PiscinaPool from 'piscina'; import yargs from 'yargs'; import { hideBin } from 'yargs/helpers'; -import { debug } from './util.js'; import { parseConfig, ParsedConfig, TransformConfig } from './config.js'; -import path from 'path'; - -import WorkerPool from 'piscina'; +import { TypedSqlTagTransformer } from './typedSqlTagTransformer.js'; +import { TypescriptAndSqlTransformer } from './typescriptAndSqlTransformer.js'; +import { debug } from './util.js'; // tslint:disable:no-console nun.configure({ autoescape: false }); -interface TransformJob { +export interface TransformJob { files: string[]; - transform: TransformConfig; } -class FileProcessor { - private readonly pool: WorkerPool; - public readonly workQueue: Promise[] = []; +export class WorkerPool { + private pool: PiscinaPool; constructor(private readonly config: ParsedConfig) { - this.pool = new WorkerPool({ + this.pool = new PiscinaPool({ filename: new URL('./worker.js', import.meta.url).href, maxThreads: config.maxWorkerThreads, workerData: config, @@ -39,46 +37,24 @@ class FileProcessor { await this.pool.destroy(); } - public push(job: TransformJob) { - this.workQueue.push( - ...job.files.map(async (fileName) => { - try { - fileName = path.relative(process.cwd(), fileName); - console.log(`Processing ${fileName}`); - const result = await this.pool.run({ - fileName, - transform: job.transform, - }); - if (result.skipped) { - console.log( - `Skipped ${fileName}: no changes or no queries detected`, - ); - } else { - console.log( - `Saved ${result.typeDecsLength} query types from ${fileName} to ${result.relativePath}`, - ); - } - } catch (err) { - if (err instanceof Error) { - const isWorkerTermination = - err.message === 'Terminating worker thread'; - if (isWorkerTermination) { - return; - } - - console.log( - `Error processing file: ${err.stack || JSON.stringify(err)}`, - ); - } else { - console.log(`Error processing file: ${JSON.stringify(err)}`); - } - if (this.config.failOnError) { - await this.pool.destroy(); - process.exit(1); - } + public async run(opts: T, functionName: string) { + try { + return this.pool.run(opts, { name: functionName }); + } catch (err) { + if (err instanceof Error) { + const isWorkerTermination = err.message === 'Terminating worker thread'; + if (isWorkerTermination) { + return; } - }), - ); + console.log( + `Error processing file: ${err.stack || JSON.stringify(err)}`, + ); + if (this.config.failOnError) { + await this.pool.destroy(); + process.exit(1); + } + } + } } } @@ -96,49 +72,32 @@ async function main( debug('connected to database %o', config.db.dbName); - const fileProcessor = new FileProcessor(config); - let fileOverrideUsed = false; - for (const transform of config.transforms) { - const pattern = `${config.srcDir}/**/${transform.include}`; - if (isWatchMode) { - const cb = (filePath: string) => { - fileProcessor.push({ - files: [filePath], - transform, - }); - }; - chokidar - .watch(pattern, { persistent: true }) - .on('add', cb) - .on('change', cb); + const pool = new WorkerPool(config); + + const transformTask = async (transform: TransformConfig) => { + if (transform.mode === 'ts-implicit') { + const transformer = new TypedSqlTagTransformer(pool, config, transform); + return transformer.start(isWatchMode); } else { - /** - * If the user didn't provide the -f paramter, we're using the list of files we got from glob. - * If he did, we're using glob file list to detect if his provided file should be used with this transform. - */ - let fileList = globSync(pattern); - if (fileOverride) { - fileList = fileList.includes(fileOverride) ? [fileOverride] : []; - if (fileList.length > 0) { - fileOverrideUsed = true; - } - } - debug('found query files %o', fileList); - const transformJob = { - files: fileList, + const transformer = new TypescriptAndSqlTransformer( + pool, + config, transform, - }; - fileProcessor.push(transformJob); + ); + return transformer.start(isWatchMode); } - } - if (fileOverride && !fileOverrideUsed) { - console.log( - 'File override specified, but file was not found in provided transforms', - ); - } + }; + + const tasks = config.transforms.map(transformTask); + if (!isWatchMode) { - await Promise.all(fileProcessor.workQueue); - await fileProcessor.shutdown(); + const transforms = await Promise.all(tasks); + if (fileOverride && !transforms.some((x) => x)) { + console.log( + 'File override specified, but file was not found in provided transforms', + ); + } + await pool.shutdown(); process.exit(0); } } diff --git a/packages/cli/src/parseTypescript.ts b/packages/cli/src/parseTypescript.ts index b15efdee..a09b38f1 100644 --- a/packages/cli/src/parseTypescript.ts +++ b/packages/cli/src/parseTypescript.ts @@ -1,19 +1,38 @@ +import { ParseEvent, parseTSQuery, TSQueryAST } from '@pgtyped/parser'; import ts from 'typescript'; +import { TransformConfig } from './config.js'; interface INode { queryName: string; queryText: string; } -import { parseTSQuery, TSQueryAST, ParseEvent } from '@pgtyped/parser'; - export type TSParseResult = { queries: TSQueryAST[]; events: ParseEvent[] }; -export function parseFile(sourceFile: ts.SourceFile): TSParseResult { +export function parseFile( + sourceFile: ts.SourceFile, + transformConfig: TransformConfig | undefined, +): TSParseResult { const foundNodes: INode[] = []; parseNode(sourceFile); function parseNode(node: ts.Node) { + if ( + transformConfig?.mode === 'ts-implicit' && + node.kind === ts.SyntaxKind.CallExpression + ) { + const callNode = node as ts.CallExpression; + const functionName = callNode.expression.getText(); + if (functionName === transformConfig.functionName) { + const queryName = callNode.parent.getChildren()[0].getText(); + const queryText = callNode.arguments[0].getText().slice(1, -1).trim(); + foundNodes.push({ + queryName, + queryText, + }); + } + } + if (node.kind === ts.SyntaxKind.TaggedTemplateExpression) { const queryName = node.parent.getChildren()[0].getText(); const taggedTemplateNode = node as ts.TaggedTemplateExpression; @@ -48,12 +67,16 @@ export function parseFile(sourceFile: ts.SourceFile): TSParseResult { return { queries, events }; } -export const parseCode = (fileContent: string, fileName = 'unnamed.ts') => { +export const parseCode = ( + fileContent: string, + fileName = 'unnamed.ts', + transformConfig?: TransformConfig, +) => { const sourceFile = ts.createSourceFile( fileName, fileContent, ts.ScriptTarget.ES2015, true, ); - return parseFile(sourceFile); + return parseFile(sourceFile, transformConfig); }; diff --git a/packages/cli/src/typedSqlTagTransformer.ts b/packages/cli/src/typedSqlTagTransformer.ts new file mode 100644 index 00000000..d6a13d75 --- /dev/null +++ b/packages/cli/src/typedSqlTagTransformer.ts @@ -0,0 +1,165 @@ +import chokidar from 'chokidar'; +import fs from 'fs-extra'; +import { globSync } from 'glob'; +import path from 'path'; +import { ParsedConfig, TSTypedSQLTagTransformConfig } from './config.js'; +import { + generateDeclarations, + genTypedSQLOverloadFunctions, + TSTypedQuery, + TypeDeclarationSet, +} from './generator.js'; +import { TransformJob, WorkerPool } from './index.js'; +import { TypeAllocator } from './types.js'; +import { debug } from './util.js'; +import { getTypeDecsFnResult } from './worker.js'; + +type TypedSQLTagTransformResult = TypeDeclarationSet | undefined; + +// tslint:disable:no-console +export class TypedSqlTagTransformer { + public readonly workQueue: Promise[] = []; + private readonly cache: Record = {}; + private readonly includePattern: string; + private readonly localFileName: string; + private readonly fullFileName: string; + + constructor( + private readonly pool: WorkerPool, + private readonly config: ParsedConfig, + private readonly transform: TSTypedSQLTagTransformConfig, + ) { + this.includePattern = `${this.config.srcDir}/**/${transform.include}`; + this.localFileName = this.transform.emitFileName; + this.fullFileName = path.relative(process.cwd(), this.localFileName); + } + + private async watch() { + let initialized = false; + + const cb = async (fileName: string) => { + const job = { + files: [fileName], + }; + !initialized + ? this.pushToQueue(job) + : await this.generateTypedSQLTagFileForJob(job, true); + }; + + chokidar + .watch(this.includePattern, { + persistent: true, + ignored: [this.localFileName], + }) + .on('add', cb) + .on('change', cb) + .on('unlink', async (file) => await this.removeFileFromCache(file)) + .on('ready', async () => { + initialized = true; + await this.waitForTypedSQLQueueAndGenerate(true); + }); + } + + public async start(watch: boolean) { + if (watch) { + return this.watch(); + } + + const fileList = globSync(this.includePattern, { + ignore: [this.localFileName], + }); + + debug('found query files %o', fileList); + + await this.generateTypedSQLTagFileForJob({ + files: fileList, + }); + } + + private pushToQueue(job: TransformJob) { + this.workQueue.push( + ...job.files.map((fileName) => this.getTsTypeDecs(fileName)), + ); + } + + private async getTsTypeDecs( + fileName: string, + ): Promise { + console.log(`Processing ${fileName}`); + return (await this.pool.run( + { + fileName, + transform: this.transform, + }, + 'getTypeDecs', + )) as Awaited; + // Result should be serializable! + } + + private async generateTypedSQLTagFileForJob( + job: TransformJob, + useCache?: boolean, + ) { + this.pushToQueue(job); + return this.waitForTypedSQLQueueAndGenerate(useCache); + } + + private async waitForTypedSQLQueueAndGenerate(useCache?: boolean) { + const queueResults = await Promise.all(this.workQueue); + this.workQueue.length = 0; + + const typeDecsSets: TypeDeclarationSet[] = []; + + for (const result of queueResults) { + if (result?.typedQueries.length) { + typeDecsSets.push(result); + if (useCache) this.cache[result.fileName] = result; + } + } + + return this.generateTypedSQLTagFile( + useCache ? Object.values(this.cache) : typeDecsSets, + ); + } + + private async removeFileFromCache(fileToRemove: string) { + delete this.cache[fileToRemove]; + return this.generateTypedSQLTagFile(Object.values(this.cache)); + } + + private contentStart = `import { ${this.transform.functionName} as sourceSql } from '@pgtyped/runtime';\n\n`; + private contentEnd = [ + `export function ${this.transform.functionName}(s: string): unknown;`, + `export function ${this.transform.functionName}(s: string): unknown {`, + ` return sourceSql([s] as any);`, + `}`, + ]; + + private async generateTypedSQLTagFile(typeDecsSets: TypeDeclarationSet[]) { + console.log(`Generating ${this.fullFileName}...`); + let typeDefinitions = ''; + let queryTypes = ''; + let typedSQLOverloadFns = ''; + + for (const typeDecSet of typeDecsSets) { + typeDefinitions += TypeAllocator.typeDefinitionDeclarations( + this.transform.emitFileName, + typeDecSet.typeDefinitions, + ); + queryTypes += generateDeclarations(typeDecSet.typedQueries); + typedSQLOverloadFns += genTypedSQLOverloadFunctions( + this.transform.functionName, + typeDecSet.typedQueries as TSTypedQuery[], + ); + } + + let content = this.contentStart; + content += typeDefinitions; + content += queryTypes; + content += typedSQLOverloadFns; + content += '\n\n'; + content += this.contentEnd.join('\n'); + await fs.outputFile(this.fullFileName, content); + console.log(`Saved ${this.fullFileName}`); + } +} diff --git a/packages/cli/src/types.ts b/packages/cli/src/types.ts index 737dd422..4e61f43e 100644 --- a/packages/cli/src/types.ts +++ b/packages/cli/src/types.ts @@ -1,14 +1,15 @@ // Default types import { + ImportedType, isAlias, isEnum, isEnumArray, isImport, MappableType, Type, - ImportedType, } from '@pgtyped/query'; import os from 'os'; +import { AliasedType, EnumType } from '@pgtyped/query/lib/type.js'; import path from 'path'; const String: Type = { name: 'string' }; @@ -234,6 +235,14 @@ export enum TypeScope { Return = 'return', } +type importsType = { [k: string]: ImportedType[] }; + +export type TypeDefinitions = { + imports: importsType; + enums: EnumType[]; + aliases: AliasedType[]; +}; + /** Wraps a TypeMapping to track which types have been used, to accumulate errors, * and emit necessary type definitions. */ export class TypeAllocator { @@ -309,26 +318,43 @@ export class TypeAllocator { return typ.name; } - /** Emit a typescript definition for all types that have been used */ - declaration(decsFileName: string): string { - const imports = Object.values(this.imports) - .map((imports) => declareImport(imports, decsFileName)) - .sort() - .join('\n'); - - // Declare database enums as string unions to maintain assignability of their values between query files - const enums = Object.values(this.types) - .filter(isEnum) - .map((t) => declareStringUnion(t.name, t.enumValues)) - .sort() - .join('\n'); + // In order to get the results out of the Piscina pool, we need to have + // a serializable variant + public toTypeDefinitions(): TypeDefinitions { + return { + imports: this.imports, + enums: Object.values(this.types).filter(isEnum), + aliases: Object.values(this.types).filter(isAlias), + }; + } - const aliases = Object.values(this.types) - .filter(isAlias) - .map((t) => declareAlias(t.name, t.definition)) - .sort() + // Static so we can also use this for serialized typeDefinitions + public static typeDefinitionDeclarations( + decsFileName: string, + types: TypeDefinitions, + ): string { + return [ + Object.values(types.imports) + .map((i) => declareImport(i, decsFileName)) + .join('\n'), + types.enums + .map((t) => declareStringUnion(t.name, t.enumValues)) + .sort() + .join('\n'), + types.aliases + .map((t) => declareAlias(t.name, t.definition)) + .sort() + .join('\n'), + ] + .filter((s) => s) .join('\n'); + } - return [imports, enums, aliases].filter((s) => s).join('\n'); + /** Emit a typescript definition for all types that have been used */ + declaration(decsFileName: string): string { + return TypeAllocator.typeDefinitionDeclarations( + decsFileName, + this.toTypeDefinitions(), + ); } } diff --git a/packages/cli/src/typescriptAndSqlTransformer.ts b/packages/cli/src/typescriptAndSqlTransformer.ts new file mode 100644 index 00000000..8fac8e7d --- /dev/null +++ b/packages/cli/src/typescriptAndSqlTransformer.ts @@ -0,0 +1,97 @@ +import chokidar from 'chokidar'; +import { globSync } from 'glob'; +import path from 'path'; +import { ParsedConfig, TransformConfig } from './config.js'; +import { TransformJob, WorkerPool } from './index.js'; +import { debug } from './util.js'; +import { processFileFnResult } from './worker.js'; + +// tslint:disable:no-console + +export class TypescriptAndSqlTransformer { + public readonly workQueue: Promise[] = []; + private readonly includePattern: string; + private fileOverrideUsed = false; + + constructor( + private readonly pool: WorkerPool, + private readonly config: ParsedConfig, + private readonly transform: TransformConfig, + ) { + this.includePattern = `${this.config.srcDir}/**/${transform.include}`; + } + + private async watch() { + const cb = async (fileName: string) => { + // we will not push it to the queue to not consume more memory + return this.processFile(fileName); + }; + + chokidar + .watch(this.includePattern, { + persistent: true, + }) + .on('add', cb) + .on('change', cb); + } + + public async start(watch: boolean, fileOverride?: string) { + if (watch) { + return this.watch(); + } + + /** + * If the user didn't provide the -f paramter, we're using the list of files we got from glob. + * If he did, we're using glob file list to detect if his provided file should be used with this transform. + */ + let fileList = globSync(this.includePattern, { + ...(this.transform.emitFileName && { + ignore: [`${this.config.srcDir}${this.transform.emitFileName}`], + }), + }); + if (fileOverride) { + fileList = fileList.includes(fileOverride) ? [fileOverride] : []; + if (fileList.length > 0) { + this.fileOverrideUsed = true; + } + } + debug('found query files %o', fileList); + + this.pushToQueue({ + files: fileList, + }); + + await Promise.all(this.workQueue); + return this.fileOverrideUsed; + } + + private async processFile(fileName: string) { + fileName = path.relative(process.cwd(), fileName); + console.log(`Processing ${fileName}`); + const result = (await this.pool.run( + { + fileName, + transform: this.transform, + }, + 'processFile', + )) as Awaited; + + if ('skipped' in result && result.skipped) { + console.log(`Skipped ${fileName}: no changes or no queries detected`); + } else if ('error' in result) { + console.error( + `Error processing ${fileName}: ${result.error.message}\n${result.error.stack}`, + ); + } else { + console.log( + `Saved ${result.typeDecsLength} query types from ${fileName} to ${result.relativePath}`, + ); + } + } + + public pushToQueue(job: TransformJob) { + this.workQueue.push( + ...job.files.map((fileName) => this.processFile(fileName)), + ); + } +} diff --git a/packages/cli/src/worker.ts b/packages/cli/src/worker.ts index a39fb074..240e74ad 100644 --- a/packages/cli/src/worker.ts +++ b/packages/cli/src/worker.ts @@ -1,11 +1,15 @@ -import nun from 'nunjucks'; -import path from 'path'; -import fs from 'fs-extra'; -import { generateDeclarationFile } from './generator.js'; import { startup } from '@pgtyped/query'; -import { ParsedConfig, TransformConfig } from './config.js'; import { AsyncQueue } from '@pgtyped/wire'; +import fs from 'fs-extra'; +import nun from 'nunjucks'; +import path from 'path'; import worker from 'piscina'; +import { ParsedConfig, TransformConfig } from './config.js'; +import { + generateDeclarationFile, + generateTypedecsFromFile, +} from './generator.js'; +import { TypeAllocator, TypeMapping, TypeScope } from './types.js'; // disable autoescape as it breaks windows paths // see https://github.com/adelsz/pgtyped/issues/519 for details @@ -19,45 +23,86 @@ interface ExtendedParsedPath extends path.ParsedPath { dir_base: string; } -export default async function processFile({ +export type IWorkerResult = + | { + skipped: boolean; + typeDecsLength: number; + relativePath: string; + } + | { + error: any; + relativePath: string; + }; + +async function connectAndGetFileContents(fileName: string) { + if (!connected) { + await startup(config.db, connection); + connected = true; + } + + // last part fixes https://github.com/adelsz/pgtyped/issues/390 + return fs.readFileSync(fileName).toString().replace(/\r\n/g, '\n'); +} + +export async function getTypeDecs({ fileName, transform, }: { fileName: string; transform: TransformConfig; -}): Promise<{ - skipped: boolean; - typeDecsLength: number; - relativePath: string; -}> { - if (!connected) { - await startup(config.db, connection); - connected = true; +}) { + const contents = await connectAndGetFileContents(fileName); + const types = new TypeAllocator(TypeMapping(config.typesOverrides)); + + if (transform.mode === 'sql') { + // Second parameter has no effect here, we could have used any value + types.use( + { name: 'PreparedQuery', from: '@pgtyped/runtime' }, + TypeScope.Return, + ); } + return await generateTypedecsFromFile( + contents, + fileName, + connection, + transform, + types, + config, + ); +} + +export type getTypeDecsFnResult = ReturnType; + +export async function processFile({ + fileName, + transform, +}: { + fileName: string; + transform: TransformConfig; +}): Promise { const ppath = path.parse(fileName) as ExtendedParsedPath; ppath.dir_base = path.basename(ppath.dir); - let decsFileName; - if (transform.emitTemplate) { + if ('emitTemplate' in transform && transform.emitTemplate) { decsFileName = nun.renderString(transform.emitTemplate, ppath); } else { const suffix = transform.mode === 'ts' ? 'types.ts' : 'ts'; decsFileName = path.resolve(ppath.dir, `${ppath.name}.${suffix}`); } - // last part fixes https://github.com/adelsz/pgtyped/issues/390 - const contents = fs.readFileSync(fileName).toString().replace(/\r\n/g, '\n'); - - const { declarationFileContents, typeDecs } = await generateDeclarationFile( - contents, - fileName, - connection, - transform.mode, - config, - decsFileName, - ); + let typeDecSet; + try { + typeDecSet = await getTypeDecs({ fileName, transform }); + } catch (e) { + return { + error: e, + relativePath: path.relative(process.cwd(), fileName), + }; + } const relativePath = path.relative(process.cwd(), decsFileName); - if (typeDecs.length > 0) { + + if (typeDecSet.typedQueries.length > 0) { + const declarationFileContents = await generateDeclarationFile(typeDecSet); const oldDeclarationFileContents = (await fs.pathExists(decsFileName)) ? await fs.readFile(decsFileName, { encoding: 'utf-8' }) : null; @@ -65,7 +110,7 @@ export default async function processFile({ await fs.outputFile(decsFileName, declarationFileContents); return { skipped: false, - typeDecsLength: typeDecs.length, + typeDecsLength: typeDecSet.typedQueries.length, relativePath, }; } @@ -76,3 +121,5 @@ export default async function processFile({ relativePath, }; } + +export type processFileFnResult = ReturnType; diff --git a/packages/example/config.json b/packages/example/config.json index 482b3b9f..6dc053dd 100644 --- a/packages/example/config.json +++ b/packages/example/config.json @@ -9,6 +9,12 @@ "mode": "ts", "include": "**/*.ts", "emitTemplate": "{{dir}}/{{name}}.types.ts" + }, + { + "mode": "ts-implicit", + "include": "index.test.ts", + "functionName": "sql", + "emitFileName": "./src/sql/index.ts" } ], "typesOverrides": { @@ -16,7 +22,7 @@ "return": "string" }, "int8": "BigInt", - "category": { "return": "./src/customTypes#Category" } + "category": { "return": "./src/customTypes.js#Category" } }, "srcDir": "./src/", "dbUrl": "postgres://postgres:password@localhost/postgres" diff --git a/packages/example/src/__snapshots__/index.test.ts.snap b/packages/example/src/__snapshots__/index.test.ts.snap index a8179b60..f98c6f6d 100644 --- a/packages/example/src/__snapshots__/index.test.ts.snap +++ b/packages/example/src/__snapshots__/index.test.ts.snap @@ -79,3 +79,15 @@ Array [ `; exports[`select query with unicode characters 1`] = `Array []`; + +exports[`ts-implicit mode query 1`] = ` +Array [ + Object { + "author_id": 1, + "categories": null, + "id": 1, + "name": "Black Swan", + "rank": 1, + }, +] +`; diff --git a/packages/example/src/books/books.queries.ts b/packages/example/src/books/books.queries.ts index ad783802..c3a675ea 100644 --- a/packages/example/src/books/books.queries.ts +++ b/packages/example/src/books/books.queries.ts @@ -1,8 +1,8 @@ /** Types generated for queries found in "src/books/books.sql" */ -import type { Category } from '../customTypes'; - import { PreparedQuery } from '@pgtyped/runtime'; +import type { Category } from '../customTypes.js'; + export type Iso31661Alpha2 = 'AD' | 'AE' | 'AF' | 'AG' | 'AI' | 'AL' | 'AM' | 'AO' | 'AQ' | 'AR' | 'AS' | 'AT' | 'AU' | 'AW' | 'AX' | 'AZ' | 'BA' | 'BB' | 'BD' | 'BE' | 'BF' | 'BG' | 'BH' | 'BI' | 'BJ' | 'BL' | 'BM' | 'BN' | 'BO' | 'BQ' | 'BR' | 'BS' | 'BT' | 'BV' | 'BW' | 'BY' | 'BZ' | 'CA' | 'CC' | 'CD' | 'CF' | 'CG' | 'CH' | 'CI' | 'CK' | 'CL' | 'CM' | 'CN' | 'CO' | 'CR' | 'CU' | 'CV' | 'CW' | 'CX' | 'CY' | 'CZ' | 'DE' | 'DJ' | 'DK' | 'DM' | 'DO' | 'DZ' | 'EC' | 'EE' | 'EG' | 'EH' | 'ER' | 'ES' | 'ET' | 'FI' | 'FJ' | 'FK' | 'FM' | 'FO' | 'FR' | 'GA' | 'GB' | 'GD' | 'GE' | 'GF' | 'GG' | 'GH' | 'GI' | 'GL' | 'GM' | 'GN' | 'GP' | 'GQ' | 'GR' | 'GS' | 'GT' | 'GU' | 'GW' | 'GY' | 'HK' | 'HM' | 'HN' | 'HR' | 'HT' | 'HU' | 'ID' | 'IE' | 'IL' | 'IM' | 'IN' | 'IO' | 'IQ' | 'IR' | 'IS' | 'IT' | 'JE' | 'JM' | 'JO' | 'JP' | 'KE' | 'KG' | 'KH' | 'KI' | 'KM' | 'KN' | 'KP' | 'KR' | 'KW' | 'KY' | 'KZ' | 'LA' | 'LB' | 'LC' | 'LI' | 'LK' | 'LR' | 'LS' | 'LT' | 'LU' | 'LV' | 'LY' | 'MA' | 'MC' | 'MD' | 'ME' | 'MF' | 'MG' | 'MH' | 'MK' | 'ML' | 'MM' | 'MN' | 'MO' | 'MP' | 'MQ' | 'MR' | 'MS' | 'MT' | 'MU' | 'MV' | 'MW' | 'MX' | 'MY' | 'MZ' | 'NA' | 'NC' | 'NE' | 'NF' | 'NG' | 'NI' | 'NL' | 'NO' | 'NP' | 'NR' | 'NU' | 'NZ' | 'OM' | 'PA' | 'PE' | 'PF' | 'PG' | 'PH' | 'PK' | 'PL' | 'PM' | 'PN' | 'PR' | 'PS' | 'PT' | 'PW' | 'PY' | 'QA' | 'RE' | 'RO' | 'RS' | 'RU' | 'RW' | 'SA' | 'SB' | 'SC' | 'SD' | 'SE' | 'SG' | 'SH' | 'SI' | 'SJ' | 'SK' | 'SL' | 'SM' | 'SN' | 'SO' | 'SR' | 'SS' | 'ST' | 'SV' | 'SX' | 'SY' | 'SZ' | 'TC' | 'TD' | 'TF' | 'TG' | 'TH' | 'TJ' | 'TK' | 'TL' | 'TM'; export type category = 'novel' | 'science-fiction' | 'thriller'; diff --git a/packages/example/src/index.test.ts b/packages/example/src/index.test.ts index 0947e7b7..dd6629cd 100644 --- a/packages/example/src/index.test.ts +++ b/packages/example/src/index.test.ts @@ -28,7 +28,8 @@ import { thresholdFrogs, } from './notifications/notifications.queries.js'; import { getUsersWithComment } from './users/sample.js'; -import { Category } from './customTypes'; +import { Category } from './customTypes.js'; +import { sql } from './sql/index.js' const { Client } = pg; @@ -230,3 +231,9 @@ test('select query with a bigint field', async () => { expect(typeof row.book_count).toBe('bigint'); expect(row.book_count).toBe(BigInt(4)); }); + + +test('ts-implicit mode query', async () => { + const books = await sql(`SELECT * FROM books WHERE id = $id`).run({id: 1}, client); + expect(books).toMatchSnapshot(); +}); diff --git a/packages/example/src/sql/index.ts b/packages/example/src/sql/index.ts new file mode 100644 index 00000000..f8016d21 --- /dev/null +++ b/packages/example/src/sql/index.ts @@ -0,0 +1,31 @@ +import { sql as sourceSql } from '@pgtyped/runtime'; + +import type { Category } from '../customTypes.js'; + +export type categoryArray = (Category)[]; +/** 'SqlSelectFromBooksWhereIdId' parameters type */ +export interface ISqlSelectFromBooksWhereIdIdParams { + id?: number | null | void; +} + +/** 'SqlSelectFromBooksWhereIdId' return type */ +export interface ISqlSelectFromBooksWhereIdIdResult { + author_id: number | null; + categories: categoryArray | null; + id: number; + name: string | null; + rank: number | null; +} + +/** 'SqlSelectFromBooksWhereIdId' query type */ +export interface ISqlSelectFromBooksWhereIdIdQuery { + params: ISqlSelectFromBooksWhereIdIdParams; + result: ISqlSelectFromBooksWhereIdIdResult; +} + +export function sql(s: `SELECT * FROM books WHERE id = $id`): ReturnType>; + +export function sql(s: string): unknown; +export function sql(s: string): unknown { + return sourceSql([s] as any); +} \ No newline at end of file