From 802eb2fee8a5aef2bb508375b3bb1d75bf19aaf8 Mon Sep 17 00:00:00 2001 From: Tobias Bosch Date: Wed, 24 May 2017 08:10:30 -0700 Subject: [PATCH] poc: typescript transformer support for tsickle --- src/decorator-annotator.ts | 252 ++++++++++-------- src/es5processor.ts | 72 +++-- src/jsdoc.ts | 5 +- src/main.ts | 20 +- src/rewriter.ts | 162 ++++++++--- src/source_map_utils.ts | 45 ++++ src/transformer.ts | 234 ++++++++++++++++ src/tsickle.ts | 237 +++++++++------- src/tsickle_compiler_host.ts | 97 ++----- src/type-translator.ts | 2 +- test/decorator-annotator_test.ts | 8 +- test/e2e_source_map_test.ts | 23 +- test/e2e_tsickle_compiler_host_test.ts | 4 +- test/es5processor_test.ts | 18 +- test/source_map_test.ts | 17 +- test/test_support.ts | 20 +- test/tsickle_test.ts | 45 ++-- test/tsickle_transformer_test.ts | 168 ++++++++++++ test_files/decorator/decorator.js | 4 +- test_files/decorator/decorator.tsickle.ts | 4 +- test_files/enum.untyped/enum.untyped.js | 11 +- .../enum.untyped/enum.untyped.tsickle.ts | 5 +- test_files/enum/enum.js | 11 +- test_files/enum/enum.tsickle.ts | 5 +- 24 files changed, 1062 insertions(+), 407 deletions(-) create mode 100644 src/transformer.ts create mode 100644 test/tsickle_transformer_test.ts diff --git a/src/decorator-annotator.ts b/src/decorator-annotator.ts index 5d612c940..a5710a3d4 100644 --- a/src/decorator-annotator.ts +++ b/src/decorator-annotator.ts @@ -6,18 +6,18 @@ * found in the LICENSE file at https://angular.io/license */ -import {SourceMapGenerator} from 'source-map'; import * as ts from 'typescript'; import {getDecoratorDeclarations} from './decorators'; -import {Rewriter} from './rewriter'; +import {getIdentifierText, Rewriter} from './rewriter'; +import {SourceMapper, SourcePosition} from './source_map_utils'; import {assertTypeChecked, TypeTranslator} from './type-translator'; import {toArray} from './util'; -// ClassRewriter rewrites a single "class Foo {...}" declaration. +// DecoratorClassVisitor rewrites a single "class Foo {...}" declaration. // It's its own object because we collect decorators on the class and the ctor // separately for each class we encounter. -class ClassRewriter extends Rewriter { +export class DecoratorClassVisitor { /** Decorators on the class itself. */ decorators: ts.Decorator[]; /** The constructor parameter list and decorators on each param. */ @@ -25,8 +25,13 @@ class ClassRewriter extends Rewriter { /** Per-method decorators. */ propDecorators: Map; - constructor(private typeChecker: ts.TypeChecker, sourceFile: ts.SourceFile) { - super(sourceFile); + constructor( + private typeChecker: ts.TypeChecker, private rewriter: Rewriter, + private classDecl: ts.ClassDeclaration) { + if (classDecl.decorators) { + let toLower = this.decoratorsToLower(classDecl); + if (toLower.length > 0) this.decorators = toLower; + } } /** @@ -69,40 +74,6 @@ class ClassRewriter extends Rewriter { return []; } - /** - * process is the main entry point, rewriting a single class node. - */ - process(node: ts.ClassDeclaration): {output: string, diagnostics: ts.Diagnostic[]} { - if (node.decorators) { - let toLower = this.decoratorsToLower(node); - if (toLower.length > 0) this.decorators = toLower; - } - - // Emit the class contents, but stop just before emitting the closing curly brace. - // (This code is the same as Rewriter.writeNode except for the curly brace handling.) - let pos = node.getFullStart(); - ts.forEachChild(node, child => { - // This forEachChild handles emitting the text between each child, while child.visit - // recursively emits the children themselves. - this.writeRange(pos, child.getFullStart()); - this.visit(child); - pos = child.getEnd(); - }); - - // At this point, we've emitted up through the final child of the class, so all that - // remains is the trailing whitespace and closing curly brace. - // The final character owned by the class node should always be a '}', - // or we somehow got the AST wrong and should report an error. - // (Any whitespace or semicolon following the '}' will be part of the next Node.) - if (this.file.text[node.getEnd() - 1] !== '}') { - this.error(node, 'unexpected class terminator'); - } - this.writeRange(pos, node.getEnd() - 1); - this.emitMetadata(); - this.emit('}'); - return this.getOutput(); - } - /** * gatherConstructor grabs the parameter list and decorators off the class * constructor, and emits nothing. @@ -147,7 +118,7 @@ class ClassRewriter extends Rewriter { if (!method.name || method.name.kind !== ts.SyntaxKind.Identifier) { // Method has a weird name, e.g. // [Symbol.foo]() {...} - this.error(method, 'cannot process decorators on strangely named method'); + this.rewriter.error(method, 'cannot process decorators on strangely named method'); return; } @@ -158,144 +129,172 @@ class ClassRewriter extends Rewriter { this.propDecorators.set(name, decorators); } - /** - * maybeProcess is called by the traversal of the AST. - * @return True if the node was handled, false to have the node emitted as normal. - */ - protected maybeProcess(node: ts.Node): boolean { + beforeProcessNode(node: ts.Node) { switch (node.kind) { - case ts.SyntaxKind.ClassDeclaration: - // Encountered a new class while processing this class; use a new separate - // rewriter to gather+emit its metadata. - let {output, diagnostics} = - new ClassRewriter(this.typeChecker, this.file).process(node as ts.ClassDeclaration); - this.diagnostics.push(...diagnostics); - this.emit(output); - return true; case ts.SyntaxKind.Constructor: this.gatherConstructor(node as ts.ConstructorDeclaration); - return false; // Proceed with ordinary emit of the ctor. case ts.SyntaxKind.PropertyDeclaration: case ts.SyntaxKind.SetAccessor: case ts.SyntaxKind.GetAccessor: case ts.SyntaxKind.MethodDeclaration: this.gatherMethodOrProperty(node as ts.Declaration); - return false; // Proceed with ordinary emit of the method. - case ts.SyntaxKind.Decorator: - if (this.shouldLower(node as ts.Decorator)) { - // Return true to signal that this node should not be emitted, - // but still emit the whitespace *before* the node. - this.writeRange(node.getFullStart(), node.getStart()); - return true; - } - return false; - default: - return false; } } + maybeProcessDecorator(node: ts.Node, start?: number): boolean { + if (this.shouldLower(node as ts.Decorator)) { + // Return true to signal that this node should not be emitted, + // but still emit the whitespace *before* the node. + if (!start) { + start = node.getFullStart(); + } + this.rewriter.writeRange(node, start, node.getStart()); + return true; + } + return false; + } + /** - * emitMetadata emits the various gathered metadata, as static fields. + * emits the types for the various gathered metadata to be used + * in the tsickle type annotations helper. */ - private emitMetadata() { + emitMetadataTypeAnnotationsHelpers() { + if (!this.classDecl.name) return; + let className = getIdentifierText(this.classDecl.name); + if (this.decorators) { + this.rewriter.emit(`/** @type {!Array<{type: !Function, args: (undefined|!Array)}>} */\n`); + this.rewriter.emit(`${className}.decorators;\n`); + } + if (this.decorators || this.ctorParameters) { + this.rewriter.emit(`/**\n`); + this.rewriter.emit(` * @nocollapse\n`); + this.rewriter.emit( + ` * @type {function(): !Array<(null|{type: ?, decorators: (undefined|!Array<{type: !Function, args: (undefined|!Array)}>)})>}\n`); + this.rewriter.emit(` */\n`); + this.rewriter.emit(`${className}.ctorParameters;\n`); + } + if (this.propDecorators) { + this.rewriter.emit( + `/** @type {!Object)}>>} */\n`); + this.rewriter.emit(`${className}.propDecorators;\n`); + } + } + + /** + * emits the various gathered metadata, as static fields. + */ + emitMetadataAsStaticProperties() { const decoratorInvocations = '{type: Function, args?: any[]}[]'; if (this.decorators) { - this.emit(`static decorators: ${decoratorInvocations} = [\n`); + this.rewriter.emit(`static decorators: ${decoratorInvocations} = [\n`); for (let annotation of this.decorators) { this.emitDecorator(annotation); - this.emit(',\n'); + this.rewriter.emit(',\n'); } - this.emit('];\n'); + this.rewriter.emit('];\n'); } if (this.decorators || this.ctorParameters) { - this.emit(`/** @nocollapse */\n`); + this.rewriter.emit(`/** @nocollapse */\n`); // ctorParameters may contain forward references in the type: field, so wrap in a function // closure - this.emit( + this.rewriter.emit( `static ctorParameters: () => ({type: any, decorators?: ` + decoratorInvocations + `}|null)[] = () => [\n`); for (let param of this.ctorParameters || []) { if (!param) { - this.emit('null,\n'); + this.rewriter.emit('null,\n'); continue; } let [ctor, decorators] = param; - this.emit(`{type: ${ctor}, `); + this.rewriter.emit(`{type: ${ctor}, `); if (decorators) { - this.emit('decorators: ['); + this.rewriter.emit('decorators: ['); for (let decorator of decorators) { this.emitDecorator(decorator); - this.emit(', '); + this.rewriter.emit(', '); } - this.emit(']'); + this.rewriter.emit(']'); } - this.emit('},\n'); + this.rewriter.emit('},\n'); } - this.emit(`];\n`); + this.rewriter.emit(`];\n`); } if (this.propDecorators) { - this.emit(`static propDecorators: {[key: string]: ` + decoratorInvocations + `} = {\n`); + this.rewriter.emit( + `static propDecorators: {[key: string]: ` + decoratorInvocations + `} = {\n`); for (let name of toArray(this.propDecorators.keys())) { - this.emit(`'${name}': [`); + this.rewriter.emit(`'${name}': [`); for (let decorator of this.propDecorators.get(name)!) { this.emitDecorator(decorator); - this.emit(','); + this.rewriter.emit(','); } - this.emit('],\n'); + this.rewriter.emit('],\n'); } - this.emit('};\n'); + this.rewriter.emit('};\n'); } } private emitDecorator(decorator: ts.Decorator) { - this.emit('{ type: '); + this.rewriter.emit('{ type: '); let expr = decorator.expression; switch (expr.kind) { case ts.SyntaxKind.Identifier: // The decorator was a plain @Foo. - this.visit(expr); + this.rewriter.visit(expr); break; case ts.SyntaxKind.CallExpression: // The decorator was a call, like @Foo(bar). let call = expr as ts.CallExpression; - this.visit(call.expression); + this.rewriter.visit(call.expression); if (call.arguments.length) { - this.emit(', args: ['); + this.rewriter.emit(', args: ['); for (let arg of call.arguments) { - this.emit(arg.getText()); - this.emit(', '); + this.rewriter.emit(arg.getText()); + this.rewriter.emit(', '); } - this.emit(']'); + this.rewriter.emit(']'); } break; default: - this.errorUnimplementedKind(expr, 'gathering metadata'); - this.emit('undefined'); + this.rewriter.errorUnimplementedKind(expr, 'gathering metadata'); + this.rewriter.emit('undefined'); } - this.emit(' }'); + this.rewriter.emit(' }'); } } class DecoratorRewriter extends Rewriter { - constructor(private typeChecker: ts.TypeChecker, sourceFile: ts.SourceFile) { - super(sourceFile); + /** ComposableDecoratorRewriter when using tsickle as a TS transformer */ + private currentDecoratorConverter: DecoratorClassVisitor; + + constructor( + private typeChecker: ts.TypeChecker, file: ts.SourceFile, sourceMapper?: SourceMapper) { + super(file, sourceMapper); } - process(): {output: string, diagnostics: ts.Diagnostic[], sourceMap: SourceMapGenerator} { + process(): {output: string, diagnostics: ts.Diagnostic[]} { this.visit(this.file); return this.getOutput(); } protected maybeProcess(node: ts.Node): boolean { + if (this.currentDecoratorConverter) { + this.currentDecoratorConverter.beforeProcessNode(node); + } switch (node.kind) { + case ts.SyntaxKind.Decorator: + return this.currentDecoratorConverter && + this.currentDecoratorConverter.maybeProcessDecorator(node); case ts.SyntaxKind.ClassDeclaration: - let {output, diagnostics} = - new ClassRewriter(this.typeChecker, this.file).process(node as ts.ClassDeclaration); - this.diagnostics.push(...diagnostics); - this.emit(output); + const oldDecoratorConverter = this.currentDecoratorConverter; + this.currentDecoratorConverter = + new DecoratorClassVisitor(this.typeChecker, this, node as ts.ClassDeclaration); + this.writeRange(node, node.getFullStart(), node.getStart()); + visitClassContent(node as ts.ClassDeclaration, this, this.currentDecoratorConverter); + this.currentDecoratorConverter = oldDecoratorConverter; return true; default: return false; @@ -303,8 +302,51 @@ class DecoratorRewriter extends Rewriter { } } -export function convertDecorators(typeChecker: ts.TypeChecker, sourceFile: ts.SourceFile): - {output: string, diagnostics: ts.Diagnostic[], sourceMap: SourceMapGenerator} { +export function convertDecorators( + typeChecker: ts.TypeChecker, sourceFile: ts.SourceFile, + sourceMapper?: SourceMapper): {output: string, diagnostics: ts.Diagnostic[]} { assertTypeChecked(sourceFile); - return new DecoratorRewriter(typeChecker, sourceFile).process(); + return new DecoratorRewriter(typeChecker, sourceFile, sourceMapper).process(); +} + +export function visitClassContent( + classDecl: ts.ClassDeclaration, rewriter: Rewriter, decoratorVisitor?: DecoratorClassVisitor) { + let pos = classDecl.getStart(); + if (decoratorVisitor) { + // strip out decorators if needed + ts.forEachChild(classDecl, child => { + if (child.kind !== ts.SyntaxKind.Decorator) { + return; + } + // Note: The getFullStart() of the first decorator is the same + // as the getFullStart() of the class declaration. + // Therefore, we need to use Math.max to not print the whitespace + // of the class again. + const childStart = Math.max(pos, child.getFullStart()); + rewriter.writeRange(classDecl, pos, childStart); + if (decoratorVisitor.maybeProcessDecorator(child, childStart)) { + pos = child.getEnd(); + } + }); + } + if (classDecl.members.length > 0) { + rewriter.writeRange(classDecl, pos, classDecl.members[0].getFullStart()); + for (let member of classDecl.members) { + rewriter.visit(member); + } + pos = classDecl.getLastToken().getFullStart(); + } + // At this point, we've emitted up through the final child of the class, so all that + // remains is the trailing whitespace and closing curly brace. + // The final character owned by the class node should always be a '}', + // or we somehow got the AST wrong and should report an error. + // (Any whitespace or semicolon following the '}' will be part of the next Node.) + if (rewriter.file.text[classDecl.getEnd() - 1] !== '}') { + rewriter.error(classDecl, 'unexpected class terminator'); + } + rewriter.writeRange(classDecl, pos, classDecl.getEnd() - 1); + if (decoratorVisitor) { + decoratorVisitor.emitMetadataAsStaticProperties(); + } + rewriter.emit('}'); } diff --git a/src/es5processor.ts b/src/es5processor.ts index a8c8c3ba8..529790b58 100644 --- a/src/es5processor.ts +++ b/src/es5processor.ts @@ -8,7 +8,9 @@ import * as ts from 'typescript'; +import {ModulesManifest} from './modules_manifest'; import {getIdentifierText, Rewriter} from './rewriter'; +import {SourceMapper} from './source_map_utils'; import {toArray} from './util'; /** @@ -52,24 +54,24 @@ class ES5Processor extends Rewriter { unusedIndex: number = 0; constructor( - file: ts.SourceFile, private pathToModuleName: (context: string, fileName: string) => string, - private prelude: string) { - super(file); + private host: Host, private options: Options, file: ts.SourceFile, + sourceMapper?: SourceMapper) { + super(file, sourceMapper); } - process(moduleId: string, isES5: boolean): {output: string, referencedModules: string[]} { + process(moduleId: string): {output: string, referencedModules: string[]} { // TODO(evanm): only emit the goog.module *after* the first comment, // so that @suppress statements work. - const moduleName = this.pathToModuleName('', this.file.fileName); + const moduleName = this.host.pathToModuleName('', this.file.fileName); // NB: No linebreak after module call so sourcemaps are not offset. this.emit(`goog.module('${moduleName}');`); - if (this.prelude) this.emit(this.prelude); + if (this.options.prelude) this.emit(this.options.prelude); // Allow code to use `module.id` to discover its module URL, e.g. to resolve // a template URL against. // Uses 'var', as this code is inserted in ES6 and ES5 modes. // The following pattern ensures closure doesn't throw an error in advanced // optimizations mode. - if (isES5) { + if (this.options.es5Mode) { this.emit(`var module = module || {id: '${moduleId}'};`); } else { // The `exports = {}` serves as a default export to disable Closure Compiler's error checking @@ -82,11 +84,11 @@ class ES5Processor extends Rewriter { let pos = 0; for (let stmt of this.file.statements) { - this.writeRange(pos, stmt.getFullStart()); + this.writeRange(this.file, pos, stmt.getFullStart()); this.visitTopLevel(stmt); pos = stmt.getEnd(); } - this.writeRange(pos, this.file.getEnd()); + this.writeRange(this.file, pos, this.file.getEnd()); let referencedModules = toArray(this.moduleVariables.keys()); // Note: don't sort referencedModules, as the keys are in the same order @@ -140,7 +142,7 @@ class ES5Processor extends Rewriter { * comment(s). */ emitCommentWithoutStatementBody(node: ts.Node) { - this.writeRange(node.getFullStart(), node.getStart()); + this.writeRange(node, node.getFullStart(), node.getStart()); } /** isUseStrict returns true if node is a "use strict"; statement. */ @@ -177,7 +179,7 @@ class ES5Processor extends Rewriter { let call = decl.initializer as ts.CallExpression; let require = this.isRequire(call); if (!require) return false; - this.writeRange(node.getFullStart(), node.getStart()); + this.writeRange(node, node.getFullStart(), node.getStart()); this.emitGoogRequire(varName, require); return true; } else if (node.kind === ts.SyntaxKind.ExpressionStatement) { @@ -203,7 +205,7 @@ class ES5Processor extends Rewriter { } if (!require) return false; - this.writeRange(node.getFullStart(), node.getStart()); + this.writeRange(node, node.getFullStart(), node.getStart()); let varName = this.emitGoogRequire(null, require); if (isExport) { @@ -239,7 +241,7 @@ class ES5Processor extends Rewriter { modName = nsImport; isNamespaceImport = true; } else { - modName = this.pathToModuleName(this.file.fileName, tsImport); + modName = this.host.pathToModuleName(this.file.fileName, tsImport); } if (!varName) { @@ -321,7 +323,7 @@ class ES5Processor extends Rewriter { if (!this.namespaceImports.has(lhs)) break; // Emit the same expression, with spaces to replace the ".default" part // so that source maps still line up. - this.writeRange(node.getFullStart(), node.getStart()); + this.writeRange(node, node.getFullStart(), node.getStart()); this.emit(`${lhs} `); return true; default: @@ -336,6 +338,25 @@ class ES5Processor extends Rewriter { } } +export interface Host { + /** + * Takes a context (the current file) and the path of the file to import + * and generates a googmodule module name + */ + pathToModuleName(context: string, importPath: string): string; + /** + * If we do googmodule processing, we polyfill module.id, since that's + * part of ES6 modules. This function determines what the module.id will be + * for each file. + */ + fileNameToModuleId(fileName: string): string; +} + +export interface Options { + es5Mode?: boolean; + prelude?: string; +} + /** * Converts TypeScript's JS+CommonJS output to Closure goog.module etc. * For use as a postprocessing step *after* TypeScript emits JavaScript. @@ -350,10 +371,23 @@ class ES5Processor extends Rewriter { * @param prelude An additional prelude to insert after the `goog.module` call, * e.g. with additional imports or requires. */ -export function processES5( - fileName: string, moduleId: string, content: string, - pathToModuleName: (context: string, fileName: string) => string, isES5 = true, - prelude = ''): {output: string, referencedModules: string[]} { +export function processES5(host: Host, options: Options, fileName: string, content: string): + {output: string, referencedModules: string[]} { let file = ts.createSourceFile(fileName, content, ts.ScriptTarget.ES5, true); - return new ES5Processor(file, pathToModuleName, prelude).process(moduleId, isES5); + const moduleId = host.fileNameToModuleId(fileName); + return new ES5Processor(host, options, file).process(moduleId); +} + +export function convertCommonJsToGoogModule( + host: Host, options: Options, modulesManifest: ModulesManifest, fileName: string, + content: string): string { + let {output, referencedModules} = processES5(host, options, fileName, content); + + const moduleName = host.pathToModuleName('', fileName); + modulesManifest.addModule(fileName, moduleName); + for (let referenced of referencedModules) { + modulesManifest.addReferencedModule(fileName, referenced); + } + + return output; } diff --git a/src/jsdoc.ts b/src/jsdoc.ts index 54e8b7208..a1c6bff18 100644 --- a/src/jsdoc.ts +++ b/src/jsdoc.ts @@ -245,8 +245,9 @@ export function toString(tags: Tag[], escapeExtraTags: Set = new Set; - tsicklePasses: tsickle.Pass[]; + tsicklePasses: tsickleCompilerHost.Pass[]; } function getDefaultClosureJSOptions(fileNames: string[], settings: Settings): ClosureJSOptions { @@ -144,7 +146,7 @@ function getDefaultClosureJSOptions(fileNames: string[], settings: Settings): Cl fileNameToModuleId: (fileName) => fileName, }, files: new Map(), - tsicklePasses: [tsickle.Pass.CLOSURIZE], + tsicklePasses: [tsickleCompilerHost.Pass.CLOSURIZE], }; } @@ -171,7 +173,7 @@ export function toClosureJS( const sourceReplacingHost = createSourceReplacingCompilerHost(closureJSOptions.files, outputRetainingHost); - const tch = new tsickle.TsickleCompilerHost( + const tch = new tsickleCompilerHost.TsickleCompilerHost( sourceReplacingHost, options, closureJSOptions.tsickleCompilerHostOptions, closureJSOptions.tsickleHost); @@ -186,13 +188,13 @@ export function toClosureJS( // Reparse and reload the program, inserting the tsickle output in // place of the original source. - if (closureJSOptions.tsicklePasses.indexOf(tsickle.Pass.DECORATOR_DOWNLEVEL) !== -1) { - tch.reconfigureForRun(program, tsickle.Pass.DECORATOR_DOWNLEVEL); + if (closureJSOptions.tsicklePasses.indexOf(tsickleCompilerHost.Pass.DECORATOR_DOWNLEVEL) !== -1) { + tch.reconfigureForRun(program, tsickleCompilerHost.Pass.DECORATOR_DOWNLEVEL); program = ts.createProgram(fileNames, options, tch); } - if (closureJSOptions.tsicklePasses.indexOf(tsickle.Pass.CLOSURIZE) !== -1) { - tch.reconfigureForRun(program, tsickle.Pass.CLOSURIZE); + if (closureJSOptions.tsicklePasses.indexOf(tsickleCompilerHost.Pass.CLOSURIZE) !== -1) { + tch.reconfigureForRun(program, tsickleCompilerHost.Pass.CLOSURIZE); program = ts.createProgram(fileNames, options, tch); } diff --git a/src/rewriter.ts b/src/rewriter.ts index 2e4e878e8..fb91ca7e3 100644 --- a/src/rewriter.ts +++ b/src/rewriter.ts @@ -6,45 +6,42 @@ * found in the LICENSE file at https://angular.io/license */ -import {SourceMapGenerator} from 'source-map'; import * as ts from 'typescript'; +import {NOOP_SOURCE_MAPPER, SourceMapper, SourcePosition} from './source_map_utils'; + /** * A Rewriter manages iterating through a ts.SourceFile, copying input * to output while letting the subclass potentially alter some nodes * along the way by implementing maybeProcess(). */ export abstract class Rewriter { - private output: string[] = []; - /** Errors found while examining the code. */ - protected diagnostics: ts.Diagnostic[] = []; - /** The source map that's generated while rewriting this file. */ - private sourceMap: SourceMapGenerator; - /** Current position in the output. */ - private position = {line: 1, column: 1}; /** * The current level of recursion through TypeScript Nodes. Used in formatting internal debug * print statements. */ private indent: number = 0; + private output: string[] = []; + /** Errors found while examining the code. */ + private diagnostics: ts.Diagnostic[] = []; + /** Current position in the output. */ + private position: SourcePosition = {line: 0, column: 0, position: 0}; + /** + * Skip emitting any code before the given offset. Used to avoid emitting @fileoverview comments + * twice. + */ + public skipUpToOffset = 0; - constructor(protected file: ts.SourceFile) { - this.sourceMap = new SourceMapGenerator({file: file.fileName}); - this.sourceMap.addMapping({ - original: this.position, - generated: this.position, - source: file.fileName, - }); + constructor(public file: ts.SourceFile, private sourceMapper: SourceMapper = NOOP_SOURCE_MAPPER) { } - getOutput(): {output: string, diagnostics: ts.Diagnostic[], sourceMap: SourceMapGenerator} { + getOutput(): {output: string, diagnostics: ts.Diagnostic[]} { if (this.indent !== 0) { throw new Error('visit() failed to track nesting'); } return { output: this.output.join(''), diagnostics: this.diagnostics, - sourceMap: this.sourceMap, }; } @@ -71,7 +68,7 @@ export abstract class Rewriter { } /** writeNode writes a ts.Node, calling this.visit() on its children. */ - writeNode(node: ts.Node, skipComments = false) { + writeNode(node: ts.Node, skipComments = false, newLineIfCommentsStripped = true) { let pos = node.getFullStart(); if (skipComments) { // To skip comments, we skip all whitespace/comments preceding @@ -79,50 +76,51 @@ export abstract class Rewriter { // a newline in its place so that the node remains separated // from the previous node. TODO: don't skip anything here if // there wasn't any comment. - if (node.getFullStart() < node.getStart()) { + if (newLineIfCommentsStripped && node.getFullStart() < node.getStart()) { this.emit('\n'); } pos = node.getStart(); } + this.writeNodeStartingFrom(node, pos); + } + + writeNodeStartingFrom(node: ts.Node, pos: number) { ts.forEachChild(node, child => { - this.writeRange(pos, child.getFullStart()); + this.writeRange(node, pos, child.getFullStart()); this.visit(child); pos = child.getEnd(); }); - this.writeRange(pos, node.getEnd()); + this.writeRange(node, pos, node.getEnd()); } - /** - * Skip emitting any code before the given offset. Used to avoid emitting @fileoverview comments - * twice. - */ - protected skipUpToOffset = 0; - /** * Write a span of the input file as expressed by absolute offsets. * These offsets are found in attributes like node.getFullStart() and * node.getEnd(). */ - writeRange(from: number, to: number) { + writeRange(node: ts.Node, from: number, to: number) { from = Math.max(from, this.skipUpToOffset); + if (this.skipUpToOffset > 0 && to <= this.skipUpToOffset) { + return; + } + // Add a source mapping. writeRange(from, to) always corresponds to + // original source code, so add a mapping at the current location that + // points back to the location at `from`. The additional code generated + // by tsickle will then be considered part of the last mapped code + // section preceding it. That's arguably incorrect (e.g. for the fake + // methods defining properties), but is good enough for stack traces. + const pos = this.file.getLineAndCharacterOfPosition(from); + this.sourceMapper.addMapping( + node, {line: pos.line, column: pos.character, position: from}, this.position, to - from); // getSourceFile().getText() is wrong here because it has the text of // the SourceFile node of the AST, which doesn't contain the comments // preceding that node. Semantically these ranges are just offsets // into the original source file text, so slice from that. let text = this.file.text.slice(from, to); if (text) { - // Add a source mapping. writeRange(from, to) always corresponds to - // original source code, so add a mapping at the current location that - // points back to the location at `from`. The additional code generated - // by tsickle will then be considered part of the last mapped code - // section preceding it. That's arguably incorrect (e.g. for the fake - // methods defining properties), but is good enough for stack traces. - const pos = this.file.getLineAndCharacterOfPosition(from); - this.sourceMap.addMapping({ - original: {line: pos.line + 1, column: pos.character + 1}, - generated: this.position, - source: this.file.fileName, - }); + if (text.indexOf('A Used by implement_import.ts') !== -1) { + console.log('>>> now', node.kind, new Error().stack); + } this.emit(text); } } @@ -133,9 +131,10 @@ export abstract class Rewriter { this.position.column++; if (c === '\n') { this.position.line++; - this.position.column = 1; + this.position.column = 0; } } + this.position.position += str.length; } /** Removes comment metacharacters from a string, to make it safe to embed in a comment. */ @@ -159,8 +158,12 @@ export abstract class Rewriter { this.error(node, `${ts.SyntaxKind[node.kind]} not implemented in ${where}`); } + addDiagnostic(d: ts.Diagnostic) { + this.diagnostics.push(d); + } + error(node: ts.Node, messageText: string) { - this.diagnostics.push({ + this.addDiagnostic({ file: this.file, start: node.getStart(), length: node.getEnd() - node.getStart(), @@ -171,6 +174,81 @@ export abstract class Rewriter { } } +export class Emitter { + private output: string[] = []; + /** Errors found while examining the code. */ + private diagnostics: ts.Diagnostic[] = []; + /** Current position in the output. */ + private position: SourcePosition = {line: 0, column: 0, position: 0}; + /** + * Skip emitting any code before the given offset. Used to avoid emitting @fileoverview comments + * twice. + */ + private skipUpToOffset = 0; + + constructor(public file: ts.SourceFile, private sourceMapper: SourceMapper = NOOP_SOURCE_MAPPER) { + } + + getOutput(): {output: string, diagnostics: ts.Diagnostic[]} { + return { + output: this.output.join(''), + diagnostics: this.diagnostics, + }; + } + + addDiagnostic(d: ts.Diagnostic) { + this.diagnostics.push(d); + } + + setSkipUpToOffset(offset: number) { + this.skipUpToOffset = offset; + } + + /** + * Write a span of the input file as expressed by absolute offsets. + * These offsets are found in attributes like node.getFullStart() and + * node.getEnd(). + */ + writeRange(node: ts.Node, from: number, to: number) { + from = Math.max(from, this.skipUpToOffset); + if (this.skipUpToOffset > 0 && to <= this.skipUpToOffset) { + return; + } + // Add a source mapping. writeRange(from, to) always corresponds to + // original source code, so add a mapping at the current location that + // points back to the location at `from`. The additional code generated + // by tsickle will then be considered part of the last mapped code + // section preceding it. That's arguably incorrect (e.g. for the fake + // methods defining properties), but is good enough for stack traces. + const pos = this.file.getLineAndCharacterOfPosition(from); + this.sourceMapper.addMapping( + node, {line: pos.line, column: pos.character, position: from}, this.position, to - from); + // getSourceFile().getText() is wrong here because it has the text of + // the SourceFile node of the AST, which doesn't contain the comments + // preceding that node. Semantically these ranges are just offsets + // into the original source file text, so slice from that. + let text = this.file.text.slice(from, to); + if (text) { + if (text.indexOf('A Used by implement_import.ts') !== -1) { + console.log('>>> now', node.kind, new Error().stack); + } + this.emit(text); + } + } + + emit(str: string) { + this.output.push(str); + for (const c of str) { + this.position.column++; + if (c === '\n') { + this.position.line++; + this.position.column = 0; + } + } + this.position.position += str.length; + } +} + /** Returns the string contents of a ts.Identifier. */ export function getIdentifierText(identifier: ts.Identifier): string { // NOTE: the 'text' property on an Identifier may be escaped if it starts diff --git a/src/source_map_utils.ts b/src/source_map_utils.ts index 9f57b378f..7a6b8d4bf 100644 --- a/src/source_map_utils.ts +++ b/src/source_map_utils.ts @@ -7,6 +7,7 @@ */ import {SourceMapConsumer, SourceMapGenerator} from 'source-map'; +import * as ts from 'typescript'; /** * Return a new RegExp object every time we want one because the @@ -94,3 +95,47 @@ export function sourceMapTextToGenerator(sourceMapText: string): SourceMapGenera const sourceMapJson: any = sourceMapText; return SourceMapGenerator.fromSourceMap(sourceMapTextToConsumer(sourceMapJson)); } + +export interface SourcePosition { + // 0 based + column: number; + // 0 based + line: number; + // 0 based + position: number; +} + +export interface SourceMapper { + addMapping( + originalNode: ts.Node, original: SourcePosition, generated: SourcePosition, + length: number): void; +} + +export const NOOP_SOURCE_MAPPER: SourceMapper = { + // tslint:disable-next-line:no-empty + addMapping: () => {} +}; + +export class DefaultSourceMapper implements SourceMapper { + /** The source map that's generated while rewriting this file. */ + public sourceMap = new SourceMapGenerator(); + + constructor(private fileName: string) { + this.sourceMap.addMapping({ + original: {line: 1, column: 1}, + generated: {line: 1, column: 1}, + source: this.fileName, + }); + } + + addMapping(node: ts.Node, original: SourcePosition, generated: SourcePosition, length: number): + void { + if (length > 0) { + this.sourceMap.addMapping({ + original: {line: original.line + 1, column: original.column + 1}, + generated: {line: generated.line + 1, column: generated.column + 1}, + source: this.fileName, + }); + } + } +} diff --git a/src/transformer.ts b/src/transformer.ts new file mode 100644 index 000000000..aa9b61436 --- /dev/null +++ b/src/transformer.ts @@ -0,0 +1,234 @@ +/** + * @license + * Copyright Google Inc. All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import * as ts from 'typescript'; + +import * as decorator from './decorator-annotator'; +import * as es5processor from './es5processor'; +import {ModulesManifest} from './modules_manifest'; +import {SourceMapper, SourcePosition} from './source_map_utils'; +import * as tsickle from './tsickle'; + +export {getGeneratedExterns} from './tsickle'; + +export interface TransformerOptions extends es5processor.Options, tsickle.Options { + /** + * Whether to downlevel decorators + */ + transformDecorators?: boolean; + /** + * Whether to convers types to closure + */ + transformTypesToClosure?: boolean; +} + +export interface TransformerHost extends es5processor.Host, tsickle.Host { + /** + * If true, tsickle and decorator downlevel processing will be skipped for + * that file. + */ + shouldSkipTsickleProcessing(fileName: string): boolean; + /** + * Tsickle treats warnings as errors, if true, ignore warnings. This might be + * useful for e.g. third party code. + */ + shouldIgnoreWarningsForPath(filePath: string): boolean; +} + +export interface EmitResult extends ts.EmitResult { + // The manifest of JS modules output by the compiler. + modulesManifest: ModulesManifest; + /** externs.js files produced by tsickle, if any. */ + externs: {[fileName: string]: string}; +} + +export interface EmitTransformers { + beforeTsickle?: ts.TransformerFactory[]; + beforeTs?: ts.TransformerFactory[]; + afterTs?: ts.TransformerFactory[]; +} + +export function emitWithTsickle( + program: ts.Program, host: TransformerHost, options: TransformerOptions, + tsHost: ts.CompilerHost, tsOptions: ts.CompilerOptions, targetSourceFile?: ts.SourceFile, + writeFile?: ts.WriteFileCallback, cancellationToken?: ts.CancellationToken, + emitOnlyDtsFiles?: boolean, customTransformers?: EmitTransformers): EmitResult { + let tsickleDiagnostics: ts.Diagnostic[] = []; + const typeChecker = createOriginalNodeTypeChecker(program.getTypeChecker()); + const beforeTsTransformers: ts.TransformerFactory[] = []; + if (options.transformTypesToClosure) { + // Note: tsickle.annotate can also lower decorators in the same run. + beforeTsTransformers.push(createTransformer(host, (sourceFile, sourceMapper) => { + const tisckleOptions: tsickle.Options = {...options, filterTypesForExportStar: true}; + const {output, diagnostics} = tsickle.annotate( + typeChecker, sourceFile, host, tisckleOptions, tsHost, tsOptions, sourceMapper); + tsickleDiagnostics.push(...diagnostics); + return output; + })); + } else if (options.transformDecorators) { + beforeTsTransformers.push(createTransformer(host, (sourceFile, sourceMapper) => { + const {output, diagnostics} = + decorator.convertDecorators(typeChecker, sourceFile, sourceMapper); + tsickleDiagnostics.push(...diagnostics); + return output; + })); + } + const afterTsTransformers: ts.TransformerFactory[] = []; + if (customTransformers) { + if (customTransformers.beforeTsickle) { + beforeTsTransformers.unshift(...customTransformers.beforeTsickle); + } + + if (customTransformers.beforeTs) { + beforeTsTransformers.push(...customTransformers.beforeTs); + } + if (customTransformers.afterTs) { + afterTsTransformers.push(...customTransformers.afterTs); + } + } + let writeFileImpl = writeFile; + const modulesManifest = new ModulesManifest(); + if (options.googmodule) { + writeFileImpl = + (fileName: string, content: string, writeByteOrderMark: boolean, + onError?: (message: string) => void, sourceFiles?: ts.SourceFile[]) => { + if (!tsickle.isDtsFileName(fileName) && !fileName.endsWith('.map')) { + content = es5processor.convertCommonJsToGoogModule( + host, options, modulesManifest, fileName, content); + } + if (writeFile) { + writeFile(fileName, content, writeByteOrderMark, onError, sourceFiles); + } else { + tsHost.writeFile(fileName, content, writeByteOrderMark, onError, sourceFiles); + } + }; + } + + const {diagnostics: tsDiagnostics, emitSkipped, emittedFiles} = program.emit( + targetSourceFile, writeFileImpl, cancellationToken, emitOnlyDtsFiles, + {before: beforeTsTransformers, after: afterTsTransformers}); + + const externs: {[fileName: string]: string} = {}; + // Note: we also need to collect externs on .d.ts files, + // so we can't do this in the ts transformer pipeline. + program.getSourceFiles().forEach(sf => { + if (host.shouldSkipTsickleProcessing(sf.fileName)) { + return; + } + const {output, diagnostics} = tsickle.writeExterns(typeChecker, sf, host, options); + if (output) { + externs[sf.fileName] = output; + } + if (diagnostics) { + tsickleDiagnostics.push(...diagnostics); + } + }); + // All diagnostics (including warnings) are treated as errors. + // If we've decided to ignore them, just discard them. + // Warnings include stuff like "don't use @type in your jsdoc"; tsickle + // warns and then fixes up the code to be Closure-compatible anyway. + tsickleDiagnostics = tsickleDiagnostics.filter( + d => d.category === ts.DiagnosticCategory.Error || + !host.shouldIgnoreWarningsForPath(d.file.fileName)); + + return { + modulesManifest, + emitSkipped, + emittedFiles, + diagnostics: [...tsDiagnostics, ...tsickleDiagnostics], + externs + }; +} + +function createTransformer( + host: TransformerHost, + operator: (sourceFile: ts.SourceFile, sourceMapper: SourceMapper) => + string): ts.TransformerFactory { + return (context: ts.TransformationContext) => (sourceFile: ts.SourceFile): ts.SourceFile => { + if (host.shouldSkipTsickleProcessing(sourceFile.fileName)) { + return sourceFile; + } + const genStartPositions = new Map(); + const originalNodeByGeneratedRange = new Map(); + + const addFullNodeRange = (node: ts.Node, genStartPos: number) => { + originalNodeByGeneratedRange.set( + nodeCacheKey(node.kind, genStartPos, genStartPos + (node.getEnd() - node.getStart())), + node); + node.forEachChild( + child => addFullNodeRange(child, genStartPos + (child.getStart() - node.getStart()))); + }; + + const sourceMapper = { + addMapping: ( + originalNode: ts.Node, original: SourcePosition, generated: SourcePosition, + length: number) => { + let originalStartPos = original.position; + let genStartPos = generated.position; + if (originalStartPos >= originalNode.getFullStart() && + originalStartPos <= originalNode.getStart()) { + // always use the node.getStart() for the index, + // as comments and whitespaces might differ between the original and transformed code. + const diffToStart = originalNode.getStart() - originalStartPos; + originalStartPos += diffToStart; + genStartPos += diffToStart; + length -= diffToStart; + genStartPositions.set(originalNode, genStartPos); + } + if (originalStartPos + length === originalNode.getEnd()) { + originalNodeByGeneratedRange.set( + nodeCacheKey( + originalNode.kind, genStartPositions.get(originalNode)!, genStartPos + length), + originalNode); + } + originalNode.forEachChild((child) => { + if (child.getStart() >= originalStartPos && child.getEnd() <= originalStartPos + length) { + addFullNodeRange(child, genStartPos + (child.getStart() - originalStartPos)); + } + }); + } + }; + + const newSource = operator(sourceFile, sourceMapper); + const newFile = + ts.createSourceFile(sourceFile.fileName, newSource, ts.ScriptTarget.Latest, true); + + ts.setOriginalNode(newFile, sourceFile); + setOriginalNode(newFile, originalNodeByGeneratedRange); + + return newFile; + }; +} + +function setOriginalNode(node: ts.Node, originalNodeByGeneratedRange: Map) { + node.flags |= ts.NodeFlags.Synthesized; + let originalNode = + originalNodeByGeneratedRange.get(nodeCacheKey(node.kind, node.getStart(), node.getEnd())); + if (originalNode) { + ts.setOriginalNode(node, originalNode); + } + ts.forEachChild(node, child => setOriginalNode(child, originalNodeByGeneratedRange)); +} + +function createOriginalNodeTypeChecker(tc: ts.TypeChecker): ts.TypeChecker { + const result = Object.create(tc); + result.getTypeAtLocation = (node: ts.Node) => tc.getTypeAtLocation(ts.getOriginalNode(node)); + result.getSymbolAtLocation = (node: ts.Node) => tc.getSymbolAtLocation(ts.getOriginalNode(node)); + result.getSignatureFromDeclaration = (declaration: ts.SignatureDeclaration) => + tc.getSignatureFromDeclaration(ts.getOriginalNode(declaration) as ts.SignatureDeclaration); + result.getSymbolsInScope = (location: ts.Node, meaning: ts.SymbolFlags) => + tc.getSymbolsInScope(ts.getOriginalNode(location), meaning); + result.getConstantValue = (node: ts.Node) => tc.getConstantValue(ts.getOriginalNode(node) as any); + result.getTypeOfSymbolAtLocation = (symbol: ts.Symbol, node: ts.Node) => + tc.getTypeOfSymbolAtLocation(symbol, ts.getOriginalNode(node)); + return result; +} + +function nodeCacheKey(kind: ts.SyntaxKind, start: number, end: number): string { + return `${kind}#${start}#${end}`; +} diff --git a/src/tsickle.ts b/src/tsickle.ts index b14863d72..1c4a1f77c 100644 --- a/src/tsickle.ts +++ b/src/tsickle.ts @@ -7,31 +7,55 @@ */ import * as path from 'path'; -import {SourceMapGenerator} from 'source-map'; import * as ts from 'typescript'; +import {DecoratorClassVisitor, visitClassContent} from './decorator-annotator'; import {hasExportingDecorator} from './decorators'; import {extractGoogNamespaceImport} from './es5processor'; import * as jsdoc from './jsdoc'; import {getIdentifierText, Rewriter, unescapeName} from './rewriter'; -import {Options} from './tsickle_compiler_host'; +import {SourceMapper} from './source_map_utils'; import * as typeTranslator from './type-translator'; import {toArray} from './util'; export {convertDecorators} from './decorator-annotator'; -export {processES5} from './es5processor'; export {FileMap, ModulesManifest} from './modules_manifest'; -export {Options, Pass, TsickleCompilerHost, TsickleHost} from './tsickle_compiler_host'; - -export interface Output { - /** The TypeScript source with Closure annotations inserted. */ - output: string; - /** Generated externs declarations, if any. */ - externs: string|null; - /** Error messages, if any. */ - diagnostics: ts.Diagnostic[]; - /** A source map mapping back into the original sources. */ - sourceMap: SourceMapGenerator; + +export interface Host { + /** + * If provided a function that logs an internal warning. + * These warnings are not actionable by an end user and should be hidden + * by default. + */ + logWarning?: (warning: ts.Diagnostic) => void; + pathToModuleName: (context: string, importPath: string) => string; +} + +export interface Options { + googmodule?: boolean; + /** + * If true, convert every type to the Closure {?} type, which means + * "don't check types". + */ + untyped?: boolean; + /** If provided, a set of paths whose types should always generate as {?}. */ + typeBlackListPaths?: Set; + /** + * Convert shorthand "/index" imports to full path (include the "/index"). + * Annotation will be slower because every import must be resolved. + */ + convertIndexImportShorthand?: boolean; + /** + * Whether to downlevel decorators as well. + */ + transformDecorators?: boolean; + /** + * Whether to filter out exports for types when expanding an `export * ...`. + * Needed e.g. for transformer mode, as + * the typechecker has no information of the new symbols, + * and therefore typescript won't automatically remove them. + */ + filterTypesForExportStar?: boolean; } /** @@ -161,8 +185,10 @@ class ClosureRewriter extends Rewriter { */ symbolsToAliasedNames = new Map(); - constructor(protected program: ts.Program, file: ts.SourceFile, protected options: Options) { - super(file); + constructor( + protected typeChecker: ts.TypeChecker, protected host: Host, protected options: Options, + file: ts.SourceFile, sourceMapper?: SourceMapper) { + super(file, sourceMapper); } /** @@ -177,7 +203,7 @@ class ClosureRewriter extends Rewriter { * function statement; for overloads, name will have been merged. */ emitFunctionType(fnDecls: ts.SignatureDeclaration[], extraTags: jsdoc.Tag[] = []): string[] { - const typeChecker = this.program.getTypeChecker(); + const typeChecker = this.typeChecker; let newDoc = extraTags; const lens = fnDecls.map(fnDecl => fnDecl.parameters.length); const minArgsCount = Math.min(...lens); @@ -329,7 +355,7 @@ class ClosureRewriter extends Rewriter { if (!parsed) return null; if (parsed.warnings) { const start = node.getFullStart() + pos; - this.diagnostics.push({ + this.addDiagnostic({ file: this.file, start, length: node.getStart() - start, @@ -376,7 +402,7 @@ class ClosureRewriter extends Rewriter { // We can only @implements an interface, not a class. // But it's fine to translate TS "implements Class" into Closure // "@extends {Class}" because this is just a type hint. - let typeChecker = this.program.getTypeChecker(); + let typeChecker = this.typeChecker; let sym = typeChecker.getSymbolAtLocation(impl.expression); if (sym.flags & ts.SymbolFlags.TypeAlias) { // It's implementing a type alias. Follow the type alias back @@ -431,7 +457,7 @@ class ClosureRewriter extends Rewriter { return '?'; } - let typeChecker = this.program.getTypeChecker(); + let typeChecker = this.typeChecker; if (!type) { type = typeChecker.getTypeAtLocation(context); } @@ -440,7 +466,7 @@ class ClosureRewriter extends Rewriter { newTypeTranslator(context: ts.Node) { const translator = new typeTranslator.TypeTranslator( - this.program.getTypeChecker(), context, this.options.typeBlackListPaths, + this.typeChecker, ts.getOriginalNode(context), this.options.typeBlackListPaths, this.symbolsToAliasedNames); translator.warn = msg => this.debugWarn(context, msg); return translator; @@ -454,7 +480,7 @@ class ClosureRewriter extends Rewriter { * for tsickle to debug itself. */ debugWarn(node: ts.Node, messageText: string) { - if (!this.options.logWarning) return; + if (!this.host.logWarning) return; // Use a ts.Diagnosic so that the warning includes context and file offets. let diagnostic: ts.Diagnostic = { file: this.file, @@ -464,7 +490,7 @@ class ClosureRewriter extends Rewriter { category: ts.DiagnosticCategory.Warning, code: 0, }; - this.options.logWarning(diagnostic); + this.host.logWarning(diagnostic); } } @@ -479,40 +505,21 @@ const FILEOVERVIEW_COMMENTS: ReadonlySet = /** Annotator translates a .ts to a .ts with Closure annotations. */ class Annotator extends ClosureRewriter { - /** - * Generated externs, if any. Any "declare" blocks encountered in the source - * are forwarded to the ExternsWriter to be translated into externs. - */ - private externsWriter: ExternsWriter; - /** Exported symbol names that have been generated by expanding an "export * from ...". */ private generatedExports = new Set(); - - private typeChecker: ts.TypeChecker; + /** ComposableDecoratorRewriter when using tsickle as a TS transformer */ + private currentDecoratorConverter: DecoratorClassVisitor|undefined; constructor( - program: ts.Program, file: ts.SourceFile, options: Options, - private pathToModuleName: (context: string, importPath: string) => string, - private host?: ts.ModuleResolutionHost, private tsOpts?: ts.CompilerOptions) { - super(program, file, options); - this.externsWriter = new ExternsWriter(program, file, options); - this.typeChecker = program.getTypeChecker(); + typeChecker: ts.TypeChecker, host: Host, options: Options, file: ts.SourceFile, + sourceMapper?: SourceMapper, private tsHost?: ts.ModuleResolutionHost, + private tsOpts?: ts.CompilerOptions) { + super(typeChecker, host, options, file, sourceMapper); } - annotate(): Output { + annotate(): {output: string, diagnostics: ts.Diagnostic[]} { this.visit(this.file); - - let annotated = this.getOutput(); - - let externs = this.externsWriter.getOutput(); - let externsSource = externs.output.length > 0 ? externs.output : null; - - return { - output: annotated.output, - externs: externsSource, - diagnostics: externs.diagnostics.concat(annotated.diagnostics), - sourceMap: annotated.sourceMap, - }; + return this.getOutput(); } getExportDeclarationNames(node: ts.Node): ts.Identifier[] { @@ -582,15 +589,16 @@ class Annotator extends ClosureRewriter { */ maybeProcess(node: ts.Node): boolean { if (hasModifierFlag(node, ts.ModifierFlags.Ambient) || isDtsFileName(this.file.fileName)) { - this.externsWriter.visit(node); // An ambient declaration declares types for TypeScript's benefit, so we want to skip Tsickle // conversion of its contents. - this.writeRange(node.getFullStart(), node.getEnd()); + this.writeRange(node, node.getFullStart(), node.getEnd()); // ... but it might need to be exported for downstream importing code. this.maybeEmitAmbientDeclarationExport(node); return true; } - + if (this.currentDecoratorConverter) { + this.currentDecoratorConverter.beforeProcessNode(node); + } switch (node.kind) { case ts.SyntaxKind.SourceFile: this.handleSourceFile(node as ts.SourceFile); @@ -599,14 +607,21 @@ class Annotator extends ClosureRewriter { return this.emitImportDeclaration(node as ts.ImportDeclaration); case ts.SyntaxKind.ExportDeclaration: let exportDecl = node; - this.writeRange(node.getFullStart(), node.getStart()); + this.writeRange(node, node.getFullStart(), node.getStart()); this.emit('export'); let exportedSymbols: NamedSymbol[] = []; if (!exportDecl.exportClause && exportDecl.moduleSpecifier) { // It's an "export * from ..." statement. // Rewrite it to re-export each exported symbol directly. exportedSymbols = this.expandSymbolsFromExportStar(exportDecl); - this.emit(` {${exportedSymbols.map(e => unescapeName(e.name)).join(',')}}`); + const exportedValueSymbols = this.options.filterTypesForExportStar ? + exportedSymbols.filter( + s => + ((s.sym.flags & ts.SymbolFlags.Value) || + (s.sym.declarations || + []).some(d => d.kind === ts.SyntaxKind.InterfaceDeclaration))) : + exportedSymbols; + this.emit(` {${exportedValueSymbols.map(e => unescapeName(e.name)).join(',')}}`); } else { if (exportDecl.exportClause) { exportedSymbols = this.getNamedSymbols(exportDecl.exportClause.elements); @@ -627,7 +642,7 @@ class Annotator extends ClosureRewriter { case ts.SyntaxKind.InterfaceDeclaration: this.emitInterface(node as ts.InterfaceDeclaration); // Emit the TS interface verbatim, with no tsickle processing of properties. - this.writeRange(node.getFullStart(), node.getEnd()); + this.writeRange(node, node.getFullStart(), node.getEnd()); return true; case ts.SyntaxKind.VariableDeclaration: let varDecl = node as ts.VariableDeclaration; @@ -662,12 +677,12 @@ class Annotator extends ClosureRewriter { let offset = ctor.getStart(); if (ctor.parameters.length) { for (let param of ctor.parameters) { - this.writeRange(offset, param.getFullStart()); + this.writeRange(node, offset, param.getFullStart()); this.visit(param); offset = param.getEnd(); } } - this.writeRange(offset, node.getEnd()); + this.writeRange(node, offset, node.getEnd()); return true; case ts.SyntaxKind.ArrowFunction: // It's difficult to annotate arrow functions due to a bug in @@ -697,7 +712,7 @@ class Annotator extends ClosureRewriter { this.error(fnDecl, 'anonymous abstract function'); return false; } - this.writeRange(fnDecl.name.getStart(), fnDecl.parameters.end); + this.writeRange(fnDecl, fnDecl.name.getStart(), fnDecl.parameters.end); this.emit(') {}'); return true; } @@ -708,7 +723,7 @@ class Annotator extends ClosureRewriter { } this.emitFunctionType([fnDecl], tags); - this.writeRange(fnDecl.getStart(), fnDecl.body.getFullStart()); + this.writeRange(fnDecl, fnDecl.getStart(), fnDecl.body.getFullStart()); this.visit(fnDecl.body); return true; case ts.SyntaxKind.TypeAliasDeclaration: @@ -759,7 +774,7 @@ class Annotator extends ClosureRewriter { if (docTags.length > 0 && node.getFirstToken()) { this.emit('\n'); this.emit(jsdoc.toString(docTags)); - this.writeRange(node.getFirstToken().getStart(), node.getEnd()); + this.writeRange(node, node.getFirstToken().getStart(), node.getEnd()); return true; } break; @@ -815,6 +830,9 @@ class Annotator extends ClosureRewriter { this.writeNode(pae.expression); this.emit(`['${getIdentifierText(pae.name)}']`); return true; + case ts.SyntaxKind.Decorator: + return !!this.currentDecoratorConverter && + this.currentDecoratorConverter.maybeProcessDecorator(node); default: break; } @@ -827,7 +845,7 @@ class Annotator extends ClosureRewriter { this.visit(stmt); } if (sf.statements.length) { - this.writeRange(sf.statements[sf.statements.length - 1].getEnd(), sf.getEnd()); + this.writeRange(sf, sf.statements[sf.statements.length - 1].getEnd(), sf.getEnd()); } } @@ -861,7 +879,7 @@ class Annotator extends ClosureRewriter { return; } const comment = comments[fileoverviewIdx]; - this.writeRange(0, comment.pos); + this.writeRange(sf, 0, comment.pos); this.skipUpToOffset = comment.end; const parsed = jsdoc.parse(sf.getFullText().substring(comment.pos, comment.end)); @@ -982,11 +1000,11 @@ class Annotator extends ClosureRewriter { } let moduleId = (moduleSpecifier as ts.StringLiteral).text; if (this.options.convertIndexImportShorthand) { - if (!this.tsOpts || !this.host) { + if (!this.tsOpts || !this.tsHost) { throw new Error( 'option convertIndexImportShorthand requires that annotate be called with a TypeScript host and options.'); } - const resolved = ts.resolveModuleName(moduleId, this.file.fileName, this.tsOpts, this.host); + const resolved = ts.resolveModuleName(moduleId, this.file.fileName, this.tsOpts, this.tsHost); if (resolved && resolved.resolvedModule) { const requestedModule = moduleId.replace(extension, ''); const resolvedModule = resolved.resolvedModule.resolvedFileName.replace(extension, ''); @@ -1009,13 +1027,14 @@ class Annotator extends ClosureRewriter { * @return true if the decl was handled, false to allow default processing. */ private emitImportDeclaration(decl: ts.ImportDeclaration): boolean { - this.writeRange(decl.getFullStart(), decl.getStart()); + this.writeRange(decl, decl.getFullStart(), decl.getStart()); this.emit('import'); const importPath = this.resolveModuleSpecifier(decl.moduleSpecifier); const importClause = decl.importClause; if (!importClause) { // import './foo'; this.emit(`'${importPath}';`); + this.writeRange(decl, decl.getEnd(), decl.getEnd()); return true; } else if ( importClause.name || @@ -1023,6 +1042,7 @@ class Annotator extends ClosureRewriter { importClause.namedBindings.kind === ts.SyntaxKind.NamedImports)) { this.visit(importClause); this.emit(` from '${importPath}';`); + this.writeRange(decl, decl.getEnd(), decl.getEnd()); // importClause.name implies // import a from ...; @@ -1061,6 +1081,7 @@ class Annotator extends ClosureRewriter { // import * as foo from ...; this.visit(importClause); this.emit(` from '${importPath}';`); + this.writeRange(decl, decl.getEnd(), decl.getEnd()); return true; } else { this.errorUnimplementedKind(decl, 'unexpected kind of import'); @@ -1092,7 +1113,7 @@ class Annotator extends ClosureRewriter { const nsImport = extractGoogNamespaceImport(importPath); const forwardDeclarePrefix = `tsickle_forward_declare_${++this.forwardDeclareCounter}`; const moduleNamespace = - nsImport !== null ? nsImport : this.pathToModuleName(this.file.fileName, importPath); + nsImport !== null ? nsImport : this.host.pathToModuleName(this.file.fileName, importPath); const exports = this.typeChecker.getExportsOfModule(this.typeChecker.getSymbolAtLocation(specifier)); // In TypeScript, importing a module for use in a type annotation does not cause a runtime load. @@ -1126,6 +1147,11 @@ class Annotator extends ClosureRewriter { } private visitClassDeclaration(classDecl: ts.ClassDeclaration) { + const oldDecoratorConverter = this.currentDecoratorConverter; + if (this.options.transformDecorators) { + this.currentDecoratorConverter = new DecoratorClassVisitor(this.typeChecker, this, classDecl); + } + let docTags = this.getJSDoc(classDecl) || []; if (hasModifierFlag(classDecl, ts.ModifierFlags.Abstract)) { docTags.push({tagName: 'abstract'}); @@ -1138,19 +1164,10 @@ class Annotator extends ClosureRewriter { this.emit('\n'); if (docTags.length > 0) this.emit(jsdoc.toString(docTags)); - if (classDecl.members.length > 0) { - // We must visit all members individually, to strip out any - // /** @export */ annotations that show up in the constructor - // and to annotate methods. - this.writeRange(classDecl.getStart(), classDecl.members[0].getFullStart()); - for (let member of classDecl.members) { - this.visit(member); - } - } else { - this.writeRange(classDecl.getStart(), classDecl.getLastToken().getFullStart()); - } - this.writeNode(classDecl.getLastToken()); + visitClassContent(classDecl, this, this.currentDecoratorConverter); this.emitTypeAnnotationsHelper(classDecl); + + this.currentDecoratorConverter = oldDecoratorConverter; return true; } @@ -1221,6 +1238,9 @@ class Annotator extends ClosureRewriter { let className = getIdentifierText(classDecl.name); this.emit(`\n\nfunction ${className}_tsickle_Closure_declarations() {\n`); + if (this.currentDecoratorConverter) { + this.currentDecoratorConverter.emitMetadataTypeAnnotationsHelpers(); + } staticProps.forEach(p => this.visitProperty([className], p)); let memberNamespace = [className, 'prototype']; nonStaticProps.forEach((p) => this.visitProperty(memberNamespace, p)); @@ -1339,11 +1359,12 @@ class Annotator extends ClosureRewriter { // both a typedef and an indexable object if we export it. this.emit('\n'); let name = node.name.getText(); - const isExported = hasModifierFlag(node, ts.ModifierFlags.Export); - if (isExported) this.emit('export '); this.emit(`type ${name} = number;\n`); - if (isExported) this.emit('export '); this.emit(`let ${name}: any = {};\n`); + const isExported = hasModifierFlag(node, ts.ModifierFlags.Export); + if (isExported) { + this.emit(`export {${name}};\n`); + } // Emit foo.BAR = 0; lines. for (let member of toArray(members.keys())) { @@ -1369,6 +1390,25 @@ class Annotator extends ClosureRewriter { /** ExternsWriter generates Closure externs from TypeScript source. */ class ExternsWriter extends ClosureRewriter { + process(): {output: string, diagnostics: ts.Diagnostic[]} { + if (isDtsFileName(this.file.fileName)) { + this.visit(this.file); + } else { + this.visitAmbientChildren(this.file); + } + return this.getOutput(); + } + + private visitAmbientChildren(node: ts.Node) { + node.forEachChild(child => { + if (hasModifierFlag(child, ts.ModifierFlags.Ambient)) { + this.visit(child); + } else { + this.visitAmbientChildren(child); + } + }); + } + /** visit is the main entry point. It generates externs from a ts.Node. */ public visit(node: ts.Node, namespace: string[] = []) { switch (node.kind) { @@ -1443,11 +1483,11 @@ class ExternsWriter extends ClosureRewriter { break; } // Gather up all overloads of this function. - const sym = this.program.getTypeChecker().getSymbolAtLocation(name); + const sym = this.typeChecker.getSymbolAtLocation(name); const decls = sym.declarations!.filter(d => d.kind === ts.SyntaxKind.FunctionDeclaration) as ts.FunctionDeclaration[]; // Only emit the first declaration of each overloaded function. - if (fnDecl !== decls[0]) break; + if (ts.getOriginalNode(fnDecl) !== decls[0]) break; const params = this.emitFunctionType(decls); this.writeExternsFunction(name.getText(), params, namespace); break; @@ -1477,10 +1517,10 @@ class ExternsWriter extends ClosureRewriter { */ private isFirstDeclaration(decl: ts.DeclarationStatement): boolean { if (!decl.name) return true; - const typeChecker = this.program.getTypeChecker(); + const typeChecker = this.typeChecker; const sym = typeChecker.getSymbolAtLocation(decl.name); if (!sym.declarations || sym.declarations.length < 2) return true; - return decl === sym.declarations[0]; + return ts.getOriginalNode(decl) === sym.declarations[0]; } private writeExternsType(decl: ts.InterfaceDeclaration|ts.ClassDeclaration, namespace: string[]) { @@ -1643,9 +1683,26 @@ class ExternsWriter extends ClosureRewriter { } export function annotate( - program: ts.Program, file: ts.SourceFile, - pathToModuleName: (context: string, importPath: string) => string, options: Options = {}, - host?: ts.ModuleResolutionHost, tsOpts?: ts.CompilerOptions): Output { + typeChecker: ts.TypeChecker, file: ts.SourceFile, host: Host, options: Options = {}, + tsHost?: ts.ModuleResolutionHost, tsOpts?: ts.CompilerOptions, + sourceMapper?: SourceMapper): {output: string, diagnostics: ts.Diagnostic[]} { typeTranslator.assertTypeChecked(file); - return new Annotator(program, file, options, pathToModuleName, host, tsOpts).annotate(); + return new Annotator(typeChecker, host, options, file, sourceMapper, tsHost, tsOpts).annotate(); +} + +export function writeExterns( + typeChecker: ts.TypeChecker, file: ts.SourceFile, host: Host, + options: Options = {}): {output: string, diagnostics: ts.Diagnostic[]} { + typeTranslator.assertTypeChecked(file); + return new ExternsWriter(typeChecker, host, options, file).process(); +} + +/** Concatenate all generated externs definitions together into a string. */ +export function getGeneratedExterns(externs: {[fileName: string]: string}): string { + let allExterns = EXTERNS_HEADER; + for (let fileName of Object.keys(externs)) { + allExterns += `// externs from ${fileName}:\n`; + allExterns += externs[fileName]; + } + return allExterns; } diff --git a/src/tsickle_compiler_host.ts b/src/tsickle_compiler_host.ts index 0897d58c0..2e4dc10b4 100644 --- a/src/tsickle_compiler_host.ts +++ b/src/tsickle_compiler_host.ts @@ -11,7 +11,7 @@ import {SourceMapGenerator} from 'source-map'; import * as ts from 'typescript'; import {convertDecorators} from './decorator-annotator'; -import {processES5} from './es5processor'; +import * as es5processor from './es5processor'; import {ModulesManifest} from './modules_manifest'; import * as sourceMapUtils from './source_map_utils'; import * as tsickle from './tsickle'; @@ -30,56 +30,27 @@ export enum Pass { CLOSURIZE } -export interface Options { - googmodule?: boolean; - es5Mode?: boolean; - prelude?: string; - /** - * If true, convert every type to the Closure {?} type, which means - * "don't check types". - */ - untyped?: boolean; - /** - * If provided a function that logs an internal warning. - * These warnings are not actionable by an end user and should be hidden - * by default. - */ - logWarning?: (warning: ts.Diagnostic) => void; - /** If provided, a set of paths whose types should always generate as {?}. */ - typeBlackListPaths?: Set; - /** - * Convert shorthand "/index" imports to full path (include the "/index"). - * Annotation will be slower because every import must be resolved. - */ - convertIndexImportShorthand?: boolean; +export interface TsickleOptions extends es5processor.Options, tsickle.Options { + // This method here for backwards compatibility. Don't use it anymore, + // but rather use the corresponding method in the TsickleHost. + logWarning?: TsickleHost['logWarning']; } /** * Provides hooks to customize TsickleCompilerHost's behavior for different * compilation environments. */ -export interface TsickleHost { +export interface TsickleHost extends es5processor.Host, tsickle.Host { /** * If true, tsickle and decorator downlevel processing will be skipped for * that file. */ shouldSkipTsickleProcessing(fileName: string): boolean; - /** - * Takes a context (the current file) and the path of the file to import - * and generates a googmodule module name - */ - pathToModuleName(context: string, importPath: string): string; /** * Tsickle treats warnings as errors, if true, ignore warnings. This might be * useful for e.g. third party code. */ shouldIgnoreWarningsForPath(filePath: string): boolean; - /** - * If we do googmodule processing, we polyfill module.id, since that's - * part of ES6 modules. This function determines what the module.id will be - * for each file. - */ - fileNameToModuleId(fileName: string): string; } /** @@ -106,7 +77,10 @@ export class TsickleCompilerHost implements ts.CompilerHost { constructor( private delegate: ts.CompilerHost, private tscOptions: ts.CompilerOptions, - private options: Options, private environment: TsickleHost) { + private options: TsickleOptions, private environment: TsickleHost) { + if (options.logWarning && !environment.logWarning) { + environment.logWarning = options.logWarning; + } // ts.CompilerHost includes a bunch of optional methods. If they're // present on the delegate host, we want to delegate them. if (this.delegate.getCancellationToken) { @@ -178,7 +152,8 @@ export class TsickleCompilerHost implements ts.CompilerHost { content = this.combineInlineSourceMaps(fileName, content); } if (this.options.googmodule && !isDtsFileName(fileName)) { - content = this.convertCommonJsToGoogModule(fileName, content); + content = es5processor.convertCommonJsToGoogModule( + this.environment, this.options, this.modulesManifest, fileName, content); } } else { content = this.combineSourceMaps(fileName, content); @@ -275,22 +250,6 @@ export class TsickleCompilerHost implements ts.CompilerHost { return sourceMapUtils.setInlineSourceMap(compiledJsWithInlineSourceMap, composedSourceMap); } - convertCommonJsToGoogModule(fileName: string, content: string): string { - const moduleId = this.environment.fileNameToModuleId(fileName); - - let {output, referencedModules} = processES5( - fileName, moduleId, content, this.environment.pathToModuleName.bind(this.environment), - this.options.es5Mode, this.options.prelude); - - const moduleName = this.environment.pathToModuleName('', fileName); - this.modulesManifest.addModule(fileName, moduleName); - for (let referenced of referencedModules) { - this.modulesManifest.addReferencedModule(fileName, referenced); - } - - return output; - } - private downlevelDecorators( sourceFile: ts.SourceFile, program: ts.Program, fileName: string, languageVersion: ts.ScriptTarget): ts.SourceFile { @@ -298,7 +257,8 @@ export class TsickleCompilerHost implements ts.CompilerHost { this.getSourceMapKeyForSourceFile(sourceFile), new SourceMapGenerator()); if (this.environment.shouldSkipTsickleProcessing(fileName)) return sourceFile; let fileContent = sourceFile.text; - const converted = convertDecorators(program.getTypeChecker(), sourceFile); + const sourceMapper = new sourceMapUtils.DefaultSourceMapper(sourceFile.fileName); + const converted = convertDecorators(program.getTypeChecker(), sourceFile, sourceMapper); if (converted.diagnostics) { this.diagnostics.push(...converted.diagnostics); } @@ -308,7 +268,7 @@ export class TsickleCompilerHost implements ts.CompilerHost { } fileContent = converted.output; this.decoratorDownlevelSourceMaps.set( - this.getSourceMapKeyForSourceFile(sourceFile), converted.sourceMap); + this.getSourceMapKeyForSourceFile(sourceFile), sourceMapper.sourceMap); return ts.createSourceFile(fileName, fileContent, languageVersion, true); } @@ -322,11 +282,16 @@ export class TsickleCompilerHost implements ts.CompilerHost { // this means we don't process e.g. lib.d.ts. if (isDefinitions && this.environment.shouldSkipTsickleProcessing(fileName)) return sourceFile; - let {output, externs, diagnostics, sourceMap} = tsickle.annotate( - program, sourceFile, this.environment.pathToModuleName.bind(this.environment), this.options, - this.delegate, this.tscOptions); - if (externs) { - this.externs[fileName] = externs; + const sourceMapper = new sourceMapUtils.DefaultSourceMapper(sourceFile.fileName); + const annotated = tsickle.annotate( + program.getTypeChecker(), sourceFile, this.environment, this.options, this.delegate, + this.tscOptions, sourceMapper); + const externs = + tsickle.writeExterns(program.getTypeChecker(), sourceFile, this.environment, this.options); + let diagnostics = externs.diagnostics.concat(annotated.diagnostics); + + if (externs.output.length > 0) { + this.externs[fileName] = externs.output; } if (this.environment.shouldIgnoreWarningsForPath(sourceFile.fileName)) { // All diagnostics (including warnings) are treated as errors. @@ -336,18 +301,14 @@ export class TsickleCompilerHost implements ts.CompilerHost { diagnostics = diagnostics.filter(d => d.category === ts.DiagnosticCategory.Error); } this.diagnostics = diagnostics; - this.tsickleSourceMaps.set(this.getSourceMapKeyForSourceFile(sourceFile), sourceMap); - return ts.createSourceFile(fileName, output, languageVersion, true); + this.tsickleSourceMaps.set( + this.getSourceMapKeyForSourceFile(sourceFile), sourceMapper.sourceMap); + return ts.createSourceFile(fileName, annotated.output, languageVersion, true); } /** Concatenate all generated externs definitions together into a string. */ getGeneratedExterns(): string { - let allExterns = tsickle.EXTERNS_HEADER; - for (let fileName of Object.keys(this.externs)) { - allExterns += `// externs from ${fileName}:\n`; - allExterns += this.externs[fileName]; - } - return allExterns; + return tsickle.getGeneratedExterns(this.externs); } // Delegate everything else to the original compiler host. diff --git a/src/type-translator.ts b/src/type-translator.ts index b4eef7f3a..c49a127cf 100644 --- a/src/type-translator.ts +++ b/src/type-translator.ts @@ -11,7 +11,7 @@ import * as ts from 'typescript'; import {toArray} from './util'; export function assertTypeChecked(sourceFile: ts.SourceFile) { - if (!('resolvedModules' in sourceFile)) { + if (!('resolvedModules' in ts.getOriginalNode(sourceFile))) { throw new Error('must provide typechecked program'); } } diff --git a/test/decorator-annotator_test.ts b/test/decorator-annotator_test.ts index c207779f2..e0df0c806 100644 --- a/test/decorator-annotator_test.ts +++ b/test/decorator-annotator_test.ts @@ -11,6 +11,7 @@ import {SourceMapConsumer} from 'source-map'; import * as ts from 'typescript'; import {convertDecorators} from '../src/decorator-annotator'; +import {DefaultSourceMapper} from '../src/source_map_utils'; import * as tsickle from '../src/tsickle'; import * as testSupport from './test_support'; @@ -34,11 +35,12 @@ describe( 'decorator-annotator', () => { function translate(sourceText: string, allowErrors = false) { let program = testSupport.createProgram(sources(sourceText)); - let {output, diagnostics, sourceMap} = - convertDecorators(program.getTypeChecker(), program.getSourceFile(testCaseFileName)); + const sourceMapper = new DefaultSourceMapper(testCaseFileName); + let {output, diagnostics} = convertDecorators( + program.getTypeChecker(), program.getSourceFile(testCaseFileName), sourceMapper); if (!allowErrors) expect(diagnostics).to.be.empty; verifyCompiles(output); - return {output, diagnostics, sourceMap}; + return {output, diagnostics, sourceMap: sourceMapper.sourceMap}; } function expectUnchanged(sourceText: string) { diff --git a/test/e2e_source_map_test.ts b/test/e2e_source_map_test.ts index e8ccec96f..d9ea1533f 100644 --- a/test/e2e_source_map_test.ts +++ b/test/e2e_source_map_test.ts @@ -14,8 +14,9 @@ import * as ts from 'typescript'; import * as cliSupport from '../src/cli_support'; import {convertDecorators} from '../src/decorator-annotator'; import {toClosureJS} from '../src/main'; -import {getInlineSourceMapCount, setInlineSourceMap,} from '../src/source_map_utils'; +import {DefaultSourceMapper, getInlineSourceMapCount, setInlineSourceMap} from '../src/source_map_utils'; import * as tsickle from '../src/tsickle'; +import * as tsickleCompilerHost from '../src/tsickle_compiler_host'; import {toArray} from '../src/util'; import * as testSupport from './test_support'; @@ -234,8 +235,9 @@ describe('source maps', () => { const closurizeSources = decoratorDownlevelAndAddInlineSourceMaps(decoratorDownlevelSources); - const {compiledJS, sourceMap} = - compile(closurizeSources, {inlineSourceMap: true, tsicklePasses: [tsickle.Pass.CLOSURIZE]}); + const {compiledJS, sourceMap} = compile( + closurizeSources, + {inlineSourceMap: true, tsicklePasses: [tsickleCompilerHost.Pass.CLOSURIZE]}); expect(getInlineSourceMapCount(compiledJS)).to.equal(1); const {line, column} = getLineAndColumn(compiledJS, 'methodName'); @@ -287,7 +289,7 @@ describe('source maps', () => { let z : string = x + y;`); const {compiledJS, sourceMap} = - compile(closurizeSources, {tsicklePasses: [tsickle.Pass.CLOSURIZE]}); + compile(closurizeSources, {tsicklePasses: [tsickleCompilerHost.Pass.CLOSURIZE]}); { const {line, column} = getLineAndColumn(compiledJS, 'methodName'); @@ -308,9 +310,10 @@ function decoratorDownlevelAndAddInlineSourceMaps(sources: Map): const transformedSources = new Map(); let program = testSupport.createProgram(sources); for (const fileName of toArray(sources.keys())) { - let {output, sourceMap: preexistingSourceMap} = - convertDecorators(program.getTypeChecker(), program.getSourceFile(fileName)); - transformedSources.set(fileName, setInlineSourceMap(output, preexistingSourceMap.toString())); + const sourceMapper = new DefaultSourceMapper(fileName); + let {output} = + convertDecorators(program.getTypeChecker(), program.getSourceFile(fileName), sourceMapper); + transformedSources.set(fileName, setInlineSourceMap(output, sourceMapper.sourceMap.toString())); } return transformedSources; } @@ -329,7 +332,7 @@ interface CompilerOptions { outFile: string; filesNotToProcess: Set; inlineSourceMap: boolean; - tsicklePasses: tsickle.Pass[]; + tsicklePasses: tsickleCompilerHost.Pass[]; generateDTS: boolean; } @@ -337,7 +340,7 @@ const DEFAULT_COMPILER_OPTIONS = { outFile: 'output.js', filesNotToProcess: new Set(), inlineSourceMap: false, - tsicklePasses: [tsickle.Pass.DECORATOR_DOWNLEVEL, tsickle.Pass.CLOSURIZE], + tsicklePasses: [tsickleCompilerHost.Pass.DECORATOR_DOWNLEVEL, tsickleCompilerHost.Pass.CLOSURIZE], generateDTS: false, }; @@ -370,7 +373,7 @@ function compile(sources: Map, partialOptions = {} as Partial fileNames.indexOf(fileName) === -1 || options.filesNotToProcess.has(fileName), pathToModuleName: cliSupport.pathToModuleName, diff --git a/test/e2e_tsickle_compiler_host_test.ts b/test/e2e_tsickle_compiler_host_test.ts index eab2e4bb0..0e6006f06 100644 --- a/test/e2e_tsickle_compiler_host_test.ts +++ b/test/e2e_tsickle_compiler_host_test.ts @@ -3,7 +3,7 @@ import * as ts from 'typescript'; import {pathToModuleName} from '../src/cli_support'; import {formatDiagnostics} from '../src/tsickle'; -import {Options, Pass, TsickleCompilerHost} from '../src/tsickle_compiler_host'; +import {Pass, TsickleCompilerHost, TsickleOptions} from '../src/tsickle_compiler_host'; import {createOutputRetainingCompilerHost, createSourceReplacingCompilerHost} from '../src/util'; const tsickleHost = { @@ -14,7 +14,7 @@ const tsickleHost = { }; describe('tsickle compiler host', () => { - let tsickleCompilerHostOptions: Options; + let tsickleCompilerHostOptions: TsickleOptions; beforeEach(() => { tsickleCompilerHostOptions = { diff --git a/test/es5processor_test.ts b/test/es5processor_test.ts index b60d8f202..a087fe949 100644 --- a/test/es5processor_test.ts +++ b/test/es5processor_test.ts @@ -12,11 +12,17 @@ import * as cliSupport from '../src/cli_support'; import * as es5processor from '../src/es5processor'; describe('convertCommonJsToGoogModule', () => { + function processES5(fileName: string, content: string, isES5 = true, prelude = '') { + const options: es5processor.Options = {es5Mode: isES5, prelude}; + const host: es5processor.Host = { + fileNameToModuleId: (fileName) => fileName, + pathToModuleName: cliSupport.pathToModuleName + }; + return es5processor.processES5(host, options, fileName, content); + } + function expectCommonJs(fileName: string, content: string, isES5 = true, prelude = '') { - return expect( - es5processor - .processES5(fileName, fileName, content, cliSupport.pathToModuleName, isES5, prelude) - .output); + return expect(processES5(fileName, content, isES5, prelude).output); } it('adds a goog.module call', () => { @@ -172,13 +178,13 @@ foo_1.A, foo_2.B, foo_2 , foo_3.default; }); it('gathers referenced modules', () => { - let {referencedModules} = es5processor.processES5('a/b', 'a/b', ` + let {referencedModules} = processES5('a/b', ` require('../foo/bare_require'); var googRequire = require('goog:foo.bar'); var es6RelativeRequire = require('./relative'); var es6NonRelativeRequire = require('non/relative'); __export(require('./export_star'); -`, cliSupport.pathToModuleName); +`); return expect(referencedModules).to.deep.equal([ 'foo.bare_require', diff --git a/test/source_map_test.ts b/test/source_map_test.ts index 3800c3812..2599aaac9 100644 --- a/test/source_map_test.ts +++ b/test/source_map_test.ts @@ -8,10 +8,12 @@ import {expect} from 'chai'; import {SourceMapConsumer} from 'source-map'; +import * as ts from 'typescript'; -import {annotate} from '../src/tsickle'; +import {DefaultSourceMapper} from '../src/source_map_utils'; +import * as tsickle from '../src/tsickle'; -import {createProgram} from './test_support'; +import * as testSupport from './test_support'; describe('source maps', () => { it('generates a source map', () => { @@ -19,9 +21,14 @@ describe('source maps', () => { sources.set('input.ts', ` class X { field: number; } class Y { field2: string; }`); - const program = createProgram(sources); - const annotated = annotate(program, program.getSourceFile('input.ts'), () => 'input'); - const rawMap = annotated.sourceMap.toJSON(); + const tsHost = ts.createCompilerHost(testSupport.compilerOptions); + const program = testSupport.createProgram(sources, tsHost); + const sourceMapper = new DefaultSourceMapper('input.ts'); + const tsickleHost: tsickle.Host = {pathToModuleName: () => 'input'}; + const annotated = tsickle.annotate( + program.getTypeChecker(), program.getSourceFile('input.ts'), tsickleHost, {}, tsHost, + testSupport.compilerOptions, sourceMapper); + const rawMap = sourceMapper.sourceMap.toJSON(); const consumer = new SourceMapConsumer(rawMap); const lines = annotated.output.split('\n'); // Uncomment to debug contents: diff --git a/test/test_support.ts b/test/test_support.ts index 0050c9f5b..d3229002c 100644 --- a/test/test_support.ts +++ b/test/test_support.ts @@ -12,6 +12,7 @@ import * as path from 'path'; import * as ts from 'typescript'; import * as cliSupport from '../src/cli_support'; +import * as es5processor from '../src/es5processor'; import * as tsickle from '../src/tsickle'; import {toArray} from '../src/util'; @@ -46,8 +47,10 @@ const {cachedLibPath, cachedLib} = (function() { })(); /** Creates a ts.Program from a set of input files. */ -export function createProgram(sources: Map): ts.Program { - let host = ts.createCompilerHost(compilerOptions); +export function createProgram(sources: Map, host?: ts.CompilerHost): ts.Program { + if (!host) { + host = ts.createCompilerHost(compilerOptions); + } host.getSourceFile = function( fileName: string, languageVersion: ts.ScriptTarget, @@ -68,13 +71,18 @@ export function createProgram(sources: Map): ts.Program { } /** Emits transpiled output with tsickle postprocessing. Throws an exception on errors. */ -export function emit(program: ts.Program): {[fileName: string]: string} { +export function emit(program: ts.Program, transformers: ts.TransformerFactory[] = [ +]): {[fileName: string]: string} { let transformed: {[fileName: string]: string} = {}; let {diagnostics} = program.emit(undefined, (fileName: string, data: string) => { + const options: es5processor.Options = {es5Mode: true, prelude: ''}; + const host: es5processor.Host = { + fileNameToModuleId: (fileName) => fileName.replace(/^\.\//, ''), + pathToModuleName: cliSupport.pathToModuleName + }; const moduleId = fileName.replace(/^\.\//, ''); - transformed[fileName] = - tsickle.processES5(fileName, moduleId, data, cliSupport.pathToModuleName).output; - }); + transformed[fileName] = es5processor.processES5(host, options, fileName, data).output; + }, undefined, undefined, {before: transformers}); if (diagnostics.length > 0) { throw new Error(tsickle.formatDiagnostics(diagnostics)); } diff --git a/test/tsickle_test.ts b/test/tsickle_test.ts index 0535ca706..858d208b4 100644 --- a/test/tsickle_test.ts +++ b/test/tsickle_test.ts @@ -131,30 +131,37 @@ testFn('golden tests', () => { let tsickleSources = new Map(); for (let tsPath of toArray(tsSources.keys())) { let warnings: ts.Diagnostic[] = []; - options.logWarning = (diag: ts.Diagnostic) => { - warnings.push(diag); - }; // Run TypeScript through tsickle and compare against goldens. - let {output, externs, diagnostics} = tsickle.annotate( - program, program.getSourceFile(tsPath), - (context, importPath) => { - importPath = importPath.replace(/(\.d)?\.[tj]s$/, ''); - if (importPath[0] === '.') importPath = path.join(path.dirname(context), importPath); - return importPath.replace(/\/|\\/g, '.'); - }, - options, { - fileExists: ts.sys.fileExists, - readFile: ts.sys.readFile, - }, + const tsHost = { + fileExists: ts.sys.fileExists, + readFile: ts.sys.readFile, + }; + const tsickleHost: tsickle.Host = { + logWarning: (diag: ts.Diagnostic) => { + warnings.push(diag); + }, + pathToModuleName: (context, importPath) => { + importPath = importPath.replace(/(\.d)?\.[tj]s$/, ''); + if (importPath[0] === '.') importPath = path.join(path.dirname(context), importPath); + return importPath.replace(/\/|\\/g, '.'); + } + }; + const sourceFile = program.getSourceFile(tsPath); + const annotated = tsickle.annotate( + program.getTypeChecker(), sourceFile, tsickleHost, options, tsHost, testSupport.compilerOptions); - if (externs && !test.name.endsWith('.no_externs')) { + const externs = + tsickle.writeExterns(program.getTypeChecker(), sourceFile, tsickleHost, options); + let diagnostics = externs.diagnostics.concat(annotated.diagnostics); + + if (externs.output && !test.name.endsWith('.no_externs')) { if (!allExterns) allExterns = tsickle.EXTERNS_HEADER; - allExterns += externs; + allExterns += externs.output; } // If there were any diagnostics, convert them into strings for // the golden output. - let fileOutput = output; + let fileOutput = annotated.output; diagnostics.push(...warnings); if (diagnostics.length > 0) { // Munge the filenames in the diagnostics so that they don't include @@ -163,12 +170,12 @@ testFn('golden tests', () => { let fileName = diag.file.fileName; diag.file.fileName = fileName.substr(fileName.indexOf('test_files')); } - fileOutput = tsickle.formatDiagnostics(diagnostics) + '\n====\n' + output; + fileOutput = tsickle.formatDiagnostics(diagnostics) + '\n====\n' + annotated.output; } let tsicklePath = tsPath.replace(/((\.d)?\.tsx?)$/, '.tsickle$1'); expect(tsicklePath).to.not.equal(tsPath); compareAgainstGolden(fileOutput, tsicklePath); - tsickleSources.set(tsPath, output); + tsickleSources.set(tsPath, annotated.output); } compareAgainstGolden(allExterns, test.externsPath); diff --git a/test/tsickle_transformer_test.ts b/test/tsickle_transformer_test.ts new file mode 100644 index 000000000..8c4b2e98b --- /dev/null +++ b/test/tsickle_transformer_test.ts @@ -0,0 +1,168 @@ +/** + * @license + * Copyright Google Inc. All Rights Reserved. + * + * Use of this source code is governed by an MIT-style license that can be + * found in the LICENSE file at https://angular.io/license + */ + +import {expect} from 'chai'; +import * as fs from 'fs'; +import * as path from 'path'; +import * as ts from 'typescript'; + +import * as transformer from '../src/transformer' +import * as tsickle from '../src/tsickle'; +import {toArray} from '../src/util'; + +import * as testSupport from './test_support'; + +const TEST_FILTER: RegExp|null = + process.env.TEST_FILTER ? new RegExp(process.env.TEST_FILTER) : null; + +function readGolden(path: string) { + return fs.readFileSync(path, 'utf-8'); +} + +function compareGoldenData(output: string|null, golden: string|null, path: string) { + // Make sure we have proper line endings when testing on Windows. + if (golden != null) golden = golden.replace(/\r\n/g, '\n'); + if (output != null) output = output.replace(/\r\n/g, '\n'); + expect(output).to.equal(golden, path); +} + +const DIAGONSTIC_FILE_REGEX = /(test_files.*?):\s/; + +function compareAgainstGoldenDiagnostics(diagnostics: ts.Diagnostic[], path: string) { + // Munge the filenames in the diagnostics so that they don't include + // the tsickle checkout path. + for (let diag of diagnostics) { + let fileName = diag.file.fileName; + diag.file.fileName = fileName.substr(fileName.indexOf('test_files')); + } + let tsicklePath = path.replace(/((\.d)?\.tsx?)$/, '.tsickle$1'); + expect(tsicklePath).to.not.equal(path); + const golden = readGolden(tsicklePath); + const goldenFormattedDiagnostics = + sortDiagnostics(golden.substring(0, golden.indexOf('\n====\n'))); + const outputFormattedDiagnostics = sortDiagnostics( + tsickle.formatDiagnostics(diagnostics.filter(diag => diag.file.fileName === path))); + + compareGoldenData(outputFormattedDiagnostics, goldenFormattedDiagnostics, path); + + function sortDiagnostics(diagnostics: string): string { + const lines = diagnostics.split('\n'); + return lines + .sort((l1, l2) => { + const m1 = DIAGONSTIC_FILE_REGEX.exec(l1) || [l1]; + const m2 = DIAGONSTIC_FILE_REGEX.exec(l2) || [l2]; + return m1[0].localeCompare(m2[0]); + }) + .join('\n'); + } +} + +/** + * compareAgainstGoldens compares a test output against the content in a golden + * path, updating the content of the golden when UPDATE_GOLDENS is true. + * + * @param output The expected output, where the empty string indicates + * the file is expected to exist and be empty, while null indicates + * the file is expected to not exist. (This subtlety is used for + * externs files, where the majority of tests are not expected to + * produce one.) + */ +function compareAgainstGolden(output: string|null, path: string) { + let golden: string|null = null; + try { + golden = readGolden(path); + } catch (e) { + if (e.code !== 'ENOENT' || output !== null) { + // a missing file + throw e; + } + } + compareGoldenData(output, golden, path); +} + + +// Only run golden tests if we filter for a specific one. +const testFn = TEST_FILTER ? describe.only : describe; + +testFn('golden tests', () => { + testSupport.goldenTests().forEach((test) => { + if (TEST_FILTER && !TEST_FILTER.exec(test.name)) { + it.skip(test.name); + return; + } + const options: transformer.TransformerOptions = { + // See test_files/jsdoc_types/nevertyped.ts. + es5Mode: true, + prelude: '', + googmodule: true, + typeBlackListPaths: new Set(['test_files/jsdoc_types/nevertyped.ts']), + convertIndexImportShorthand: true, + transformDecorators: true, + transformTypesToClosure: true, + }; + if (/\.untyped\b/.test(test.name)) { + options.untyped = true; + } + it(test.name, () => { + // Read all the inputs into a map, and create a ts.Program from them. + const tsSources = new Map(); + for (let tsFile of test.tsFiles) { + const tsPath = path.join(test.path, tsFile); + let tsSource = fs.readFileSync(tsPath, 'utf-8'); + tsSource = tsSource.replace(/\r\n/g, '\n'); + tsSources.set(tsPath, tsSource); + } + const tsHost = ts.createCompilerHost(testSupport.compilerOptions); + const program = testSupport.createProgram(tsSources, tsHost); + { + const diagnostics = ts.getPreEmitDiagnostics(program); + if (diagnostics.length) { + throw new Error(tsickle.formatDiagnostics(diagnostics)); + } + } + const typeChecker = program.getTypeChecker(); + const allDiagnostics: ts.Diagnostic[] = []; + const host: transformer.TransformerHost = { + logWarning: (diag: ts.Diagnostic) => { + allDiagnostics.push(diag); + }, + shouldSkipTsickleProcessing: (fileName) => !tsSources.has(fileName), + shouldIgnoreWarningsForPath: () => false, + pathToModuleName: (context, importPath) => { + importPath = importPath.replace(/(\.d)?\.[tj]s$/, ''); + if (importPath[0] === '.') importPath = path.join(path.dirname(context), importPath); + return importPath.replace(/\/|\\/g, '.'); + }, + fileNameToModuleId: (fileName) => fileName.replace(/^\.\//, ''), + }; + const jsSources: {[fileName: string]: string} = {}; + const {diagnostics, externs} = transformer.emitWithTsickle( + program, host, options, tsHost, testSupport.compilerOptions, undefined, + (fileName: string, data: string) => { + jsSources[fileName] = data; + }); + allDiagnostics.push(...diagnostics); + let allExterns: string|null = null; + if (!test.name.endsWith('.no_externs')) { + for (let tsPath of toArray(tsSources.keys())) { + if (externs[tsPath]) { + if (!allExterns) allExterns = tsickle.EXTERNS_HEADER; + allExterns += externs[tsPath]; + } + } + } + compareAgainstGolden(allExterns, test.externsPath); + for (let jsPath of Object.keys(jsSources)) { + compareAgainstGolden(jsSources[jsPath], jsPath); + } + for (let tsPath of toArray(tsSources.keys())) { + compareAgainstGoldenDiagnostics(allDiagnostics, tsPath); + } + }); + }); +}); diff --git a/test_files/decorator/decorator.js b/test_files/decorator/decorator.js index f850586f6..ed468f16b 100644 --- a/test_files/decorator/decorator.js +++ b/test_files/decorator/decorator.js @@ -31,9 +31,7 @@ class DecoratorTest { DecoratorTest.decorators = [ { type: classAnnotation }, ]; -/** - * @nocollapse - */ +/** @nocollapse */ DecoratorTest.ctorParameters = () => []; DecoratorTest.propDecorators = { 'y': [{ type: annotationDecorator },], diff --git a/test_files/decorator/decorator.tsickle.ts b/test_files/decorator/decorator.tsickle.ts index eb0a22809..eac8d1ba6 100644 --- a/test_files/decorator/decorator.tsickle.ts +++ b/test_files/decorator/decorator.tsickle.ts @@ -37,9 +37,7 @@ private y: number; static decorators: {type: Function, args?: any[]}[] = [ { type: classAnnotation }, ]; -/** - * @nocollapse - */ +/** @nocollapse */ static ctorParameters: () => ({type: any, decorators?: {type: Function, args?: any[]}[]}|null)[] = () => [ ]; static propDecorators: {[key: string]: {type: Function, args?: any[]}[]} = { diff --git a/test_files/enum.untyped/enum.untyped.js b/test_files/enum.untyped/enum.untyped.js index fd86c1475..13db2422a 100644 --- a/test_files/enum.untyped/enum.untyped.js +++ b/test_files/enum.untyped/enum.untyped.js @@ -8,8 +8,9 @@ EnumUntypedTest1.XYZ = 0; EnumUntypedTest1.PI = 3.14159; EnumUntypedTest1[EnumUntypedTest1.XYZ] = "XYZ"; EnumUntypedTest1[EnumUntypedTest1.PI] = "PI"; -exports.EnumUntypedTest2 = {}; -exports.EnumUntypedTest2.XYZ = 0; -exports.EnumUntypedTest2.PI = 3.14159; -exports.EnumUntypedTest2[exports.EnumUntypedTest2.XYZ] = "XYZ"; -exports.EnumUntypedTest2[exports.EnumUntypedTest2.PI] = "PI"; +let EnumUntypedTest2 = {}; +exports.EnumUntypedTest2 = EnumUntypedTest2; +EnumUntypedTest2.XYZ = 0; +EnumUntypedTest2.PI = 3.14159; +EnumUntypedTest2[EnumUntypedTest2.XYZ] = "XYZ"; +EnumUntypedTest2[EnumUntypedTest2.PI] = "PI"; diff --git a/test_files/enum.untyped/enum.untyped.tsickle.ts b/test_files/enum.untyped/enum.untyped.tsickle.ts index 1e1540d0a..112d3db3c 100644 --- a/test_files/enum.untyped/enum.untyped.tsickle.ts +++ b/test_files/enum.untyped/enum.untyped.tsickle.ts @@ -11,8 +11,9 @@ EnumUntypedTest1.PI = 3.14159; EnumUntypedTest1[EnumUntypedTest1.XYZ] = "XYZ"; EnumUntypedTest1[EnumUntypedTest1.PI] = "PI"; -export type EnumUntypedTest2 = number; -export let EnumUntypedTest2: any = {}; +type EnumUntypedTest2 = number; +let EnumUntypedTest2: any = {}; +export {EnumUntypedTest2}; EnumUntypedTest2.XYZ = 0; EnumUntypedTest2.PI = 3.14159; EnumUntypedTest2[EnumUntypedTest2.XYZ] = "XYZ"; diff --git a/test_files/enum/enum.js b/test_files/enum/enum.js index f022a3fa9..fd00525ad 100644 --- a/test_files/enum/enum.js +++ b/test_files/enum/enum.js @@ -27,13 +27,14 @@ function enumTestFunction(val) { } enumTestFunction(enumTestValue); let /** @type {number} */ enumTestLookup = EnumTest1["XYZ"]; let /** @type {?} */ enumTestLookup2 = EnumTest1["xyz".toUpperCase()]; -exports.EnumTest2 = {}; +let EnumTest2 = {}; +exports.EnumTest2 = EnumTest2; /** @type {number} */ -exports.EnumTest2.XYZ = 0; +EnumTest2.XYZ = 0; /** @type {number} */ -exports.EnumTest2.PI = 3.14159; -exports.EnumTest2[exports.EnumTest2.XYZ] = "XYZ"; -exports.EnumTest2[exports.EnumTest2.PI] = "PI"; +EnumTest2.PI = 3.14159; +EnumTest2[EnumTest2.XYZ] = "XYZ"; +EnumTest2[EnumTest2.PI] = "PI"; let ComponentIndex = {}; /** @type {number} */ ComponentIndex.Scheme = 1; diff --git a/test_files/enum/enum.tsickle.ts b/test_files/enum/enum.tsickle.ts index 36c61b503..6078994a4 100644 --- a/test_files/enum/enum.tsickle.ts +++ b/test_files/enum/enum.tsickle.ts @@ -35,8 +35,9 @@ enumTestFunction(enumTestValue); let /** @type {number} */ enumTestLookup = EnumTest1["XYZ"]; let /** @type {?} */ enumTestLookup2 = EnumTest1["xyz".toUpperCase()]; -export type EnumTest2 = number; -export let EnumTest2: any = {}; +type EnumTest2 = number; +let EnumTest2: any = {}; +export {EnumTest2}; /** @type {number} */ EnumTest2.XYZ = 0; /** @type {number} */