Limit deep generic type instantiations

src/compiler/checker.ts

@@ -47,6 +47,7 @@ namespace ts {
 
         let typeCount = 0;
         let symbolCount = 0;
+        let symbolInstantiationDepth = 0;
 
         const emptyArray: any[] = [];
         const emptySymbols = createMap<Symbol>();
@@ -4440,14 +4441,22 @@ namespace ts {
         function getTypeOfInstantiatedSymbol(symbol: Symbol): Type {
             const links = getSymbolLinks(symbol);
             if (!links.type) {
-                if (!pushTypeResolution(symbol, TypeSystemPropertyName.Type)) {
-                    return unknownType;
+                if (symbolInstantiationDepth === 100) {
+                    error(symbol.valueDeclaration, Diagnostics.Generic_type_instantiation_is_excessively_deep_and_possibly_infinite);
+                    links.type = unknownType;
                 }
-                let type = instantiateType(getTypeOfSymbol(links.target), links.mapper);
-                if (!popTypeResolution()) {
-                    type = reportCircularityError(symbol);
+                else {
+                    if (!pushTypeResolution(symbol, TypeSystemPropertyName.Type)) {
+                        return unknownType;
+                    }
+                    symbolInstantiationDepth++;
+                    let type = instantiateType(getTypeOfSymbol(links.target), links.mapper);
+                    symbolInstantiationDepth--;
+                    if (!popTypeResolution()) {
+                        type = reportCircularityError(symbol);
+                    }
+                    links.type = type;
                 }
-                links.type = type;
             }
             return links.type;
         }
@@ -7257,8 +7266,9 @@ namespace ts {
                 else {
                     error(indexNode, Diagnostics.Type_0_cannot_be_used_as_an_index_type, typeToString(indexType));
                 }
+                return unknownType;
             }
-            return unknownType;
+            return anyType;
         }
 
         function getIndexedAccessForMappedType(type: MappedType, indexType: Type, accessNode?: ElementAccessExpression | IndexedAccessTypeNode) {

src/compiler/diagnosticMessages.json

@@ -1827,6 +1827,10 @@
         "category": "Error",
         "code": 2549
     },
+    "Generic type instantiation is excessively deep and possibly infinite.": {
+        "category": "Error",
+        "code": 2550
+    },
     "JSX element attributes type '{0}' may not be a union type.": {
         "category": "Error",
         "code": 2600

tests/baselines/reference/limitDeepInstantiations.errors.txt

@@ -0,0 +1,15 @@
+tests/cases/compiler/limitDeepInstantiations.ts(3,35): error TS2550: Generic type instantiation is excessively deep and possibly infinite.
+tests/cases/compiler/limitDeepInstantiations.ts(5,13): error TS2344: Type '"false"' does not satisfy the constraint '"true"'.
+
+
+==== tests/cases/compiler/limitDeepInstantiations.ts (2 errors) ====
+    // Repro from #14837
+
+    type Foo<T extends "true", B> = { "true": Foo<T, Foo<T, B>> }[T];
+                                      ~~~~~~~~~~~~~~~~~~~~~~~~~
+!!! error TS2550: Generic type instantiation is excessively deep and possibly infinite.
+    let f1: Foo<"true", {}>;
+    let f2: Foo<"false", {}>;
+                ~~~~~~~
+!!! error TS2344: Type '"false"' does not satisfy the constraint '"true"'.
+

tests/baselines/reference/limitDeepInstantiations.js

@@ -0,0 +1,12 @@
+//// [limitDeepInstantiations.ts]
+// Repro from #14837
+
+type Foo<T extends "true", B> = { "true": Foo<T, Foo<T, B>> }[T];
+let f1: Foo<"true", {}>;
+let f2: Foo<"false", {}>;
+
+
+//// [limitDeepInstantiations.js]
+// Repro from #14837
+var f1;
+var f2;

tests/cases/compiler/limitDeepInstantiations.ts

@@ -0,0 +1,5 @@
+// Repro from #14837
+
+type Foo<T extends "true", B> = { "true": Foo<T, Foo<T, B>> }[T];
+let f1: Foo<"true", {}>;
+let f2: Foo<"false", {}>;