micro-optimizations for typemap

src/dump.c

@@ -1993,9 +1993,9 @@ static void jl_restore_system_image_from_stream(ios_t *f)
     jl_top_module = (jl_module_t*)jl_deserialize_value(f, NULL);
     jl_internal_main_module = jl_main_module;
     jl_typeinf_func = (jl_function_t*)jl_deserialize_value(f, NULL);
-    jl_type_type->name->mt = (jl_methtable_t*)jl_deserialize_value(f, NULL);
-    jl_typector_type->name->mt = jl_uniontype_type->name->mt = jl_datatype_type->name->mt =
-        jl_type_type->name->mt;
+    jl_type_type_mt = (jl_methtable_t*)jl_deserialize_value(f, NULL);
+    jl_type_type->name->mt = jl_typector_type->name->mt = jl_uniontype_type->name->mt = jl_datatype_type->name->mt =
+        jl_type_type_mt;
 
     jl_core_module = (jl_module_t*)jl_get_global(jl_main_module,
                                                  jl_symbol("Core"));

src/gf.c

@@ -106,7 +106,7 @@ const struct jl_typemap_info tfunc_cache = {
 
 static int8_t jl_cachearg_offset(jl_methtable_t *mt)
 {
-    return (mt == jl_type_type->name->mt) ? 0 : 1;
+    return (mt == jl_type_type_mt) ? 0 : 1;
 }
 
 /// ----- Insertion logic for special entries ----- ///
@@ -1581,10 +1581,20 @@ JL_DLLEXPORT jl_value_t *jl_apply_generic(jl_value_t **args, uint32_t nargs)
             jl_method_error((jl_function_t*)F, args, nargs);
             // unreachable
         }
+        jl_value_t *res;
+        if (mfunc->inInference || mfunc->inCompile) {
+            // if inference is running on this function, return a copy
+            // of the function to be compiled without inference and run.
+            res = jl_call_unspecialized(mfunc->sparam_vals, jl_get_unspecialized(mfunc), args, nargs);
+        }
+        else {
+            res = jl_call_method_internal(mfunc, args, nargs);
+        }
+        JL_GC_POP();
+        return verify_type(res);
     }
-    else {
-        mfunc = entry->func.linfo;
-    }
+
+    mfunc = entry->func.linfo;
 #ifdef JL_TRACE
     if (traceen)
         jl_printf(JL_STDOUT, " at %s:%d\n", jl_symbol_name(mfunc->file), mfunc->line);
@@ -1598,7 +1608,6 @@ JL_DLLEXPORT jl_value_t *jl_apply_generic(jl_value_t **args, uint32_t nargs)
     else {
         res = jl_call_method_internal(mfunc, args, nargs);
     }
-    if (tt) JL_GC_POP();
     return verify_type(res);
 }
 

src/jltypes.c

@@ -22,6 +22,7 @@ extern "C" {
 
 jl_datatype_t *jl_any_type;
 jl_datatype_t *jl_type_type;
+jl_methtable_t *jl_type_type_mt;
 jl_datatype_t *jl_typename_type;
 jl_datatype_t *jl_sym_type;
 jl_datatype_t *jl_symbol_type;
@@ -3263,7 +3264,8 @@ void jl_init_types(void)
     jl_any_type = jl_new_abstracttype((jl_value_t*)jl_symbol("Any"), NULL, jl_emptysvec);
     jl_any_type->super = jl_any_type;
     jl_type_type = jl_new_abstracttype((jl_value_t*)jl_symbol("Type"), jl_any_type, jl_emptysvec);
-    jl_type_type->name->mt = jl_new_method_table(jl_type_type->name->name, jl_current_module);
+    jl_type_type_mt = jl_new_method_table(jl_type_type->name->name, jl_current_module);
+    jl_type_type->name->mt = jl_type_type_mt;
 
     // initialize them. lots of cycles.
     jl_datatype_type->name = jl_new_typename(jl_symbol("DataType"));

src/julia_internal.h

@@ -16,6 +16,9 @@
 extern "C" {
 #endif
 
+// useful constants
+extern jl_methtable_t *jl_type_type_mt;
+
 // execution of certain certain unpure
 // statements is prohibited from certain
 // callbacks (such as generated functions)
@@ -623,7 +626,19 @@ jl_typemap_entry_t *jl_typemap_insert(union jl_typemap_t *cache, jl_value_t *par
 jl_typemap_entry_t *jl_typemap_assoc_by_type(union jl_typemap_t ml_or_cache, jl_tupletype_t *types, jl_svec_t **penv,
         int8_t subtype_inexact__sigseq_useenv, int8_t subtype, int8_t offs);
 static jl_typemap_entry_t *const INEXACT_ENTRY = (jl_typemap_entry_t*)(uintptr_t)-1;
-jl_typemap_entry_t *jl_typemap_assoc_exact(union jl_typemap_t ml_or_cache, jl_value_t **args, size_t n, int8_t offs);
+jl_typemap_entry_t *jl_typemap_level_assoc_exact(jl_typemap_level_t *cache, jl_value_t **args, size_t n, int8_t offs);
+jl_typemap_entry_t *jl_typemap_entry_assoc_exact(jl_typemap_entry_t *mn, jl_value_t **args, size_t n);
+STATIC_INLINE jl_typemap_entry_t *jl_typemap_assoc_exact(union jl_typemap_t ml_or_cache, jl_value_t **args, size_t n, int8_t offs)
+{
+    // NOTE: This function is a huge performance hot spot!!
+    if (jl_typeof(ml_or_cache.unknown) == (jl_value_t*)jl_typemap_entry_type) {
+        return jl_typemap_entry_assoc_exact(ml_or_cache.leaf, args, n);
+    }
+    else if (jl_typeof(ml_or_cache.unknown) == (jl_value_t*)jl_typemap_level_type) {
+        return jl_typemap_level_assoc_exact(ml_or_cache.node, args, n, offs);
+    }
+    return NULL;
+}
 
 typedef int (*jl_typemap_visitor_fptr)(jl_typemap_entry_t *l, void *closure);
 int jl_typemap_visitor(union jl_typemap_t a, jl_typemap_visitor_fptr fptr, void *closure);

src/typemap.c

@@ -492,10 +492,11 @@ int jl_typemap_intersection_visitor(union jl_typemap_t map, int offs,
                     if (!jl_typemap_intersection_array_visitor(cache->arg1, ty, 0, offs, closure)) return 0;
                 }
             }
-            if (!jl_typemap_intersection_node_visitor(map.node->linear, closure))
-                return 0;
-            return jl_typemap_intersection_visitor(map.node->any, offs+1, closure);
         }
+        if (!jl_typemap_intersection_node_visitor(map.node->linear, closure))
+            return 0;
+        if (ty)
+            return jl_typemap_intersection_visitor(map.node->any, offs+1, closure);
         return 1;
     }
     else {
@@ -685,104 +686,121 @@ jl_typemap_entry_t *jl_typemap_assoc_by_type(union jl_typemap_t ml_or_cache, jl_
     }
 }
 
-static jl_typemap_entry_t *jl_typemap_node_assoc_exact(jl_typemap_entry_t *ml, jl_value_t **args, size_t n)
+jl_typemap_entry_t *jl_typemap_entry_assoc_exact(jl_typemap_entry_t *ml, jl_value_t **args, size_t n)
 {
+    // some manually-unrolled common special cases
+    jl_typemap_entry_t *prev = NULL, *first = ml;
+    while (ml->simplesig == (void*)jl_nothing && ml->guardsigs == jl_emptysvec && ml->isleafsig) {
+        // use a tight loop for a long as possible
+        if (n == jl_datatype_nfields(ml->sig) && jl_typeof(args[0]) == jl_tparam(ml->sig, 0)) {
+            if (n == 1)
+                goto matchnm;
+            if (n == 2) {
+                if (jl_typeof(args[1]) == jl_tparam(ml->sig, 1))
+                    goto matchnm;
+            }
+            else if (n == 3) {
+                if (jl_typeof(args[1]) == jl_tparam(ml->sig, 1) &&
+                    jl_typeof(args[2]) == jl_tparam(ml->sig, 2))
+                    goto matchnm;
+            }
+            else {
+                if (sig_match_leaf(args, jl_svec_data(ml->sig->parameters), n))
+                    goto matchnm;
+            }
+        }
+        prev = ml;
+        ml = ml->next;
+        if (ml == (void*)jl_nothing)
+            return NULL;
+    }
+
     while (ml != (void*)jl_nothing) {
         size_t lensig = jl_datatype_nfields(ml->sig);
         if (lensig == n || (ml->va && lensig <= n+1)) {
-            int ismatch = 1;
             if (ml->simplesig != (void*)jl_nothing) {
                 size_t lensimplesig = jl_datatype_nfields(ml->simplesig);
                 int isva = lensimplesig > 0 && jl_is_vararg_type(jl_tparam(ml->simplesig, lensimplesig - 1));
-                if (lensig == n || (isva && lensimplesig <= n + 1))
-                    ismatch = sig_match_simple(args, n, jl_svec_data(ml->simplesig->parameters), isva, lensimplesig);
-                else
-                    ismatch = 0;
+                if (lensig == n || (isva && lensimplesig <= n + 1)) {
+                    if (!sig_match_simple(args, n, jl_svec_data(ml->simplesig->parameters), isva, lensimplesig))
+                        goto nomatch;
+                }
+                else {
+                    goto nomatch;
+                }
             }
 
-            if (ismatch == 0)
-                ; // nothing
-            else if (ml->isleafsig)
-                ismatch = sig_match_leaf(args, jl_svec_data(ml->sig->parameters), n);
-            else if (ml->issimplesig)
-                ismatch = sig_match_simple(args, n, jl_svec_data(ml->sig->parameters), ml->va, lensig);
-            else
-                ismatch = jl_tuple_subtype(args, n, ml->sig, 1);
+            if (ml->isleafsig) {
+                if (!sig_match_leaf(args, jl_svec_data(ml->sig->parameters), n))
+                    goto nomatch;
+            }
+            else if (ml->issimplesig) {
+                if (!sig_match_simple(args, n, jl_svec_data(ml->sig->parameters), ml->va, lensig))
+                    goto nomatch;
+            }
+            else {
+                if (!jl_tuple_subtype(args, n, ml->sig, 1))
+                    goto nomatch;
+            }
 
-            if (ismatch) {
-                size_t i, l;
+            size_t i, l;
+            if (ml->guardsigs != jl_emptysvec) {
                 for (i = 0, l = jl_svec_len(ml->guardsigs); i < l; i++) {
                     // checking guard entries require a more
                     // expensive subtype check, since guard entries added for ANY might be
                     // abstract. this fixed issue #12967.
                     if (jl_tuple_subtype(args, n, (jl_tupletype_t*)jl_svecref(ml->guardsigs, i), 1)) {
-                        break;
+                        goto nomatch;
                     }
                 }
-                if (i == l)
-                    return ml;
             }
+            goto matchnm;
         }
+nomatch:
+        prev = ml;
         ml = ml->next;
     }
     return NULL;
+matchnm:
+    if (prev != NULL && ml->isleafsig && first->next != ml) {
+        // LRU queue: move ml from prev->next to first->next
+        prev->next = ml->next;
+        jl_gc_wb(prev, prev->next);
+        ml->next = first->next;
+        jl_gc_wb(ml, ml->next);
+        first->next = ml;
+        jl_gc_wb(first, first->next);
+    }
+    return ml;
 }
 
-jl_typemap_entry_t *jl_typemap_assoc_exact(union jl_typemap_t ml_or_cache, jl_value_t **args, size_t n, int8_t offs)
+jl_typemap_entry_t *jl_typemap_level_assoc_exact(jl_typemap_level_t *cache, jl_value_t **args, size_t n, int8_t offs)
 {
-    // NOTE: This function is a huge performance hot spot!!
-    if (jl_typeof(ml_or_cache.unknown) == (jl_value_t*)jl_typemap_level_type) {
-        jl_typemap_level_t *cache = ml_or_cache.node;
-        if (n > offs) {
-            jl_value_t *a1 = args[offs];
-            jl_value_t *ty = (jl_value_t*)jl_typeof(a1);
-            assert(jl_is_datatype(ty));
-            if (ty == (jl_value_t*)jl_datatype_type && cache->targ != (void*)jl_nothing) {
-                ml_or_cache = mtcache_hash_lookup(cache->targ, a1, 1, offs);
-                jl_typemap_entry_t *ml = jl_typemap_assoc_exact(ml_or_cache, args, n, offs+1);
-                if (ml) return ml;
-            }
-            if (cache->arg1 != (void*)jl_nothing) {
-                ml_or_cache = mtcache_hash_lookup(cache->arg1, ty, 0, offs);
-                if (jl_typeof(ml_or_cache.unknown) == (jl_value_t*)jl_typemap_entry_type &&
-                        ml_or_cache.leaf->simplesig == (void*)jl_nothing && offs < 2 && n > 1) {
-                    jl_value_t *a0 = args[1-offs];
-                    jl_value_t *t0 = (jl_value_t*)jl_typeof(a0);
-                    if (ml_or_cache.leaf->next==(void*)jl_nothing && n==2 && jl_datatype_nfields(ml_or_cache.leaf->sig)==2 &&
-                        jl_tparam(ml_or_cache.leaf->sig, 1 - offs) == t0)
-                        return ml_or_cache.leaf;
-                    if (n==3) {
-                        // some manually-unrolled common special cases
-                        jl_value_t *a2 = args[2];
-                        if (!jl_is_tuple(a2)) {  // issue #6426
-                            jl_typemap_entry_t *mn = ml_or_cache.leaf;
-                            if (jl_datatype_nfields(mn->sig)==3 &&
-                                jl_tparam(mn->sig,1-offs)==t0 &&
-                                jl_tparam(mn->sig,2)==(jl_value_t*)jl_typeof(a2))
-                                return mn;
-                            mn = mn->next;
-                            if (mn!=(void*)jl_nothing && jl_datatype_nfields(mn->sig)==3 &&
-                                jl_tparam(mn->sig,1-offs)==t0 &&
-                                jl_tparam(mn->sig,2)==(jl_value_t*)jl_typeof(a2))
-                                return mn;
-                        }
-                    }
-                }
-                jl_typemap_entry_t *ml = jl_typemap_assoc_exact(ml_or_cache, args, n, offs+1);
-                if (ml) return ml;
-            }
+    if (n > offs) {
+        jl_value_t *a1 = args[offs];
+        jl_value_t *ty = (jl_value_t*)jl_typeof(a1);
+        assert(jl_is_datatype(ty));
+        if (ty == (jl_value_t*)jl_datatype_type && cache->targ != (void*)jl_nothing) {
+            union jl_typemap_t ml_or_cache = mtcache_hash_lookup(cache->targ, a1, 1, offs);
+            jl_typemap_entry_t *ml = jl_typemap_assoc_exact(ml_or_cache, args, n, offs+1);
+            if (ml) return ml;
+        }
+        if (cache->arg1 != (void*)jl_nothing) {
+            union jl_typemap_t ml_or_cache = mtcache_hash_lookup(cache->arg1, ty, 0, offs);
+            jl_typemap_entry_t *ml = jl_typemap_assoc_exact(ml_or_cache, args, n, offs+1);
+            if (ml) return ml;
         }
-        jl_typemap_entry_t *ml = jl_typemap_node_assoc_exact(cache->linear, args, n);
-        if (ml) return ml;
-        return jl_typemap_assoc_exact(cache->any, args, n, offs+1);
     }
-    else {
-        return jl_typemap_node_assoc_exact(ml_or_cache.leaf, args, n);
+    if (jl_typeof(cache->linear) != (jl_value_t*)jl_nothing) {
+        jl_typemap_entry_t *ml = jl_typemap_entry_assoc_exact(cache->linear, args, n);
+        if (ml) return ml;
     }
+    if (jl_typeof(cache->any.unknown) != (jl_value_t*)jl_nothing)
+        return jl_typemap_assoc_exact(cache->any, args, n, offs+1);
+    return NULL;
 }
 
 
-
 // ----- Method List Insertion Management ----- //
 
 static unsigned jl_typemap_list_count(jl_typemap_entry_t *ml)

test/reflection.jl

@@ -415,7 +415,9 @@ for i = 1:100; @eval fLargeTable(::Any, ::Val{$i}) = 2; end
 fLargeTable(::Any...) = 3
 fLargeTable(::Complex, ::Complex) = 4
 fLargeTable(::Union{Complex64, Complex128}...) = 5
-@test length(methods(fLargeTable)) == 203
+fLargeTable() = 4
+@test length(methods(fLargeTable)) == 204
+@test length(methods(fLargeTable, Tuple{})) == 1
 @test fLargeTable(1im, 2im) == 4
 @test fLargeTable(1.0im, 2.0im) == 5
 @test_throws MethodError fLargeTable(Val{1}(), Val{1}())