From ce3ba249cb52350a5f5f070bce8e31312d87e13a Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 18 May 2016 17:15:44 -0400 Subject: [PATCH 1/5] fix over-eager prunning of the typemap intersection search --- src/typemap.c | 7 ++++--- test/reflection.jl | 4 +++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/typemap.c b/src/typemap.c index b2ac0ac21e2d3..23a8540232f30 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -492,10 +492,11 @@ int jl_typemap_intersection_visitor(union jl_typemap_t map, int offs, if (!jl_typemap_intersection_array_visitor(cache->arg1, ty, 0, offs, closure)) return 0; } } - if (!jl_typemap_intersection_node_visitor(map.node->linear, closure)) - return 0; - return jl_typemap_intersection_visitor(map.node->any, offs+1, closure); } + if (!jl_typemap_intersection_node_visitor(map.node->linear, closure)) + return 0; + if (ty) + return jl_typemap_intersection_visitor(map.node->any, offs+1, closure); return 1; } else { diff --git a/test/reflection.jl b/test/reflection.jl index 0f8c34e205c1c..21d22b709e2f9 100644 --- a/test/reflection.jl +++ b/test/reflection.jl @@ -415,7 +415,9 @@ for i = 1:100; @eval fLargeTable(::Any, ::Val{$i}) = 2; end fLargeTable(::Any...) = 3 fLargeTable(::Complex, ::Complex) = 4 fLargeTable(::Union{Complex64, Complex128}...) = 5 -@test length(methods(fLargeTable)) == 203 +fLargeTable() = 4 +@test length(methods(fLargeTable)) == 204 +@test length(methods(fLargeTable, Tuple{})) == 1 @test fLargeTable(1im, 2im) == 4 @test fLargeTable(1.0im, 2.0im) == 5 @test_throws MethodError fLargeTable(Val{1}(), Val{1}()) From 3f35f2d9886ccca20d0669526bbfe642f0cdab97 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 18 May 2016 19:13:22 -0400 Subject: [PATCH 2/5] micro-optimizations for jl_typemap_assoc_exact, including making it an LRU --- src/typemap.c | 71 ++++++++++++++++++++++++++++++++------------------- 1 file changed, 45 insertions(+), 26 deletions(-) diff --git a/src/typemap.c b/src/typemap.c index 23a8540232f30..dd7d5b400e1d3 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -9,7 +9,7 @@ #include #endif -#define MAX_METHLIST_COUNT 12 // this can strongly affect the sysimg size and speed! +#define MAX_METHLIST_COUNT 24 // this can strongly affect the sysimg size and speed! #define INIT_CACHE_SIZE 8 // must be a power-of-two #ifdef __cplusplus @@ -688,6 +688,7 @@ jl_typemap_entry_t *jl_typemap_assoc_by_type(union jl_typemap_t ml_or_cache, jl_ static jl_typemap_entry_t *jl_typemap_node_assoc_exact(jl_typemap_entry_t *ml, jl_value_t **args, size_t n) { + jl_typemap_entry_t *prev = NULL, *first = ml; while (ml != (void*)jl_nothing) { size_t lensig = jl_datatype_nfields(ml->sig); if (lensig == n || (ml->va && lensig <= n+1)) { @@ -720,10 +721,21 @@ static jl_typemap_entry_t *jl_typemap_node_assoc_exact(jl_typemap_entry_t *ml, j break; } } - if (i == l) + if (i == l) { + if (prev != NULL && ml->isleafsig && first->next != ml) { + // LRU queue: move ml from prev->next to first->next + prev->next = ml->next; + jl_gc_wb(prev, prev->next); + ml->next = first->next; + jl_gc_wb(ml, ml->next); + first->next = ml; + jl_gc_wb(first, first->next); + } return ml; + } } } + prev = ml; ml = ml->next; } return NULL; @@ -745,37 +757,44 @@ jl_typemap_entry_t *jl_typemap_assoc_exact(union jl_typemap_t ml_or_cache, jl_va } if (cache->arg1 != (void*)jl_nothing) { ml_or_cache = mtcache_hash_lookup(cache->arg1, ty, 0, offs); - if (jl_typeof(ml_or_cache.unknown) == (jl_value_t*)jl_typemap_entry_type && - ml_or_cache.leaf->simplesig == (void*)jl_nothing && offs < 2 && n > 1) { - jl_value_t *a0 = args[1-offs]; - jl_value_t *t0 = (jl_value_t*)jl_typeof(a0); - if (ml_or_cache.leaf->next==(void*)jl_nothing && n==2 && jl_datatype_nfields(ml_or_cache.leaf->sig)==2 && - jl_tparam(ml_or_cache.leaf->sig, 1 - offs) == t0) - return ml_or_cache.leaf; - if (n==3) { - // some manually-unrolled common special cases - jl_value_t *a2 = args[2]; - if (!jl_is_tuple(a2)) { // issue #6426 - jl_typemap_entry_t *mn = ml_or_cache.leaf; - if (jl_datatype_nfields(mn->sig)==3 && - jl_tparam(mn->sig,1-offs)==t0 && - jl_tparam(mn->sig,2)==(jl_value_t*)jl_typeof(a2)) - return mn; - mn = mn->next; - if (mn!=(void*)jl_nothing && jl_datatype_nfields(mn->sig)==3 && - jl_tparam(mn->sig,1-offs)==t0 && - jl_tparam(mn->sig,2)==(jl_value_t*)jl_typeof(a2)) - return mn; + if (jl_typeof(ml_or_cache.unknown) == (jl_value_t*)jl_typemap_entry_type) { + if (ml_or_cache.leaf->simplesig == (void*)jl_nothing && offs < 2 && n > 1) { + jl_value_t *a0 = args[1-offs]; + jl_value_t *t0 = (jl_value_t*)jl_typeof(a0); + if (n==2 && jl_datatype_nfields(ml_or_cache.leaf->sig)==2 && + jl_tparam(ml_or_cache.leaf->sig, 1 - offs) == t0) + return ml_or_cache.leaf; + if (n==3) { + // some manually-unrolled common special cases + jl_value_t *a2 = args[2]; + if (!jl_is_tuple(a2)) { // issue #6426 + jl_typemap_entry_t *mn = ml_or_cache.leaf; + if (jl_datatype_nfields(mn->sig)==3 && + jl_tparam(mn->sig,1-offs)==t0 && + jl_tparam(mn->sig,2)==(jl_value_t*)jl_typeof(a2)) + return mn; + mn = mn->next; + if (mn!=(void*)jl_nothing && jl_datatype_nfields(mn->sig)==3 && + jl_tparam(mn->sig,1-offs)==t0 && + jl_tparam(mn->sig,2)==(jl_value_t*)jl_typeof(a2)) + return mn; + } } } + jl_typemap_entry_t *ml = jl_typemap_node_assoc_exact(ml_or_cache.leaf, args, n); + if (ml) return ml; + } + else if (jl_typeof(ml_or_cache.unknown) == (jl_value_t*)jl_typemap_level_type) { + jl_typemap_entry_t *ml = jl_typemap_assoc_exact(ml_or_cache, args, n, offs+1); + if (ml) return ml; } - jl_typemap_entry_t *ml = jl_typemap_assoc_exact(ml_or_cache, args, n, offs+1); - if (ml) return ml; } } jl_typemap_entry_t *ml = jl_typemap_node_assoc_exact(cache->linear, args, n); if (ml) return ml; - return jl_typemap_assoc_exact(cache->any, args, n, offs+1); + if (jl_typeof(ml_or_cache.unknown) != (jl_value_t*)jl_nothing) + return jl_typemap_assoc_exact(cache->any, args, n, offs+1); + return NULL; } else { return jl_typemap_node_assoc_exact(ml_or_cache.leaf, args, n); From 0d36174eabb7d8c9495872371f4570b2c5840aa5 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 18 May 2016 19:54:55 -0400 Subject: [PATCH 3/5] unroll TypeMap more carefully this lets us catch an extra case, deletes an out-of-date condition, and is a bugfix if the entry was supposed to have extra attributes to test (like guardsig) that it was ignoring --- src/typemap.c | 42 +++++++++++++++++++++++++----------------- 1 file changed, 25 insertions(+), 17 deletions(-) diff --git a/src/typemap.c b/src/typemap.c index dd7d5b400e1d3..97f2cddb0f22c 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -758,25 +758,33 @@ jl_typemap_entry_t *jl_typemap_assoc_exact(union jl_typemap_t ml_or_cache, jl_va if (cache->arg1 != (void*)jl_nothing) { ml_or_cache = mtcache_hash_lookup(cache->arg1, ty, 0, offs); if (jl_typeof(ml_or_cache.unknown) == (jl_value_t*)jl_typemap_entry_type) { - if (ml_or_cache.leaf->simplesig == (void*)jl_nothing && offs < 2 && n > 1) { - jl_value_t *a0 = args[1-offs]; + if (offs < 2 && n > 1) { + // some manually-unrolled common special cases + jl_value_t *a0 = args[1 - offs]; jl_value_t *t0 = (jl_value_t*)jl_typeof(a0); - if (n==2 && jl_datatype_nfields(ml_or_cache.leaf->sig)==2 && - jl_tparam(ml_or_cache.leaf->sig, 1 - offs) == t0) - return ml_or_cache.leaf; - if (n==3) { - // some manually-unrolled common special cases - jl_value_t *a2 = args[2]; - if (!jl_is_tuple(a2)) { // issue #6426 - jl_typemap_entry_t *mn = ml_or_cache.leaf; - if (jl_datatype_nfields(mn->sig)==3 && - jl_tparam(mn->sig,1-offs)==t0 && - jl_tparam(mn->sig,2)==(jl_value_t*)jl_typeof(a2)) + jl_typemap_entry_t *mn = ml_or_cache.leaf; + if (mn->simplesig == (void*)jl_nothing && mn->guardsigs == jl_emptysvec && + n == jl_datatype_nfields(mn->sig) && mn->isleafsig) { + if (n == 2) { + if (jl_tparam(ml_or_cache.leaf->sig, 1 - offs) == t0) + return ml_or_cache.leaf; + } + else if (n == 3) { + jl_value_t *a2 = args[2]; + if (jl_tparam(mn->sig, 1 - offs) == t0 && + jl_tparam(mn->sig, 2) == (jl_value_t*)jl_typeof(a2)) + return mn; + } + else { + if (sig_match_leaf(args, jl_svec_data(mn->sig->parameters), n)) return mn; - mn = mn->next; - if (mn!=(void*)jl_nothing && jl_datatype_nfields(mn->sig)==3 && - jl_tparam(mn->sig,1-offs)==t0 && - jl_tparam(mn->sig,2)==(jl_value_t*)jl_typeof(a2)) + } + mn = mn->next; + if (n == 3 && mn != (void*)jl_nothing && + mn->simplesig == (void*)jl_nothing && mn->guardsigs == jl_emptysvec && + n == jl_datatype_nfields(mn->sig) && mn->isleafsig) { + if (jl_tparam(mn->sig, 1 - offs) == t0 && + jl_tparam(mn->sig, 2) == (jl_value_t*)jl_typeof(args[2])) return mn; } } From 84983638cf98e929e423fe58f273ccb1175a2f45 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 18 May 2016 20:41:55 -0400 Subject: [PATCH 4/5] manually jump-thread the guardsigs test this was showing up in the profile, despite being very cold in branches not taken this is also closer to the code we want the compiler to emit --- src/typemap.c | 61 ++++++++++++++++++++++++++++----------------------- 1 file changed, 33 insertions(+), 28 deletions(-) diff --git a/src/typemap.c b/src/typemap.c index 97f2cddb0f22c..638ebca0c1cbf 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -692,49 +692,54 @@ static jl_typemap_entry_t *jl_typemap_node_assoc_exact(jl_typemap_entry_t *ml, j while (ml != (void*)jl_nothing) { size_t lensig = jl_datatype_nfields(ml->sig); if (lensig == n || (ml->va && lensig <= n+1)) { - int ismatch = 1; if (ml->simplesig != (void*)jl_nothing) { size_t lensimplesig = jl_datatype_nfields(ml->simplesig); int isva = lensimplesig > 0 && jl_is_vararg_type(jl_tparam(ml->simplesig, lensimplesig - 1)); - if (lensig == n || (isva && lensimplesig <= n + 1)) - ismatch = sig_match_simple(args, n, jl_svec_data(ml->simplesig->parameters), isva, lensimplesig); - else - ismatch = 0; + if (lensig == n || (isva && lensimplesig <= n + 1)) { + if (!sig_match_simple(args, n, jl_svec_data(ml->simplesig->parameters), isva, lensimplesig)) + goto nomatch; + } + else { + goto nomatch; + } } - if (ismatch == 0) - ; // nothing - else if (ml->isleafsig) - ismatch = sig_match_leaf(args, jl_svec_data(ml->sig->parameters), n); - else if (ml->issimplesig) - ismatch = sig_match_simple(args, n, jl_svec_data(ml->sig->parameters), ml->va, lensig); - else - ismatch = jl_tuple_subtype(args, n, ml->sig, 1); + if (ml->isleafsig) { + if (!sig_match_leaf(args, jl_svec_data(ml->sig->parameters), n)) + goto nomatch; + } + else if (ml->issimplesig) { + if (!sig_match_simple(args, n, jl_svec_data(ml->sig->parameters), ml->va, lensig)) + goto nomatch; + } + else { + if (!jl_tuple_subtype(args, n, ml->sig, 1)) + goto nomatch; + } - if (ismatch) { - size_t i, l; + size_t i, l; + if (ml->guardsigs != jl_emptysvec) { for (i = 0, l = jl_svec_len(ml->guardsigs); i < l; i++) { // checking guard entries require a more // expensive subtype check, since guard entries added for ANY might be // abstract. this fixed issue #12967. if (jl_tuple_subtype(args, n, (jl_tupletype_t*)jl_svecref(ml->guardsigs, i), 1)) { - break; + goto nomatch; } } - if (i == l) { - if (prev != NULL && ml->isleafsig && first->next != ml) { - // LRU queue: move ml from prev->next to first->next - prev->next = ml->next; - jl_gc_wb(prev, prev->next); - ml->next = first->next; - jl_gc_wb(ml, ml->next); - first->next = ml; - jl_gc_wb(first, first->next); - } - return ml; - } } + if (prev != NULL && ml->isleafsig && first->next != ml) { + // LRU queue: move ml from prev->next to first->next + prev->next = ml->next; + jl_gc_wb(prev, prev->next); + ml->next = first->next; + jl_gc_wb(ml, ml->next); + first->next = ml; + jl_gc_wb(first, first->next); + } + return ml; } +nomatch: prev = ml; ml = ml->next; } From f6ae2276ac5777aabc4e98c0891c9d2b6f98ce13 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Wed, 18 May 2016 23:38:45 -0400 Subject: [PATCH 5/5] re-structure micro-optimizations of typemap I believe this should better match the usage patterns and is able to unroll more cases and more effectively use that information --- src/dump.c | 6 +- src/gf.c | 19 ++++-- src/jltypes.c | 4 +- src/julia_internal.h | 17 +++++- src/typemap.c | 135 +++++++++++++++++++------------------------ 5 files changed, 96 insertions(+), 85 deletions(-) diff --git a/src/dump.c b/src/dump.c index 32493ff37690c..4a07a3657d13c 100644 --- a/src/dump.c +++ b/src/dump.c @@ -1993,9 +1993,9 @@ static void jl_restore_system_image_from_stream(ios_t *f) jl_top_module = (jl_module_t*)jl_deserialize_value(f, NULL); jl_internal_main_module = jl_main_module; jl_typeinf_func = (jl_function_t*)jl_deserialize_value(f, NULL); - jl_type_type->name->mt = (jl_methtable_t*)jl_deserialize_value(f, NULL); - jl_typector_type->name->mt = jl_uniontype_type->name->mt = jl_datatype_type->name->mt = - jl_type_type->name->mt; + jl_type_type_mt = (jl_methtable_t*)jl_deserialize_value(f, NULL); + jl_type_type->name->mt = jl_typector_type->name->mt = jl_uniontype_type->name->mt = jl_datatype_type->name->mt = + jl_type_type_mt; jl_core_module = (jl_module_t*)jl_get_global(jl_main_module, jl_symbol("Core")); diff --git a/src/gf.c b/src/gf.c index 6e8ff93b51d98..31d8f5a4b9893 100644 --- a/src/gf.c +++ b/src/gf.c @@ -106,7 +106,7 @@ const struct jl_typemap_info tfunc_cache = { static int8_t jl_cachearg_offset(jl_methtable_t *mt) { - return (mt == jl_type_type->name->mt) ? 0 : 1; + return (mt == jl_type_type_mt) ? 0 : 1; } /// ----- Insertion logic for special entries ----- /// @@ -1581,10 +1581,20 @@ JL_DLLEXPORT jl_value_t *jl_apply_generic(jl_value_t **args, uint32_t nargs) jl_method_error((jl_function_t*)F, args, nargs); // unreachable } + jl_value_t *res; + if (mfunc->inInference || mfunc->inCompile) { + // if inference is running on this function, return a copy + // of the function to be compiled without inference and run. + res = jl_call_unspecialized(mfunc->sparam_vals, jl_get_unspecialized(mfunc), args, nargs); + } + else { + res = jl_call_method_internal(mfunc, args, nargs); + } + JL_GC_POP(); + return verify_type(res); } - else { - mfunc = entry->func.linfo; - } + + mfunc = entry->func.linfo; #ifdef JL_TRACE if (traceen) jl_printf(JL_STDOUT, " at %s:%d\n", jl_symbol_name(mfunc->file), mfunc->line); @@ -1598,7 +1608,6 @@ JL_DLLEXPORT jl_value_t *jl_apply_generic(jl_value_t **args, uint32_t nargs) else { res = jl_call_method_internal(mfunc, args, nargs); } - if (tt) JL_GC_POP(); return verify_type(res); } diff --git a/src/jltypes.c b/src/jltypes.c index 6293ebde84f48..7410d2b2de2b1 100644 --- a/src/jltypes.c +++ b/src/jltypes.c @@ -22,6 +22,7 @@ extern "C" { jl_datatype_t *jl_any_type; jl_datatype_t *jl_type_type; +jl_methtable_t *jl_type_type_mt; jl_datatype_t *jl_typename_type; jl_datatype_t *jl_sym_type; jl_datatype_t *jl_symbol_type; @@ -3263,7 +3264,8 @@ void jl_init_types(void) jl_any_type = jl_new_abstracttype((jl_value_t*)jl_symbol("Any"), NULL, jl_emptysvec); jl_any_type->super = jl_any_type; jl_type_type = jl_new_abstracttype((jl_value_t*)jl_symbol("Type"), jl_any_type, jl_emptysvec); - jl_type_type->name->mt = jl_new_method_table(jl_type_type->name->name, jl_current_module); + jl_type_type_mt = jl_new_method_table(jl_type_type->name->name, jl_current_module); + jl_type_type->name->mt = jl_type_type_mt; // initialize them. lots of cycles. jl_datatype_type->name = jl_new_typename(jl_symbol("DataType")); diff --git a/src/julia_internal.h b/src/julia_internal.h index a70db1649a3f7..fbe9dace4b175 100644 --- a/src/julia_internal.h +++ b/src/julia_internal.h @@ -16,6 +16,9 @@ extern "C" { #endif +// useful constants +extern jl_methtable_t *jl_type_type_mt; + // execution of certain certain unpure // statements is prohibited from certain // callbacks (such as generated functions) @@ -623,7 +626,19 @@ jl_typemap_entry_t *jl_typemap_insert(union jl_typemap_t *cache, jl_value_t *par jl_typemap_entry_t *jl_typemap_assoc_by_type(union jl_typemap_t ml_or_cache, jl_tupletype_t *types, jl_svec_t **penv, int8_t subtype_inexact__sigseq_useenv, int8_t subtype, int8_t offs); static jl_typemap_entry_t *const INEXACT_ENTRY = (jl_typemap_entry_t*)(uintptr_t)-1; -jl_typemap_entry_t *jl_typemap_assoc_exact(union jl_typemap_t ml_or_cache, jl_value_t **args, size_t n, int8_t offs); +jl_typemap_entry_t *jl_typemap_level_assoc_exact(jl_typemap_level_t *cache, jl_value_t **args, size_t n, int8_t offs); +jl_typemap_entry_t *jl_typemap_entry_assoc_exact(jl_typemap_entry_t *mn, jl_value_t **args, size_t n); +STATIC_INLINE jl_typemap_entry_t *jl_typemap_assoc_exact(union jl_typemap_t ml_or_cache, jl_value_t **args, size_t n, int8_t offs) +{ + // NOTE: This function is a huge performance hot spot!! + if (jl_typeof(ml_or_cache.unknown) == (jl_value_t*)jl_typemap_entry_type) { + return jl_typemap_entry_assoc_exact(ml_or_cache.leaf, args, n); + } + else if (jl_typeof(ml_or_cache.unknown) == (jl_value_t*)jl_typemap_level_type) { + return jl_typemap_level_assoc_exact(ml_or_cache.node, args, n, offs); + } + return NULL; +} typedef int (*jl_typemap_visitor_fptr)(jl_typemap_entry_t *l, void *closure); int jl_typemap_visitor(union jl_typemap_t a, jl_typemap_visitor_fptr fptr, void *closure); diff --git a/src/typemap.c b/src/typemap.c index 638ebca0c1cbf..5993014836bab 100644 --- a/src/typemap.c +++ b/src/typemap.c @@ -9,7 +9,7 @@ #include #endif -#define MAX_METHLIST_COUNT 24 // this can strongly affect the sysimg size and speed! +#define MAX_METHLIST_COUNT 12 // this can strongly affect the sysimg size and speed! #define INIT_CACHE_SIZE 8 // must be a power-of-two #ifdef __cplusplus @@ -686,9 +686,35 @@ jl_typemap_entry_t *jl_typemap_assoc_by_type(union jl_typemap_t ml_or_cache, jl_ } } -static jl_typemap_entry_t *jl_typemap_node_assoc_exact(jl_typemap_entry_t *ml, jl_value_t **args, size_t n) +jl_typemap_entry_t *jl_typemap_entry_assoc_exact(jl_typemap_entry_t *ml, jl_value_t **args, size_t n) { + // some manually-unrolled common special cases jl_typemap_entry_t *prev = NULL, *first = ml; + while (ml->simplesig == (void*)jl_nothing && ml->guardsigs == jl_emptysvec && ml->isleafsig) { + // use a tight loop for a long as possible + if (n == jl_datatype_nfields(ml->sig) && jl_typeof(args[0]) == jl_tparam(ml->sig, 0)) { + if (n == 1) + goto matchnm; + if (n == 2) { + if (jl_typeof(args[1]) == jl_tparam(ml->sig, 1)) + goto matchnm; + } + else if (n == 3) { + if (jl_typeof(args[1]) == jl_tparam(ml->sig, 1) && + jl_typeof(args[2]) == jl_tparam(ml->sig, 2)) + goto matchnm; + } + else { + if (sig_match_leaf(args, jl_svec_data(ml->sig->parameters), n)) + goto matchnm; + } + } + prev = ml; + ml = ml->next; + if (ml == (void*)jl_nothing) + return NULL; + } + while (ml != (void*)jl_nothing) { size_t lensig = jl_datatype_nfields(ml->sig); if (lensig == n || (ml->va && lensig <= n+1)) { @@ -728,94 +754,53 @@ static jl_typemap_entry_t *jl_typemap_node_assoc_exact(jl_typemap_entry_t *ml, j } } } - if (prev != NULL && ml->isleafsig && first->next != ml) { - // LRU queue: move ml from prev->next to first->next - prev->next = ml->next; - jl_gc_wb(prev, prev->next); - ml->next = first->next; - jl_gc_wb(ml, ml->next); - first->next = ml; - jl_gc_wb(first, first->next); - } - return ml; + goto matchnm; } nomatch: prev = ml; ml = ml->next; } return NULL; +matchnm: + if (prev != NULL && ml->isleafsig && first->next != ml) { + // LRU queue: move ml from prev->next to first->next + prev->next = ml->next; + jl_gc_wb(prev, prev->next); + ml->next = first->next; + jl_gc_wb(ml, ml->next); + first->next = ml; + jl_gc_wb(first, first->next); + } + return ml; } -jl_typemap_entry_t *jl_typemap_assoc_exact(union jl_typemap_t ml_or_cache, jl_value_t **args, size_t n, int8_t offs) +jl_typemap_entry_t *jl_typemap_level_assoc_exact(jl_typemap_level_t *cache, jl_value_t **args, size_t n, int8_t offs) { - // NOTE: This function is a huge performance hot spot!! - if (jl_typeof(ml_or_cache.unknown) == (jl_value_t*)jl_typemap_level_type) { - jl_typemap_level_t *cache = ml_or_cache.node; - if (n > offs) { - jl_value_t *a1 = args[offs]; - jl_value_t *ty = (jl_value_t*)jl_typeof(a1); - assert(jl_is_datatype(ty)); - if (ty == (jl_value_t*)jl_datatype_type && cache->targ != (void*)jl_nothing) { - ml_or_cache = mtcache_hash_lookup(cache->targ, a1, 1, offs); - jl_typemap_entry_t *ml = jl_typemap_assoc_exact(ml_or_cache, args, n, offs+1); - if (ml) return ml; - } - if (cache->arg1 != (void*)jl_nothing) { - ml_or_cache = mtcache_hash_lookup(cache->arg1, ty, 0, offs); - if (jl_typeof(ml_or_cache.unknown) == (jl_value_t*)jl_typemap_entry_type) { - if (offs < 2 && n > 1) { - // some manually-unrolled common special cases - jl_value_t *a0 = args[1 - offs]; - jl_value_t *t0 = (jl_value_t*)jl_typeof(a0); - jl_typemap_entry_t *mn = ml_or_cache.leaf; - if (mn->simplesig == (void*)jl_nothing && mn->guardsigs == jl_emptysvec && - n == jl_datatype_nfields(mn->sig) && mn->isleafsig) { - if (n == 2) { - if (jl_tparam(ml_or_cache.leaf->sig, 1 - offs) == t0) - return ml_or_cache.leaf; - } - else if (n == 3) { - jl_value_t *a2 = args[2]; - if (jl_tparam(mn->sig, 1 - offs) == t0 && - jl_tparam(mn->sig, 2) == (jl_value_t*)jl_typeof(a2)) - return mn; - } - else { - if (sig_match_leaf(args, jl_svec_data(mn->sig->parameters), n)) - return mn; - } - mn = mn->next; - if (n == 3 && mn != (void*)jl_nothing && - mn->simplesig == (void*)jl_nothing && mn->guardsigs == jl_emptysvec && - n == jl_datatype_nfields(mn->sig) && mn->isleafsig) { - if (jl_tparam(mn->sig, 1 - offs) == t0 && - jl_tparam(mn->sig, 2) == (jl_value_t*)jl_typeof(args[2])) - return mn; - } - } - } - jl_typemap_entry_t *ml = jl_typemap_node_assoc_exact(ml_or_cache.leaf, args, n); - if (ml) return ml; - } - else if (jl_typeof(ml_or_cache.unknown) == (jl_value_t*)jl_typemap_level_type) { - jl_typemap_entry_t *ml = jl_typemap_assoc_exact(ml_or_cache, args, n, offs+1); - if (ml) return ml; - } - } + if (n > offs) { + jl_value_t *a1 = args[offs]; + jl_value_t *ty = (jl_value_t*)jl_typeof(a1); + assert(jl_is_datatype(ty)); + if (ty == (jl_value_t*)jl_datatype_type && cache->targ != (void*)jl_nothing) { + union jl_typemap_t ml_or_cache = mtcache_hash_lookup(cache->targ, a1, 1, offs); + jl_typemap_entry_t *ml = jl_typemap_assoc_exact(ml_or_cache, args, n, offs+1); + if (ml) return ml; + } + if (cache->arg1 != (void*)jl_nothing) { + union jl_typemap_t ml_or_cache = mtcache_hash_lookup(cache->arg1, ty, 0, offs); + jl_typemap_entry_t *ml = jl_typemap_assoc_exact(ml_or_cache, args, n, offs+1); + if (ml) return ml; } - jl_typemap_entry_t *ml = jl_typemap_node_assoc_exact(cache->linear, args, n); - if (ml) return ml; - if (jl_typeof(ml_or_cache.unknown) != (jl_value_t*)jl_nothing) - return jl_typemap_assoc_exact(cache->any, args, n, offs+1); - return NULL; } - else { - return jl_typemap_node_assoc_exact(ml_or_cache.leaf, args, n); + if (jl_typeof(cache->linear) != (jl_value_t*)jl_nothing) { + jl_typemap_entry_t *ml = jl_typemap_entry_assoc_exact(cache->linear, args, n); + if (ml) return ml; } + if (jl_typeof(cache->any.unknown) != (jl_value_t*)jl_nothing) + return jl_typemap_assoc_exact(cache->any, args, n, offs+1); + return NULL; } - // ----- Method List Insertion Management ----- // static unsigned jl_typemap_list_count(jl_typemap_entry_t *ml)