Skip to content

Commit

Permalink
fix sort / fallback order for incremental precompile
Browse files Browse the repository at this point in the history 
fix #18069

(cherry picked from PR #18150 commit 94ce57d)
  • Loading branch information
@vtjnash
vtjnash committed Aug 30, 2016
1 parent 355445a commit 917acce
Show file tree
Hide file tree
Showing 2 changed files with 51 additions and 41 deletions.
83 changes: 46 additions & 37 deletions base/loading.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,9 +28,9 @@ end

function find_in_node_path(name, srcpath, node::Int=1)
if myid() == node
find_in_path(name, srcpath)
return find_in_path(name, srcpath)
else
remotecall_fetch(node, find_in_path, name, srcpath)
return remotecall_fetch(node, find_in_path, name, srcpath)
end
end

Expand All @@ -39,7 +39,7 @@ function find_source_file(file)
file2 = find_in_path(file)
file2 !== nothing && return file2
file2 = joinpath(JULIA_HOME, DATAROOTDIR, "julia", "base", file)
isfile(file2) ? file2 : nothing
return isfile(file2) ? file2 : nothing
end

function find_all_in_cache_path(mod::Symbol)
Expand All @@ -51,18 +51,22 @@ function find_all_in_cache_path(mod::Symbol)
push!(paths, path)
end
end
paths
return paths
end

function _include_from_serialized(content::Vector{UInt8})
return ccall(:jl_restore_incremental_from_buf, Any, (Ptr{UInt8},Int), content, sizeof(content))
return ccall(:jl_restore_incremental_from_buf, Any, (Ptr{UInt8}, Int), content, sizeof(content))
end

function _include_from_serialized(path::ByteString)
return ccall(:jl_restore_incremental, Any, (Cstring,), path)
end

# returns an array of modules loaded, or nothing if failed
function _require_from_serialized(node::Int, mod::Symbol, path_to_try::ByteString, toplevel_load::Bool)
restored = nothing
local restored = nothing
local content::Vector{UInt8}
if toplevel_load && myid() == 1 && nprocs() > 1
recompile_stale(mod, path_to_try)
# broadcast top-level import/using from node 1 (only)
if node == myid()
content = open(readbytes, path_to_try)
Expand All @@ -80,13 +84,11 @@ function _require_from_serialized(node::Int, mod::Symbol, path_to_try::ByteStrin
end
end
elseif node == myid()
myid() == 1 && recompile_stale(mod, path_to_try)
restored = ccall(:jl_restore_incremental, Any, (Ptr{UInt8},), path_to_try)
restored = _include_from_serialized(path_to_try)
else
content = remotecall_fetch(node, open, readbytes, path_to_try)
restored = _include_from_serialized(content)
end
# otherwise, continue search

if restored !== nothing
for M in restored
Expand All @@ -98,22 +100,28 @@ function _require_from_serialized(node::Int, mod::Symbol, path_to_try::ByteStrin
return restored
end

function _require_from_serialized(node::Int, mod::Symbol, toplevel_load::Bool)
# returns `true` if require found a precompile cache for this mod, but couldn't load it
# returns `false` if the module isn't known to be precompilable
# returns the set of modules restored if the cache load succeeded
function _require_search_from_serialized(node::Int, mod::Symbol, sourcepath::ByteString, toplevel_load::Bool)
if node == myid()
paths = find_all_in_cache_path(mod)
else
paths = @fetchfrom node find_all_in_cache_path(mod)
end
sort!(paths, by=mtime, rev=true) # try newest cachefiles first

for path_to_try in paths
if stale_cachefile(sourcepath, path_to_try)
continue
end
restored = _require_from_serialized(node, mod, path_to_try, toplevel_load)
if restored === nothing
warn("deserialization checks failed while attempting to load cache from $path_to_try")
else
return restored
end
end
return nothing
return !isempty(paths)
end

# to synchronize multiple tasks trying to import/using something
Expand Down Expand Up @@ -232,21 +240,27 @@ function require(mod::Symbol)
last = toplevel_load::Bool
try
toplevel_load = false
if nothing !== _require_from_serialized(1, mod, last)
return
name = string(mod)
path = find_in_node_path(name, nothing, 1)
if path === nothing
throw(ArgumentError("module $name not found in current path.\nRun `Pkg.add(\"$name\")` to install the $name package."))
end
if JLOptions().incremental != 0
# spawn off a new incremental precompile task from node 1 for recursive `require` calls

doneprecompile = _require_search_from_serialized(1, mod, path, last)
if !isa(doneprecompile, Bool)
return # success
elseif doneprecompile === true || JLOptions().incremental != 0
# spawn off a new incremental pre-compile task from node 1 for recursive `require` calls
# or if the require search declared it was pre-compiled before (and therefore is expected to still be pre-compilable)
cachefile = compilecache(mod)
if nothing === _require_from_serialized(1, mod, cachefile, last)
warn("require failed to create a precompiled cache file")
warn("compilecache failed to create a usable precompiled cache file for module $name.")
else
return # success
end
return
end
# fall-through to attempting to load the source file

name = string(mod)
path = find_in_node_path(name, nothing, 1)
path === nothing && throw(ArgumentError("$name not found in path"))
try
if last && myid() == 1 && nprocs() > 1
# include on node 1 first to check for PrecompilableErrors
Expand All @@ -259,13 +273,12 @@ function require(mod::Symbol)
eval(Main, :(Base.include_from_node1($path)))
end
catch ex
if !precompilableerror(ex, true)
if doneprecompile === true || !precompilableerror(ex, true)
rethrow() # rethrow non-precompilable=true errors
end
isinteractive() && info("Precompiling module $mod...")
cachefile = compilecache(mod)
if nothing === _require_from_serialized(1, mod, cachefile, last)
error("__precompile__(true) but require failed to create a precompiled cache file")
error("module $mod declares __precompile__(true) but require failed to create a usable precompiled cache file.")
end
end
finally
Expand Down Expand Up @@ -396,6 +409,13 @@ function compilecache(name::ByteString)
mkpath(cachepath)
end
cachefile = abspath(cachepath, name*".ji")
if isinteractive()
if isfile(cachepath)
info("Recompiling stale cache file $cachefile for module $name.")
else
info("Precompiling module $name.")
end
end
if !success(create_expr_cache(path, cachefile))
error("Failed to precompile $name to $cachefile")
end
Expand Down Expand Up @@ -445,7 +465,7 @@ function stale_cachefile(modpath, cachefile)
if files[1][1] != modpath
return true # cache file was compiled from a different path
end
for (f,ftime) in files
for (f, ftime) in files
# Issue #13606: compensate for Docker images rounding mtimes
if mtime(f) ∉ (ftime, floor(ftime))
return true
Expand All @@ -465,14 +485,3 @@ function stale_cachefile(modpath, cachefile)
close(io)
end
end

function recompile_stale(mod, cachefile)
path = find_in_path(string(mod), nothing)
if path === nothing
error("module $mod not found in current path; you should rm(\"$(escape_string(cachefile))\") to remove the orphaned cache file")
end
if stale_cachefile(path, cachefile)
info("Recompiling stale cache file $cachefile for module $mod.")
compilecache(mod)
end
end
9 changes: 5 additions & 4 deletions test/compile.jl
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@ try
# the module doesn't load from the image:
try
redirected_stderr()
@test nothing !== Base._require_from_serialized(myid(), Foo_module, #=broadcast-load=#false)
@test nothing !== Base._require_search_from_serialized(myid(), Foo_module, Foo_file, #=broadcast-load=#false)
finally
redirect_stderr(olderr)
end
Expand Down Expand Up @@ -88,14 +88,15 @@ try
end
""")
end
Base.compilecache("FooBar")
sleep(2)

Base.compilecache("FooBar")
@test isfile(joinpath(dir, "FooBar.ji"))
@test !Base.stale_cachefile(FooBar_file, joinpath(dir, "FooBar.ji"))

touch(FooBar_file)
insert!(Base.LOAD_CACHE_PATH, 1, dir2)
Base.recompile_stale(:FooBar, joinpath(dir, "FooBar.ji"))
sleep(2)
Base.compilecache("FooBar")
@test isfile(joinpath(dir2, "FooBar.ji"))
@test Base.stale_cachefile(FooBar_file, joinpath(dir, "FooBar.ji"))
@test !Base.stale_cachefile(FooBar_file, joinpath(dir2, "FooBar.ji"))
Expand Down

0 comments on commit 917acce

Please sign in to comment.