From 47fe1bdc697705140b159c1ae252ed73276100c1 Mon Sep 17 00:00:00 2001 From: Jameson Nash Date: Thu, 17 Oct 2024 14:02:51 +0000 Subject: [PATCH] fix precompile process flags CacheFlags could get set, but were never propagated to the target process, so the result would be unusable. Additionally, the debug and optimization levels were not synchronized with the sysimg, causing a regression in pkgimage usability after moving out stdlibs. Fixes #56207 Fixes #56054 Fixes #56206 --- base/loading.jl | 50 ++++++++++++++++++++++++++++++------------------- pkgimage.mk | 3 ++- test/loading.jl | 5 +---- 3 files changed, 34 insertions(+), 24 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index b396c7897c1fd3..1658b7c70bc4a5 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -1668,7 +1668,7 @@ function CacheFlags(f::UInt8) CacheFlags(use_pkgimages, debug_level, check_bounds, inline, opt_level) end CacheFlags(f::Int) = CacheFlags(UInt8(f)) -function CacheFlags(cf::CacheFlags=CacheFlags(ccall(:jl_cache_flags, UInt8, ())); +function CacheFlags(cf::CacheFlags=CacheFlags(ccall(:jl_cache_flag, UInt8, ())); use_pkgimages::Union{Nothing,Bool}=nothing, debug_level::Union{Nothing,Int}=nothing, check_bounds::Union{Nothing,Int}=nothing, @@ -1695,11 +1695,13 @@ function _cacheflag_to_uint8(cf::CacheFlags)::UInt8 end function show(io::IO, cf::CacheFlags) - print(io, "use_pkgimages = ", cf.use_pkgimages) - print(io, ", debug_level = ", cf.debug_level) - print(io, ", check_bounds = ", cf.check_bounds) - print(io, ", inline = ", cf.inline) - print(io, ", opt_level = ", cf.opt_level) + print(io, "CacheFlags(") + print(io, "; use_pkgimages=", cf.use_pkgimages) + print(io, ", debug_level=", cf.debug_level) + print(io, ", check_bounds=", cf.check_bounds) + print(io, ", inline=", cf.inline) + print(io, ", opt_level=", cf.opt_level) + print(io, ")") end struct ImageTarget @@ -2952,7 +2954,8 @@ end const PRECOMPILE_TRACE_COMPILE = Ref{String}() function create_expr_cache(pkg::PkgId, input::String, output::String, output_o::Union{Nothing, String}, - concrete_deps::typeof(_concrete_dependencies), flags::Cmd=``, internal_stderr::IO = stderr, internal_stdout::IO = stdout, isext::Bool=false) + concrete_deps::typeof(_concrete_dependencies), flags::Cmd=``, cacheflags::CacheFlags, + internal_stderr::IO = stderr, internal_stdout::IO = stdout, isext::Bool=false) @nospecialize internal_stderr internal_stdout rm(output, force=true) # Remove file if it exists output_o === nothing || rm(output_o, force=true) @@ -2995,24 +2998,33 @@ function create_expr_cache(pkg::PkgId, input::String, output::String, output_o:: deps = deps_eltype * "[" * join(deps_strs, ",") * "]" precomp_stack = "Base.PkgId[$(join(map(pkg_str, vcat(Base.precompilation_stack, pkg)), ", "))]" + opts = String[] + nativeflags = CacheFlags() # julia_cmd is generated for the running system, and must be fixed if running for precompile instead + cacheflags.use_pkgimages != nativeflags.use_pkgimages && push!(opts, cacheflags.use_pkgimages ? "--pkgimages=yes" : "--pkgimages=no") + cacheflags.debug_level != nativeflags.debug_level && push!(opts, "-g$(cacheflags.debug_level)") + cacheflags.check_bounds != nativeflags.check_bounds && push!(opts, ("--check-bounds=auto", "--check-bounds=yes" "--check-bounds=no")[cacheflags.check_bounds + 1]) + cacheflags.inline != nativeflags.inline && push!(opts, cacheflags.inline ? "--inline=yes" : "--inline=no") + isassigned(PRECOMPILE_TRACE_COMPILE) && push!(opts, "--trace-compile=$(PRECOMPILE_TRACE_COMPILE[])") if output_o !== nothing @debug "Generating object cache file for $(repr("text/plain", pkg))" cpu_target = get(ENV, "JULIA_CPU_TARGET", nothing) - opts = `--output-o $(output_o) --output-ji $(output) --output-incremental=yes` + cacheflags.opt_level != nativeflags.opt_level && push!(opts, "-O$(cacheflags.opt_level)") + push!(opts, "--output-o", output_o) + push!(opts, "--output-ji", output) else @debug "Generating cache file for $(repr("text/plain", pkg))" cpu_target = nothing - opts = `-O0 --output-ji $(output) --output-incremental=yes` + push!(opts, "-O0", output) + push!(opts, "--output-ji", output) end - trace = isassigned(PRECOMPILE_TRACE_COMPILE) ? `--trace-compile=$(PRECOMPILE_TRACE_COMPILE[]) --trace-compile-timing` : `` io = open(pipeline(addenv(`$(julia_cmd(;cpu_target)::Cmd) - $(flags) - $(opts) - --startup-file=no --history-file=no --warn-overwrite=yes - --color=$(have_color === nothing ? "auto" : have_color ? "yes" : "no") - $trace - -`, + $(flags) + $(opts) + --output-incremental=yes + --startup-file=no --history-file=no --warn-overwrite=yes + $(have_color === nothing ? "--color=auto" : have_color ? "--color=yes" : "--color=no") + -`, "OPENBLAS_NUM_THREADS" => 1, "JULIA_NUM_THREADS" => 1), stderr = internal_stderr, stdout = internal_stdout), @@ -3130,7 +3142,7 @@ function compilecache(pkg::PkgId, path::String, internal_stderr::IO = stderr, in close(tmpio_o) close(tmpio_so) end - p = create_expr_cache(pkg, path, tmppath, tmppath_o, concrete_deps, flags, internal_stderr, internal_stdout, isext) + p = create_expr_cache(pkg, path, tmppath, tmppath_o, concrete_deps, flags, cacheflags, internal_stderr, internal_stdout, isext) if success(p) if cache_objects @@ -4135,5 +4147,5 @@ end precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), Nothing)) || @assert false precompile(include_package_for_output, (PkgId, String, Vector{String}, Vector{String}, Vector{String}, typeof(_concrete_dependencies), String)) || @assert false -precompile(create_expr_cache, (PkgId, String, String, String, typeof(_concrete_dependencies), Cmd, IO, IO)) || @assert false -precompile(create_expr_cache, (PkgId, String, String, Nothing, typeof(_concrete_dependencies), Cmd, IO, IO)) || @assert false +precompile(create_expr_cache, (PkgId, String, String, String, typeof(_concrete_dependencies), CacheFlags, Cmd, IO, IO)) || @assert false +precompile(create_expr_cache, (PkgId, String, String, Nothing, typeof(_concrete_dependencies), CacheFlags, Cmd, IO, IO)) || @assert false diff --git a/pkgimage.mk b/pkgimage.mk index 0bc035ee03b089..78b2618be549f0 100644 --- a/pkgimage.mk +++ b/pkgimage.mk @@ -25,7 +25,8 @@ print-depot-path: @$(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no -e '@show Base.DEPOT_PATH') $(BUILDDIR)/stdlib/%.image: $(JULIAHOME)/stdlib/Project.toml $(JULIAHOME)/stdlib/Manifest.toml $(INDEPENDENT_STDLIBS_SRCS) $(JULIA_DEPOT_PATH)/compiled - @$(call PRINT_JULIA, JULIA_CPU_TARGET="$(JULIA_CPU_TARGET)" $(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no -e 'Base.Precompilation.precompilepkgs(;configs=[``=>Base.CacheFlags(), `--check-bounds=yes`=>Base.CacheFlags(;check_bounds=1)])') + @$(call PRINT_JULIA, JULIA_CPU_TARGET="$(JULIA_CPU_TARGET)" $(call spawn,$(JULIA_EXECUTABLE)) --startup-file=no -e \ + 'Base.Precompilation.precompilepkgs(configs=[``=>Base.CacheFlags(debug_level=2, opt_level=3), ``=>Base.CacheFlags(check_bounds=1, debug_level=2, opt_level=3)])') touch $@ $(BUILDDIR)/stdlib/release.image: $(build_private_libdir)/sys.$(SHLIB_EXT) diff --git a/test/loading.jl b/test/loading.jl index 9e7e40ff3b50aa..f359efff981c04 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -1225,10 +1225,7 @@ end @test cf.check_bounds == 3 @test cf.inline @test cf.opt_level == 3 - - io = PipeBuffer() - show(io, cf) - @test read(io, String) == "use_pkgimages = true, debug_level = 3, check_bounds = 3, inline = true, opt_level = 3" + @test repr(cf) == "CacheFlags(; use_pkgimages=true, debug_level=3, check_bounds=3, inline=true, opt_level=3)" end empty!(Base.DEPOT_PATH)