From 74301e46d5cba5b9af24e79f70da6d09e701eddc Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Thu, 4 Jan 2024 22:48:57 +0100 Subject: [PATCH 1/6] remove test case that was made obsolote by #51989 --- test/relocatedepot.jl | 15 --------------- 1 file changed, 15 deletions(-) diff --git a/test/relocatedepot.jl b/test/relocatedepot.jl index 7aaeda1174e29..e326920527b93 100644 --- a/test/relocatedepot.jl +++ b/test/relocatedepot.jl @@ -95,21 +95,6 @@ if !test_relocated_depot else - # must come before any of the load tests, because the will recompile and generate new cache files - @testset "attempt loading precompiled pkgs when depot is missing" begin - test_harness() do - empty!(LOAD_PATH) - push!(LOAD_PATH, joinpath(@__DIR__, "relocatedepot")) - for pkgname in ("RelocationTestPkg1", "RelocationTestPkg2") - pkg = Base.identify_package(pkgname) - cachefile = only(Base.find_all_in_cache_path(pkg)) - @test_throws ArgumentError(""" - Failed to determine depot from srctext files in cache file $cachefile. - - Make sure you have adjusted DEPOT_PATH in case you relocated depots.""") Base.isprecompiled(pkg) - end - end - end - @testset "load stdlib from test/relocatedepot" begin test_harness() do push!(LOAD_PATH, joinpath(@__DIR__, "relocatedepot")) From b7266a8952766ef7f9fb665c59aa9e8505ace306 Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Thu, 4 Jan 2024 23:04:31 +0100 Subject: [PATCH 2/6] refactor @depot tag resolution logic Allow include() and include_dependency() files to resolve to different depots. We use one and the same depot for all include() files, but include_dependency() files can each resolve to a different depot. Co-authored-by: staticfloat staticfloat@gmail.com --- base/loading.jl | 60 ++++++++++++++++++++++++++----------------------- 1 file changed, 32 insertions(+), 28 deletions(-) diff --git a/base/loading.jl b/base/loading.jl index fdb6b0e8fff57..351af58673619 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -2778,22 +2778,10 @@ function restore_depot_path(path::AbstractString, depot::AbstractString) replace(path, r"^@depot" => depot; count=1) end -# Find depot in DEPOT_PATH for which all @depot tags from the `includes` -# can be replaced so that they point to a file on disk each. -function resolve_depot(includes::Union{AbstractVector,AbstractSet}) - # `all` because it's possible to have a mixture of includes inside and outside of the depot - if all(includes) do inc - !startswith(inc, "@depot") - end - return :fully_outside_depot - end +function resolve_depot(inc::AbstractString) + startswith(inc, "@depot") || return :not_relocatable for depot in DEPOT_PATH - # `any` because it's possible to have a mixture of includes inside and outside of the depot - if any(includes) do inc - isfile(restore_depot_path(inc, depot)) - end - return depot - end + isfile(restore_depot_path(inc, depot)) && return depot end return :no_depot_found end @@ -2881,25 +2869,41 @@ function parse_cache_header(f::IO, cachefile::AbstractString) l = read(f, Int32) clone_targets = read(f, l) - # determine path for @depot replacement from srctext files only, e.g. ignore any include_dependency files srcfiles = srctext_files(f, srctextpos, includes) - depot = resolve_depot(srcfiles) - keepidx = Int[] - for (i, chi) in enumerate(includes) - chi.filename ∈ srcfiles && push!(keepidx, i) - end - if depot === :no_depot_found - @debug("Unable to resolve @depot tag in cache file $cachefile", srcfiles) - elseif depot === :fully_outside_depot - @debug("All include dependencies in cache file $cachefile are outside of a depot.", srcfiles) + + includes_srcfiles = CacheHeaderIncludes[] + includes_depfiles = CacheHeaderIncludes[] + for (i, inc) in enumerate(includes) + if inc.filename ∈ srcfiles + push!(includes_srcfiles, inc) + else + push!(includes_depfiles, inc) + end + end + + # determine depot for @depot replacement for include() files and include_dependency() files separately + srcfiles_depot = resolve_depot(first(srcfiles)) + if srcfiles_depot === :no_depot_found + @debug("Unable to resolve @depot tag include() files from cache file $cachefile", srcfiles) + elseif srcfiles_depot === :not_relocatable + @debug("include() files from $cachefile are not relocatable", srcfiles) else - for inc in includes + for inc in includes_srcfiles + inc.filename = restore_depot_path(inc.filename, srcfiles_depot) + end + end + for inc in includes_depfiles + depot = resolve_depot(inc.filename) + if depot === :no_depot_found + @debug("Unable to resolve @depot tag for include_dependency() file $(inc.filename) from cache file $cachefile", srcfiles) + elseif depot === :not_relocatable + @debug("include_dependency() file $(inc.filename) from $cachefile is not relocatable", srcfiles) + else inc.filename = restore_depot_path(inc.filename, depot) end end - includes_srcfiles_only = includes[keepidx] - return modules, (includes, includes_srcfiles_only, requires), required_modules, srctextpos, prefs, prefs_hash, clone_targets, flags + return modules, (includes, includes_srcfiles, requires), required_modules, srctextpos, prefs, prefs_hash, clone_targets, flags end function parse_cache_header(cachefile::String) From 84a4772d74c774e57a3d72f689256e3011bfdc73 Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Sat, 6 Jan 2024 13:36:46 +0100 Subject: [PATCH 3/6] test: non-existent include_dependency() files are not relocatable --- test/precompile.jl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/test/precompile.jl b/test/precompile.jl index a717d543bc0ce..d87e86ab4911c 100644 --- a/test/precompile.jl +++ b/test/precompile.jl @@ -436,7 +436,8 @@ precompile_test_harness(false) do dir modules, (deps, _, requires), required_modules, _... = Base.parse_cache_header(cachefile) discard_module = mod_fl_mt -> mod_fl_mt.filename @test modules == [ Base.PkgId(Foo) => Base.module_build_id(Foo) % UInt64 ] - @test map(x -> x.filename, deps) == [ Foo_file, joinpath(dir, "foo.jl"), joinpath(dir, "bar.jl") ] + # foo.jl and bar.jl are never written to disk, so they are not relocatable + @test map(x -> x.filename, deps) == [ Foo_file, joinpath("@depot", "foo.jl"), joinpath("@depot", "bar.jl") ] @test requires == [ Base.PkgId(Foo) => Base.PkgId(string(FooBase_module)), Base.PkgId(Foo) => Base.PkgId(Foo2), Base.PkgId(Foo) => Base.PkgId(Test), From edb66b8b1462eff10683e05e21243cefb79a12cc Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Sat, 6 Jan 2024 14:20:40 +0100 Subject: [PATCH 4/6] add test for #52161 --- test/relocatedepot.jl | 52 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 52 insertions(+) diff --git a/test/relocatedepot.jl b/test/relocatedepot.jl index e326920527b93..5dfa636603bb4 100644 --- a/test/relocatedepot.jl +++ b/test/relocatedepot.jl @@ -93,6 +93,58 @@ if !test_relocated_depot end end + @testset "#52161" begin + # Take the src files from two RelocationTestPkg1 and RelocationTestPkg2, + # which are each located in depot1 and depot2, respectively, and + # add them as include_dependency()s to a new pkg Foo, which will be precompiled into depot3. + # After loading the include_dependency()s of Foo should refer to depot1 depot2 each. + test_harness() do + mktempdir() do depot1 + # precompile RelocationTestPkg1 in depot1 + cp(joinpath(@__DIR__,"RelocationTestPkg1"),joinpath(depot1,"RelocationTestPkg1")) + pushfirst!(LOAD_PATH, depot1) + pushfirst!(DEPOT_PATH, depot1) + pkg = Base.identify_package("RelocationTestPkg1") + Base.require(pkg) + mktempdir() do depot2 + cp(joinpath(@__DIR__,"RelocationTestPkg2"),joinpath(depot2,"RelocationTestPkg2")) + # precompile RelocationTestPkg2 in depot2 + pushfirst!(LOAD_PATH, depot2) + pushfirst!(DEPOT_PATH, depot2) + pkg = Base.identify_package("RelocationTestPkg2") + Base.require(pkg) + # precompile Foo into in depot3 + mktempdir() do depot3 + pushfirst!(LOAD_PATH, depot3) + pushfirst!(DEPOT_PATH, depot3) + foofile = joinpath(depot3, "Foo.jl") + write(foofile, """ + module Foo + using RelocationTestPkg1 + using RelocationTestPkg2 + srcfile1 = joinpath(pkgdir(RelocationTestPkg1), "src", "RelocationTestPkg1.jl") + srcfile2 = joinpath(pkgdir(RelocationTestPkg2), "src", "RelocationTestPkg2.jl") + @show srcfile1 + @show srcfile2 + include_dependency(srcfile1) + include_dependency(srcfile2) + end + """) + pkg = Base.identify_package("Foo") + Base.require(pkg) + cachefile = joinpath(depot3, "compiled", "v1.11", "Foo.ji") + _, (deps, _, _), _... = Base.parse_cache_header(cachefile) + @test map(x -> x.filename, deps) == + [ joinpath(depot3, "Foo.jl"), + joinpath(depot1, "RelocationTestPkg1", "src", "RelocationTestPkg1.jl"), + joinpath(depot2, "RelocationTestPkg2", "src", "RelocationTestPkg2.jl") ] + end + end + end + end + end + + else @testset "load stdlib from test/relocatedepot" begin From 7e0e7d4babb5c76dcbaf56195e9c3bab05d70bf3 Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Sun, 7 Jan 2024 20:43:03 +0100 Subject: [PATCH 5/6] update test to use temporarily generated Example pkgs to avoid loading of already loaded pkg fixup version tag; use open() instead of write() --- test/relocatedepot.jl | 76 ++++++++++++++++++++++++++++++------------- 1 file changed, 53 insertions(+), 23 deletions(-) diff --git a/test/relocatedepot.jl b/test/relocatedepot.jl index 5dfa636603bb4..257708aa23d7a 100644 --- a/test/relocatedepot.jl +++ b/test/relocatedepot.jl @@ -94,50 +94,80 @@ if !test_relocated_depot end @testset "#52161" begin - # Take the src files from two RelocationTestPkg1 and RelocationTestPkg2, + # Take the src files from two pkgs Example1 and Example2, # which are each located in depot1 and depot2, respectively, and # add them as include_dependency()s to a new pkg Foo, which will be precompiled into depot3. # After loading the include_dependency()s of Foo should refer to depot1 depot2 each. test_harness() do mktempdir() do depot1 - # precompile RelocationTestPkg1 in depot1 - cp(joinpath(@__DIR__,"RelocationTestPkg1"),joinpath(depot1,"RelocationTestPkg1")) - pushfirst!(LOAD_PATH, depot1) - pushfirst!(DEPOT_PATH, depot1) - pkg = Base.identify_package("RelocationTestPkg1") - Base.require(pkg) + # precompile Example in depot1 + example1_root = joinpath(depot1, "Example1") + mkpath(joinpath(example1_root, "src")) + open(joinpath(example1_root, "src", "Example1.jl"); write=true) do io + println(io, """ + module Example1 + greet() = println("Hello from Example1!") + end + """) + end + open(joinpath(example1_root, "Project.toml"); write=true) do io + println(io, """ + name = "Example1" + uuid = "00000000-0000-0000-0000-000000000001" + version = "1.0.0" + """) + end + pushfirst!(LOAD_PATH, depot1); pushfirst!(DEPOT_PATH, depot1) + pkg = Base.identify_package("Example1"); Base.require(pkg) mktempdir() do depot2 - cp(joinpath(@__DIR__,"RelocationTestPkg2"),joinpath(depot2,"RelocationTestPkg2")) - # precompile RelocationTestPkg2 in depot2 + # precompile Example in depot2 + example2_root = joinpath(depot2, "Example2") + mkpath(joinpath(example2_root, "src")) + open(joinpath(example2_root, "src", "Example2.jl"); write=true) do io + println(io, """ + module Example2 + greet() = println("Hello from Example2!") + end + """) + end + open(joinpath(example2_root, "Project.toml"); write=true) do io + println(io, """ + name = "Example2" + uuid = "00000000-0000-0000-0000-000000000002" + version = "1.0.0" + """) + end pushfirst!(LOAD_PATH, depot2) pushfirst!(DEPOT_PATH, depot2) - pkg = Base.identify_package("RelocationTestPkg2") + pkg = Base.identify_package("Example2") Base.require(pkg) - # precompile Foo into in depot3 mktempdir() do depot3 - pushfirst!(LOAD_PATH, depot3) - pushfirst!(DEPOT_PATH, depot3) - foofile = joinpath(depot3, "Foo.jl") - write(foofile, """ + # precompile Foo in depot3 + open(joinpath(depot3, "Foo.jl"), write=true) do io + println(io, """ module Foo - using RelocationTestPkg1 - using RelocationTestPkg2 - srcfile1 = joinpath(pkgdir(RelocationTestPkg1), "src", "RelocationTestPkg1.jl") - srcfile2 = joinpath(pkgdir(RelocationTestPkg2), "src", "RelocationTestPkg2.jl") + using Example1 + using Example2 + srcfile1 = joinpath(pkgdir(Example1), "src", "Example1.jl") + srcfile2 = joinpath(pkgdir(Example2), "src", "Example2.jl") @show srcfile1 @show srcfile2 include_dependency(srcfile1) include_dependency(srcfile2) end - """) + """) + end + pushfirst!(LOAD_PATH, depot3) + pushfirst!(DEPOT_PATH, depot3) pkg = Base.identify_package("Foo") Base.require(pkg) - cachefile = joinpath(depot3, "compiled", "v1.11", "Foo.ji") + cachefile = joinpath(depot3, "compiled", + "v$(VERSION.major).$(VERSION.minor)", "Foo.ji") _, (deps, _, _), _... = Base.parse_cache_header(cachefile) @test map(x -> x.filename, deps) == [ joinpath(depot3, "Foo.jl"), - joinpath(depot1, "RelocationTestPkg1", "src", "RelocationTestPkg1.jl"), - joinpath(depot2, "RelocationTestPkg2", "src", "RelocationTestPkg2.jl") ] + joinpath(depot1, "Example1", "src", "Example1.jl"), + joinpath(depot2, "Example2", "src", "Example2.jl") ] end end end From d29b365dda4b52db9690f83e14ce992905d36f9d Mon Sep 17 00:00:00 2001 From: Florian Atteneder Date: Sun, 7 Jan 2024 23:39:03 +0100 Subject: [PATCH 6/6] relocation test: tune load_path/depot_path setup to better exercise functionality --- test/Makefile | 4 ++-- test/relocatedepot.jl | 22 +++++++++++++--------- 2 files changed, 15 insertions(+), 11 deletions(-) diff --git a/test/Makefile b/test/Makefile index c3b877793f4b5..d2d6b39747cae 100644 --- a/test/Makefile +++ b/test/Makefile @@ -43,7 +43,7 @@ relocatedepot: @cp -R $(SRCDIR)/RelocationTestPkg1 $(SRCDIR)/relocatedepot @cp -R $(SRCDIR)/RelocationTestPkg2 $(SRCDIR)/relocatedepot @cd $(SRCDIR) && \ - $(call PRINT_JULIA, $(call spawn,JULIA_DEBUG=loading RELOCATEDEPOT="" JULIA_DEPOT_PATH=$(SRCDIR)/relocatedepot/julia $(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl $@) + $(call PRINT_JULIA, $(call spawn,JULIA_DEBUG=loading RELOCATEDEPOT="" $(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl $@) revise-relocatedepot: revise-% : @rm -rf $(SRCDIR)/relocatedepot @@ -54,7 +54,7 @@ revise-relocatedepot: revise-% : @cp -R $(SRCDIR)/RelocationTestPkg1 $(SRCDIR)/relocatedepot @cp -R $(SRCDIR)/RelocationTestPkg2 $(SRCDIR)/relocatedepot @cd $(SRCDIR) && \ - $(call PRINT_JULIA, $(call spawn,JULIA_DEBUG=loading RELOCATEDEPOT="" JULIA_DEPOT_PATH=$(SRCDIR)/relocatedepot/julia $(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl --revise $*) + $(call PRINT_JULIA, $(call spawn,JULIA_DEBUG=loading RELOCATEDEPOT="" $(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl --revise $*) embedding: @$(MAKE) -C $(SRCDIR)/$@ check $(EMBEDDING_ARGS) diff --git a/test/relocatedepot.jl b/test/relocatedepot.jl index 257708aa23d7a..2f33b8503670d 100644 --- a/test/relocatedepot.jl +++ b/test/relocatedepot.jl @@ -5,10 +5,12 @@ using Logging include("testenv.jl") -function test_harness(@nospecialize(fn)) +function test_harness(@nospecialize(fn); empty_load_path=true, empty_depot_path=true) load_path = copy(LOAD_PATH) depot_path = copy(DEPOT_PATH) try + empty_load_path && empty!(LOAD_PATH) + empty_depot_path && empty!(DEPOT_PATH) fn() finally copy!(LOAD_PATH, load_path) @@ -66,9 +68,9 @@ if !test_relocated_depot @testset "precompile RelocationTestPkg1" begin pkgname = "RelocationTestPkg1" - test_harness() do + test_harness(empty_depot_path=false) do push!(LOAD_PATH, @__DIR__) - push!(DEPOT_PATH, @__DIR__) + push!(DEPOT_PATH, @__DIR__) # required to make relocatable, but cache is written to DEPOT_PATH[1] pkg = Base.identify_package(pkgname) cachefiles = Base.find_all_in_cache_path(pkg) rm.(cachefiles, force=true) @@ -80,9 +82,9 @@ if !test_relocated_depot @testset "precompile RelocationTestPkg2 (contains include_dependency)" begin pkgname = "RelocationTestPkg2" - test_harness() do + test_harness(empty_depot_path=false) do push!(LOAD_PATH, @__DIR__) - push!(DEPOT_PATH, string(@__DIR__, "/")) + push!(DEPOT_PATH, @__DIR__) # required to make relocatable, but cache is written to DEPOT_PATH[1] pkg = Base.identify_package(pkgname) cachefiles = Base.find_all_in_cache_path(pkg) rm.(cachefiles, force=true) @@ -179,8 +181,8 @@ else @testset "load stdlib from test/relocatedepot" begin test_harness() do - push!(LOAD_PATH, joinpath(@__DIR__, "relocatedepot")) - push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot")) + push!(LOAD_PATH, "@stdlib") + push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot", "julia")) # stdlib should be already precompiled pkg = Base.identify_package("DelimitedFiles") @test Base.isprecompiled(pkg) == true @@ -191,7 +193,8 @@ else pkgname = "RelocationTestPkg1" test_harness() do push!(LOAD_PATH, joinpath(@__DIR__, "relocatedepot")) - push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot")) + push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot")) # required to find src files + push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot", "julia")) # contains cache file pkg = Base.identify_package(pkgname) @test Base.isprecompiled(pkg) == true Base.require(pkg) # re-precompile @@ -203,7 +206,8 @@ else pkgname = "RelocationTestPkg2" test_harness() do push!(LOAD_PATH, joinpath(@__DIR__, "relocatedepot")) - push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot")) + push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot")) # required to find src files + push!(DEPOT_PATH, joinpath(@__DIR__, "relocatedepot", "julia")) # contains cache file pkg = Base.identify_package(pkgname) @test Base.isprecompiled(pkg) == false # moving depot changes mtime of include_dependency Base.require(pkg) # re-precompile