From 4a2c59333946c5450c73f4c5ba7d2d28529db8de Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Sat, 23 Mar 2024 22:32:02 +0100 Subject: [PATCH] add code loading + precompilation support for workspaces (#53653) This is similar to workspaces in cargo where multiple projects share a manifest https://doc.rust-lang.org/book/ch14-03-cargo-workspaces.html and upon resolving the dependencies and compat of all projects in the workspace is adhered to. The idea is to use this for e.g. test, doc environments where you want to "overlay" a dependency graph on top of a base one. The code change in Base adds support for the code loading and precompilation part of this, those changes are: - Finding the manifest from any active project in the workspace - Merge preferences among projects in a workspace. - Allowing one to pass `manifest=true` to `precompilepkgs` to compile every package in the manifest. - The effect of giving no packages to `precompilepkgs` was changed from compiling all packages in the manifest to only those in the active project (which is equivalent in case of no workspace being used but different when it is used). --- base/loading.jl | 69 ++++++++++++++++--- base/precompilation.jl | 50 ++++++++------ doc/src/manual/code-loading.md | 22 ++++++ test/loading.jl | 58 ++++++++++++++++ test/project/SubProject/Devved/Project.toml | 3 + test/project/SubProject/Devved/src/Devved.jl | 5 ++ test/project/SubProject/Devved2/Project.toml | 3 + .../project/SubProject/Devved2/src/Devved2.jl | 5 ++ test/project/SubProject/Manifest.toml | 68 ++++++++++++++++++ .../SubProject/PackageThatIsSub/Project.toml | 14 ++++ .../PackageThatIsSub/src/PackageThatIsSub.jl | 5 ++ .../PackageThatIsSub/test/Project.toml | 8 +++ test/project/SubProject/Project.toml | 13 ++++ test/project/SubProject/src/MyPkg.jl | 3 + test/project/SubProject/sub/Project.toml | 3 + test/project/SubProject/test/Project.toml | 4 ++ 16 files changed, 304 insertions(+), 29 deletions(-) create mode 100644 test/project/SubProject/Devved/Project.toml create mode 100644 test/project/SubProject/Devved/src/Devved.jl create mode 100644 test/project/SubProject/Devved2/Project.toml create mode 100644 test/project/SubProject/Devved2/src/Devved2.jl create mode 100644 test/project/SubProject/Manifest.toml create mode 100644 test/project/SubProject/PackageThatIsSub/Project.toml create mode 100644 test/project/SubProject/PackageThatIsSub/src/PackageThatIsSub.jl create mode 100644 test/project/SubProject/PackageThatIsSub/test/Project.toml create mode 100644 test/project/SubProject/Project.toml create mode 100644 test/project/SubProject/src/MyPkg.jl create mode 100644 test/project/SubProject/sub/Project.toml create mode 100644 test/project/SubProject/test/Project.toml diff --git a/base/loading.jl b/base/loading.jl index 986efc1e1e5fb..fd8eba6bed460 100644 --- a/base/loading.jl +++ b/base/loading.jl @@ -611,6 +611,23 @@ function env_project_file(env::String)::Union{Bool,String} end end +function base_project(project_file) + base_dir = abspath(joinpath(dirname(project_file), "..")) + base_project_file = env_project_file(base_dir) + base_project_file isa String || return nothing + d = parsed_toml(base_project_file) + workspace = get(d, "workspace", nothing)::Union{Dict{String, Any}, Nothing} + if workspace === nothing + return nothing + end + projects = get(workspace, "projects", nothing)::Union{Vector{String}, Nothing, String} + projects === nothing && return nothing + if projects isa Vector && basename(dirname(project_file)) in projects + return base_project_file + end + return nothing +end + function project_deps_get(env::String, name::String)::Union{Nothing,PkgId} project_file = env_project_file(env) if project_file isa String @@ -622,21 +639,27 @@ function project_deps_get(env::String, name::String)::Union{Nothing,PkgId} return nothing end +function package_get(project_file, where::PkgId, name::String) + proj = project_file_name_uuid(project_file, where.name) + if proj == where + # if `where` matches the project, use [deps] section as manifest, and stop searching + pkg_uuid = explicit_project_deps_get(project_file, name) + return PkgId(pkg_uuid, name) + end + return nothing +end + function manifest_deps_get(env::String, where::PkgId, name::String)::Union{Nothing,PkgId} uuid = where.uuid @assert uuid !== nothing project_file = env_project_file(env) if project_file isa String - # first check if `where` names the Project itself - proj = project_file_name_uuid(project_file, where.name) - if proj == where - # if `where` matches the project, use [deps] section as manifest, and stop searching - pkg_uuid = explicit_project_deps_get(project_file, name) - return PkgId(pkg_uuid, name) - end + pkg = package_get(project_file, where, name) + pkg === nothing || return pkg d = parsed_toml(project_file) exts = get(d, "extensions", nothing)::Union{Dict{String, Any}, Nothing} if exts !== nothing + proj = project_file_name_uuid(project_file, where.name) # Check if `where` is an extension of the project if where.name in keys(exts) && where.uuid == uuid5(proj.uuid::UUID, where.name) # Extensions can load weak deps... @@ -726,6 +749,14 @@ function project_file_path(project_file::String) joinpath(dirname(project_file), get(d, "path", "")::String) end +function workspace_manifest(project_file) + base = base_project(project_file) + if base !== nothing + return project_file_manifest_path(base) + end + return nothing +end + # find project file's corresponding manifest file function project_file_manifest_path(project_file::String)::Union{Nothing,String} @lock require_lock begin @@ -736,6 +767,10 @@ function project_file_manifest_path(project_file::String)::Union{Nothing,String} end dir = abspath(dirname(project_file)) d = parsed_toml(project_file) + base_manifest = workspace_manifest(project_file) + if base_manifest !== nothing + return base_manifest + end explicit_manifest = get(d, "manifest", nothing)::Union{String, Nothing} manifest_path = nothing if explicit_manifest !== nothing @@ -3355,9 +3390,27 @@ function recursive_prefs_merge(base::Dict{String, Any}, overrides::Dict{String, return new_base end +function get_projects_workspace_to_root(project_file) + projects = String[project_file] + while true + project_file = base_project(project_file) + if project_file === nothing + return projects + end + push!(projects, project_file) + end +end + function get_preferences(uuid::Union{UUID,Nothing} = nothing) merged_prefs = Dict{String,Any}() - for env in reverse(load_path()) + loadpath = load_path() + projects_to_merge_prefs = String[] + append!(projects_to_merge_prefs, Iterators.drop(loadpath, 1)) + if length(loadpath) >= 1 + prepend!(projects_to_merge_prefs, get_projects_workspace_to_root(first(loadpath))) + end + + for env in reverse(projects_to_merge_prefs) project_toml = env_project_file(env) if !isa(project_toml, String) continue diff --git a/base/precompilation.jl b/base/precompilation.jl index 50b84ad829d0a..a823b2b579852 100644 --- a/base/precompilation.jl +++ b/base/precompilation.jl @@ -1,7 +1,8 @@ module Precompilation using Base: PkgId, UUID, SHA1, parsed_toml, project_file_name_uuid, project_names, - project_file_manifest_path, get_deps, preferences_names, isaccessibledir, isfile_casesensitive + project_file_manifest_path, get_deps, preferences_names, isaccessibledir, isfile_casesensitive, + base_project # This is currently only used for pkgprecompile but the plan is to use this in code loading in the future # see the `kc/codeloading2.0` branch @@ -59,6 +60,19 @@ function ExplicitEnv(envpath::String=Base.active_project()) delete!(project_deps, name) end + # This project might be a package, in that case, that is also a "dependency" + # of the project. + proj_name = get(project_d, "name", nothing)::Union{String, Nothing} + _proj_uuid = get(project_d, "uuid", nothing)::Union{String, Nothing} + proj_uuid = _proj_uuid === nothing ? nothing : UUID(_proj_uuid) + + project_is_package = proj_name !== nothing && proj_uuid !== nothing + if project_is_package + # TODO: Error on missing uuid? + project_deps[proj_name] = UUID(proj_uuid) + names[UUID(proj_uuid)] = proj_name + end + project_extensions = Dict{String, Vector{UUID}}() # Collect all extensions of the project for (name, triggers::Union{String, Vector{String}}) in get(Dict{String, Any}, project_d, "extensions")::Dict{String, Any} @@ -76,18 +90,6 @@ function ExplicitEnv(envpath::String=Base.active_project()) project_extensions[name] = uuids end - # This project might be a package, in that case, that is also a "dependency" - # of the project. - proj_name = get(project_d, "name", nothing)::Union{String, Nothing} - _proj_uuid = get(project_d, "uuid", nothing)::Union{String, Nothing} - proj_uuid = _proj_uuid === nothing ? nothing : UUID(_proj_uuid) - - if proj_name !== nothing && proj_uuid !== nothing - # TODO: Error on missing uuid? - project_deps[proj_name] = UUID(proj_uuid) - names[UUID(proj_uuid)] = proj_name - end - manifest = project_file_manifest_path(envpath) manifest_d = manifest === nothing ? Dict{String, Any}() : parsed_toml(manifest) @@ -355,8 +357,8 @@ function precompilepkgs(pkgs::Vector{String}=String[]; configs::Union{Config,Vector{Config}}=(``=>Base.CacheFlags()), io::IO=stderr, # asking for timing disables fancy mode, as timing is shown in non-fancy mode - fancyprint::Bool = can_fancyprint(io) && !timing - ) + fancyprint::Bool = can_fancyprint(io) && !timing, + manifest::Bool=false,) configs = configs isa Config ? [configs] : configs @@ -512,9 +514,15 @@ function precompilepkgs(pkgs::Vector{String}=String[]; end @debug "precompile: circular dep check done" - # if a list of packages is given, restrict to dependencies of given packages - if !isempty(pkgs) - function collect_all_deps(depsmap, dep, alldeps=Set{Base.PkgId}()) + if !manifest + if isempty(pkgs) + pkgs = [pkg.name for pkg in direct_deps] + target = "all packages" + else + target = join(pkgs, ", ") + end + # restrict to dependencies of given packages + function collect_all_deps(depsmap, dep, alldeps=Set{Base.PkgId}()) for _dep in depsmap[dep] if !(_dep in alldeps) push!(alldeps, _dep) @@ -544,13 +552,13 @@ function precompilepkgs(pkgs::Vector{String}=String[]; # TODO: actually handle packages from other envs in the stack return else - error("No direct dependencies outside of the sysimage found matching $(repr(pkgs))") + return end end - target = join(pkgs, ", ") else - target = "project" + target = "manifest" end + nconfigs = length(configs) if nconfigs == 1 if !isempty(only(configs)[1]) diff --git a/doc/src/manual/code-loading.md b/doc/src/manual/code-loading.md index 25ebf3475f34c..9e753d2a7de33 100644 --- a/doc/src/manual/code-loading.md +++ b/doc/src/manual/code-loading.md @@ -394,6 +394,28 @@ are stored in the manifest file in the section for that package. The dependency a package are the same as for its "parent" except that the listed extension dependencies are also considered as dependencies. +### [Workspaces](@id workspaces) + +A project file can define a workspace by giving a set of projects that is part of that workspace: + +```toml +[workspace] +projects = ["test", "benchmarks", "docs", "SomePackage"] +``` + +Each subfolder contains its own `Project.toml` file, which may include additional dependencies and compatibility constraints. In such cases, the package manager gathers all dependency information from all the projects in the workspace generating a single manifest file that combines the versions of all dependencies. + +Furthermore, workspaces can be "nested", meaning a project defining a workspace can also be part of another workspace. In this scenario, a single manifest file is still utilized, stored alongside the "root project" (the project that doesn't have another workspace including it). An example file structure could look like this: + +``` +Project.toml # projects = ["MyPackage"] +Manifest.toml +MyPackage/ + Project.toml # projects = ["test"] + test/ + Project.toml +``` + ### [Package/Environment Preferences](@id preferences) Preferences are dictionaries of metadata that influence package behavior within an environment. diff --git a/test/loading.jl b/test/loading.jl index 8ba2cf3026120..872ccd197f0f8 100644 --- a/test/loading.jl +++ b/test/loading.jl @@ -1550,3 +1550,61 @@ end rot13proj = joinpath(@__DIR__, "project", "Rot13") @test readchomp(`$(Base.julia_cmd()) --startup-file=no --project=$rot13proj -m Rot13 --project nowhere ABJURER`) == "--cebwrpg abjurer NOWHERE " end + +@testset "workspace loading" begin + old_load_path = copy(LOAD_PATH) + try + empty!(LOAD_PATH) + push!(LOAD_PATH, joinpath(@__DIR__, "project", "SubProject")) + @test Base.get_preferences()["value"] == 1 + @test Base.get_preferences()["x"] == 1 + + empty!(LOAD_PATH) + push!(LOAD_PATH, joinpath(@__DIR__, "project", "SubProject", "sub")) + id = Base.identify_package("Devved") + @test isfile(Base.locate_package(id)) + @test Base.identify_package("Devved2") === nothing + id3 = Base.identify_package("MyPkg") + @test isfile(Base.locate_package(id3)) + + empty!(LOAD_PATH) + push!(LOAD_PATH, joinpath(@__DIR__, "project", "SubProject", "PackageThatIsSub")) + id_pkg = Base.identify_package("PackageThatIsSub") + @test Base.identify_package(id_pkg, "Devved") === nothing + id_dev2 = Base.identify_package(id_pkg, "Devved2") + @test isfile(Base.locate_package(id_dev2)) + id_mypkg = Base.identify_package("MyPkg") + @test isfile(Base.locate_package(id_mypkg)) + id_dev = Base.identify_package(id_mypkg, "Devved") + @test isfile(Base.locate_package(id_dev)) + @test Base.get_preferences()["value"] == 2 + @test Base.get_preferences()["x"] == 1 + @test Base.get_preferences()["y"] == 2 + + empty!(LOAD_PATH) + push!(LOAD_PATH, joinpath(@__DIR__, "project", "SubProject", "PackageThatIsSub", "test")) + id_pkg = Base.identify_package("PackageThatIsSub") + @test isfile(Base.locate_package(id_pkg)) + @test Base.identify_package(id_pkg, "Devved") === nothing + id_dev2 = Base.identify_package(id_pkg, "Devved2") + @test isfile(Base.locate_package(id_dev2)) + id_mypkg = Base.identify_package("MyPkg") + @test isfile(Base.locate_package(id_mypkg)) + id_dev = Base.identify_package(id_mypkg, "Devved") + @test isfile(Base.locate_package(id_dev)) + @test Base.get_preferences()["value"] == 3 + @test Base.get_preferences()["x"] == 1 + @test Base.get_preferences()["y"] == 2 + @test Base.get_preferences()["z"] == 3 + + empty!(LOAD_PATH) + push!(LOAD_PATH, joinpath(@__DIR__, "project", "SubProject", "test")) + id_mypkg = Base.identify_package("MyPkg") + id_dev = Base.identify_package(id_mypkg, "Devved") + @test isfile(Base.locate_package(id_dev)) + @test Base.identify_package("Devved2") === nothing + + finally + copy!(LOAD_PATH, old_load_path) + end +end diff --git a/test/project/SubProject/Devved/Project.toml b/test/project/SubProject/Devved/Project.toml new file mode 100644 index 0000000000000..63088a132cb77 --- /dev/null +++ b/test/project/SubProject/Devved/Project.toml @@ -0,0 +1,3 @@ +name = "Devved" +uuid = "cbce3a6e-7a3d-4e84-8e6d-b87208df7599" +version = "0.1.0" diff --git a/test/project/SubProject/Devved/src/Devved.jl b/test/project/SubProject/Devved/src/Devved.jl new file mode 100644 index 0000000000000..f3eb267409ece --- /dev/null +++ b/test/project/SubProject/Devved/src/Devved.jl @@ -0,0 +1,5 @@ +module Devved + +greet() = print("Hello World!") + +end # module Devved diff --git a/test/project/SubProject/Devved2/Project.toml b/test/project/SubProject/Devved2/Project.toml new file mode 100644 index 0000000000000..c761630566116 --- /dev/null +++ b/test/project/SubProject/Devved2/Project.toml @@ -0,0 +1,3 @@ +name = "Devved2" +uuid = "08f74b90-50f5-462f-80b9-a72b1258a17b" +version = "0.1.0" diff --git a/test/project/SubProject/Devved2/src/Devved2.jl b/test/project/SubProject/Devved2/src/Devved2.jl new file mode 100644 index 0000000000000..9bd5df2793671 --- /dev/null +++ b/test/project/SubProject/Devved2/src/Devved2.jl @@ -0,0 +1,5 @@ +module Devved2 + +greet() = print("Hello World!") + +end # module Devved2 diff --git a/test/project/SubProject/Manifest.toml b/test/project/SubProject/Manifest.toml new file mode 100644 index 0000000000000..5d791a74652d4 --- /dev/null +++ b/test/project/SubProject/Manifest.toml @@ -0,0 +1,68 @@ +# This file is machine-generated - editing it directly is not advised + +julia_version = "1.12.0-DEV" +manifest_format = "2.0" +project_hash = "620b9377bc807ff657e6618c8ccc24887eb40285" + +[[deps.Base64]] +uuid = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f" +version = "1.11.0" + +[[deps.Devved]] +path = "Devved" +uuid = "cbce3a6e-7a3d-4e84-8e6d-b87208df7599" +version = "0.1.0" + +[[deps.Devved2]] +path = "Devved2" +uuid = "08f74b90-50f5-462f-80b9-a72b1258a17b" +version = "0.1.0" + +[[deps.InteractiveUtils]] +deps = ["Markdown"] +uuid = "b77e0a4c-d291-57a0-90e8-8db25a27a240" +version = "1.11.0" + +[[deps.Logging]] +deps = ["StyledStrings"] +uuid = "56ddb016-857b-54e1-b83d-db4d58db5568" +version = "1.11.0" + +[[deps.Markdown]] +deps = ["Base64"] +uuid = "d6f4376e-aef5-505a-96c1-9c027394607a" +version = "1.11.0" + +[[deps.MyPkg]] +deps = ["Devved", "Devved2"] +path = "." +uuid = "0cafdeb2-d7a2-40d0-8d22-4411fcc2c4ee" +version = "0.0.0" + +[[deps.PackageThatIsSub]] +deps = ["Devved2", "MyPkg"] +path = "PackageThatIsSub" +uuid = "1efb588c-9412-4e40-90a4-710420bd84aa" +version = "0.1.0" + +[[deps.Random]] +deps = ["SHA"] +uuid = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c" +version = "1.11.0" + +[[deps.SHA]] +uuid = "ea8e919c-243c-51af-8825-aaa63cd721ce" +version = "0.7.0" + +[[deps.Serialization]] +uuid = "9e88b42a-f829-5b0c-bbe9-9e923198166b" +version = "1.11.0" + +[[deps.StyledStrings]] +uuid = "f489334b-da3d-4c2e-b8f0-e476e12c162b" +version = "1.11.0" + +[[deps.Test]] +deps = ["InteractiveUtils", "Logging", "Random", "Serialization"] +uuid = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +version = "1.11.0" diff --git a/test/project/SubProject/PackageThatIsSub/Project.toml b/test/project/SubProject/PackageThatIsSub/Project.toml new file mode 100644 index 0000000000000..e41dd998c5a1c --- /dev/null +++ b/test/project/SubProject/PackageThatIsSub/Project.toml @@ -0,0 +1,14 @@ +name = "PackageThatIsSub" +uuid = "1efb588c-9412-4e40-90a4-710420bd84aa" +version = "0.1.0" + +[workspace] +projects = ["test"] + +[deps] +Devved2 = "08f74b90-50f5-462f-80b9-a72b1258a17b" +MyPkg = "0cafdeb2-d7a2-40d0-8d22-4411fcc2c4ee" + +[preferences] +value = 2 +y = 2 diff --git a/test/project/SubProject/PackageThatIsSub/src/PackageThatIsSub.jl b/test/project/SubProject/PackageThatIsSub/src/PackageThatIsSub.jl new file mode 100644 index 0000000000000..7f9ea94ccb156 --- /dev/null +++ b/test/project/SubProject/PackageThatIsSub/src/PackageThatIsSub.jl @@ -0,0 +1,5 @@ +module PackageThatIsSub + +greet() = print("Hello World!") + +end # module PackageThatIsSub diff --git a/test/project/SubProject/PackageThatIsSub/test/Project.toml b/test/project/SubProject/PackageThatIsSub/test/Project.toml new file mode 100644 index 0000000000000..dc8186e2b735e --- /dev/null +++ b/test/project/SubProject/PackageThatIsSub/test/Project.toml @@ -0,0 +1,8 @@ +[deps] +MyPkg = "0cafdeb2-d7a2-40d0-8d22-4411fcc2c4ee" +PackageThatIsSub = "1efb588c-9412-4e40-90a4-710420bd84aa" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" + +[preferences] +value = 3 +z = 3 diff --git a/test/project/SubProject/Project.toml b/test/project/SubProject/Project.toml new file mode 100644 index 0000000000000..dcb84d865ac85 --- /dev/null +++ b/test/project/SubProject/Project.toml @@ -0,0 +1,13 @@ +name = "MyPkg" +uuid = "0cafdeb2-d7a2-40d0-8d22-4411fcc2c4ee" + +[workspace] +projects = ["sub", "PackageThatIsSub", "test"] + +[deps] +Devved = "cbce3a6e-7a3d-4e84-8e6d-b87208df7599" +Devved2 = "08f74b90-50f5-462f-80b9-a72b1258a17b" + +[preferences] +value = 1 +x = 1 diff --git a/test/project/SubProject/src/MyPkg.jl b/test/project/SubProject/src/MyPkg.jl new file mode 100644 index 0000000000000..6d84954645d55 --- /dev/null +++ b/test/project/SubProject/src/MyPkg.jl @@ -0,0 +1,3 @@ +module MyPkg + +end diff --git a/test/project/SubProject/sub/Project.toml b/test/project/SubProject/sub/Project.toml new file mode 100644 index 0000000000000..50aa238e91d57 --- /dev/null +++ b/test/project/SubProject/sub/Project.toml @@ -0,0 +1,3 @@ +[deps] +Devved = "cbce3a6e-7a3d-4e84-8e6d-b87208df7599" +MyPkg = "0cafdeb2-d7a2-40d0-8d22-4411fcc2c4ee" diff --git a/test/project/SubProject/test/Project.toml b/test/project/SubProject/test/Project.toml new file mode 100644 index 0000000000000..b64312e4b1ee2 --- /dev/null +++ b/test/project/SubProject/test/Project.toml @@ -0,0 +1,4 @@ +[deps] +MyPkg = "0cafdeb2-d7a2-40d0-8d22-4411fcc2c4ee" +Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40" +Devved = "cbce3a6e-7a3d-4e84-8e6d-b87208df7599"