RFC: major LOAD_PATH machinery simplification

base/initdefs.jl

@@ -54,77 +54,73 @@ function init_depot_path(BINDIR::String = Sys.BINDIR)
     end
 end
 
-## load-path types ##
+## LOAD_PATH ##
 
-abstract type AbstractEnv end
+# split on `:` (or `;` on Windows)
+# first empty entry is replaced with DEFAULT_LOAD_PATH, the rest are skipped
+# entries starting with `@` are named environments:
+#  - the first three `#`s in a named environment are replaced with version numbers
+#  - `@stdlib` is a special name for the standard library and expands to its path
 
-struct CurrentEnv <: AbstractEnv
-    create::Bool
-    CurrentEnv(; create::Bool=false) = new(create)
-end
-
-struct NamedEnv <: AbstractEnv
-    name::String
-    create::Bool
-    NamedEnv(name::String; create::Bool=false) = new(name, create)
-end
-
-function show(io::IO, env::CurrentEnv)
-    print(io, CurrentEnv, "(")
-    env.create && print(io, "create=true")
-    print(io, ")")
-end
+# if you want a current env setup, use direnv and
+# have your .envrc do something like this:
+#
+#   export JULIA_LOAD_PATH="$(pwd):$JULIA_LOAD_PATH"
+#
+# this will inherit an existing JULIA_LOAD_PATH value or if there is none, leave
+# a trailing empty entry in JULIA_LOAD_PATH which will be replaced with defaults.
 
-function show(io::IO, env::NamedEnv)
-    print(io, NamedEnv, "(", repr(env.name))
-    env.create && print(io, ", create=true")
-    print(io, ")")
-end
-
-function parse_env(env::Union{String,SubString{String}})
-    isempty(env) && return Any[]
-    env == "@" && return CurrentEnv()
-    env == "@!" && return CurrentEnv(create=true)
-    if env[1] == '@'
-        create = env[2] == '!'
-        name = env[2+create:end]
-        name = replace(name, '#' => VERSION.major, count=1)
-        name = replace(name, '#' => VERSION.minor, count=1)
-        name = replace(name, '#' => VERSION.patch, count=1)
-        return NamedEnv(name, create=create)
-    end
-    return env # literal path
-end
+const DEFAULT_LOAD_PATH = ["@v#.#", "@stdlib"]
 
 """
     LOAD_PATH
 
-An array of paths as strings or custom loader objects for the `require`
-function and `using` and `import` statements to consider when loading
-code.
+An array of paths for `using` and `import` statements to consdier as project
+environments or package directories when loading code. See Code Loading.
 """
-const LOAD_PATH = Any[]
+const LOAD_PATH = copy(DEFAULT_LOAD_PATH)
+
+function current_env(dir::AbstractString = pwd())
+    # look for project file in current dir and parents
+    home = homedir()
+    while true
+        for proj in project_names
+            file = joinpath(dir, proj)
+            isfile_casesensitive(file) && return file
+        end
+        # bail at home directory or top of git repo
+        (dir == home || ispath(joinpath(dir, ".git"))) && break
+        old, dir = dir, dirname(dir)
+        dir == old && break
+    end
+end
 
 function parse_load_path(str::String)
-    envs = Any[split(str, Sys.iswindows() ? ';' : ':');]
-    for (i, env) in enumerate(envs)
-        if '|' in env
-            envs[i] = Any[parse_env(e) for e in split(env, '|')]
+    envs = String[]
+    isempty(str) && return envs
+    first_empty = true
+    for env in split(str, Sys.iswindows() ? ';' : ':')
+        if isempty(env)
+            first_empty && append!(envs, DEFAULT_LOAD_PATH)
+            first_empty = false
+        elseif env == "@"
+            dir = current_env()
+            dir !== nothing && push!(envs, dir)
         else
-            envs[i] = parse_env(env)
+            push!(envs, env)
         end
     end
     return envs
 end
 
-const default_named = parse_load_path("@v#.#.#|@v#.#|@v#|@default|@!v#.#")
-
 function init_load_path(BINDIR::String = Sys.BINDIR)
-    vers = "v$(VERSION.major).$(VERSION.minor)"
-    stdlib = abspath(BINDIR, "..", "share", "julia", "stdlib", vers)
-    load_path = Base.creating_sysimg           ? Any[stdlib] :
-                haskey(ENV, "JULIA_LOAD_PATH") ? parse_load_path(ENV["JULIA_LOAD_PATH"]) :
-                                                 Any[CurrentEnv(); default_named; stdlib]
+    if Base.creating_sysimg
+        load_path = ["@stdlib"]
+    elseif haskey(ENV, "JULIA_LOAD_PATH")
+        load_path = parse_load_path(ENV["JULIA_LOAD_PATH"])
+    else
+        load_path = DEFAULT_LOAD_PATH
+    end
     append!(empty!(LOAD_PATH), load_path)
 end
 

base/loading.jl

@@ -147,58 +147,44 @@ end
 
 ## load path expansion: turn LOAD_PATH entries into concrete paths ##
 
-function find_env(envs::Vector)
-    for env in envs
-        path = find_env(env)
-        path != nothing && return path
+function load_path_expand(env::AbstractString)::Union{String, Nothing}
+    # named environment?
+    if startswith(env, '@')
+        # `@` in JULIA_LOAD_PATH is expanded early (at startup time)
+        # if you put a `@` in LOAD_PATH manually, it's expanded late
+        env == "@" && return current_env()
+        env == "@stdlib" && return Sys.STDLIB
+        env = replace(env, '#' => VERSION.major, count=1)
+        env = replace(env, '#' => VERSION.minor, count=1)
+        env = replace(env, '#' => VERSION.patch, count=1)
+        name = env[2:end]
+        # look for named env in each depot
+        for depot in DEPOT_PATH
+            path = joinpath(depot, "environments", name)
+            isdir(path) || continue
+            for proj in project_names
+                file = abspath(path, proj)
+                isfile_casesensitive(file) && return file
+            end
+            return path
+        end
+        isempty(DEPOT_PATH) && return nothing
+        return abspath(DEPOT_PATH[1], "environments", name, project_names[end])
     end
-end
-
-function find_env(env::AbstractString)
+    # otherwise, it's a path
     path = abspath(env)
     if isdir(path)
         # directory with a project file?
-        for name in project_names
-            file = abspath(path, name)
+        for proj in project_names
+            file = joinpath(path, proj)
             isfile_casesensitive(file) && return file
         end
     end
     # package dir or path to project file
     return path
 end
 
-function find_env(env::NamedEnv)
-    # look for named env in each depot
-    for depot in DEPOT_PATH
-        isdir(depot) || continue
-        file = nothing
-        for name in project_names
-            file = abspath(depot, "environments", env.name, name)
-            isfile_casesensitive(file) && return file
-        end
-        file != nothing && env.create && return file
-    end
-end
-
-function find_env(env::CurrentEnv, dir::AbstractString = pwd())
-    # look for project file in current dir and parents
-    home = homedir()
-    while true
-        for name in project_names
-            file = joinpath(dir, name)
-            isfile_casesensitive(file) && return file
-        end
-        # bail at home directory or top of git repo
-        (dir == home || ispath(joinpath(dir, ".git"))) && break
-        old, dir = dir, dirname(dir)
-        dir == old && break
-    end
-    env.create ? joinpath(pwd(), project_names[end]) : nothing
-end
-
-find_env(env::Function) = find_env(env())
-
-load_path() = String[env for env in map(find_env, LOAD_PATH) if env ≠ nothing]
+load_path() = String[env for env in map(load_path_expand, LOAD_PATH) if env ≠ nothing]
 
 ## package identification: determine unique identity of package to be loaded ##
 

base/precompile.jl

@@ -20,14 +20,12 @@ end
 precompile(Tuple{Type{Array{Base.StackTraces.StackFrame, 1}}, UndefInitializer, Int64})
 precompile(Tuple{Type{Array{Union{Nothing, String}, 1}}, UndefInitializer, Int64})
 precompile(Tuple{Type{Base.CoreLogging.LogState}, Logging.ConsoleLogger})
-precompile(Tuple{Type{Base.CurrentEnv}})
 precompile(Tuple{Type{Base.GC_Diff}, Base.GC_Num, Base.GC_Num})
 precompile(Tuple{Type{Base.Generator{I, F} where F where I}, typeof(Base.names), Array{Any, 1}})
 precompile(Tuple{Type{Base.IOContext{IO_t} where IO_t<:IO}, Base.GenericIOBuffer{Array{UInt8, 1}}, Base.IOStream})
 precompile(Tuple{Type{Base.IOContext{IO_t} where IO_t<:IO}, REPL.Terminals.TTYTerminal, Base.Pair{Symbol, Bool}})
 precompile(Tuple{Type{Base.MIME{Symbol("text/plain")}}})
 precompile(Tuple{Type{Base.Multimedia.TextDisplay}, Base.TTY})
-precompile(Tuple{Type{Base.NamedEnv}, String})
 precompile(Tuple{Type{Base.Order.Perm{O, V} where V<:(AbstractArray{T, 1} where T) where O<:Base.Order.Ordering}, Base.Order.ForwardOrdering, Array{Tuple{Float64, Int64}, 1}})
 precompile(Tuple{Type{Base.Pair{A, B} where B where A}, Base.PkgId, UInt64})
 precompile(Tuple{Type{Base.Pair{A, B} where B where A}, Int64, Int64})
@@ -67,7 +65,6 @@ precompile(Tuple{getfield(Base.Cartesian, Symbol("#@ncall")), LineNumberNode, Mo
 precompile(Tuple{getfield(Base.Cartesian, Symbol("#@nexprs")), LineNumberNode, Module, Int64, Expr})
 precompile(Tuple{getfield(Base.Meta, Symbol("#kw##parse")), NamedTuple{(:raise, :depwarn), Tuple{Bool, Bool}}, typeof(Base.Meta.parse), String, Int64})
 precompile(Tuple{getfield(Core, Symbol("#@doc")), LineNumberNode, Module, Symbol})
-precompile(Tuple{getfield(Core, Symbol("#kw#Type")), NamedTuple{(:create,), Tuple{Bool}}, Type{Base.NamedEnv}, String})
 precompile(Tuple{getfield(Core, Symbol("#kw#Type")), NamedTuple{(:prompt_prefix, :prompt_suffix, :complete, :sticky), Tuple{String, String, Pkg.REPLMode.PkgCompletionProvider, Bool}}, Type{REPL.LineEdit.Prompt}, typeof(Pkg.REPLMode.promptf)})
 precompile(Tuple{getfield(Core, Symbol("#kw#Type")), NamedTuple{(:prompt_prefix, :prompt_suffix, :repl, :complete, :on_enter), Tuple{String, typeof(Base.input_color), REPL.LineEditREPL, REPL.REPLCompletionProvider, typeof(REPL.return_callback)}}, Type{REPL.LineEdit.Prompt}, String})
 precompile(Tuple{getfield(REPL, Symbol("#@repl")), LineNumberNode, Module, Base.TTY, Symbol})
@@ -136,10 +133,8 @@ precompile(Tuple{typeof(Base.bytesavailable), Base.GenericIOBuffer{Array{UInt8,
 precompile(Tuple{typeof(Base.check_open), Base.TTY})
 precompile(Tuple{typeof(Base.close), Base.Pipe})
 precompile(Tuple{typeof(Base.collect_similar), Array{Any, 1}, Base.Generator{Array{Any, 1}, typeof(Base.names)}})
-precompile(Tuple{typeof(Base.collect_to!), Array{Union{Nothing, String}, 1}, Base.Generator{Array{Any, 1}, typeof(Base.find_env)}, Int64, Int64})
 precompile(Tuple{typeof(Base.collect_to_with_first!), Array{Array{Base.StackTraces.StackFrame, 1}, 1}, Array{Base.StackTraces.StackFrame, 1}, Base.Generator{Array{Union{Ptr{Nothing}, Base.InterpreterIP}, 1}, typeof(Base.StackTraces.lookup)}, Int64})
 precompile(Tuple{typeof(Base.collect_to_with_first!), Array{Markdown.MD, 1}, Markdown.MD, Base.Generator{Array{Base.Docs.DocStr, 1}, typeof(Base.Docs.parsedoc)}, Int64})
-precompile(Tuple{typeof(Base.collect_to_with_first!), Array{Nothing, 1}, Nothing, Base.Generator{Array{Any, 1}, typeof(Base.find_env)}, Int64})
 precompile(Tuple{typeof(Base.collect_to_with_first!), Array{String, 1}, String, Base.Generator{Array{Any, 1}, typeof(Base.string)}, Int64})
 precompile(Tuple{typeof(Base.convert), Type{Array{String, 1}}, Array{Any, 1}})
 precompile(Tuple{typeof(Base.convert), Type{Base.Dict{Char, V} where V}, Base.Dict{Char, Any}})
@@ -168,10 +163,6 @@ precompile(Tuple{typeof(Base.divrem), Int64, Int64})
 precompile(Tuple{typeof(Base.empty!), Array{Base.Pair{Base.PkgId, UInt64}, 1}})
 precompile(Tuple{typeof(Base.eof), Base.PipeEndpoint})
 precompile(Tuple{typeof(Base.eof), Base.TTY})
-precompile(Tuple{typeof(Base.find_env), Array{Any, 1}})
-precompile(Tuple{typeof(Base.find_env), Base.CurrentEnv})
-precompile(Tuple{typeof(Base.find_env), Base.NamedEnv})
-precompile(Tuple{typeof(Base.find_env), typeof(OldPkg.dir)})
 precompile(Tuple{typeof(Base.findlast), String, String})
 precompile(Tuple{typeof(Base.findprev), String, String, Int64})
 precompile(Tuple{typeof(Base.first), Base.OneTo{Int64}})
@@ -191,8 +182,6 @@ precompile(Tuple{typeof(Base.getindex), Tuple{Int64, Int64}, Int64})
 precompile(Tuple{typeof(Base.getindex), Tuple{Symbol, Symbol, Symbol}, Base.UnitRange{Int64}})
 precompile(Tuple{typeof(Base.getindex), Tuple{Symbol}, Base.UnitRange{Int64}})
 precompile(Tuple{typeof(Base.getindex), Tuple{Symbol}, Int64})
-precompile(Tuple{typeof(Base.getindex), Type{Any}, Base.CurrentEnv, Array{Any, 1}, String, typeof(OldPkg.dir)})
-precompile(Tuple{typeof(Base.getindex), Type{Any}, Base.NamedEnv, Base.NamedEnv, Base.NamedEnv, Base.NamedEnv, Base.NamedEnv})
 precompile(Tuple{typeof(Base.getproperty), Base.CoreLogging.LogState, Symbol})
 precompile(Tuple{typeof(Base.getproperty), Base.GC_Diff, Symbol})
 precompile(Tuple{typeof(Base.getproperty), Base.GenericIOBuffer{Array{UInt8, 1}}, Symbol})
@@ -294,7 +283,6 @@ precompile(Tuple{typeof(Base.print), Base.IOContext{Base.GenericIOBuffer{Array{U
 precompile(Tuple{typeof(Base.print), Base.IOContext{Base.GenericIOBuffer{Array{UInt8, 1}}}, Module, String, Symbol})
 precompile(Tuple{typeof(Base.print), Base.IOContext{Base.GenericIOBuffer{Array{UInt8, 1}}}, String})
 precompile(Tuple{typeof(Base.print), Base.IOContext{Base.GenericIOBuffer{Array{UInt8, 1}}}, Symbol})
-precompile(Tuple{typeof(Base.print), Base.IOContext{Base.GenericIOBuffer{Array{UInt8, 1}}}, Type{Base.CurrentEnv}})
 precompile(Tuple{typeof(Base.print), Base.IOContext{REPL.Terminals.TTYTerminal}, Char, String, String})
 precompile(Tuple{typeof(Base.print), Base.IOContext{REPL.Terminals.TTYTerminal}, String, Type{Module}})
 precompile(Tuple{typeof(Base.print), Base.IOContext{REPL.Terminals.TTYTerminal}, String})
@@ -370,11 +358,8 @@ precompile(Tuple{typeof(Base.show), Base.GenericIOBuffer{Array{UInt8, 1}}, Strin
 precompile(Tuple{typeof(Base.show), Base.GenericIOBuffer{Array{UInt8, 1}}, Type{Any}})
 precompile(Tuple{typeof(Base.show), Base.GenericIOBuffer{Array{UInt8, 1}}, UInt64})
 precompile(Tuple{typeof(Base.show), Base.IOContext{Base.GenericIOBuffer{Array{UInt8, 1}}}, Array{Any, 1}})
-precompile(Tuple{typeof(Base.show), Base.IOContext{Base.GenericIOBuffer{Array{UInt8, 1}}}, Base.CurrentEnv})
-precompile(Tuple{typeof(Base.show), Base.IOContext{Base.GenericIOBuffer{Array{UInt8, 1}}}, Base.NamedEnv})
 precompile(Tuple{typeof(Base.show), Base.IOContext{Base.GenericIOBuffer{Array{UInt8, 1}}}, Int64})
 precompile(Tuple{typeof(Base.show), Base.IOContext{Base.GenericIOBuffer{Array{UInt8, 1}}}, String})
-precompile(Tuple{typeof(Base.show), Base.IOContext{Base.GenericIOBuffer{Array{UInt8, 1}}}, Type{Base.CurrentEnv}})
 precompile(Tuple{typeof(Base.show), Base.IOContext{Base.GenericIOBuffer{Array{UInt8, 1}}}, typeof(OldPkg.dir)})
 precompile(Tuple{typeof(Base.show), Base.IOContext{REPL.Terminals.TTYTerminal}, Base.MIME{Symbol("text/plain")}, Array{Float64, 1}})
 precompile(Tuple{typeof(Base.show), Base.IOContext{REPL.Terminals.TTYTerminal}, Base.MIME{Symbol("text/plain")}, Array{Float64, 2}})

base/sysinfo.jl

@@ -6,6 +6,7 @@ Provide methods for retrieving information about hardware and the operating syst
 """ Sys
 
 export BINDIR,
+       STDLIB,
        CPU_CORES,
        CPU_NAME,
        WORD_SIZE,
@@ -37,6 +38,13 @@ A string containing the full path to the directory containing the `julia` execut
 """
 :BINDIR
 
+"""
+    Sys.STDLIB
+
+A string containing the full path to the directory containing the `stdlib` packages.
+"""
+STDLIB = "$BINDIR/../../stdlib" # for bootstrap
+
 # helper to avoid triggering precompile warnings
 
 global CPU_CORES
@@ -94,6 +102,8 @@ function __init__()
     global CPU_NAME = ccall(:jl_get_cpu_name, Ref{String}, ())
     global JIT = ccall(:jl_get_JIT, Ref{String}, ())
     global BINDIR = ccall(:jl_get_julia_bindir, Any, ())::String
+    vers = "v$(VERSION.major).$(VERSION.minor)"
+    global STDLIB = abspath(BINDIR, "..", "share", "julia", "stdlib", vers)
     nothing
 end
 

stdlib/OldPkg/test/pkg.jl

@@ -546,7 +546,7 @@ temp_pkg_dir() do
 
         OldPkg.add(package)
         msg = read(ignorestatus(`$(Base.julia_cmd()) --startup-file=no -e
-            "import OldPkg; push!(LOAD_PATH, OldPkg.dir); redirect_stderr(stdout); using Logging; global_logger(SimpleLogger(stdout)); using Example; OldPkg.update(\"$package\")"`), String)
+            "import OldPkg; push!(LOAD_PATH, OldPkg.dir()); redirect_stderr(stdout); using Logging; global_logger(SimpleLogger(stdout)); using Example; OldPkg.update(\"$package\")"`), String)
         @test occursin(Regex("- $package.*Restart Julia to use the updated versions","s"), msg)
     end
 

stdlib/Pkg/src/Types.jl

@@ -13,7 +13,7 @@ using ..TOML
 import ..Pkg
 import Pkg: GitTools, depots, logdir
 
-import Base: SHA1, AbstractEnv
+import Base: SHA1
 using SHA
 
 export UUID, pkgID, SHA1, VersionRange, VersionSpec, empty_versionspec,
@@ -207,7 +207,7 @@ const default_envs = [
 
 mutable struct EnvCache
     # environment info:
-    env::Union{Nothing,String,AbstractEnv}
+    env::Union{Nothing,String}
     git::Union{Nothing,LibGit2.GitRepo}
 
     # paths for files:
@@ -225,26 +225,26 @@ mutable struct EnvCache
     uuids::Dict{String,Vector{UUID}}
     paths::Dict{UUID,Vector{String}}
 
-    function EnvCache(env::Union{Nothing,String,AbstractEnv}=nothing)
+    function EnvCache(env::Union{Nothing,String}=nothing)
         if env isa Nothing
             project_file = nothing
             for entry in LOAD_PATH
-                project_file = Base.find_env(entry)
+                project_file = Base.load_path_expand(entry)
                 project_file isa String && !isdir(project_file) && break
                 project_file = nothing
             end
             if project_file == nothing
                 project_dir = nothing
                 for entry in LOAD_PATH
-                    project_dir = Base.find_env(entry)
+                    project_dir = Base.load_path_expand(entry)
                     project_dir isa String && isdir(project_dir) && break
                     project_dir = nothing
                 end
                 project_dir == nothing && error("No Pkg environment found in LOAD_PATH")
                 project_file = joinpath(project_dir, Base.project_names[end])
             end
-        elseif env isa AbstractEnv
-            project_file = Base.find_env(env)
+        elseif startswith(env, '@')
+            project_file = Base.load_path_expand(env)
             project_file === nothing && error("package environment does not exist: $env")
         elseif env isa String
             if isdir(env)

stdlib/Pkg/test/pkg.jl

@@ -238,7 +238,7 @@ temp_pkg_dir() do project_path
             cp(joinpath(@__DIR__, "test_packages", "UnregisteredWithProject"), joinpath(dir, "UnregisteredWithProject"))
             cd(joinpath(dir, "UnregisteredWithProject")) do
                 try
-                    pushfirst!(LOAD_PATH, Base.parse_load_path("@"))
+                    pushfirst!(LOAD_PATH, Base.current_env())
                     Pkg.up()
                     @test haskey(Pkg.installed(), "Example")
                 finally
@@ -278,7 +278,7 @@ temp_pkg_dir() do project_path
             cp(joinpath(@__DIR__, "test_packages", "UnregisteredWithProject"), joinpath(dir, "UnregisteredWithProject"))
             cd(joinpath(dir, "UnregisteredWithProject")) do
                 try
-                    pushfirst!(LOAD_PATH, Base.parse_load_path("@"))
+                    pushfirst!(LOAD_PATH, Base.current_env())
                     Pkg.add("Test") # test https://github.com/JuliaLang/Pkg.jl/issues/324
                     Pkg.test()
                 finally