diff --git a/.travis.yml b/.travis.yml index 1967440e21..e62ca4e5f4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -16,8 +16,5 @@ before_script: script: - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi - - julia --check-bounds=yes -e 'Pkg.clone(pwd()); Pkg.build("Pkg3"); Pkg.test("Pkg3"; coverage=true)' - -after_success: - - julia -e 'cd(Pkg.dir("Pkg3")); Pkg.add("Coverage"); using Coverage; Coveralls.submit(Coveralls.process_folder())' + - julia --check-bounds=yes -e 'Pkg.add("SHA"); include("src/Pkg3.jl"); Pkg3.clone(pwd()); Pkg3.test("Pkg3"; coverage=true)' diff --git a/REQUIRE b/REQUIRE index 07f7a93008..9c16d24b92 100644 --- a/REQUIRE +++ b/REQUIRE @@ -1,3 +1,2 @@ julia 0.6 SHA -# Pkg.clone("https://github.com/StefanKarpinski/TOML.jl.git") diff --git a/bin/loadmeta.jl b/bin/loadmeta.jl index 599ab4fb2f..035fcf0d1a 100644 --- a/bin/loadmeta.jl +++ b/bin/loadmeta.jl @@ -6,9 +6,9 @@ using Base.Random: UUID using Pkg3.Types using Pkg3.Types: uuid_package, uuid_registry, uuid5 -include("Pkg2/Pkg2.jl") -import .Pkg2.Reqs: Reqs, Requirement -import .Pkg2.Types: VersionInterval +import Pkg3: Pkg2 +import Pkg3.Pkg2.Reqs: Reqs, Requirement +import Pkg3.Pkg2.Types: VersionInterval ## Loading data into various data structures ## diff --git a/ext/BinaryProvider/src/BinaryProvider.jl b/ext/BinaryProvider/src/BinaryProvider.jl index 6b6c3b364c..0d218096f2 100644 --- a/ext/BinaryProvider/src/BinaryProvider.jl +++ b/ext/BinaryProvider/src/BinaryProvider.jl @@ -1,19 +1,19 @@ module BinaryProvider -import Pkg3: iswindows, isapple, islinux +import ..Pkg3: iswindows, isapple, islinux # Include our subprocess running funtionality -include("OutputCollector.jl") +#include("OutputCollector.jl") # External utilities such as downloading/decompressing tarballs include("PlatformEngines.jl") # Platform naming -include("PlatformNames.jl") +#include("PlatformNames.jl") # Everything related to file/path management -include("Prefix.jl") +#include("Prefix.jl") # Abstraction of "needing" a file, that would trigger an install -include("Products.jl") +#include("Products.jl") # Abstraction of bundled binary package -include("BinaryPackage.jl") +#include("BinaryPackage.jl") # BinDeps support, disabled for now because I don't particularly want to force # users to install BinDeps to install this package. That seems counter-productive diff --git a/ext/TerminalMenus/src/TerminalMenus.jl b/ext/TerminalMenus/src/TerminalMenus.jl index bbc4a96e10..6b48bdd28a 100644 --- a/ext/TerminalMenus/src/TerminalMenus.jl +++ b/ext/TerminalMenus/src/TerminalMenus.jl @@ -1,6 +1,6 @@ module TerminalMenus -import Pkg3.iswindows +import ..Pkg3.iswindows terminal = nothing # The user terminal diff --git a/src/API.jl b/src/API.jl index 2da2307f08..b23aefcf05 100644 --- a/src/API.jl +++ b/src/API.jl @@ -1,9 +1,9 @@ module API -import Pkg3 -using Pkg3.Display.DiffEntry -import Pkg3: depots, logdir, TOML -using Pkg3: Types, Dates +import ..Pkg3 +using ..Pkg3: Display.DiffEntry +import ..Pkg3: depots, logdir, default_dev_path, TOML +using ..Pkg3: Dates, Types using Base.Random.UUID previewmode_info() = info("In preview mode") @@ -128,7 +128,7 @@ function test(env::EnvCache, pkgs::Vector{PackageSpec}; coverage=false, preview= end -function convert(::Type{Dict{String, VersionNumber}}, diffs::Union{Array{DiffEntry}, Void}) +function convert(::Type{Dict{String, VersionNumber}}, diffs::Union{Array{DiffEntry}, Void}) version_status = Dict{String, VersionNumber}() diffs == nothing && return version_status for entry in diffs @@ -239,18 +239,52 @@ function gc(env::EnvCache=EnvCache(); period = Week(6), preview=env.preview[]) info("Deleted $(length(paths_to_delete)) package installations", byte_save_str) end -function _get_deps!(env::EnvCache, pkgs::Vector{PackageSpec}, uuids::Vector{Base.Random.UUID}) - for pkg in pkgs - info = manifest_info(env, pkg.uuid) - pkg.uuid in uuids && continue - push!(uuids, pkg.uuid) - if haskey(info, "deps") - pkgs = [PackageSpec(name, UUID(uuid)) for (name, uuid) in info["deps"]] - _get_deps!(env, pkgs, uuids) - end - end + + +function url_and_pkg(url_or_pkg::AbstractString) + # try to parse as URL or local path + m = match(r"(?:^|[/\\])(\w+?)(?:\.jl)?(?:\.git)?$", url_or_pkg) + m === nothing && cmderror("can't determine package name from URL: $url_or_pkg") + return url_or_pkg, m.captures[1] +end +clone(pkg::String; kwargs...) = clone(EnvCache(), pkg; kwargs...) + +function clone(env::EnvCache, url::AbstractString; name=nothing, basepath=get(ENV, "JULIA_DEVDIR", default_dev_path()), preview=env.preview[]) + preview && cmderror("Preview mode not implemented for cloning") + if name == nothing + url, name = url_and_pkg(url) + end + pkg = PackageSpec(name=name, path=joinpath(basepath, name), url=url) + registry_resolve!(env, [pkg]) + project_resolve!(env, [pkg]) + manifest_resolve!(env, [pkg]) + Pkg3.Operations.clone(env, [pkg]) +end + +free(;kwargs...) = free(PackageSpec[], kwargs...) +free(pkg::String; kwargs...) = free([pkg]; kwargs...) +free(pkgs::Vector{String}; kwargs...) = free([PackageSpec(pkg) for pkg in pkgs]; kwargs...) +free(pkgs::Vector{PackageSpec}; kwargs...) = free(EnvCache(), pkgs; kwargs...) +function free(env::EnvCache, pkgs::Vector{PackageSpec}; preview = env.preview[]) + env.preview[] = preview + preview && previewmode_info() + project_resolve!(env, pkgs) + manifest_resolve!(env, pkgs) + ensure_resolved(env, pkgs) + Pkg3.Operations.free(env, pkgs) end +function _get_deps!(env::EnvCache, pkgs::Vector{PackageSpec}, uuids::Vector{Base.Random.UUID}) + for pkg in pkgs + info = manifest_info(env, pkg.uuid) + pkg.uuid in uuids && continue + push!(uuids, pkg.uuid) + if haskey(info, "deps") + pkgs = [PackageSpec(name, UUID(uuid)) for (name, uuid) in info["deps"]] + _get_deps!(env, pkgs, uuids) + end + end + end build(pkgs...) = build([PackageSpec(pkg) for pkg in pkgs]) build(pkg::Array{Union{}, 1}) = build(PackageSpec[]) @@ -275,7 +309,6 @@ function build(env::EnvCache, pkgs::Vector{PackageSpec}) Pkg3.Operations.build_versions(env, uuids) end - function init(path = pwd()) Pkg3.Operations.init(path) end diff --git a/src/Display.jl b/src/Display.jl index 07925c2493..37c656767e 100644 --- a/src/Display.jl +++ b/src/Display.jl @@ -1,7 +1,7 @@ module Display using Base.Random: UUID -using Pkg3.Types +using ..Pkg3.Types export print_project_diff, print_manifest_diff @@ -12,7 +12,7 @@ const colors = Dict( '↑' => :light_yellow, '~' => :light_yellow, '↓' => :light_magenta, - '?' => :red, + '?' => :white, ) const color_dark = :light_black @@ -20,7 +20,7 @@ function status(env::EnvCache, mode::Symbol, use_as_api=false) project₀ = project₁ = env.project manifest₀ = manifest₁ = env.manifest diff = nothing - + if env.git != nothing git_path = LibGit2.path(env.git) project_path = relpath(env.project_file, git_path) @@ -76,17 +76,23 @@ function print_manifest_diff(env₀::EnvCache, env₁::EnvCache) end struct VerInfo - hash::SHA1 + hash_or_path::Union{SHA1, String} ver::Union{VersionNumber,Void} end +islocal(v::VerInfo) = v.hash_or_path isa String -vstring(a::VerInfo) = - a.ver == nothing ? "[$(string(a.hash)[1:16])]" : "v$(a.ver)" +function vstring(a::VerInfo) + if islocal(a) + return "[$(a.hash_or_path)]" + else + return a.ver == nothing ? "[$(string(a.hash_or_path)[1:16])]" : "v$(a.ver)" + end +end Base.:(==)(a::VerInfo, b::VerInfo) = - a.hash == b.hash && a.ver == b.ver + a.hash_or_path == b.hash_or_path && a.ver == b.ver -≈(a::VerInfo, b::VerInfo) = a.hash == b.hash && +≈(a::VerInfo, b::VerInfo) = a.hash_or_path isa SHA1 && a.hash_or_path == b.hash_or_path && (a.ver == nothing || b.ver == nothing || a.ver == b.ver) struct DiffEntry @@ -105,19 +111,24 @@ function print_diff(io::IO, diff::Vector{DiffEntry}) verb = ' ' vstr = vstring(x.new) else - if x.old.hash != x.new.hash && x.old.ver != x.new.ver + if x.old.hash_or_path != x.new.hash_or_path && x.old.ver != x.new.ver verb = x.old.ver == nothing || x.new.ver == nothing || x.old.ver == x.new.ver ? '~' : x.old.ver < x.new.ver ? '↑' : '↓' else verb = '?' - msg = x.old.hash == x.new.hash ? + msg = x.old.hash_or_path isa SHA1 && x.old.hash_or_path == x.new.hash_or_path ? "hashes match but versions don't: $(x.old.ver) ≠ $(x.new.ver)" : - "versions match but hashes don't: $(x.old.hash) ≠ $(x.new.hash)" + "versions match but hashes don't: $(x.old.hash_or_path) ≠ $(x.new.hash_or_path)" push!(warnings, msg) end - vstr = x.old.ver == x.new.ver ? vstring(x.new) : - vstring(x.old) * " ⇒ " * vstring(x.new) + # Moving from hash -> path + if typeof(x.old.hash_or_path) != typeof(x.new.hash_or_path) + vstr = vstring(x.old) * " ⇒ " * vstring(x.new) + else + vstr = x.old.ver == x.new.ver ? vstring(x.new) : + vstring(x.old) * " ⇒ " * vstring(x.new) + end end elseif x.new != nothing verb = '+' @@ -148,9 +159,9 @@ end function name_ver_info(info::Dict) name = info["name"] - hash = haskey(info, "git-tree-sha1") ? SHA1(info["git-tree-sha1"]) : nothing + hash_or_path = haskey(info, "path") ? info["path"] : haskey(info, "git-tree-sha1") ? SHA1(info["git-tree-sha1"]) : nothing ver = haskey(info, "version") ? VersionNumber(info["version"]) : nothing - name, VerInfo(hash, ver) + name, VerInfo(hash_or_path, ver) end function manifest_diff(manifest₀::Dict, manifest₁::Dict) diff --git a/src/GraphType.jl b/src/GraphType.jl index c2fb3b9fae..a37c26cf16 100644 --- a/src/GraphType.jl +++ b/src/GraphType.jl @@ -285,6 +285,7 @@ mutable struct Graph adjdict = [Dict{Int,Int}() for p0 = 1:np] for p0 = 1:np, v0 = 1:(spp[p0]-1), (p1,rmsk1) in extended_deps[p0][v0] + @assert p0 ≠ p1 j0 = get(adjdict[p1], p0, length(gadj[p0]) + 1) j1 = get(adjdict[p0], p1, length(gadj[p1]) + 1) diff --git a/src/Operations.jl b/src/Operations.jl index e352d5c761..e58ce911ee 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -2,9 +2,10 @@ module Operations using Base.Random: UUID using Base: LibGit2 -using Pkg3: TerminalMenus, Types, GraphType, Resolve -import Pkg3: GLOBAL_SETTINGS, depots, BinaryProvider -import Pkg3.Types: uuid_julia +using ..Pkg3: TerminalMenus, Types, GraphType, Resolve +import ..Pkg3: GLOBAL_SETTINGS, depots, BinaryProvider, BinaryProvider +import ..Pkg3.Types: uuid_julia, VersionBound +import ..Pkg3: Pkg2 const SlugInt = UInt32 # max p = 4 const chars = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789" @@ -160,11 +161,13 @@ function deps_graph(env::EnvCache, uuid_to_name::Dict{UUID,String}, reqs::Requir for (vr,dd) in deps_data all_deps_u_vr = get_or_make!(all_deps_u, vr) for (name,other_uuid) in dd + other_uuid in keys(fixed) && continue # check conflicts?? all_deps_u_vr[name] = other_uuid other_uuid in uuids || push!(uuids, other_uuid) end end + uuid in keys(fixed) && continue for (vr,cd) in compatibility_data all_compat_u_vr = get_or_make!(all_compat_u, vr) for (name,vs) in cd @@ -190,6 +193,51 @@ function deps_graph(env::EnvCache, uuid_to_name::Dict{UUID,String}, reqs::Requir return Graph(all_versions, all_deps, all_compat, uuid_to_name, reqs, fixed) end +# This also sets the .path field for fixed packages in `pkgs` +function collect_fixed!(env, pkgs, uuids, uuid_to_name) + fix_deps = PackageSpec[] + fixed_pkgs = PackageSpec[] + fix_deps_map = Dict{UUID,Vector{PackageSpec}}() + uuid_to_pkg = Dict{UUID,PackageSpec}() + for pkg in pkgs + !has_path(pkg) && continue + # A package with a path should have a version + @assert pkg.version != nothing + + uuid_to_pkg[pkg.uuid] = pkg + uuid_to_name[pkg.uuid] = pkg.name + # Load the dependencies if this package has a REQUIRE + reqfile = joinpath(pkg.path, "REQUIRE") + fix_deps_map[pkg.uuid] = valtype(fix_deps_map)() + !isfile(reqfile) && continue + for r in filter!(r->r isa Pkg2.Reqs.Requirement, Pkg2.Reqs.read(reqfile)) + pkg_name, vspec = r.package, VersionSpec(VersionRange[r.versions.intervals...]) + deppkg = PackageSpec(pkg_name, vspec) + push!(fix_deps_map[pkg.uuid], deppkg) + push!(fix_deps, deppkg) + end + end + + # Look up the UUIDS for all the fixed dependencies in the registry in one shot + registry_resolve!(env, fix_deps) + fixed = Dict{UUID,Fixed}() + # Collect the dependencies for the fixed packages + for (uuid, fixed_pkgs) in fix_deps_map + fix_pkg = uuid_to_pkg[uuid] + v = Dict{VersionNumber,Dict{UUID,VersionSpec}}() + q = Dict{UUID, VersionSpec}() + for deppkg in fixed_pkgs + if !has_uuid(deppkg) + cmderror("Could not find a UUID for package $(pkg.name) in the registry") + end + uuid_to_name[deppkg.uuid] = deppkg.name + q[deppkg.uuid] = deppkg.version + end + fixed[uuid] = Fixed(fix_pkg.version, q) + end + return fixed +end + "Resolve a set of versions given package version specs" function resolve_versions!(env::EnvCache, pkgs::Vector{PackageSpec})::Dict{UUID,VersionNumber} info("Resolving package versions") @@ -204,9 +252,12 @@ function resolve_versions!(env::EnvCache, pkgs::Vector{PackageSpec})::Dict{UUID, ver = VersionNumber(info["version"]) push!(pkgs, PackageSpec(name, uuid, ver)) end - # construct data structures for resolver and call it + path_resolve!(env, pkgs) + reqs = Requires(pkg.uuid => pkg.version for pkg in pkgs if pkg.uuid ≠ uuid_julia) - fixed = Dict([uuid_julia => Fixed(VERSION)]) + # Collect all fixed packages (have a path) and their dependencies + fixed = collect_fixed!(env, pkgs, uuids, uuid_to_name) + fixed[uuid_julia] = Fixed(VERSION) # fix julia to current version graph = deps_graph(env, uuid_to_name, reqs, fixed) simplify_graph!(graph) @@ -257,7 +308,6 @@ function version_data(env::EnvCache, pkgs::Vector{PackageSpec}) end end end - @assert haskey(hashes, uuid) end foreach(sort!, values(upstreams)) return names, hashes, upstreams @@ -352,7 +402,7 @@ function install( return version_path, true end -function update_manifest(env::EnvCache, uuid::UUID, name::String, hash::SHA1, version::VersionNumber) +function update_manifest(env::EnvCache, uuid::UUID, name::String, hash_or_path::Union{String, SHA1}, version::VersionNumber) infos = get!(env.manifest, name, Dict{String,Any}[]) info = nothing for i in infos @@ -365,13 +415,44 @@ function update_manifest(env::EnvCache, uuid::UUID, name::String, hash::SHA1, ve push!(infos, info) end info["version"] = string(version) - info["git-tree-sha1"] = string(hash) + if hash_or_path isa SHA1 + hash = hash_or_path + @assert hash != nothing + haskey(info, "path") && delete!(info, "path") + info["git-tree-sha1"] = string(hash) + else + path = hash_or_path + haskey(info, "git-tree-sha1") && delete!(info, "git-tree-sha1") + info["path"] = path + end delete!(info, "deps") - for path in registered_paths(env, uuid) - data = load_package_data(UUID, joinpath(path, "dependencies.toml"), version) - data == nothing && continue - info["deps"] = convert(Dict{String,String}, data) - break + if hash_or_path isa String + path = hash_or_path + reqfile = joinpath(path, "REQUIRE") + + !isfile(reqfile) && return + deps = Dict{String,String}() + pkgs = PackageSpec[] + for r in filter!(r->r isa Pkg2.Reqs.Requirement, Pkg2.Reqs.read(reqfile)) + push!(pkgs, PackageSpec(r.package)) + end + # TODO: Get rid of this one? + registry_resolve!(env, pkgs) + for pkg in pkgs + pkg.name == "julia" && continue + @assert pkg.uuid != UUID(zero(UInt128)) + deps[pkg.name] = pkg.uuid + end + if !isempty(deps) + info["deps"] = deps + end + else + for path in registered_paths(env, uuid) + data = load_package_data(UUID, joinpath(path, "dependencies.toml"), version) + data == nothing && continue + info["deps"] = convert(Dict{String,String}, data) + break + end end return info end @@ -425,11 +506,16 @@ function apply_versions(env::EnvCache, pkgs::Vector{PackageSpec})::Vector{UUID} for i in 1:GLOBAL_SETTINGS.num_concurrent_downloads @schedule begin for pkg in jobs + # Check if this is a local package + if has_path(pkg) + put!(results, (pkg, pkg.path, pkg.version, nothing, false)) + continue + end uuid = pkg.uuid version = pkg.version::VersionNumber - name, hash = names[uuid], hashes[uuid] + name, hash, url = names[uuid], hashes[uuid], urls[uuid] try - version_path, new = install(env, uuid, name, hash, urls[uuid], version) + version_path, new = install(env, uuid, name, hash, url, version) put!(results, (pkg, version_path, version, hash, new)) catch e put!(results, e) @@ -440,7 +526,7 @@ function apply_versions(env::EnvCache, pkgs::Vector{PackageSpec})::Vector{UUID} textwidth = VERSION < v"0.7.0-DEV.1930" ? Base.strwidth : Base.textwidth widths = [textwidth(names[pkg.uuid]) for pkg in pkgs if haskey(names, pkg.uuid)] - max_name = length(widths) == 0 ? 0 : maximum(widths) + max_name = (length(widths) == 0) ? 0 : maximum(widths) for _ in 1:length(pkgs) r = take!(results) @@ -450,10 +536,13 @@ function apply_versions(env::EnvCache, pkgs::Vector{PackageSpec})::Vector{UUID} vstr = version != nothing ? "v$version" : "[$h]" new && info("Installed $(rpad(names[pkg.uuid] * " ", max_name + 2, "─")) $vstr") end - uuid = pkg.uuid + uuid, path = pkg.uuid, pkg.path version = pkg.version::VersionNumber - name, hash = names[uuid], hashes[uuid] - update_manifest(env, uuid, name, hash, version) + if haskey(hashes, uuid) + update_manifest(env, uuid, names[uuid], hashes[uuid], version) + else + update_manifest(env, uuid, pkg.name, path, version) + end new && push!(new_versions, uuid) end prune_manifest(env) @@ -483,17 +572,20 @@ end function build_versions(env::EnvCache, uuids::Vector{UUID}) # collect builds for UUIDs with `deps/build.jl` files env.preview[] && (info("Skipping building in preview mode"); return) - builds = Tuple{UUID,String,SHA1,String}[] + builds = Tuple{UUID,String,Union{SHA1, String},String}[] for uuid in uuids info = manifest_info(env, uuid) name = info["name"] - # TODO: handle development packages? - haskey(info, "git-tree-sha1") || continue - hash = SHA1(info["git-tree-sha1"]) - path = find_installed(uuid, hash) + path, hash = if haskey(info, "path") + info["path"], nothing + else + hash = SHA1(info["git-tree-sha1"]) + path = find_installed(uuid, hash) + path, hash + end ispath(path) || error("Build path for $name does not exist: $path") build_file = joinpath(path, "deps", "build.jl") - ispath(build_file) && push!(builds, (uuid, name, hash, build_file)) + ispath(build_file) && push!(builds, (uuid, name, hash == nothing ? path : hash, build_file)) end # toposort builds by dependencies order = dependency_order_uuids(env, map(first, builds)) @@ -501,9 +593,13 @@ function build_versions(env::EnvCache, uuids::Vector{UUID}) # build each package verions in a child process withenv("JULIA_ENV" => env.project_file) do LOAD_PATH = filter(x -> x isa AbstractString, Base.LOAD_PATH) - for (uuid, name, hash, build_file) in builds + for (uuid, name, hash_or_path, build_file) in builds log_file = splitext(build_file)[1] * ".log" - Base.info("Building $name [$(string(hash)[1:16])]...") + if hash_or_path isa SHA1 + Base.info("Building $name [$(string(hash_or_path)[1:16])]...") + else + Base.info("Building $name [$hash_or_path]...") + end Base.info(" → $log_file") code = """ empty!(Base.LOAD_PATH) @@ -658,8 +754,12 @@ function test(env::EnvCache, pkgs::Vector{PackageSpec}; coverage=false) version_paths = String[] for pkg in pkgs info = manifest_info(env, pkg.uuid) - haskey(info, "git-tree-sha1") || cmderror("Could not find git-tree-sha1 for package $(pkg.name)") - version_path = find_installed(pkg.uuid, SHA1(info["git-tree-sha1"])) + version_path = if haskey(info, "path") + info["path"] + else + haskey(info, "git-tree-sha1") || cmderror("Could not find git-tree-sha1 for package $(pkg.name)") + find_installed(pkg.uuid, SHA1(info["git-tree-sha1"])) + end testfile = joinpath(version_path, "test", "runtests.jl") if !isfile(testfile) push!(missing_runtests, pkg.name) @@ -726,5 +826,64 @@ function init(path::String) info("Initialized environment in $path by creating the file Project.toml") end +function clone(env::EnvCache, pkgs::Vector{PackageSpec}) + new_repos = Dict{UUID, String}() + for pkg in pkgs + if has_uuid(pkg) + # This is a registered package, set its version to the maximum version + # since we will clone master + uuid = pkg.uuid + max_version = typemin(VersionNumber) + for path in registered_paths(env, uuid) + version_info = load_versions(path) + max_version = max(max_version, maximum(keys(version_info))) + end + pkg.version = VersionNumber(max_version.major, max_version.minor, max_version.patch, max_version.prerelease, ("",)) + else + # We are cloning a packages that doesn't exist in the registry. + # Give it a new UUID and some version + pkg.version = v"0.0" + pkg.uuid = Base.Random.UUID(rand(UInt128)) + info("Cloning an unregistered package, giving it a random UUID of: $(pkg.uuid)") + end + if isdir(joinpath(pkg.path)) + if !isfile(joinpath(pkg.path, "src", pkg.name * ".jl")) + cmderror("Path $(pkg.path) exists but it does not contain `src/$(pkg).jl.") + end + info("Path $(pkg.path) exists and looks like a package, using that.") + else + info("Cloning $(pkg.name) from $(pkg.url) to $(pkg.path)") + mkpath(pkg.path) + new_repos[pkg.uuid] = pkg.path + LibGit2.clone(pkg.url, pkg.path) + end + env.project["deps"][pkg.name] = string(pkg.uuid) + end + try + resolve_versions!(env, pkgs) + new = apply_versions(env, pkgs) + write_env(env) + build_versions(env, new) + catch e + for pkg in pkgs + haskey(new_repos, pkg.uuid) && Base.rm(new_repos[pkg.uuid]; recursive = true) + end + rethrow(e) + end +end + +function free(env::EnvCache, pkgs::Vector{PackageSpec}) + for pkg in pkgs + if isempty(registered_name(env, pkg.uuid)) + cmderror("can only free registered packages") + end + pkg.beingfreed = true + end + resolve_versions!(env, pkgs) + new = apply_versions(env, pkgs) + write_env(env) # write env before building + build_versions(env, new) +end + end # module diff --git a/bin/Pkg2/Pkg2.jl b/src/Pkg2/Pkg2.jl similarity index 100% rename from bin/Pkg2/Pkg2.jl rename to src/Pkg2/Pkg2.jl diff --git a/bin/Pkg2/reqs.jl b/src/Pkg2/reqs.jl similarity index 100% rename from bin/Pkg2/reqs.jl rename to src/Pkg2/reqs.jl diff --git a/bin/Pkg2/types.jl b/src/Pkg2/types.jl similarity index 86% rename from bin/Pkg2/types.jl rename to src/Pkg2/types.jl index 4a52b29405..05ce5731db 100644 --- a/bin/Pkg2/types.jl +++ b/src/Pkg2/types.jl @@ -5,7 +5,8 @@ module Types export VersionInterval, VersionSet import Base: show, isempty, in, intersect, union!, union, ==, hash, copy, deepcopy_internal -import Pkg3.iswindows +import ...Pkg3.iswindows +import ...Pkg3.Types: VersionBound, VersionRange struct VersionInterval lower::VersionNumber @@ -21,6 +22,27 @@ intersect(a::VersionInterval, b::VersionInterval) = VersionInterval(max(a.lower, ==(a::VersionInterval, b::VersionInterval) = a.lower == b.lower && a.upper == b.upper hash(i::VersionInterval, h::UInt) = hash((i.lower, i.upper), h + (0x0f870a92db508386 % UInt)) +# Used to translate from Pkg2 intervals to Pkg3 ranges +function Base.convert(::Type{VersionRange}, a::VersionInterval) + lower, upper = a.lower, a.upper + + lb = VersionBound(lower.major, lower.minor, lower.patch) + + vb = Int[upper.major, upper.minor, upper.patch] + i = 3 + while i > 0 && vb[i] == 0 + pop!(vb) + i -= 1 + end + # NOTE: an upper bound of 0 could happen in principle, e.g. [v"0.0.0-", v"0.0.0") + # but we just ignore this here... + i > 0 || error("invalid interval upper bound v$upper") + vb[i] -= 1 + ub = VersionBound(vb...) + + return VersionRange(lb, ub) +end + function normalize!(ivals::Vector{VersionInterval}) # VersionSet internal normalization: # removes empty intervals and fuses intervals without gaps diff --git a/src/Pkg3.jl b/src/Pkg3.jl index e7e63a8e12..73959e16a2 100644 --- a/src/Pkg3.jl +++ b/src/Pkg3.jl @@ -1,7 +1,6 @@ __precompile__(true) module Pkg3 - if VERSION < v"0.7.0-DEV.2575" const Dates = Base.Dates else @@ -21,6 +20,7 @@ const GLOBAL_SETTINGS = GlobalSettings() depots() = GLOBAL_SETTINGS.depots logdir() = joinpath(depots()[1], "logs") +default_dev_path() = joinpath(depots()[1], "dev") iswindows() = @static VERSION < v"0.7-" ? Sys.is_windows() : Sys.iswindows() isapple() = @static VERSION < v"0.7-" ? Sys.is_apple() : Sys.isapple() @@ -37,6 +37,7 @@ if !isdefined(Base, :EqualTo) end # load snapshotted dependencies +# include("../ext/SHA/src/SHA.jl") include("../ext/BinaryProvider/src/BinaryProvider.jl") include("../ext/TOML/src/TOML.jl") include("../ext/TerminalMenus/src/TerminalMenus.jl") @@ -45,15 +46,29 @@ include("Types.jl") include("GraphType.jl") include("Resolve.jl") include("Display.jl") +include("Pkg2/Pkg2.jl") include("Operations.jl") include("REPLMode.jl") include("API.jl") -import .API: add, rm, up, test, gc, init, build, installed +import .API: add, rm, up, test, gc, init, build, installed, clone, free const update = up +if VERSION.minor == 6 + struct Pkg3Loader end + + Base._str(::Pkg3Loader) = Pkg3Loader() + + function Base.load_hook(::Pkg3Loader, name::String, ::Any) + return _find_package(name) + end +else + Base.find_package(name::String) = _find_package(name) +end + function __init__() push!(empty!(LOAD_PATH), dirname(dirname(@__DIR__))) + VERSION.minor == 6 && push!(LOAD_PATH, Pkg3Loader()) if isdefined(Base, :active_repl) REPLMode.repl_init(Base.active_repl) @@ -71,13 +86,6 @@ function Base.julia_cmd(julia::AbstractString) return cmd end -if VERSION < v"0.7.0-DEV.2303" - Base.find_in_path(name::String, wd::Void) = _find_package(name) - Base.find_in_path(name::String, wd::String) = _find_package(name) -else - Base.find_package(name::String) = _find_package(name) -end - function _find_package(name::String) isabspath(name) && return name base = name @@ -88,6 +96,7 @@ function _find_package(name::String) end info = Pkg3.Operations.package_env_info(base, verb = "use") info == nothing && @goto find_global + haskey(info, "path") && (path = info["path"]; ispath(path)) && return joinpath(path, "src", name) haskey(info, "uuid") || @goto find_global haskey(info, "git-tree-sha1") || @goto find_global uuid = Base.Random.UUID(info["uuid"]) diff --git a/src/REPLMode.jl b/src/REPLMode.jl index e1ca341266..0c655296fc 100644 --- a/src/REPLMode.jl +++ b/src/REPLMode.jl @@ -1,9 +1,8 @@ module REPLMode -import Pkg3 -using Pkg3.Types -using Pkg3.Display -using Pkg3.Operations +import ..Pkg3 +import ..Pkg3: default_dev_path +using ..Pkg3: Types, Display, Operations import Base: LineEdit, REPL, REPLCompletions import Base.Random: UUID @@ -34,6 +33,8 @@ const cmds = Dict( "preview" => :preview, "init" => :init, "build" => :build, + "clone" => :clone, + "free" => :free, ) const opts = Dict( @@ -47,6 +48,8 @@ const opts = Dict( "patch" => :patch, "fixed" => :fixed, "coverage" => :coverage, + "path" => :path, + "name" => :name, ) function parse_option(word::AbstractString) @@ -58,6 +61,14 @@ function parse_option(word::AbstractString) (:opt, opts[k]) : (:opt, opts[k], String(m.captures[3])) end +function parse_string(word::AbstractString) + if !endswith(word, "\"") + cmderror("invalid string syntax, expected ending `\"`") + end + str = word[chr2ind(word, 2):chr2ind(word, length(word)-1)] + return (:string, str) +end + let uuid = raw"(?i)[0-9a-z]{8}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{4}-[0-9a-z]{12}(?-i)", name = raw"(\w+)(?:\.jl)?" global const name_re = Regex("^$name\$") @@ -69,7 +80,6 @@ const lex_re = r"^[\?\./\+\-](?!\-) | [^@\s]+\s*=\s*[^@\s]+ | @\s*[^@\s]* | [^@\ function tokenize(cmd::String)::Vector{Tuple{Symbol,Vararg{Any}}} tokens = Tuple{Symbol,Vararg{Any}}[] - # TODO: handle string-quoted values, e.g. path names words = map(m->m.match, eachmatch(lex_re, cmd)) help_mode = false while !isempty(words) @@ -88,7 +98,9 @@ function tokenize(cmd::String)::Vector{Tuple{Symbol,Vararg{Any}}} end while !isempty(words) word = shift!(words) - if word[1] == '-' + if word[1] == '"' + push!(tokens, parse_string(word)) + elseif word[1] == '-' push!(tokens, parse_option(word)) elseif word[1] == '@' push!(tokens, (:ver, VersionRange(strip(word[2:end])))) @@ -151,6 +163,8 @@ function do_cmd!(tokens, repl; preview = false) cmd == :test ? do_test!(env, tokens) : cmd == :gc ? do_gc!(env, tokens) : cmd == :build ? do_build!(env, tokens) : + cmd == :clone ? do_clone!(env, tokens) : + cmd == :free ? do_free!(env, tokens) : cmderror("`$cmd` command not yet implemented") end @@ -197,9 +211,13 @@ const help = Base.Markdown.parse(""" `gc`: garbage collect packages not used for a significant time - `init` initializes an environment in the current, or git base, directory + `init`: initializes an environment in the current, or git base, directory + + `build`: run the build script for packages - `build` run the build script for packages + `clone`: clones a package from an url to a local directory + + `free`: start using the registered version of a package instead of the cloned one """) const helps = Dict( @@ -300,7 +318,25 @@ const helps = Dict( Run the build script in deps/build.jl for each package in pkgs and all of their dependencies in depth-first recursive order. If no packages are given, runs the build scripts for all packages in the manifest. - """, + """, :clone => md""" + + clone url [--path localpath] + + Clones the package from the git repo at `url` to the path `localpath`, defaulting to `.julia/dev/` + in the home directory. A cloned package have its dependencies read from the packages local dependency file + instead of the registry. It is never be upgraded, removed, or changed in any way by + the package manager. If the package `localpath` exist, no cloning is done and the existing + package at that path is used. To `free` a package, i.e., go back to using the the versioned + package, use `free`. + """, :free => md""" + + free pkg[=uuid] + + Undo the clone command on `pkg`, i.e., instead of using the path where `pkg` + was cloned to when loading it, use the standard versioned path. + This allows the package to again be upgraded. + The directory where the package was originally cloned at is left untouched. + """ ) function do_help!( @@ -349,6 +385,8 @@ function do_rm!(env::EnvCache, tokens::Vector{Tuple{Symbol,Vararg{Any}}}) else cmderror("invalid option for `rm`: --$(token[2])") end + elseif token[1] == :string + cmderror("unexpected string encountered: $(token[2])") end end isempty(pkgs) && @@ -373,6 +411,8 @@ function do_add!(env::EnvCache, tokens::Vector{Tuple{Symbol,Vararg{Any}}}) cmderror("package name/uuid must precede version spec `@$(tokens[1][2])`") elseif token[1] == :opt cmderror("`add` doesn't take options: --$(join(token[2:end], '='))") + elseif token[1] == :string + cmderror("unexpected string encountered: $(token[2])") end end Pkg3.API.add(env, pkgs) @@ -407,6 +447,8 @@ function do_up!(env::EnvCache, tokens::Vector{Tuple{Symbol,Vararg{Any}}}) else cmderror("invalid option for `up`: --$(token[2])") end + elseif token[1] == :string + cmderror("unexpected string encountered: $(token[2])") end last_token_type = token[1] end @@ -486,6 +528,50 @@ function do_init!(tokens::Vector{Tuple{Symbol,Vararg{Any}}}) Pkg3.API.init(pwd()) end +function do_clone!(env::EnvCache, tokens::Vector{Tuple{Symbol,Vararg{Any}}}) + isempty(tokens) && cmderror("`clone` take an url to a package to clone") + local url + basepath = get(ENV, "JULIA_DEVDIR", default_dev_path()) + token = shift!(tokens) + if token[1] != :string + cmderror("expected a url given as a string") + end + url = token[2] + if !isempty(tokens) + token = shift!(tokens) + if token[1] == :opt + if token[2] == :path + if isempty(tokens) + cmderror("expected an argument to the `--path` option") + else + token = shift!(tokens) + if token[1] != :string + cmderror("expecetd a string argument to the `--path` option") + else + basepath = token[2] + end + end + else + cmderror("`clone` only support the `--path` option") + end + end + end + Pkg3.API.clone(env, url, basepath = basepath) +end + +function do_free!(env::EnvCache, tokens::Vector{Tuple{Symbol,Vararg{Any}}}) + pkgs = PackageSpec[] + while !isempty(tokens) + token = shift!(tokens) + if token[1] == :pkg + push!(pkgs, PackageSpec(token[2:end]...)) + else + cmderror("free only takes a list of packages") + end + end + Pkg3.API.free(env, pkgs) +end + function create_mode(repl, main) pkg_mode = LineEdit.Prompt("pkg> "; diff --git a/src/Types.jl b/src/Types.jl index 1dd6bca01e..0259a53a80 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -1,17 +1,18 @@ module Types using Base.Random: UUID -using SHA -using Pkg3: TOML, TerminalMenus, Dates -import Pkg3 -import Pkg3: depots, logdir, iswindows + +using SHA +import ..Pkg3 +using ..Pkg3: TOML, TerminalMenus, Dates +import ..Pkg3: depots, logdir, iswindows export UUID, pkgID, SHA1, VersionRange, VersionSpec, empty_versionspec, Requires, Fixed, merge_requires!, satisfies, PkgError, PackageSpec, UpgradeLevel, EnvCache, - CommandError, cmderror, has_name, has_uuid, write_env, parse_toml, find_registered!, - project_resolve!, manifest_resolve!, registry_resolve!, ensure_resolved, + CommandError, cmderror, has_name, has_uuid, has_path, has_url, has_version, write_env, parse_toml, find_registered!, + project_resolve!, manifest_resolve!, registry_resolve!, path_resolve!, ensure_resolved, manifest_info, registered_uuids, registered_paths, registered_uuid, registered_name, git_file_stream, git_discover, read_project, read_manifest, pathrepr, registries @@ -365,18 +366,30 @@ mutable struct PackageSpec uuid::UUID version::VersionTypes mode::Symbol - PackageSpec(name::String, uuid::UUID, version::VersionTypes) = - new(name, uuid, version, :project) + path::String + url::String + beingfreed::Bool # TODO: Get rid of this field + PackageSpec(name::AbstractString, uuid::UUID, version::VersionTypes, project::Symbol=:project, + path::AbstractString="", url::AbstractString="", beingfreed=false) = + new(name, uuid, version, project, path, url, beingfreed) end -PackageSpec(name::String, uuid::UUID) = +PackageSpec(name::AbstractString, uuid::UUID) = PackageSpec(name, uuid, VersionSpec()) PackageSpec(name::AbstractString, version::VersionTypes=VersionSpec()) = - PackageSpec(name, UUID(zero(UInt128)), version) + PackageSpec(name, name == "julia" ? uuid_julia : UUID(zero(UInt128)), version) PackageSpec(uuid::UUID, version::VersionTypes=VersionSpec()) = PackageSpec("", uuid, version) +PackageSpec(;name::AbstractString="", uuid::UUID=UUID(zero(UInt128)), version::VersionTypes=VersionSpec(), + mode::Symbol=:project, path::AbstractString="", url::AbstractString="", beingfreed::Bool=false) = + PackageSpec(name, uuid, version, mode, path, url, beingfreed) + has_name(pkg::PackageSpec) = !isempty(pkg.name) has_uuid(pkg::PackageSpec) = pkg.uuid != UUID(zero(UInt128)) +has_path(pkg::PackageSpec) = !isempty(pkg.path) +has_url(pkg::PackageSpec) = !isempty(pkg.urkl) +# TODO: Optimize:? +has_version(pkg::PackageSpec) = !(pkg.version isa VersionSpec && pkg.version.ranges == VersionSpec().ranges) function Base.show(io::IO, pkg::PackageSpec) print(io, "PackageSpec(") @@ -733,6 +746,19 @@ function registry_resolve!(env::EnvCache, pkgs::AbstractVector{PackageSpec}) return pkgs end + +function path_resolve!(env::EnvCache, pkgs::AbstractVector{PackageSpec}) + for pkg in pkgs + pkg.beingfreed && continue + info = manifest_info(env, pkg.uuid) + info == nothing && continue + if haskey(info, "path") + pkg.path = info["path"] + haskey(info, "version") && (pkg.version = VersionNumber(info["version"])) + end + end +end + "Ensure that all packages are fully resolved" function ensure_resolved( env::EnvCache, @@ -935,7 +961,7 @@ function registered_uuid(env::EnvCache, name::String)::UUID uuids = registered_uuids(env, name) length(uuids) == 0 && return UUID(zero(UInt128)) choices::Vector{String} = [] - choices_cache::Vector{Tuple{UUID, String}} = [] + choices_cache::Vector{Tuple{UUID, String}} = [] for uuid in uuids values = registered_info(env, uuid, "repo") for value in values @@ -974,7 +1000,7 @@ function registered_info(env::EnvCache, uuid::UUID, key::String) for path in paths info = parse_toml(path, "package.toml") value = get(info, key, nothing) - push!(values, (path, value)) + push!(values, (path, value)) end return values end diff --git a/test/operations.jl b/test/operations.jl index b083adbed3..c7bfbfd2ff 100644 --- a/test/operations.jl +++ b/test/operations.jl @@ -6,14 +6,17 @@ using Pkg3.Types function temp_pkg_dir(fn::Function) local project_path + local dev_dir try # TODO: Use a temporary depot project_path = joinpath(tempdir(), randstring()) - withenv("JULIA_ENV" => project_path) do + dev_dir = joinpath(tempdir(), randstring()) + withenv("JULIA_ENV" => project_path, "JULIA_DEVDIR" => dev_dir) do fn(project_path) end finally rm(project_path, recursive=true, force=true) + rm(dev_dir, recursive=true, force=true) end end @@ -52,6 +55,15 @@ temp_pkg_dir() do project_path usage = Pkg3.TOML.parse(String(read(joinpath(Pkg3.logdir(), "usage.toml")))) @test any(x -> startswith(x, joinpath(project_path, "Manifest.toml")), keys(usage)) + # Clone an unregistered packge and check that it can be imported + Pkg3.clone("https://github.com/fredrikekre/ImportMacros.jl") + @eval import ImportMacros + Pkg3.test("ImportMacros") + + # Clone a registered packge and check that it can be imported + Pkg3.clone("https://github.com/KristofferC/TimerOutputs.jl") + @eval import TimerOutputs + nonexisting_pkg = randstring(14) @test_throws CommandError Pkg3.add(nonexisting_pkg) @test_throws CommandError Pkg3.up(nonexisting_pkg) @@ -74,6 +86,10 @@ temp_pkg_dir() do project_path end Pkg3.rm(TEST_PKG) + # TimerOutputs depends on Crayons, which is therefore still installed + @test isinstalled(TEST_PKG) + # This removes also Crayons + Pkg3.rm("TimerOutputs") @test !isinstalled(TEST_PKG) end diff --git a/test/runtests.jl b/test/runtests.jl index b39539a93d..b6060df47d 100644 --- a/test/runtests.jl +++ b/test/runtests.jl @@ -7,7 +7,7 @@ else using Test end -include("resolve.jl") include("operations.jl") +include("resolve.jl") end # module