rebase of #56 (implement clone/free) onto #81 (graph refactor)

.travis.yml

@@ -16,8 +16,5 @@ before_script:
 
 script:
   - if [[ -a .git/shallow ]]; then git fetch --unshallow; fi
-  - julia --check-bounds=yes -e 'Pkg.clone(pwd()); Pkg.build("Pkg3"); Pkg.test("Pkg3"; coverage=true)'
-
-after_success:
-  - julia -e 'cd(Pkg.dir("Pkg3")); Pkg.add("Coverage"); using Coverage; Coveralls.submit(Coveralls.process_folder())'
+  - julia --check-bounds=yes -e 'Pkg.add("SHA"); include("src/Pkg3.jl"); Pkg3.clone(pwd()); Pkg3.test("Pkg3"; coverage=true)'
 

REQUIRE

@@ -1,3 +1,2 @@
 julia 0.6
 SHA
-# Pkg.clone("https://github.com/StefanKarpinski/TOML.jl.git")

bin/loadmeta.jl

@@ -6,9 +6,9 @@ using Base.Random: UUID
 using Pkg3.Types
 using Pkg3.Types: uuid_package, uuid_registry, uuid5
 
-include("Pkg2/Pkg2.jl")
-import .Pkg2.Reqs: Reqs, Requirement
-import .Pkg2.Types: VersionInterval
+import Pkg3: Pkg2
+import Pkg3.Pkg2.Reqs: Reqs, Requirement
+import Pkg3.Pkg2.Types: VersionInterval
 
 ## Loading data into various data structures ##
 

ext/BinaryProvider/src/BinaryProvider.jl

@@ -1,19 +1,19 @@
 module BinaryProvider
 
-import Pkg3: iswindows, isapple, islinux
+import ..Pkg3: iswindows, isapple, islinux
 
 # Include our subprocess running funtionality
-include("OutputCollector.jl")
+#include("OutputCollector.jl")
 # External utilities such as downloading/decompressing tarballs
 include("PlatformEngines.jl")
 # Platform naming
-include("PlatformNames.jl")
+#include("PlatformNames.jl")
 # Everything related to file/path management
-include("Prefix.jl")
+#include("Prefix.jl")
 # Abstraction of "needing" a file, that would trigger an install
-include("Products.jl")
+#include("Products.jl")
 # Abstraction of bundled binary package
-include("BinaryPackage.jl")
+#include("BinaryPackage.jl")
 
 # BinDeps support, disabled for now because I don't particularly want to force
 # users to install BinDeps to install this package.  That seems counter-productive

ext/TerminalMenus/src/TerminalMenus.jl

@@ -1,6 +1,6 @@
 module TerminalMenus
 
-import Pkg3.iswindows
+import ..Pkg3.iswindows
 
 terminal = nothing  # The user terminal
 

src/API.jl

@@ -1,9 +1,9 @@
 module API
 
-import Pkg3
-using Pkg3.Display.DiffEntry
-import Pkg3: depots, logdir, TOML
-using Pkg3: Types, Dates
+import ..Pkg3
+using ..Pkg3: Display.DiffEntry
+import ..Pkg3: depots, logdir, default_dev_path, TOML
+using ..Pkg3: Dates, Types
 using Base.Random.UUID
 
 previewmode_info() = info("In preview mode")
@@ -128,7 +128,7 @@ function test(env::EnvCache, pkgs::Vector{PackageSpec}; coverage=false, preview=
 end
 
 
-function convert(::Type{Dict{String, VersionNumber}}, diffs::Union{Array{DiffEntry}, Void})    
+function convert(::Type{Dict{String, VersionNumber}}, diffs::Union{Array{DiffEntry}, Void})
     version_status = Dict{String, VersionNumber}()
     diffs == nothing && return version_status
     for entry in diffs
@@ -239,18 +239,52 @@ function gc(env::EnvCache=EnvCache(); period = Week(6), preview=env.preview[])
     info("Deleted $(length(paths_to_delete)) package installations", byte_save_str)
 end
 
-function _get_deps!(env::EnvCache, pkgs::Vector{PackageSpec}, uuids::Vector{Base.Random.UUID})
-   for pkg in pkgs
-       info = manifest_info(env, pkg.uuid)
-       pkg.uuid in uuids && continue
-       push!(uuids, pkg.uuid)
-       if haskey(info, "deps")
-           pkgs = [PackageSpec(name, UUID(uuid)) for (name, uuid) in info["deps"]]
-           _get_deps!(env, pkgs, uuids)
-       end
-   end
+
+
+function url_and_pkg(url_or_pkg::AbstractString)
+    # try to parse as URL or local path
+    m = match(r"(?:^|[/\\])(\w+?)(?:\.jl)?(?:\.git)?$", url_or_pkg)
+    m === nothing && cmderror("can't determine package name from URL: $url_or_pkg")
+    return url_or_pkg, m.captures[1]
+end
+clone(pkg::String; kwargs...) = clone(EnvCache(), pkg; kwargs...)
+
+function clone(env::EnvCache, url::AbstractString; name=nothing, basepath=get(ENV, "JULIA_DEVDIR", default_dev_path()), preview=env.preview[])
+    preview && cmderror("Preview mode not implemented for cloning")
+    if name == nothing
+        url, name = url_and_pkg(url)
+    end
+    pkg = PackageSpec(name=name, path=joinpath(basepath, name), url=url)
+    registry_resolve!(env, [pkg])
+    project_resolve!(env, [pkg])
+    manifest_resolve!(env, [pkg])
+    Pkg3.Operations.clone(env, [pkg])
+end
+
+free(;kwargs...)                           = free(PackageSpec[], kwargs...)
+free(pkg::String; kwargs...)               = free([pkg]; kwargs...)
+free(pkgs::Vector{String}; kwargs...)      = free([PackageSpec(pkg) for pkg in pkgs]; kwargs...)
+free(pkgs::Vector{PackageSpec}; kwargs...) = free(EnvCache(), pkgs; kwargs...)
+function free(env::EnvCache, pkgs::Vector{PackageSpec}; preview = env.preview[])
+    env.preview[] = preview
+    preview && previewmode_info()
+    project_resolve!(env, pkgs)
+    manifest_resolve!(env, pkgs)
+    ensure_resolved(env, pkgs)
+    Pkg3.Operations.free(env, pkgs)
 end
 
+function _get_deps!(env::EnvCache, pkgs::Vector{PackageSpec}, uuids::Vector{Base.Random.UUID})
+    for pkg in pkgs
+        info = manifest_info(env, pkg.uuid)
+        pkg.uuid in uuids && continue
+        push!(uuids, pkg.uuid)
+        if haskey(info, "deps")
+            pkgs = [PackageSpec(name, UUID(uuid)) for (name, uuid) in info["deps"]]
+            _get_deps!(env, pkgs, uuids)
+        end
+    end
+ end
 
 build(pkgs...) = build([PackageSpec(pkg) for pkg in pkgs])
 build(pkg::Array{Union{}, 1}) = build(PackageSpec[])
@@ -275,7 +309,6 @@ function build(env::EnvCache, pkgs::Vector{PackageSpec})
    Pkg3.Operations.build_versions(env, uuids)
 end
 
-
 function init(path = pwd())
     Pkg3.Operations.init(path)
 end

src/Display.jl

@@ -1,7 +1,7 @@
 module Display
 
 using Base.Random: UUID
-using Pkg3.Types
+using ..Pkg3.Types
 
 export print_project_diff, print_manifest_diff
 
@@ -12,15 +12,15 @@ const colors = Dict(
     '↑' => :light_yellow,
     '~' => :light_yellow,
     '↓' => :light_magenta,
-    '?' => :red,
+    '?' => :white,
 )
 const color_dark = :light_black
 
 function status(env::EnvCache, mode::Symbol, use_as_api=false)
     project₀ = project₁ = env.project
     manifest₀ = manifest₁ = env.manifest
     diff = nothing
-    
+
     if env.git != nothing
         git_path = LibGit2.path(env.git)
         project_path = relpath(env.project_file, git_path)
@@ -76,17 +76,23 @@ function print_manifest_diff(env₀::EnvCache, env₁::EnvCache)
 end
 
 struct VerInfo
-    hash::SHA1
+    hash_or_path::Union{SHA1, String}
     ver::Union{VersionNumber,Void}
 end
+islocal(v::VerInfo) = v.hash_or_path isa String
 
-vstring(a::VerInfo) =
-    a.ver == nothing ? "[$(string(a.hash)[1:16])]" : "v$(a.ver)"
+function vstring(a::VerInfo)
+    if islocal(a)
+        return "[$(a.hash_or_path)]"
+    else
+        return a.ver == nothing ? "[$(string(a.hash_or_path)[1:16])]" : "v$(a.ver)"
+    end
+end
 
 Base.:(==)(a::VerInfo, b::VerInfo) =
-    a.hash == b.hash && a.ver == b.ver
+    a.hash_or_path == b.hash_or_path && a.ver == b.ver
 
-≈(a::VerInfo, b::VerInfo) = a.hash == b.hash &&
+≈(a::VerInfo, b::VerInfo) = a.hash_or_path isa SHA1 && a.hash_or_path == b.hash_or_path &&
     (a.ver == nothing || b.ver == nothing || a.ver == b.ver)
 
 struct DiffEntry
@@ -105,19 +111,24 @@ function print_diff(io::IO, diff::Vector{DiffEntry})
                 verb = ' '
                 vstr = vstring(x.new)
             else
-                if x.old.hash != x.new.hash && x.old.ver != x.new.ver
+                if x.old.hash_or_path != x.new.hash_or_path && x.old.ver != x.new.ver
                     verb = x.old.ver == nothing || x.new.ver == nothing ||
                            x.old.ver == x.new.ver ? '~' :
                            x.old.ver < x.new.ver  ? '↑' : '↓'
                 else
                     verb = '?'
-                    msg = x.old.hash == x.new.hash ?
+                    msg = x.old.hash_or_path isa SHA1 && x.old.hash_or_path == x.new.hash_or_path ?
                         "hashes match but versions don't: $(x.old.ver) ≠ $(x.new.ver)" :
-                        "versions match but hashes don't: $(x.old.hash) ≠ $(x.new.hash)"
+                        "versions match but hashes don't: $(x.old.hash_or_path) ≠ $(x.new.hash_or_path)"
                     push!(warnings, msg)
                 end
-                vstr = x.old.ver == x.new.ver ? vstring(x.new) :
-                    vstring(x.old) * " ⇒ " * vstring(x.new)
+                # Moving from hash -> path
+                if typeof(x.old.hash_or_path) != typeof(x.new.hash_or_path)
+                    vstr = vstring(x.old) * " ⇒ " * vstring(x.new)
+                else
+                    vstr = x.old.ver == x.new.ver ? vstring(x.new) :
+                        vstring(x.old) * " ⇒ " * vstring(x.new)
+                end
             end
         elseif x.new != nothing
             verb = '+'
@@ -148,9 +159,9 @@ end
 
 function name_ver_info(info::Dict)
     name = info["name"]
-    hash = haskey(info, "git-tree-sha1") ? SHA1(info["git-tree-sha1"]) : nothing
+    hash_or_path = haskey(info, "path") ? info["path"] : haskey(info, "git-tree-sha1") ? SHA1(info["git-tree-sha1"]) : nothing
     ver = haskey(info, "version") ? VersionNumber(info["version"]) : nothing
-    name, VerInfo(hash, ver)
+    name, VerInfo(hash_or_path, ver)
 end
 
 function manifest_diff(manifest₀::Dict, manifest₁::Dict)

src/GraphType.jl

@@ -285,6 +285,7 @@ mutable struct Graph
         adjdict = [Dict{Int,Int}() for p0 = 1:np]
 
         for p0 = 1:np, v0 = 1:(spp[p0]-1), (p1,rmsk1) in extended_deps[p0][v0]
+            @assert p0 ≠ p1
             j0 = get(adjdict[p1], p0, length(gadj[p0]) + 1)
             j1 = get(adjdict[p0], p1, length(gadj[p1]) + 1)