implement gc (#60)

* implement "garbage collection"

ext/TOML/src/TOML.jl

@@ -1,7 +1,9 @@
 module TOML
 
     if Base.isdeprecated(Base, :Dates)
-        import Dates
+        using Dates
+    else
+        using Base.Dates
     end
 
     include("parser.jl")

src/API.jl

@@ -1,7 +1,8 @@
 module API
 
 import Pkg3
-using Pkg3.Types
+import Pkg3: depots, logdir, TOML
+using Pkg3: Types, Dates
 using Base.Random.UUID
 
 previewmode_info() = info("In preview mode")
@@ -75,5 +76,103 @@ function test(env::EnvCache, pkgs::Vector{PackageSpec}; coverage=false, preview=
     Pkg3.Operations.test(env, pkgs; coverage=coverage)
 end
 
+
+function recursive_dir_size(path)
+    sz = 0
+    for (root, dirs, files) in walkdir(path)
+      for file in files
+          sz += stat(joinpath(root, file)).size
+      end
+    end
+    return sz
+end
+
+function gc(env::EnvCache=EnvCache(); period = Week(6), preview=env.preview[])
+    env.preview[] = preview
+    preview && previewmode_info()
+
+    # If the manifest was not used
+    gc_time = now() - period
+    usage_file = joinpath(logdir(), "usage.toml")
+
+    # Collect only the manifest that is least recently used
+    manifest_date = Dict{String, DateTime}()
+    for (manifest_file, infos) in TOML.parse(String(read(usage_file)))
+        for info in infos
+            date = info["time"]
+            manifest_date[manifest_file] = haskey(manifest_date, date) ? max(manifest_date[date], date) : date
+        end
+    end
+
+    # Find all reachable packages through manifests recently used
+    new_usage = Dict{String, Any}()
+    paths_to_keep = String[]
+    for (manifestfile, date) in manifest_date
+        !isfile(manifestfile) && continue
+        if date < gc_time
+            continue
+        end
+        infos = read_manifest(manifestfile)
+        new_usage[manifestfile] = [Dict("time" => date)]
+        for entry in infos
+            entry isa Pair || continue
+            name, _stanzas = entry
+            @assert length(_stanzas) == 1
+            stanzas = _stanzas[1]
+            if stanzas isa Dict && haskey(stanzas, "uuid") && haskey(stanzas, "hash-sha1")
+                push!(paths_to_keep, Pkg3.Operations.find_installed(UUID(stanzas["uuid"]), SHA1(stanzas["hash-sha1"])))
+            end
+        end
+    end
+
+    # Collect the paths to delete (everything that is not reachable)
+    paths_to_delete = String[]
+    for depot in depots()
+        packagedir = abspath(depot, "packages")
+        if isdir(packagedir)
+            for uuidslug in readdir(packagedir)
+                for shaslug in readdir(joinpath(packagedir, uuidslug))
+                    versiondir = joinpath(packagedir, uuidslug, shaslug)
+                    if !(versiondir in paths_to_keep)
+                        push!(paths_to_delete, versiondir)
+                    end
+                end
+            end
+        end
+    end
+
+    # Delete paths for noreachable package versions and compute size saved
+    sz = 0
+    for path in paths_to_delete
+        sz += recursive_dir_size(path)
+        if !env.preview[]
+            Base.rm(path; recursive=true)
+        end
+    end
+
+    # Delete package paths that are now empty
+    for depot in depots()
+        packagedir = abspath(depot, "packages")
+        if isdir(packagedir)
+            for uuidslug in readdir(packagedir)
+                uuidslug_path = joinpath(packagedir, uuidslug)
+                if isempty(readdir(uuidslug_path))
+                    !env.preview[] && Base.rm(uuidslug_path)
+                end
+            end
+        end
+    end
+
+    # Write the new condensed usage file
+    if !env.preview[]
+        open(usage_file, "w") do io
+            TOML.print(io, new_usage, sorted=true)
+        end
+    end
+    bytes, mb = Base.prettyprint_getunits(sz, length(Base._mem_units), Int64(1024))
+    byte_save_str = length(paths_to_delete) == 0 ? "" : (" saving " * @sprintf("%.3f %s", bytes, Base._mem_units[mb]))
+    info("Deleted $(length(paths_to_delete)) package installations", byte_save_str)
+end
+
 end # module
 

src/Pkg3.jl

@@ -1,8 +1,16 @@
 __precompile__(true)
 module Pkg3
 
+
+if VERSION < v"0.7.0-DEV.2575"
+    const Dates = Base.Dates
+else
+    import Dates
+end
+
 const DEPOTS = [joinpath(homedir(), ".julia")]
 depots() = DEPOTS
+logdir() = joinpath(DEPOTS[1], "logs")
 
 const USE_LIBGIT2_FOR_ALL_DOWNLOADS = false
 const NUM_CONCURRENT_DOWNLOADS      = 8
@@ -33,7 +41,7 @@ include("Operations.jl")
 include("REPLMode.jl")
 include("API.jl")
 
-import .API: add, rm, up, test
+import .API: add, rm, up, test, gc
 const update = up
 
 @enum LoadErrorChoice LOAD_ERROR_QUERY LOAD_ERROR_INSTALL LOAD_ERROR_ERROR

src/REPLMode.jl

@@ -144,7 +144,8 @@ function do_cmd!(env, tokens, repl)
     cmd == :add     ?     do_add!(env, tokens) :
     cmd == :up      ?      do_up!(env, tokens) :
     cmd == :status  ?  do_status!(env, tokens) :
-    cmd == :test   ?   do_test!(env, tokens) :
+    cmd == :test    ?    do_test!(env, tokens) :
+    cmd == :gc      ?      do_gc!(env, tokens) :
         cmderror("`$cmd` command not yet implemented")
 end
 
@@ -188,6 +189,8 @@ const help = Base.Markdown.parse("""
     `preview`: previews a subsequent command without affecting the current state
 
     `test`: run tests for packages
+
+    `gc`: garbage collect packages not used for a significant time
     """)
 
 const helps = Dict(
@@ -273,7 +276,10 @@ const helps = Dict(
     Run the tests for package `pkg`. This is done by running the file `test/runtests.jl`
     in the package directory. The option `--coverage` can be used to run the tests with
     coverage enabled.
-    """,
+    """, :gc => md"""
+
+    Deletes packages that are not reached from any environment used within the last 6 weeks.
+    """
 )
 
 function do_help!(
@@ -434,6 +440,12 @@ function do_test!(env::EnvCache, tokens::Vector{Tuple{Symbol,Vararg{Any}}})
     Pkg3.API.test(env, pkgs; coverage = coverage)
 end
 
+function do_gc!(env::EnvCache, tokens::Vector{Tuple{Symbol,Vararg{Any}}})
+    !isempty(tokens) && cmderror("`gc` does not take any arguments")
+    Pkg3.API.gc(env)
+end
+
+
 function create_mode(repl, main)
     pkg_mode = LineEdit.Prompt("pkg> ";
         prompt_prefix = Base.text_colors[:blue],

src/Types.jl

@@ -2,11 +2,10 @@ module Types
 
 using Base.Random: UUID
 using Pkg3.Pkg2.Types: VersionSet, Available
-using Pkg3.TOML
-using Pkg3.TerminalMenus
+using Pkg3: TOML, TerminalMenus, Dates
 
 import Pkg3
-import Pkg3: depots, iswindows
+import Pkg3: depots, logdir, iswindows
 
 export SHA1, VersionRange, VersionSpec, PackageSpec, UpgradeLevel, EnvCache,
     CommandError, cmderror, has_name, has_uuid, write_env, parse_toml, find_registered!,
@@ -245,6 +244,7 @@ struct EnvCache
                 isfile(manifest_file) && break
             end
         end
+        write_env_usage(manifest_file)
         manifest = read_manifest(manifest_file)
         uuids = Dict{String,Vector{UUID}}()
         paths = Dict{UUID,Vector{String}}()
@@ -264,6 +264,17 @@ struct EnvCache
 end
 EnvCache() = EnvCache(get(ENV, "JULIA_ENV", nothing))
 
+function write_env_usage(manifest_file::AbstractString)
+    !ispath(logdir()) && mkpath(logdir())
+    usage_file = joinpath(logdir(), "usage.toml")
+    touch(usage_file)
+    !isfile(manifest_file) && return
+    open(usage_file, "a") do io
+        println(io, "[[\"", escape_string(manifest_file), "\"]]")
+        println(io, "time = ", Dates.format(now(), "YYYY-mm-ddTHH:MM:SS.sssZ"))
+    end
+end
+
 function read_project(io::IO)
     project = TOML.parse(io)
     if !haskey(project, "deps")

test/runtests.jl

@@ -1,17 +1,18 @@
 using Pkg3
 using Pkg3.Types
-if Base.isdeprecated(Main, :Test)
-    using Test
-else
+if VERSION < v"0.7.0-DEV.2005"
     using Base.Test
+else
+    using Test
 end
 
 function temp_pkg_dir(fn::Function)
     local project_path
     try
+        # TODO: Use a temporary depot
         project_path = joinpath(tempdir(), randstring())
         withenv("JULIA_ENV" => project_path) do
-            fn()
+            fn(project_path)
         end
     finally
         rm(project_path, recursive=true, force=true)
@@ -23,7 +24,7 @@ end
 # in the meantime
 const TEST_PKG = "Crayons"
 
-temp_pkg_dir() do
+temp_pkg_dir() do project_path
     Pkg3.add(TEST_PKG; preview = true)
     @test_warn "not in project" Pkg3.API.rm("Example")
     Pkg3.add(TEST_PKG)
@@ -45,6 +46,8 @@ temp_pkg_dir() do
         @test contains(sprint(showerror, e), TEST_PKG)
     end
 
+    usage = Pkg3.TOML.parse(String(read(joinpath(Pkg3.logdir(), "usage.toml"))))
+    @test any(x -> startswith(x, joinpath(project_path, "Manifest.toml")), keys(usage))
 
     nonexisting_pkg = randstring(14)
     @test_throws CommandError Pkg3.add(nonexisting_pkg)