diff --git a/stdlib/Pkg/src/Operations.jl b/stdlib/Pkg/src/Operations.jl index d2f638e4edc5cd..1994189e68fe01 100644 --- a/stdlib/Pkg/src/Operations.jl +++ b/stdlib/Pkg/src/Operations.jl @@ -537,6 +537,9 @@ function apply_versions(ctx::Context, pkgs::Vector{PackageSpec}, hashes::Dict{UU end try success = install_archive(urls[pkg.uuid], hashes[pkg.uuid], path) + if ctx.use_only_tarballs_for_downloads && !success + cmderror("failed to get tarball from $(urls[pkg.uuid])") + end put!(results, (pkg, success, path)) catch err put!(results, (pkg, err, catch_backtrace())) diff --git a/stdlib/Pkg/src/PlatformEngines.jl b/stdlib/Pkg/src/PlatformEngines.jl index 0968c7f9a97691..8fb430d146be8b 100644 --- a/stdlib/Pkg/src/PlatformEngines.jl +++ b/stdlib/Pkg/src/PlatformEngines.jl @@ -124,7 +124,7 @@ If `verbose` is `true`, print out the various engines as they are searched. function probe_platform_engines!(;verbose::Bool = false) global already_probed global gen_download_cmd, gen_list_tarball_cmd, gen_package_cmd - global gen_unpack_cmd, parse_tarball_listing, gen_sh_cmd + global gen_unpack_cmd, parse_tarball_listing already_probed && return # download_engines is a list of (test_cmd, download_opts_functor) # The probulator will check each of them by attempting to run `$test_cmd`, @@ -133,6 +133,7 @@ function probe_platform_engines!(;verbose::Bool = false) (`curl --help`, (url, path) -> `curl -C - -\# -f -o $path -L $url`), (`wget --help`, (url, path) -> `wget -c -O $path $url`), (`fetch --help`, (url, path) -> `fetch -f $path $url`), + (`busybox wget --help`, (url, path) -> `busybox wget -c -O $path $url`), ] # 7z is rather intensely verbose. We also want to try running not only @@ -156,20 +157,36 @@ function probe_platform_engines!(;verbose::Bool = false) # Tar is rather less verbose, and we don't need to search multiple places # for it, so just rely on PATH to have `tar` available for us: - unpack_tar = (tarball_path, out_path) -> - `tar xzf $(tarball_path) --directory=$(out_path)` - package_tar = (in_path, tarball_path) -> - `tar -czvf $tarball_path -C $(in_path) .` - list_tar = (in_path) -> `tar tzf $in_path` # compression_engines is a list of (test_cmd, unpack_opts_functor, # package_opts_functor, list_opts_functor, parse_functor). The probulator # will check each of them by attempting to run `$test_cmd`, and if that # works, will set the global compression functions appropriately. gen_7z = (p) -> (unpack_7z(p), package_7z(p), list_7z(p), parse_7z_list) - compression_engines = Tuple[ - (`tar --help`, unpack_tar, package_tar, list_tar, parse_tar_list), - ] + compression_engines = Tuple[] + + for tar_cmd in [`tar`, `busybox tar`] + # Some tar's aren't smart enough to auto-guess decompression method. :( + unpack_tar = (tarball_path, out_path) -> begin + Jjz = "z" + if endswith(tarball_path, ".xz") + Jjz = "J" + elseif endswith(tarball_path, ".bz2") + Jjz = "j" + end + return `$tar_cmd -x$(Jjz)f $(tarball_path) --directory=$(out_path)` + end + package_tar = (in_path, tarball_path) -> + `$tar_cmd -czvf $tarball_path -C $(in_path) .` + list_tar = (in_path) -> `$tar_cmd -tzf $in_path` + push!(compression_engines, ( + `$tar_cmd --help`, + unpack_tar, + package_tar, + list_tar, + parse_tar_list, + )) + end # For windows, we need to tweak a few things, as the tools available differ @static if Sys.iswindows() @@ -182,6 +199,7 @@ function probe_platform_engines!(;verbose::Bool = false) [System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12; \$webclient = (New-Object System.Net.Webclient); + \$webclient.Headers.Add("user-agent", \"Pkg.jl (https://github.com/JuliaLang/Pkg.jl)\"); \$webclient.DownloadFile(\"$url\", \"$path\") """ replace(webclient_code, "\n" => " ") @@ -192,10 +210,10 @@ function probe_platform_engines!(;verbose::Bool = false) # We want to search both the `PATH`, and the direct path for powershell psh_path = "C:\\Windows\\System32\\WindowsPowerShell\\v1.0\\powershell" prepend!(download_engines, [ - (`$psh_path -Help`, psh_download(psh_path)) + (`$psh_path -Command ""`, psh_download(psh_path)) ]) prepend!(download_engines, [ - (`powershell -Help`, psh_download(`powershell`)) + (`powershell -Command ""`, psh_download(`powershell`)) ]) # We greatly prefer `7z` as a compression engine on Windows @@ -209,13 +227,14 @@ function probe_platform_engines!(;verbose::Bool = false) # Allow environment override if haskey(ENV, "BINARYPROVIDER_DOWNLOAD_ENGINE") engine = ENV["BINARYPROVIDER_DOWNLOAD_ENGINE"] - dl_ngs = filter(e -> e[1].exec[1] == engine, download_engines) + es = split(engine) + dl_ngs = filter(e -> e[1].exec[1:length(es)] == es, download_engines) if isempty(dl_ngs) all_ngs = join([d[1].exec[1] for d in download_engines], ", ") warn_msg = "Ignoring BINARYPROVIDER_DOWNLOAD_ENGINE as its value " warn_msg *= "of `$(engine)` doesn't match any known valid engines." warn_msg *= " Try one of `$(all_ngs)`." - warn(warn_msg) + @warn(warn_msg) else # If BINARYPROVIDER_DOWNLOAD_ENGINE matches one of our download engines, # then restrict ourselves to looking only at that engine @@ -225,13 +244,14 @@ function probe_platform_engines!(;verbose::Bool = false) if haskey(ENV, "BINARYPROVIDER_COMPRESSION_ENGINE") engine = ENV["BINARYPROVIDER_COMPRESSION_ENGINE"] - comp_ngs = filter(e -> e[1].exec[1] == engine, compression_engines) + es = split(engine) + comp_ngs = filter(e -> e[1].exec[1:length(es)] == es, compression_engines) if isempty(comp_ngs) all_ngs = join([c[1].exec[1] for c in compression_engines], ", ") warn_msg = "Ignoring BINARYPROVIDER_COMPRESSION_ENGINE as its " warn_msg *= "value of `$(engine)` doesn't match any known valid " warn_msg *= "engines. Try one of `$(all_ngs)`." - warn(warn_msg) + @warn(warn_msg) else # If BINARYPROVIDER_COMPRESSION_ENGINE matches one of our download # engines, then restrict ourselves to looking only at that engine diff --git a/stdlib/Pkg/src/Types.jl b/stdlib/Pkg/src/Types.jl index ec24acbb274434..fdf78c9d7a47a0 100644 --- a/stdlib/Pkg/src/Types.jl +++ b/stdlib/Pkg/src/Types.jl @@ -325,6 +325,7 @@ Base.@kwdef mutable struct Context env::EnvCache = EnvCache() preview::Bool = false use_libgit2_for_all_downloads::Bool = false + use_only_tarballs_for_downloads::Bool = false num_concurrent_downloads::Int = 8 graph_verbose::Bool = false stdlibs::Dict{UUID,String} = gather_stdlib_uuids() diff --git a/stdlib/Pkg/test/pkg.jl b/stdlib/Pkg/test/pkg.jl index 2a87c55f2a6ba7..4577bd5fff0d9a 100644 --- a/stdlib/Pkg/test/pkg.jl +++ b/stdlib/Pkg/test/pkg.jl @@ -302,6 +302,19 @@ temp_pkg_dir() do project_path end end +temp_pkg_dir() do project_path + @testset "libgit2 downloads" begin + Pkg.add(TEST_PKG.name; use_libgit2_for_all_downloads=true) + @test haskey(Pkg.installed(), TEST_PKG.name) + Pkg.rm(TEST_PKG.name) + end + @testset "tarball downloads" begin + Pkg.add("JSON"; use_only_tarballs_for_downloads=true) + @test haskey(Pkg.installed(), "JSON") + Pkg.rm("JSON") + end +end + @testset "parse package url win" begin @test typeof(Pkg.REPLMode.parse_package("https://github.com/abc/ABC.jl"; add_or_develop=true)) == PackageSpec end