diff --git a/Cargo.toml b/Cargo.toml index 477a044f688..b992dfba76d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -21,7 +21,7 @@ atty = "0.2" crates-io = { path = "src/crates-io", version = "0.16" } crossbeam = "0.3" crypto-hash = "0.3" -curl = "0.4.6" +curl = "0.4.11" docopt = "0.8.1" env_logger = "0.5" failure = "0.1.1" @@ -91,3 +91,4 @@ doc = false [[test]] name = "testsuite" path = "tests/testsuite/lib.rs" + diff --git a/src/cargo/core/package.rs b/src/cargo/core/package.rs index 6fdc1f1ef27..0f402b07ff3 100644 --- a/src/cargo/core/package.rs +++ b/src/cargo/core/package.rs @@ -197,22 +197,32 @@ impl<'cfg> PackageSet<'cfg> { Box::new(self.packages.keys()) } - pub fn get(&self, id: &PackageId) -> CargoResult<&Package> { - let slot = self.packages.get(id).ok_or_else(|| { - internal(format!("couldn't find `{}` in package set", id)) - })?; - if let Some(pkg) = slot.borrow() { - return Ok(pkg) + pub fn get(&self, ids: &[&PackageId]) -> CargoResult> { + let mut pending = BTreeMap::new(); + for &id in ids { + let slot = self.packages.get(id).ok_or_else(|| { + internal(format!("couldn't find `{}` in package set", id)) + })?; + if slot.borrow().is_none() { + let &mut (ref mut pending_ids, ref mut slots) = pending.entry(id.source_id()).or_insert_with(|| (Vec::new(), Vec::new())); + pending_ids.push(id); + slots.push(slot); + } } let mut sources = self.sources.borrow_mut(); - let source = sources.get_mut(id.source_id()).ok_or_else(|| { - internal(format!("couldn't find source for `{}`", id)) - })?; - let pkg = source.download(id).chain_err(|| { - format_err!("unable to get packages from source") - })?; - assert!(slot.fill(pkg).is_ok()); - Ok(slot.borrow().unwrap()) + for (source_id, &(ref pending_ids, ref slots)) in &pending { + let source = sources.get_mut(source_id).ok_or_else(|| { + internal(format!("couldn't find source `{}`", source_id)) + })?; + let pkgs = source.download(&*pending_ids).chain_err(|| { + format_err!("unable to get packages from source") + })?; + for (pkg, slot) in pkgs.into_iter().zip(slots) { + assert!(slot.fill(pkg).is_ok()); + } + } + + Ok(ids.iter().map(|id| self.packages.get(id).unwrap().borrow().unwrap()).collect()) } pub fn sources(&self) -> Ref> { diff --git a/src/cargo/core/source/mod.rs b/src/cargo/core/source/mod.rs index bc54fa1848f..e806ed64762 100644 --- a/src/cargo/core/source/mod.rs +++ b/src/cargo/core/source/mod.rs @@ -20,7 +20,7 @@ pub trait Source: Registry { /// The download method fetches the full package for each name and /// version specified. - fn download(&mut self, package: &PackageId) -> CargoResult; + fn download(&mut self, ids: &[&PackageId]) -> CargoResult>; /// Generates a unique string which represents the fingerprint of the /// current state of the source. @@ -57,8 +57,8 @@ impl<'a, T: Source + ?Sized + 'a> Source for Box { } /// Forwards to `Source::download` - fn download(&mut self, id: &PackageId) -> CargoResult { - (**self).download(id) + fn download(&mut self, ids: &[&PackageId]) -> CargoResult> { + (**self).download(ids) } /// Forwards to `Source::fingerprint` diff --git a/src/cargo/ops/cargo_clean.rs b/src/cargo/ops/cargo_clean.rs index 647aa1d3657..953501880e5 100644 --- a/src/cargo/ops/cargo_clean.rs +++ b/src/cargo/ops/cargo_clean.rs @@ -45,11 +45,10 @@ pub fn clean(ws: &Workspace, opts: &CleanOptions) -> CargoResult<()> { profiles)?; let mut units = Vec::new(); - for spec in opts.spec { - // Translate the spec to a Package - let pkgid = resolve.query(spec)?; - let pkg = packages.get(pkgid)?; + let pkg_ids: CargoResult> = opts.spec.iter().map(|spec| resolve.query(spec)).collect(); + let pkgs = packages.get(&*pkg_ids?)?; + for pkg in pkgs { // Generate all relevant `Unit` targets for this package for target in pkg.targets() { for kind in [Kind::Host, Kind::Target].iter() { diff --git a/src/cargo/ops/cargo_compile.rs b/src/cargo/ops/cargo_compile.rs index 484c0d32a33..6bdc5e450b0 100644 --- a/src/cargo/ops/cargo_compile.rs +++ b/src/cargo/ops/cargo_compile.rs @@ -247,12 +247,11 @@ pub fn compile_ws<'a>(ws: &Workspace<'a>, )?; let (packages, resolve_with_overrides) = resolve; - let to_builds = specs.iter().map(|p| { - let pkgid = p.query(resolve_with_overrides.iter())?; - let p = packages.get(pkgid)?; - p.manifest().print_teapot(ws.config()); - Ok(p) - }).collect::>>()?; + let pkg_ids: CargoResult> = specs.iter().map(|p| p.query(resolve_with_overrides.iter())).collect(); + let to_builds = packages.get(&*pkg_ids?)?; + for pkg in &to_builds { + pkg.manifest().print_teapot(ws.config()); + } let mut general_targets = Vec::new(); let mut package_targets = Vec::new(); diff --git a/src/cargo/ops/cargo_doc.rs b/src/cargo/ops/cargo_doc.rs index ed19b62405f..15c1474eeac 100644 --- a/src/cargo/ops/cargo_doc.rs +++ b/src/cargo/ops/cargo_doc.rs @@ -22,10 +22,8 @@ pub fn doc(ws: &Workspace, options: &DocOptions) -> CargoResult<()> { &specs)?; let (packages, resolve_with_overrides) = resolve; - let pkgs = specs.iter().map(|p| { - let pkgid = p.query(resolve_with_overrides.iter())?; - packages.get(pkgid) - }).collect::>>()?; + let pkg_ids: CargoResult> = specs.iter().map(|p| p.query(resolve_with_overrides.iter())).collect(); + let pkgs = packages.get(&*pkg_ids?)?; let mut lib_names = HashMap::new(); let mut bin_names = HashMap::new(); diff --git a/src/cargo/ops/cargo_fetch.rs b/src/cargo/ops/cargo_fetch.rs index 80dfdd0853b..290432a7f48 100644 --- a/src/cargo/ops/cargo_fetch.rs +++ b/src/cargo/ops/cargo_fetch.rs @@ -5,8 +5,9 @@ use util::CargoResult; /// Executes `cargo fetch`. pub fn fetch<'a>(ws: &Workspace<'a>) -> CargoResult<(Resolve, PackageSet<'a>)> { let (packages, resolve) = ops::resolve_ws(ws)?; - for id in resolve.iter() { - packages.get(id)?; + { + let pkg_ids: Vec<_> = resolve.iter().collect(); + packages.get(&*pkg_ids)?; } Ok((resolve, packages)) } diff --git a/src/cargo/ops/cargo_install.rs b/src/cargo/ops/cargo_install.rs index 6dce40768db..403c6e57af0 100644 --- a/src/cargo/ops/cargo_install.rs +++ b/src/cargo/ops/cargo_install.rs @@ -397,8 +397,8 @@ fn select_pkg<'a, T>(mut source: T, let deps = source.query_vec(&dep)?; match deps.iter().map(|p| p.package_id()).max() { Some(pkgid) => { - let pkg = source.download(pkgid)?; - Ok((pkg, Box::new(source))) + let pkg = source.download(&[pkgid])?; + Ok((pkg.into_iter().next().unwrap(), Box::new(source))) } None => { let vers_info = vers.map(|v| format!(" with version `{}`", v)) diff --git a/src/cargo/ops/cargo_output_metadata.rs b/src/cargo/ops/cargo_output_metadata.rs index 8bcf56e9496..d354e9d3366 100644 --- a/src/cargo/ops/cargo_output_metadata.rs +++ b/src/cargo/ops/cargo_output_metadata.rs @@ -54,9 +54,8 @@ fn metadata_full(ws: &Workspace, &specs)?; let (packages, resolve) = deps; - let packages = packages.package_ids() - .map(|i| packages.get(i).map(|p| p.clone())) - .collect::>>()?; + let package_ids: Vec<_> = packages.package_ids().collect(); + let packages: Vec<_> = packages.get(&*package_ids)?.into_iter().cloned().collect(); Ok(ExportInfo { packages, diff --git a/src/cargo/ops/cargo_rustc/context.rs b/src/cargo/ops/cargo_rustc/context.rs index 9f7766e0ef8..bfe6b31b295 100644 --- a/src/cargo/ops/cargo_rustc/context.rs +++ b/src/cargo/ops/cargo_rustc/context.rs @@ -780,7 +780,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> { let id = unit.pkg.package_id(); let deps = self.resolve.deps(id); - let mut ret = deps.filter(|dep| { + let pkg_ids: Vec<_> = deps.filter(|dep| { unit.pkg.dependencies().iter().filter(|d| { d.name() == dep.name() && d.version_req().matches(dep.version()) }).any(|d| { @@ -814,22 +814,19 @@ impl<'a, 'cfg> Context<'a, 'cfg> { // actually used! true }) - }).filter_map(|id| { - match self.get_package(id) { - Ok(pkg) => { - pkg.targets().iter().find(|t| t.is_lib()).map(|t| { - let unit = Unit { - pkg, - target: t, - profile: self.lib_or_check_profile(unit, t), - kind: unit.kind.for_target(t), - }; - Ok(unit) - }) + }).collect(); + + let pkgs = self.get_packages(&*pkg_ids)?; + let mut ret: Vec<_> = pkgs.into_iter().filter_map(|pkg| { + pkg.targets().iter().find(|t| t.is_lib()).map(|t| { + Unit { + pkg, + target: t, + profile: self.lib_or_check_profile(unit, t), + kind: unit.kind.for_target(t), } - Err(e) => Some(Err(e)) - } - }).collect::>>()?; + }) + }).collect(); // If this target is a build script, then what we've collected so far is // all we need. If this isn't a build script, then it depends on the @@ -913,7 +910,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> { /// Returns the dependencies necessary to document a package fn doc_deps(&self, unit: &Unit<'a>) -> CargoResult>> { - let deps = self.resolve.deps(unit.pkg.package_id()).filter(|dep| { + let dep_ids: Vec<_> = self.resolve.deps(unit.pkg.package_id()).filter(|dep| { unit.pkg.dependencies().iter().filter(|d| { d.name() == dep.name() }).any(|dep| { @@ -923,16 +920,15 @@ impl<'a, 'cfg> Context<'a, 'cfg> { _ => false, } }) - }).map(|dep| { - self.get_package(dep) - }); + }).collect(); + + let deps = self.get_packages(&*dep_ids)?; // To document a library, we depend on dependencies actually being // built. If we're documenting *all* libraries, then we also depend on // the documentation of the library being built. let mut ret = Vec::new(); for dep in deps { - let dep = dep?; let lib = match dep.targets().iter().find(|t| t.is_lib()) { Some(lib) => lib, None => continue, @@ -1007,7 +1003,7 @@ impl<'a, 'cfg> Context<'a, 'cfg> { } /// Gets a package for the given package id. - pub fn get_package(&self, id: &PackageId) -> CargoResult<&'a Package> { + pub fn get_packages(&self, id: &[&PackageId]) -> CargoResult> { self.packages.get(id) } diff --git a/src/cargo/ops/cargo_rustc/job_queue.rs b/src/cargo/ops/cargo_rustc/job_queue.rs index b8867ef76ef..b44837ea7d9 100644 --- a/src/cargo/ops/cargo_rustc/job_queue.rs +++ b/src/cargo/ops/cargo_rustc/job_queue.rs @@ -400,7 +400,7 @@ impl<'a> Key<'a> { fn dependencies<'cfg>(&self, cx: &Context<'a, 'cfg>) -> CargoResult>> { let unit = Unit { - pkg: cx.get_package(self.pkg)?, + pkg: cx.get_packages(&[self.pkg])?[0], target: self.target, profile: self.profile, kind: self.kind, diff --git a/src/cargo/ops/mod.rs b/src/cargo/ops/mod.rs index 5da0781ec87..a183e61d76c 100644 --- a/src/cargo/ops/mod.rs +++ b/src/cargo/ops/mod.rs @@ -16,10 +16,11 @@ pub use self::cargo_generate_lockfile::UpdateOptions; pub use self::lockfile::{load_pkg_lockfile, write_pkg_lockfile}; pub use self::cargo_test::{run_tests, run_benches, TestOptions}; pub use self::cargo_package::{package, PackageOpts}; -pub use self::registry::{publish, registry_configuration, RegistryConfig}; -pub use self::registry::{registry_login, search, needs_custom_http_transport, http_handle}; -pub use self::registry::{modify_owners, yank, OwnersOptions, PublishOpts}; -pub use self::registry::configure_http_handle; +pub use self::registry::{publish, registry_configuration, RegistryConfig, + registry_login, search, needs_custom_http_transport, + http_handle, http_easy2_handle, http_multi_handle, + modify_owners, yank, OwnersOptions, PublishOpts, + configure_http_handle, configure_http_easy2_handle}; pub use self::cargo_fetch::fetch; pub use self::cargo_pkgid::pkgid; pub use self::resolve::{resolve_ws, resolve_ws_precisely, resolve_ws_with_method, resolve_with_previous}; diff --git a/src/cargo/ops/registry.rs b/src/cargo/ops/registry.rs index fb776d61bf8..f26e3766a28 100644 --- a/src/cargo/ops/registry.rs +++ b/src/cargo/ops/registry.rs @@ -3,7 +3,8 @@ use std::fs::{self, File}; use std::iter::repeat; use std::time::Duration; -use curl::easy::{Easy, SslOpt}; +use curl::easy::{Easy, Easy2, Handler, SslOpt, HttpVersion}; +use curl::multi::Multi; use git2; use registry::{Registry, NewCrate, NewCrateDependency}; @@ -291,6 +292,60 @@ pub fn http_handle(config: &Config) -> CargoResult { Ok(handle) } +pub fn http_multi_handle(config: &Config) -> CargoResult { + if config.frozen() { + bail!("attempting to make an HTTP request, but --frozen was \ + specified") + } + if !config.network_allowed() { + bail!("can't make HTTP request in the offline mode") + } + + let mut handle = Multi::new(); + let pipelining = config.get_bool("http.pipelining")?.map(|x| x.val).unwrap_or(true); + let multiplexing = config.get_bool("http.multiplexing")?.map(|x| x.val).unwrap_or(true); + handle.pipelining(pipelining, multiplexing)?; + Ok(handle) +} + +/// Create a curl Easy2 handle with default options +pub fn http_easy2_handle(config: &Config, handler: H) -> CargoResult> { + let mut handle = Easy2::new(handler); + configure_http_easy2_handle(config, &mut handle)?; + Ok(handle) +} + +pub fn configure_http_easy2_handle(config: &Config, handle: &mut Easy2) -> CargoResult<()> { + // This is a duplicate of configure_http_handle, due to Easy and Easy2 + // being completely different types. + // The timeout option for libcurl by default times out the entire transfer, + // but we probably don't want this. Instead we only set timeouts for the + // connect phase as well as a "low speed" timeout so if we don't receive + // many bytes in a large-ish period of time then we time out. + handle.connect_timeout(Duration::new(30, 0))?; + handle.low_speed_limit(10 /* bytes per second */)?; + handle.low_speed_time(Duration::new(30, 0))?; + handle.useragent(&version().to_string())?; + // Not all cURL builds support HTTP/2, ignore any errors. + if config.get_bool("http.http2")?.map(|x| x.val).unwrap_or(true) { + let _ = handle.http_version(HttpVersion::V2TLS); + } + if let Some(proxy) = http_proxy(config)? { + handle.proxy(&proxy)?; + } + if let Some(cainfo) = config.get_path("http.cainfo")? { + handle.cainfo(&cainfo.val)?; + } + if let Some(check) = config.get_bool("http.check-revoke")? { + handle.ssl_options(SslOpt::new().no_revoke(!check.val))?; + } + if let Some(timeout) = http_timeout(config)? { + handle.connect_timeout(Duration::new(timeout as u64, 0))?; + handle.low_speed_time(Duration::new(timeout as u64, 0))?; + } + Ok(()) +} + pub fn needs_custom_http_transport(config: &Config) -> CargoResult { let proxy_exists = http_proxy_exists(config)?; let timeout = http_timeout(config)?; @@ -310,6 +365,10 @@ pub fn configure_http_handle(config: &Config, handle: &mut Easy) -> CargoResult< handle.low_speed_limit(10 /* bytes per second */)?; handle.low_speed_time(Duration::new(30, 0))?; handle.useragent(&version().to_string())?; + // Not all cURL builds support HTTP/2, ignore any errors. + if config.get_bool("http.http2")?.map(|x| x.val).unwrap_or(true) { + let _ = handle.http_version(HttpVersion::V2TLS); + } if let Some(proxy) = http_proxy(config)? { handle.proxy(&proxy)?; } diff --git a/src/cargo/sources/directory.rs b/src/cargo/sources/directory.rs index 97cf4242d0a..415e86e6580 100644 --- a/src/cargo/sources/directory.rs +++ b/src/cargo/sources/directory.rs @@ -142,10 +142,11 @@ impl<'cfg> Source for DirectorySource<'cfg> { Ok(()) } - fn download(&mut self, id: &PackageId) -> CargoResult { - self.packages.get(id).map(|p| &p.0).cloned().ok_or_else(|| { - format_err!("failed to find package with id: {}", id) - }) + fn download(&mut self, ids: &[&PackageId]) -> CargoResult> { + ids.iter().map(|id| + self.packages.get(id).map(|p| &p.0).cloned().ok_or_else(|| { + format_err!("failed to find package with id: {}", id) + })).collect() } fn fingerprint(&self, pkg: &Package) -> CargoResult { diff --git a/src/cargo/sources/git/source.rs b/src/cargo/sources/git/source.rs index aba45e53189..d0fc6e5464e 100644 --- a/src/cargo/sources/git/source.rs +++ b/src/cargo/sources/git/source.rs @@ -200,12 +200,14 @@ impl<'cfg> Source for GitSource<'cfg> { self.path_source.as_mut().unwrap().update() } - fn download(&mut self, id: &PackageId) -> CargoResult { - trace!("getting packages for package id `{}` from `{:?}`", id, - self.remote); + fn download(&mut self, ids: &[&PackageId]) -> CargoResult> { + for id in ids { + trace!("getting packages for package id `{}` from `{:?}`", id, + self.remote); + } self.path_source.as_mut() - .expect("BUG: update() must be called before get()") - .download(id) + .expect("BUG: update() must be called before get()") + .download(ids) } fn fingerprint(&self, _pkg: &Package) -> CargoResult { diff --git a/src/cargo/sources/path.rs b/src/cargo/sources/path.rs index fa7fc13455a..e11687936fa 100644 --- a/src/cargo/sources/path.rs +++ b/src/cargo/sources/path.rs @@ -508,13 +508,15 @@ impl<'cfg> Source for PathSource<'cfg> { Ok(()) } - fn download(&mut self, id: &PackageId) -> CargoResult { - trace!("getting packages; id={}", id); + fn download(&mut self, ids: &[&PackageId]) -> CargoResult> { + ids.iter().map(|&id| { + trace!("getting packages; id={}", id); - let pkg = self.packages.iter().find(|pkg| pkg.package_id() == id); - pkg.cloned().ok_or_else(|| { - internal(format!("failed to find {} in path source", id)) - }) + let pkg = self.packages.iter().find(|pkg| pkg.package_id() == id); + pkg.cloned().ok_or_else(|| { + internal(format!("failed to find {} in path source", id)) + }) + }).collect() } fn fingerprint(&self, pkg: &Package) -> CargoResult { diff --git a/src/cargo/sources/registry/local.rs b/src/cargo/sources/registry/local.rs index 73f6f719bae..574d0ffdc47 100644 --- a/src/cargo/sources/registry/local.rs +++ b/src/cargo/sources/registry/local.rs @@ -65,41 +65,43 @@ impl<'cfg> RegistryData for LocalRegistry<'cfg> { Ok(()) } - fn download(&mut self, pkg: &PackageId, checksum: &str) - -> CargoResult { - let crate_file = format!("{}-{}.crate", pkg.name(), pkg.version()); - let mut crate_file = self.root.open_ro(&crate_file, - self.config, - "crate file")?; + fn download(&mut self, pkgs_with_checksum: &[(&PackageId, &str)]) + -> CargoResult> { + pkgs_with_checksum.iter().map(|&(pkg, checksum)| { + let crate_file = format!("{}-{}.crate", pkg.name(), pkg.version()); + let mut crate_file = self.root.open_ro(&crate_file, + self.config, + "crate file")?; - // If we've already got an unpacked version of this crate, then skip the - // checksum below as it is in theory already verified. - let dst = format!("{}-{}", pkg.name(), pkg.version()); - if self.src_path.join(dst).into_path_unlocked().exists() { - return Ok(crate_file) - } + // If we've already got an unpacked version of this crate, then skip the + // checksum below as it is in theory already verified. + let dst = format!("{}-{}", pkg.name(), pkg.version()); + if self.src_path.join(dst).into_path_unlocked().exists() { + return Ok(crate_file); + } - self.config.shell().status("Unpacking", pkg)?; + self.config.shell().status("Unpacking", pkg)?; - // We don't actually need to download anything per-se, we just need to - // verify the checksum matches the .crate file itself. - let mut state = Sha256::new(); - let mut buf = [0; 64 * 1024]; - loop { - let n = crate_file.read(&mut buf).chain_err(|| { - format!("failed to read `{}`", crate_file.path().display()) - })?; - if n == 0 { - break + // We don't actually need to download anything per-se, we just need to + // verify the checksum matches the .crate file itself. + let mut state = Sha256::new(); + let mut buf = [0; 64 * 1024]; + loop { + let n = crate_file.read(&mut buf).chain_err(|| { + format!("failed to read `{}`", crate_file.path().display()) + })?; + if n == 0 { + break; + } + state.update(&buf[..n]); + } + if hex::encode(state.finish()) != checksum { + bail!("failed to verify the checksum of `{}`", pkg) } - state.update(&buf[..n]); - } - if hex::encode(state.finish()) != checksum { - bail!("failed to verify the checksum of `{}`", pkg) - } - crate_file.seek(SeekFrom::Start(0))?; + crate_file.seek(SeekFrom::Start(0))?; - Ok(crate_file) + Ok(crate_file) + }).collect() } } diff --git a/src/cargo/sources/registry/mod.rs b/src/cargo/sources/registry/mod.rs index cb2b9fc1e63..66603ad48d7 100644 --- a/src/cargo/sources/registry/mod.rs +++ b/src/cargo/sources/registry/mod.rs @@ -249,8 +249,7 @@ pub trait RegistryData { fn config(&mut self) -> CargoResult>; fn update_index(&mut self) -> CargoResult<()>; fn download(&mut self, - pkg: &PackageId, - checksum: &str) -> CargoResult; + pkgs_with_checksum: &[(&PackageId, &str)]) -> CargoResult>; fn is_crate_downloaded(&self, _pkg: &PackageId) -> bool { true } } @@ -421,27 +420,30 @@ impl<'cfg> Source for RegistrySource<'cfg> { Ok(()) } - fn download(&mut self, package: &PackageId) -> CargoResult { - let hash = self.index.hash(package, &mut *self.ops)?; - let path = self.ops.download(package, &hash)?; - let path = self.unpack_package(package, &path).chain_err(|| { - internal(format!("failed to unpack package `{}`", package)) - })?; - let mut src = PathSource::new(&path, &self.source_id, self.config); - src.update()?; - let pkg = src.download(package)?; - - // Unfortunately the index and the actual Cargo.toml in the index can - // differ due to historical Cargo bugs. To paper over these we trash the - // *summary* loaded from the Cargo.toml we just downloaded with the one - // we loaded from the index. - let summaries = self.index.summaries(&*package.name(), &mut *self.ops)?; - let summary = summaries.iter().map(|s| &s.0).find(|s| { - s.package_id() == package - }).expect("summary not found"); - let mut manifest = pkg.manifest().clone(); - manifest.set_summary(summary.clone()); - Ok(Package::new(manifest, pkg.manifest_path())) + fn download(&mut self, packages: &[&PackageId]) -> CargoResult> { + let hashes = packages.iter().map(|&package| self.index.hash(package, &mut *self.ops)).collect::>>()?; + let pkgs_with_checksum: Vec<_> = packages.iter().zip(&hashes).map(|(&pkg, &ref checksum)| (pkg, &**checksum)).collect(); + let paths = self.ops.download(&*pkgs_with_checksum)?; + paths.into_iter().zip(packages).map(|(path, &package)| { + let path = self.unpack_package(package, &path).chain_err(|| { + internal(format!("failed to unpack package `{}`", package)) + })?; + let mut src = PathSource::new(&path, &self.source_id, self.config); + src.update()?; + let pkg = src.download(&[package])?.into_iter().next().unwrap(); + + // Unfortunately the index and the actual Cargo.toml in the index can + // differ due to historical Cargo bugs. To paper over these we trash the + // *summary* loaded from the Cargo.toml we just downloaded with the one + // we loaded from the index. + let summaries = self.index.summaries(&*package.name(), &mut *self.ops)?; + let summary = summaries.iter().map(|s| &s.0).find(|s| { + s.package_id() == package + }).expect("summary not found"); + let mut manifest = pkg.manifest().clone(); + manifest.set_summary(summary.clone()); + Ok(Package::new(manifest, pkg.manifest_path())) + }).collect() } fn fingerprint(&self, pkg: &Package) -> CargoResult { diff --git a/src/cargo/sources/registry/remote.rs b/src/cargo/sources/registry/remote.rs index d09b6d53118..d5e4ea7192b 100644 --- a/src/cargo/sources/registry/remote.rs +++ b/src/cargo/sources/registry/remote.rs @@ -1,22 +1,22 @@ -use std::cell::{RefCell, Ref, Cell}; +use core::{PackageId, SourceId}; +use curl::easy::{Easy2, Handler, WriteError}; +use curl::multi::Easy2Handle; +use git2; +use hex; +use lazycell::LazyCell; +use serde_json; +use sources::git; +use sources::registry::{CRATE_TEMPLATE, INDEX_LOCK, RegistryConfig, RegistryData, VERSION_TEMPLATE}; +use std::cell::{Cell, Ref, RefCell}; use std::fmt::Write as FmtWrite; -use std::io::SeekFrom; use std::io::prelude::*; +use std::io::SeekFrom; use std::mem; +use std::time::Duration; use std::path::Path; use std::str; - -use git2; -use hex; -use serde_json; -use lazycell::LazyCell; - -use core::{PackageId, SourceId}; -use sources::git; -use sources::registry::{RegistryData, RegistryConfig, INDEX_LOCK, CRATE_TEMPLATE, VERSION_TEMPLATE}; -use util::network; use util::{FileLock, Filesystem}; -use util::{Config, Sha256, ToUrl, Progress}; +use util::{Config, Sha256, ToUrl}; use util::errors::{CargoResult, CargoResultExt, HttpNot200}; pub struct RemoteRegistry<'cfg> { @@ -187,84 +187,149 @@ impl<'cfg> RegistryData for RemoteRegistry<'cfg> { Ok(()) } - fn download(&mut self, pkg: &PackageId, checksum: &str) - -> CargoResult { - let filename = format!("{}-{}.crate", pkg.name(), pkg.version()); - let path = Path::new(&filename); + fn download(&mut self, pkgs_with_checksum: &[(&PackageId, &str)]) + -> CargoResult> { + let mut result = Vec::new(); + let mut to_download = Vec::new(); + for (idx, &(pkg, _)) in pkgs_with_checksum.iter().enumerate() { + let filename = format!("{}-{}.crate", pkg.name(), pkg.version()); + let path = Path::new(&filename); - // Attempt to open an read-only copy first to avoid an exclusive write - // lock and also work with read-only filesystems. Note that we check the - // length of the file like below to handle interrupted downloads. - // - // If this fails then we fall through to the exclusive path where we may - // have to redownload the file. - if let Ok(dst) = self.cache_path.open_ro(path, self.config, &filename) { + // Attempt to open an read-only copy first to avoid an exclusive write + // lock and also work with read-only filesystems. Note that we check the + // length of the file like below to handle interrupted downloads. + // + // If this fails then we fall through to the exclusive path where we may + // have to redownload the file. + if let Ok(dst) = self.cache_path.open_ro(path, self.config, &filename) { + let meta = dst.file().metadata()?; + if meta.len() > 0 { + result.push(dst); + continue; + } + } + let mut dst = self.cache_path.open_rw(path, self.config, &filename)?; let meta = dst.file().metadata()?; + result.push(dst); if meta.len() > 0 { - return Ok(dst) + continue; + } + + let config = self.config()?.unwrap(); + let mut url = config.dl.clone(); + if !url.contains(CRATE_TEMPLATE) && !url.contains(VERSION_TEMPLATE) { + write!(url, "/{}/{}/download", CRATE_TEMPLATE, VERSION_TEMPLATE).unwrap(); } + let url = url + .replace(CRATE_TEMPLATE, &*pkg.name()) + .replace(VERSION_TEMPLATE, &pkg.version().to_string()) + .to_url()?; + + to_download.push((idx, url.to_string())); } - let mut dst = self.cache_path.open_rw(path, self.config, &filename)?; - let meta = dst.file().metadata()?; - if meta.len() > 0 { - return Ok(dst) + + // Offline-friendly fast path + if to_download.is_empty() { return Ok(result); } + + struct MultiHandler { + body: Vec, + state: Sha256, } - self.config.shell().status("Downloading", pkg)?; - let config = self.config()?.unwrap(); - let mut url = config.dl.clone(); - if !url.contains(CRATE_TEMPLATE) && !url.contains(VERSION_TEMPLATE) { - write!(url, "/{}/{}/download", CRATE_TEMPLATE, VERSION_TEMPLATE).unwrap(); + impl MultiHandler { + fn new() -> MultiHandler { + MultiHandler { body: Vec::new(), state: Sha256::new() } + } + } + + impl Handler for MultiHandler { + // TODO: progress + fn write(&mut self, data: &[u8]) -> Result { + self.state.update(data); + self.body.extend_from_slice(data); + Ok(data.len()) + } } - let url = url - .replace(CRATE_TEMPLATE, &*pkg.name()) - .replace(VERSION_TEMPLATE, &pkg.version().to_string()) - .to_url()?; // TODO: don't download into memory, but ensure that if we ctrl-c a // download we should resume either from the start or the middle // on the next time - let url = url.to_string(); - let mut handle = self.config.http()?.borrow_mut(); - handle.get(true)?; - handle.url(&url)?; - handle.follow_location(true)?; - let mut state = Sha256::new(); - let mut body = Vec::new(); - network::with_retry(self.config, || { - state = Sha256::new(); - body = Vec::new(); - let mut pb = Progress::new("Fetch", self.config); - { - handle.progress(true)?; - let mut handle = handle.transfer(); - handle.progress_function(|dl_total, dl_cur, _, _| { - pb.tick(dl_cur as usize, dl_total as usize).is_ok() - })?; - handle.write_function(|buf| { - state.update(buf); - body.extend_from_slice(buf); - Ok(buf.len()) - })?; - handle.perform()?; - } - let code = handle.response_code()?; - if code != 200 && code != 0 { - let url = handle.effective_url()?.unwrap_or(&url); - Err(HttpNot200 { code, url: url.to_string() }.into()) - } else { - Ok(()) - } - })?; - - // Verify what we just downloaded - if hex::encode(state.finish()) != checksum { - bail!("failed to verify the checksum of `{}`", pkg) + let (multi, easy) = self.config.http_multi(to_download.len(), MultiHandler::new)?; + let mut running_handles: Vec<_> = (0..easy.len()).map(|_| None).collect(); + let mut cur_index = 0; + // Take the next file in queue and push it into the Multi queue with + // the provided Easy. Returns Ok(true) if the handle is configured, + // Ok(false) if there's no more files, Err(e) if some error occurred + // during configuration. Assumes that the states are clean. + let mut configure_handle = |mut handle: Easy2, index: usize, + running_handles: &mut Vec)>>| -> CargoResult{ + if cur_index == to_download.len() { return Ok(false); } + let (pkg_idx, ref url) = to_download[cur_index]; + let (pkg, _) = pkgs_with_checksum[pkg_idx]; + cur_index += 1; + self.config.shell().status("Downloading", pkg)?; + handle.get(true)?; + handle.url(&url)?; + handle.follow_location(true)?; + let mut running_handle = multi.add2(handle)?; + running_handle.set_token(index)?; + running_handles[index] = Some((pkg_idx, running_handle)); + Ok(true) + }; + for (index, handle) in easy.into_iter().enumerate() { + assert!(configure_handle(handle, index, &mut running_handles)?); } + loop { + // If critical error occurred in multi interface, directly return. + let mut active = multi.perform()?; + // If there are multiple errors, take the first one. + let mut error: CargoResult<()> = Ok(()); + multi.messages(|msg| match msg.result() { + Some(Ok(())) => { + // try .. catch + if let Err(e) = (|| { + let index = msg.token()?; + let (pkg_idx, done_handle) = running_handles[index].take().unwrap(); + let mut handle = multi.remove2(done_handle)?; + let code = handle.response_code()?; + if code != 200 && code != 0 { + // FIXME: can this be None? + let url = handle.effective_url()?.unwrap(); + return Err(HttpNot200 { code, url: url.to_string() }.into()) + } + { + let (pkg, checksum) = pkgs_with_checksum[pkg_idx]; + let &mut MultiHandler { ref mut body, ref mut state } = handle.get_mut(); + let dst = &mut result[pkg_idx]; + if hex::encode(state.finish()) != checksum { + bail!("failed to verify the checksum of `{}`", pkg) + } - dst.write_all(&body)?; - dst.seek(SeekFrom::Start(0))?; - Ok(dst) + dst.write_all(&body)?; + dst.seek(SeekFrom::Start(0))?; + body.clear(); + } + self.config.http_easy2_reset(&mut handle)?; + if configure_handle(handle, index, &mut running_handles)? { + // Avoid exiting too early + active += 1; + } + Ok(()) + })() { + if error.is_ok() { error = Err(e); } + } + } + Some(Err(e)) => { if error.is_ok() { error = Err(e.into()); } } + None => {} + }); + // TODO: retries + error?; + if active == 0 { break; } + // Skip the check: curl_multi_wait will return immediately if there + // is nothing to wait. + multi.wait(&mut [], Duration::new(::std::u64::MAX, 0))?; + } + Ok(result) } diff --git a/src/cargo/sources/replaced.rs b/src/cargo/sources/replaced.rs index 81980210f69..a209c5c489c 100644 --- a/src/cargo/sources/replaced.rs +++ b/src/cargo/sources/replaced.rs @@ -57,13 +57,14 @@ impl<'cfg> Source for ReplacedSource<'cfg> { Ok(()) } - fn download(&mut self, id: &PackageId) -> CargoResult { - let id = id.with_source_id(&self.replace_with); - let pkg = self.inner.download(&id).chain_err(|| { + fn download(&mut self, ids: &[&PackageId]) -> CargoResult> { + let replaced_ids: Vec<_> = ids.iter().map(|id| id.with_source_id(&self.replace_with)).collect(); + let replaced_id_refs: Vec<_> = replaced_ids.iter().collect(); + let pkgs = self.inner.download(&*replaced_id_refs).chain_err(|| { format!("failed to download replaced source {}", self.to_replace) })?; - Ok(pkg.map_source(&self.replace_with, &self.to_replace)) + Ok(pkgs.into_iter().map(|x| x.map_source(&self.replace_with, &self.to_replace)).collect()) } fn fingerprint(&self, id: &Package) -> CargoResult { diff --git a/src/cargo/util/config.rs b/src/cargo/util/config.rs index 927d0e132f1..6f962e6176d 100644 --- a/src/cargo/util/config.rs +++ b/src/cargo/util/config.rs @@ -2,6 +2,7 @@ use std::cell::{RefCell, RefMut}; use std::collections::HashSet; use std::collections::hash_map::Entry::{Occupied, Vacant}; use std::collections::hash_map::HashMap; +use std::cmp; use std::env; use std::fmt; use std::fs::{self, File}; @@ -12,7 +13,8 @@ use std::path::{Path, PathBuf}; use std::str::FromStr; use std::sync::{Once, ONCE_INIT}; -use curl::easy::Easy; +use curl::easy::{Easy, Easy2, Handler}; +use curl::multi::Multi; use jobserver; use serde::{Serialize, Serializer}; use toml; @@ -663,6 +665,22 @@ impl Config { Ok(http) } + pub fn http_multi H>(&self, max: usize, mut handler: F) -> CargoResult<(Multi, Vec>)> { + let concurrency = self.get_i64("http.concurrency")?.map(|x| x.val).unwrap_or(16); + if concurrency <= 0 { + bail!("http.concurrency must be positive, got {}", concurrency); + } + let multi = ops::http_multi_handle(self)?; + let easy: CargoResult> = (0..cmp::min(max, concurrency as usize)).map(|_| ops::http_easy2_handle(self, handler())).collect(); + Ok((multi, easy?)) + } + + pub fn http_easy2_reset(&self, handle: &mut Easy2) -> CargoResult<()> { + handle.reset(); + ops::configure_http_easy2_handle(self, handle)?; + Ok(()) + } + pub fn crates_io_source_id(&self, f: F) -> CargoResult where F: FnMut() -> CargoResult { diff --git a/src/doc/src/reference/config.md b/src/doc/src/reference/config.md index b19c8923cf9..8fcffb0afeb 100644 --- a/src/doc/src/reference/config.md +++ b/src/doc/src/reference/config.md @@ -95,6 +95,14 @@ proxy = "host:port" # HTTP proxy to use for HTTP requests (defaults to none) timeout = 60000 # Timeout for each HTTP request, in milliseconds cainfo = "cert.pem" # Path to Certificate Authority (CA) bundle (optional) check-revoke = true # Indicates whether SSL certs are checked for revocation +http2 = true # Indicates whether HTTP/2 should be negotiated. +pipelining = true # Indicates whether HTTP/1.1 pipelining should be enabled. + # Disable if the registry server doesn't implement the spec + # correctly. +multiplexing = true # Indicates whether HTTP/2 multiplexing should be enabled. + # Disable if you want to use multiple TCP connections. +concurrency = 16 # How many requests should be fired in parallel for crate + # downloads. [build] jobs = 1 # number of parallel jobs, defaults to # of CPUs