From b2e36a3c4392457f0855803ad379a8b8d6975b56 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Tue, 1 Nov 2022 19:04:24 -0400 Subject: [PATCH 01/43] Start of work --- cli/npm/registry.rs | 68 +++++- cli/npm/resolution.rs | 467 ++++++++++++++++++++++++++++++++---------- cli/npm/tarball.rs | 1 + 3 files changed, 417 insertions(+), 119 deletions(-) diff --git a/cli/npm/registry.rs b/cli/npm/registry.rs index ccbe18c7f815e5..02c408af4930c3 100644 --- a/cli/npm/registry.rs +++ b/cli/npm/registry.rs @@ -1,5 +1,6 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. +use std::cmp::Ordering; use std::collections::HashMap; use std::fs; use std::io::ErrorKind; @@ -24,6 +25,7 @@ use crate::http_cache::CACHE_PERM; use crate::progress_bar::ProgressBar; use super::cache::NpmCache; +use super::resolution::NpmVersionMatcher; use super::semver::NpmVersionReq; // npm registry docs: https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md @@ -36,10 +38,45 @@ pub struct NpmPackageInfo { pub dist_tags: HashMap, } +#[derive(Eq, PartialEq)] +pub enum NpmDependencyEntryKind { + Dep, + Peer, + OptionalPeer, +} + +#[derive(Eq, PartialEq)] pub struct NpmDependencyEntry { pub bare_specifier: String, pub name: String, pub version_req: NpmVersionReq, + pub kind: NpmDependencyEntryKind, +} + +impl PartialOrd for NpmDependencyEntry { + fn partial_cmp(&self, other: &Self) -> Option { + Some(self.cmp(other)) + } +} + +impl Ord for NpmDependencyEntry { + fn cmp(&self, other: &Self) -> std::cmp::Ordering { + // sort the dependencies alphabetically by name then by version descending + match self.name.cmp(&other.name) { + // sort by newest to oldest + Ordering::Equal => other + .version_req + .version_text() + .cmp(&self.version_req.version_text()), + ordering => ordering, + } + } +} + +#[derive(Debug, Default, Deserialize, Serialize, Clone)] +pub struct NpmPeerDependencyMeta { + #[serde(default)] + optional: bool, } #[derive(Debug, Default, Deserialize, Serialize, Clone)] @@ -50,14 +87,19 @@ pub struct NpmPackageVersionInfo { // package and version (ex. `"typescript-3.0.1": "npm:typescript@3.0.1"`). #[serde(default)] pub dependencies: HashMap, + #[serde(default)] + pub peerDependencies: HashMap, + #[serde(default)] + pub peerDependenciesMeta: HashMap, } impl NpmPackageVersionInfo { pub fn dependencies_as_entries( &self, ) -> Result, AnyError> { - fn entry_as_bare_specifier_and_reference( + fn parse_dep_entry( entry: (&String, &String), + kind: NpmDependencyEntryKind, ) -> Result { let bare_specifier = entry.0.clone(); let (name, version_req) = @@ -81,14 +123,28 @@ impl NpmPackageVersionInfo { bare_specifier, name, version_req, + kind, }) } - self - .dependencies - .iter() - .map(entry_as_bare_specifier_and_reference) - .collect::, AnyError>>() + let mut result = + Vec::with_capacity(self.dependencies.len() + self.peerDependencies.len()); + for entry in &self.dependencies { + result.push(parse_dep_entry(entry, NpmDependencyEntryKind::Dep)?); + } + for entry in &self.peerDependencies { + let is_optional = self + .peerDependenciesMeta + .get(entry.0) + .map(|d| d.optional) + .unwrap_or(false); + let kind = match is_optional { + true => NpmDependencyEntryKind::OptionalPeer, + false => NpmDependencyEntryKind::Peer, + }; + result.push(parse_dep_entry(entry, kind)?); + } + Ok(result) } } diff --git a/cli/npm/resolution.rs b/cli/npm/resolution.rs index 28a42bc333917b..d51fe083ad2025 100644 --- a/cli/npm/resolution.rs +++ b/cli/npm/resolution.rs @@ -1,9 +1,12 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. +use std::borrow::BorrowMut; +use std::cell::RefCell; use std::cmp::Ordering; use std::collections::HashMap; use std::collections::HashSet; use std::collections::VecDeque; +use std::rc::Rc; use deno_ast::ModuleSpecifier; use deno_core::anyhow::bail; @@ -18,6 +21,8 @@ use serde::Serialize; use std::sync::Arc; use crate::lockfile::Lockfile; +use crate::npm::registry::NpmDependencyEntry; +use crate::npm::registry::NpmDependencyEntryKind; use super::cache::should_sync_download; use super::registry::NpmPackageInfo; @@ -165,6 +170,7 @@ impl NpmVersionMatcher for NpmPackageReq { pub struct NpmPackageId { pub name: String, pub version: NpmVersion, + pub peer_dependencies: Vec, } impl NpmPackageId { @@ -178,6 +184,9 @@ impl NpmPackageId { } pub fn serialize_for_lock_file(&self) -> String { + if !self.peer_dependencies.is_empty() { + todo!(); + } format!("{}@{}", self.name, self.version) } @@ -192,6 +201,7 @@ impl NpmPackageId { Ok(Self { name: reference.req.name, version, + peer_dependencies: Vec::new(), // todo }) } } @@ -214,8 +224,8 @@ pub struct NpmResolutionPackage { #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct NpmResolutionSnapshot { #[serde(with = "map_to_vec")] - package_reqs: HashMap, - packages_by_name: HashMap>, + package_reqs: HashMap, + package_versions_by_name: HashMap>, #[serde(with = "map_to_vec")] packages: HashMap, } @@ -352,7 +362,7 @@ impl NpmResolutionSnapshot { version_matcher: &impl NpmVersionMatcher, ) -> Option { let mut maybe_best_version: Option<&NpmVersion> = None; - if let Some(versions) = self.packages_by_name.get(name) { + if let Some(versions) = self.package_versions_by_name.get(name) { for version in versions { if version_matcher.matches(version) { let is_best_version = maybe_best_version @@ -445,7 +455,7 @@ impl NpmResolutionSnapshot { Ok(Self { package_reqs, - packages_by_name, + package_versions_by_name: packages_by_name, packages, }) } @@ -529,6 +539,230 @@ impl NpmResolution { mut package_reqs: Vec, mut snapshot: NpmResolutionSnapshot, ) -> Result { + #[derive(Clone, PartialEq, Eq, Hash)] + enum NodeParent { + Req(NpmPackageReq), + Node(NpmPackageId), + } + + struct Node { + pub id: NpmPackageId, + pub parents: HashSet, + pub children: HashSet, + pub unresolved_peers: Vec, + } + + #[derive(Default)] + struct Graph { + package_reqs: HashMap, + packages_by_name: HashMap>, + packages: HashMap>>, + } + + impl Graph { + pub fn get_or_create_for_id( + &mut self, + id: &NpmPackageId, + ) -> (bool, Rc>) { + if let Some(node) = self.packages.get(id) { + (false, node.clone()) + } else { + let node = Rc::new(RefCell::new(Node { + id: id.clone(), + parents: Default::default(), + children: Default::default(), + unresolved_peers: Default::default(), + })); + self + .packages_by_name + .entry(id.name.clone()) + .or_default() + .push(id.clone()); + self.packages.insert(id.clone(), node.clone()); + (true, node) + } + } + + pub fn fill_with_snapshot(&mut self, snapshot: &NpmResolutionSnapshot) { + for (package_req, id) in &snapshot.package_reqs { + let node = self.fill_for_id_with_snapshot(id, snapshot); + (*node) + .borrow_mut() + .parents + .insert(NodeParent::Req(package_req.clone())); + self.package_reqs.insert(package_req.clone(), id.clone()); + } + } + + fn fill_for_id_with_snapshot( + &mut self, + id: &NpmPackageId, + snapshot: &NpmResolutionSnapshot, + ) -> Rc> { + let resolution = snapshot.packages.get(id).unwrap(); + let node = self.get_or_create_for_id(id).1; + for (name, id) in resolution.dependencies { + let child_node = self.fill_for_id_with_snapshot(&id, snapshot); + set_parent_node(&child_node, &node); + } + node + } + + pub fn resolve_best_package_version( + &self, + name: &str, + version_matcher: &impl NpmVersionMatcher, + ) -> Option { + let mut maybe_best_version: Option<&NpmVersion> = None; + if let Some(ids) = self.packages_by_name.get(name) { + for version in ids.iter().map(|id| &id.version) { + if version_matcher.matches(version) { + let is_best_version = maybe_best_version + .as_ref() + .map(|best_version| (*best_version).cmp(version).is_lt()) + .unwrap_or(true); + if is_best_version { + maybe_best_version = Some(version); + } + } + } + } + maybe_best_version.cloned() + } + + pub fn resolve_peer_dep( + &mut self, + parent_node: &Rc>, + peer_dep: &NpmDependencyEntry, + ) { + self.resolve_peer_dep_parents_with_path(parent_node, peer_dep, vec![]) + } + + fn resolve_peer_dep_parents_with_path( + &mut self, + child_node: &Rc>, + peer_dep: &NpmDependencyEntry, + mut path: Vec>>, + ) { + let parents = child_node.borrow().parents.clone(); + path.push(child_node.clone()); + for parent in parents { + match parent { + NodeParent::Node(parent_node_id) => self + .resolve_peer_dep_with_path( + // todo: this should probably be a debug panic only as it indicates + // a node in the graph was accidentally orphaned + self + .packages + .get(parent_node_id) + .cloned() + .expect("orphaned node"), + peer_dep, + path.clone(), + ), + NodeParent::Req(req) => { + todo!() + } + } + } + } + + fn resolve_peer_dep_with_path( + &mut self, + node: Rc>, + peer_dep: &NpmDependencyEntry, + path: Vec>>, + ) { + for child in node.borrow().children.clone() { + let child = child.borrow(); + if child.id.name == peer_dep.name + && peer_dep.version_req.satisfies(&child.id.version) + { + // go down the descendants creating a new path + let parents = node.borrow().parents.clone(); + self.set_new_peer_dep(parents, node, &child.id, path); + } + } + } + + fn set_new_peer_dep( + &mut self, + previous_parents: HashSet, + node: Rc>, + peer_dep_id: &NpmPackageId, + path: Vec>>, + ) { + let old_node = node.borrow(); + let old_id = old_node.id; + let mut new_id = old_node.id.clone(); + new_id.peer_dependencies.push(peer_dep_id.clone()); + // remove the previous parents from the old node + for previous_parent in &previous_parents { + old_node.parents.remove(previous_parent); + } + drop(old_node); + + let (created, new_node) = self.get_or_create_for_id(&new_id); + + // update the previous parent to point to the new node + // and this node to point at those parents + for parent in previous_parents { + match &parent { + NodeParent::Node(parent_id) => { + let mut parent = + (**self.packages.get(parent_id).unwrap()).borrow_mut(); + parent.children.remove(&old_id); + parent.children.insert(new_id.clone()); + } + NodeParent::Req(req) => { + self.package_reqs.insert(req.clone(), new_id.clone()); + } + } + (*new_node).borrow_mut().parents.insert(parent); + } + + // if this is the case, then we can have the descendant forget about us + + if created { + if let Some(next_node) = path.pop() { + self.set_new_peer_dep(next_node, peer_dep_id, path); + } + } else { + let node = (*node).borrow_mut(); + if node.parents.is_empty() { + self.forget_orphan(&mut node); + } + } + } + + fn forget_orphan(&mut self, node: &mut Node) { + assert_eq!(node.parents.len(), 0); + self.packages.remove(&node.id); + let parent = NodeParent::Node(node.id.clone()); + for child_id in &node.children { + let mut child = (**self.packages.get(child_id).unwrap()).borrow_mut(); + child.parents.remove(&node.id); + if child.parents.is_empty() { + self.forget_orphan(&mut child); + } + } + } + } + + pub fn set_parent_node( + child: &Rc>, + parent: &Rc>, + ) { + let mut child = (**child).borrow_mut(); + let mut parent = (**parent).borrow_mut(); + parent.children.insert(child.id.clone()); + child.parents.insert(NodeParent::Node(parent.id.clone())); + } + + // convert the snapshot to a traversable graph + let mut graph = Graph::default(); + graph.fill_with_snapshot(&snapshot); + // multiple packages are resolved in alphabetical order package_reqs.sort_by(|a, b| a.name.cmp(&b.name)); @@ -536,17 +770,10 @@ impl NpmResolution { // tree one level at a time through all the branches let mut unresolved_tasks = Vec::with_capacity(package_reqs.len()); for package_req in package_reqs { - if snapshot.package_reqs.contains_key(&package_req) { + if graph.package_reqs.contains_key(&package_req) { // skip analyzing this package, as there's already a matching top level package continue; } - // inspect the list of current packages - if let Some(version) = - snapshot.resolve_best_package_version(&package_req.name, &package_req) - { - snapshot.package_reqs.insert(package_req, version); - continue; // done, no need to continue - } // no existing best version, so resolve the current packages let api = self.api.clone(); @@ -566,131 +793,145 @@ impl NpmResolution { } let mut pending_dependencies = VecDeque::new(); + for result in futures::future::join_all(unresolved_tasks).await { let (package_req, info) = result??; - let version_and_info = get_resolved_package_version_and_info( - &package_req.name, - &package_req, - info, - None, - )?; + // inspect if there's a match in the list of current packages and otherwise + // fall back to looking at the registry + let version_and_info = if let Some(version) = + graph.resolve_best_package_version(&package_req.name, &package_req) + { + match info.versions.get(&version.to_string()) { + Some(version_info) => VersionAndInfo { + version, + info: version_info.clone(), + }, + None => { + bail!("could not find version '{}' for '{}'", version, package_req) + } + } + } else { + get_resolved_package_version_and_info( + &package_req.name, + &package_req, + info, + None, + )? + }; let id = NpmPackageId { name: package_req.name.clone(), version: version_and_info.version.clone(), + peer_dependencies: Vec::new(), }; + let node = graph.get_or_create_for_id(&id).1; + (*node) + .borrow_mut() + .parents + .push(NodeParent::Req(package_req.clone())); + let dependencies = version_and_info .info .dependencies_as_entries() .with_context(|| format!("npm package: {}", id))?; - pending_dependencies.push_back((id.clone(), dependencies)); - snapshot.packages.insert( - id.clone(), - NpmResolutionPackage { - id, - dist: version_and_info.info.dist, - dependencies: Default::default(), - }, - ); - snapshot - .packages_by_name - .entry(package_req.name.clone()) - .or_default() - .push(version_and_info.version.clone()); - snapshot - .package_reqs - .insert(package_req, version_and_info.version); + pending_dependencies.push_back((node, dependencies)); } - // now go down through the dependencies by tree depth - while let Some((parent_package_id, mut deps)) = - pending_dependencies.pop_front() + let mut pending_peer_dependencies = VecDeque::new(); + while !pending_dependencies.is_empty() + || !pending_peer_dependencies.is_empty() { - // sort the dependencies alphabetically by name then by version descending - deps.sort_by(|a, b| match a.name.cmp(&b.name) { - // sort by newest to oldest - Ordering::Equal => b - .version_req - .version_text() - .cmp(&a.version_req.version_text()), - ordering => ordering, - }); - - // cache all the dependencies' registry infos in parallel if should - if !should_sync_download() { - let handles = deps - .iter() - .map(|dep| { - let name = dep.name.clone(); - let api = self.api.clone(); - tokio::task::spawn(async move { - // it's ok to call this without storing the result, because - // NpmRegistryApi will cache the package info in memory - api.package_info(&name).await + // now go down through the dependencies by tree depth + while let Some((parent_node, mut deps)) = pending_dependencies.pop_front() + { + // ensure name alphabetical and then version descending + deps.sort(); + + // cache all the dependencies' registry infos in parallel if should + if !should_sync_download() { + let handles = deps + .iter() + .map(|dep| { + let name = dep.name.clone(); + let api = self.api.clone(); + tokio::task::spawn(async move { + // it's ok to call this without storing the result, because + // NpmRegistryApi will cache the package info in memory + api.package_info(&name).await + }) }) - }) - .collect::>(); - let results = futures::future::join_all(handles).await; - for result in results { - result??; // surface the first error + .collect::>(); + let results = futures::future::join_all(handles).await; + for result in results { + result??; // surface the first error + } } - } - // now resolve them - for dep in deps { - // check if an existing dependency matches this - let id = if let Some(version) = - snapshot.resolve_best_package_version(&dep.name, &dep.version_req) - { - NpmPackageId { - name: dep.name.clone(), - version, + // resolve the non-peer dependencies + for dep in deps { + if matches!( + dep.kind, + NpmDependencyEntryKind::Peer | NpmDependencyEntryKind::OptionalPeer + ) { + pending_peer_dependencies.push_back((parent_node.clone(), dep)); + continue; // handle these later } - } else { - // get the information - let info = self.api.package_info(&dep.name).await?; - let version_and_info = get_resolved_package_version_and_info( - &dep.name, - &dep.version_req, - info, - None, - )?; - let dependencies = version_and_info - .info - .dependencies_as_entries() - .with_context(|| { - format!("npm package: {}@{}", dep.name, version_and_info.version) - })?; + + // check if an existing dependency matches this + let package_info = self.api.package_info(&dep.name).await?; + let version_and_info = if let Some(version) = + snapshot.resolve_best_package_version(&dep.name, &dep.version_req) + { + match package_info.versions.get(&version.to_string()) { + Some(version_info) => VersionAndInfo { + version, + info: version_info.clone(), + }, + None => { + bail!("could not find version '{}' for '{}'", version, dep.name) + } + } + } else { + // get the information + get_resolved_package_version_and_info( + &dep.name, + &dep.version_req, + package_info, + None, + )? + }; let id = NpmPackageId { name: dep.name.clone(), version: version_and_info.version.clone(), + peer_dependencies: Vec::new(), }; - pending_dependencies.push_back((id.clone(), dependencies)); - snapshot.packages.insert( - id.clone(), - NpmResolutionPackage { - id: id.clone(), - dist: version_and_info.info.dist, - dependencies: Default::default(), - }, - ); - snapshot - .packages_by_name - .entry(dep.name.clone()) - .or_default() - .push(id.version.clone()); - - id - }; + let (created, node) = graph.get_or_create_for_id(&id); + set_parent_node(&node, &parent_node); + + if !created { + // inspect the dependencies of the package + let dependencies = version_and_info + .info + .dependencies_as_entries() + .with_context(|| { + format!( + "npm package: {}@{}", + dep.name, version_and_info.version + ) + })?; + + pending_dependencies.push_back((node, dependencies)); + } + } + } - // add this version as a dependency of the package - snapshot - .packages - .get_mut(&parent_package_id) - .unwrap() - .dependencies - .insert(dep.bare_specifier.clone(), id); + if let Some((parent_node, peer_dep)) = + pending_peer_dependencies.pop_front() + { + graph.resolve_peer_dep(&parent_node, &peer_dep); + // peer_dep.version_req.satisfies(version) + //parent_node.borrow_mut(). } } diff --git a/cli/npm/tarball.rs b/cli/npm/tarball.rs index 3971e0b07478e6..928331a71a9bb3 100644 --- a/cli/npm/tarball.rs +++ b/cli/npm/tarball.rs @@ -165,6 +165,7 @@ mod test { let package_id = NpmPackageId { name: "package".to_string(), version: NpmVersion::parse("1.0.0").unwrap(), + peer_dependencies: Vec::new(), }; let actual_checksum = "z4phnx7vul3xvchq1m2ab9yg5aulvxxcg/spidns6c5h0ne8xyxysp+dgnkhfuwvy7kxvudbeoglodj6+sfapg=="; From fa2d28d209e63d60204f9fd9a7242756a9e89884 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Tue, 1 Nov 2022 19:20:09 -0400 Subject: [PATCH 02/43] Fix. --- cli/npm/resolution.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cli/npm/resolution.rs b/cli/npm/resolution.rs index d51fe083ad2025..8073531eb09fca 100644 --- a/cli/npm/resolution.rs +++ b/cli/npm/resolution.rs @@ -741,7 +741,7 @@ impl NpmResolution { let parent = NodeParent::Node(node.id.clone()); for child_id in &node.children { let mut child = (**self.packages.get(child_id).unwrap()).borrow_mut(); - child.parents.remove(&node.id); + child.parents.remove(&parent); if child.parents.is_empty() { self.forget_orphan(&mut child); } From c725a9c0e8cc10980b92681300f281c317e30103 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 2 Nov 2022 12:01:24 -0400 Subject: [PATCH 03/43] More work. --- cli/npm/registry.rs | 6 + cli/npm/resolution.rs | 794 +++++++++++++++++++++++------------------- 2 files changed, 448 insertions(+), 352 deletions(-) diff --git a/cli/npm/registry.rs b/cli/npm/registry.rs index 02c408af4930c3..2f75d6e95373fe 100644 --- a/cli/npm/registry.rs +++ b/cli/npm/registry.rs @@ -45,6 +45,12 @@ pub enum NpmDependencyEntryKind { OptionalPeer, } +impl NpmDependencyEntryKind { + pub fn is_optional(&self) -> bool { + matches!(self, NpmDependencyEntryKind::OptionalPeer) + } +} + #[derive(Eq, PartialEq)] pub struct NpmDependencyEntry { pub bare_specifier: String, diff --git a/cli/npm/resolution.rs b/cli/npm/resolution.rs index e7d2efa184eb2d..f07261e6219d27 100644 --- a/cli/npm/resolution.rs +++ b/cli/npm/resolution.rs @@ -1,7 +1,9 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. use std::borrow::BorrowMut; +use std::cell::Ref; use std::cell::RefCell; +use std::cell::RefMut; use std::cmp::Ordering; use std::collections::HashMap; use std::collections::HashSet; @@ -565,226 +567,6 @@ impl NpmResolution { mut package_reqs: Vec, mut snapshot: NpmResolutionSnapshot, ) -> Result { - #[derive(Clone, PartialEq, Eq, Hash)] - enum NodeParent { - Req(NpmPackageReq), - Node(NpmPackageId), - } - - struct Node { - pub id: NpmPackageId, - pub parents: HashSet, - pub children: HashSet, - pub unresolved_peers: Vec, - } - - #[derive(Default)] - struct Graph { - package_reqs: HashMap, - packages_by_name: HashMap>, - packages: HashMap>>, - } - - impl Graph { - pub fn get_or_create_for_id( - &mut self, - id: &NpmPackageId, - ) -> (bool, Rc>) { - if let Some(node) = self.packages.get(id) { - (false, node.clone()) - } else { - let node = Rc::new(RefCell::new(Node { - id: id.clone(), - parents: Default::default(), - children: Default::default(), - unresolved_peers: Default::default(), - })); - self - .packages_by_name - .entry(id.name.clone()) - .or_default() - .push(id.clone()); - self.packages.insert(id.clone(), node.clone()); - (true, node) - } - } - - pub fn fill_with_snapshot(&mut self, snapshot: &NpmResolutionSnapshot) { - for (package_req, id) in &snapshot.package_reqs { - let node = self.fill_for_id_with_snapshot(id, snapshot); - (*node) - .borrow_mut() - .parents - .insert(NodeParent::Req(package_req.clone())); - self.package_reqs.insert(package_req.clone(), id.clone()); - } - } - - fn fill_for_id_with_snapshot( - &mut self, - id: &NpmPackageId, - snapshot: &NpmResolutionSnapshot, - ) -> Rc> { - let resolution = snapshot.packages.get(id).unwrap(); - let node = self.get_or_create_for_id(id).1; - for (name, id) in resolution.dependencies { - let child_node = self.fill_for_id_with_snapshot(&id, snapshot); - set_parent_node(&child_node, &node); - } - node - } - - pub fn resolve_best_package_version( - &self, - name: &str, - version_matcher: &impl NpmVersionMatcher, - ) -> Option { - let mut maybe_best_version: Option<&NpmVersion> = None; - if let Some(ids) = self.packages_by_name.get(name) { - for version in ids.iter().map(|id| &id.version) { - if version_matcher.matches(version) { - let is_best_version = maybe_best_version - .as_ref() - .map(|best_version| (*best_version).cmp(version).is_lt()) - .unwrap_or(true); - if is_best_version { - maybe_best_version = Some(version); - } - } - } - } - maybe_best_version.cloned() - } - - pub fn resolve_peer_dep( - &mut self, - parent_node: &Rc>, - peer_dep: &NpmDependencyEntry, - ) { - self.resolve_peer_dep_parents_with_path(parent_node, peer_dep, vec![]) - } - - fn resolve_peer_dep_parents_with_path( - &mut self, - child_node: &Rc>, - peer_dep: &NpmDependencyEntry, - mut path: Vec>>, - ) { - let parents = child_node.borrow().parents.clone(); - path.push(child_node.clone()); - for parent in parents { - match parent { - NodeParent::Node(parent_node_id) => self - .resolve_peer_dep_with_path( - // todo: this should probably be a debug panic only as it indicates - // a node in the graph was accidentally orphaned - self - .packages - .get(parent_node_id) - .cloned() - .expect("orphaned node"), - peer_dep, - path.clone(), - ), - NodeParent::Req(req) => { - todo!() - } - } - } - } - - fn resolve_peer_dep_with_path( - &mut self, - node: Rc>, - peer_dep: &NpmDependencyEntry, - path: Vec>>, - ) { - for child in node.borrow().children.clone() { - let child = child.borrow(); - if child.id.name == peer_dep.name - && peer_dep.version_req.satisfies(&child.id.version) - { - // go down the descendants creating a new path - let parents = node.borrow().parents.clone(); - self.set_new_peer_dep(parents, node, &child.id, path); - } - } - } - - fn set_new_peer_dep( - &mut self, - previous_parents: HashSet, - node: Rc>, - peer_dep_id: &NpmPackageId, - path: Vec>>, - ) { - let old_node = node.borrow(); - let old_id = old_node.id; - let mut new_id = old_node.id.clone(); - new_id.peer_dependencies.push(peer_dep_id.clone()); - // remove the previous parents from the old node - for previous_parent in &previous_parents { - old_node.parents.remove(previous_parent); - } - drop(old_node); - - let (created, new_node) = self.get_or_create_for_id(&new_id); - - // update the previous parent to point to the new node - // and this node to point at those parents - for parent in previous_parents { - match &parent { - NodeParent::Node(parent_id) => { - let mut parent = - (**self.packages.get(parent_id).unwrap()).borrow_mut(); - parent.children.remove(&old_id); - parent.children.insert(new_id.clone()); - } - NodeParent::Req(req) => { - self.package_reqs.insert(req.clone(), new_id.clone()); - } - } - (*new_node).borrow_mut().parents.insert(parent); - } - - // if this is the case, then we can have the descendant forget about us - - if created { - if let Some(next_node) = path.pop() { - self.set_new_peer_dep(next_node, peer_dep_id, path); - } - } else { - let node = (*node).borrow_mut(); - if node.parents.is_empty() { - self.forget_orphan(&mut node); - } - } - } - - fn forget_orphan(&mut self, node: &mut Node) { - assert_eq!(node.parents.len(), 0); - self.packages.remove(&node.id); - let parent = NodeParent::Node(node.id.clone()); - for child_id in &node.children { - let mut child = (**self.packages.get(child_id).unwrap()).borrow_mut(); - child.parents.remove(&parent); - if child.parents.is_empty() { - self.forget_orphan(&mut child); - } - } - } - } - - pub fn set_parent_node( - child: &Rc>, - parent: &Rc>, - ) { - let mut child = (**child).borrow_mut(); - let mut parent = (**parent).borrow_mut(); - parent.children.insert(child.id.clone()); - child.parents.insert(NodeParent::Node(parent.id.clone())); - } - // convert the snapshot to a traversable graph let mut graph = Graph::default(); graph.fill_with_snapshot(&snapshot); @@ -818,57 +600,317 @@ impl NpmResolution { })); } - let mut pending_dependencies = VecDeque::new(); + let mut resolver = GraphDependencyResolver { + graph: &mut graph, + api: &self.api, + pending_dependencies: Default::default(), + pending_peer_dependencies: Default::default(), + }; for result in futures::future::join_all(unresolved_tasks).await { let (package_req, info) = result??; - // inspect if there's a match in the list of current packages and otherwise - // fall back to looking at the registry - let version_and_info = if let Some(version) = - graph.resolve_best_package_version(&package_req.name, &package_req) - { - match info.versions.get(&version.to_string()) { - Some(version_info) => VersionAndInfo { - version, - info: version_info.clone(), - }, - None => { - bail!("could not find version '{}' for '{}'", version, package_req) - } - } - } else { - get_resolved_package_version_and_info( - &package_req.name, - &package_req, - info, - None, - )? - }; - let id = NpmPackageId { - name: package_req.name.clone(), - version: version_and_info.version.clone(), - peer_dependencies: Vec::new(), - }; - let node = graph.get_or_create_for_id(&id).1; + resolver.resolve_npm_package_req(&package_req, info)?; + } + + resolver.resolve_pending().await?; + + Ok(snapshot) + } + + pub fn resolve_package_from_id( + &self, + id: &NpmPackageId, + ) -> Option { + self.snapshot.read().package_from_id(id).cloned() + } + + pub fn resolve_package_from_package( + &self, + name: &str, + referrer: &NpmPackageId, + ) -> Result { + self + .snapshot + .read() + .resolve_package_from_package(name, referrer) + .cloned() + } + + /// Resolve a node package from a deno module. + pub fn resolve_package_from_deno_module( + &self, + package: &NpmPackageReq, + ) -> Result { + self + .snapshot + .read() + .resolve_package_from_deno_module(package) + .cloned() + } + + pub fn all_packages(&self) -> Vec { + self.snapshot.read().all_packages() + } + + pub fn has_packages(&self) -> bool { + !self.snapshot.read().packages.is_empty() + } + + pub fn snapshot(&self) -> NpmResolutionSnapshot { + self.snapshot.read().clone() + } + + pub fn lock( + &self, + lockfile: &mut Lockfile, + snapshot: &NpmResolutionSnapshot, + ) -> Result<(), AnyError> { + for (package_req, version) in snapshot.package_reqs.iter() { + lockfile.insert_npm_specifier(package_req, version.to_string()); + } + for package in self.all_packages() { + lockfile.check_or_insert_npm_package(&package)?; + } + Ok(()) + } +} +#[derive(Clone, PartialEq, Eq, Hash)] +enum NodeParent { + Req(NpmPackageReq), + Node(NpmPackageId), +} + +struct Node { + pub id: NpmPackageId, + pub parents: HashSet, + pub children: HashSet, + pub unresolved_peers: Vec, +} + +#[derive(Default)] +struct Graph { + package_reqs: HashMap, + packages_by_name: HashMap>, + packages: HashMap>>, +} + +impl Graph { + pub fn get_or_create_for_id( + &mut self, + id: &NpmPackageId, + ) -> (bool, Rc>) { + if let Some(node) = self.packages.get(id) { + (false, node.clone()) + } else { + let node = Rc::new(RefCell::new(Node { + id: id.clone(), + parents: Default::default(), + children: Default::default(), + unresolved_peers: Default::default(), + })); + self + .packages_by_name + .entry(id.name.clone()) + .or_default() + .push(id.clone()); + self.packages.insert(id.clone(), node.clone()); + (true, node) + } + } + + pub fn fill_with_snapshot(&mut self, snapshot: &NpmResolutionSnapshot) { + for (package_req, id) in &snapshot.package_reqs { + let node = self.fill_for_id_with_snapshot(id, snapshot); (*node) .borrow_mut() .parents - .push(NodeParent::Req(package_req.clone())); + .insert(NodeParent::Req(package_req.clone())); + self.package_reqs.insert(package_req.clone(), id.clone()); + } + } + + fn fill_for_id_with_snapshot( + &mut self, + id: &NpmPackageId, + snapshot: &NpmResolutionSnapshot, + ) -> Rc> { + let resolution = snapshot.packages.get(id).unwrap(); + let node = self.get_or_create_for_id(id).1; + for (name, child_id) in resolution.dependencies { + let child_node = self.fill_for_id_with_snapshot(&child_id, snapshot); + self.set_child_parent_node(&child_node, &id); + } + node + } + + fn borrow_node(&self, id: &NpmPackageId) -> Ref { + (**self.packages.get(id).unwrap()).borrow() + } + + fn borrow_node_mut(&self, id: &NpmPackageId) -> RefMut { + (**self.packages.get(id).unwrap()).borrow_mut() + } + + fn forget_orphan(&mut self, node: &mut Node) { + assert_eq!(node.parents.len(), 0); + self.packages.remove(&node.id); + let parent = NodeParent::Node(node.id.clone()); + for child_id in &node.children { + let mut child = (**self.packages.get(child_id).unwrap()).borrow_mut(); + child.parents.remove(&parent); + if child.parents.is_empty() { + self.forget_orphan(&mut child); + } + } + } + + pub fn set_child_parent_node( + &mut self, + child: &Rc>, + parent_id: &NpmPackageId, + ) { + let mut child = (**child).borrow_mut(); + let mut parent = (**self.packages.get(parent_id).unwrap()).borrow_mut(); + debug_assert_ne!(parent.id, child.id); + parent.children.insert(child.id.clone()); + child.parents.insert(NodeParent::Node(parent.id.clone())); + } +} + +struct GraphDependencyResolver<'a> { + graph: &'a mut Graph, + api: &'a NpmRegistryApi, + pending_dependencies: VecDeque<(NpmPackageId, Vec)>, + pending_peer_dependencies: + VecDeque<(NpmPackageId, NpmDependencyEntry, NpmPackageInfo)>, +} + +impl<'a> GraphDependencyResolver<'a> { + pub fn resolve_best_package_version_and_info( + &self, + name: &str, + version_matcher: &impl NpmVersionMatcher, + package_info: NpmPackageInfo, + ) -> Result { + if let Some(version) = + self.resolve_best_package_version(name, version_matcher) + { + match package_info.versions.get(&version.to_string()) { + Some(version_info) => Ok(VersionAndInfo { + version, + info: version_info.clone(), + }), + None => { + bail!("could not find version '{}' for '{}'", version, name) + } + } + } else { + // get the information + get_resolved_package_version_and_info( + name, + version_matcher, + package_info, + None, + ) + } + } + + pub fn resolve_best_package_version( + &self, + name: &str, + version_matcher: &impl NpmVersionMatcher, + ) -> Option { + let mut maybe_best_version: Option<&NpmVersion> = None; + if let Some(ids) = self.graph.packages_by_name.get(name) { + for version in ids.iter().map(|id| &id.version) { + if version_matcher.matches(version) { + let is_best_version = maybe_best_version + .as_ref() + .map(|best_version| (*best_version).cmp(version).is_lt()) + .unwrap_or(true); + if is_best_version { + maybe_best_version = Some(version); + } + } + } + } + maybe_best_version.cloned() + } + + pub fn resolve_npm_package_req( + &mut self, + package_req: &NpmPackageReq, + info: NpmPackageInfo, + ) -> Result<(), AnyError> { + // inspect if there's a match in the list of current packages and otherwise + // fall back to looking at the registry + let version_and_info = self.resolve_best_package_version_and_info( + &package_req.name, + package_req, + info, + )?; + let id = NpmPackageId { + name: package_req.name.clone(), + version: version_and_info.version.clone(), + peer_dependencies: Vec::new(), + }; + let node = self.graph.get_or_create_for_id(&id).1; + (*node) + .borrow_mut() + .parents + .insert(NodeParent::Req(package_req.clone())); + + let dependencies = version_and_info + .info + .dependencies_as_entries() + .with_context(|| format!("npm package: {}", id))?; + + self.pending_dependencies.push_back((id, dependencies)); + Ok(()) + } + + fn analyze_dependency( + &mut self, + name: &str, + version_matcher: &impl NpmVersionMatcher, + package_info: NpmPackageInfo, + parent_id: &NpmPackageId, + ) -> Result<(), AnyError> { + let version_and_info = self.resolve_best_package_version_and_info( + name, + version_matcher, + package_info, + )?; + + let id = NpmPackageId { + name: name.to_string(), + version: version_and_info.version.clone(), + peer_dependencies: Vec::new(), + }; + let (created, node) = self.graph.get_or_create_for_id(&id); + self.graph.set_child_parent_node(&node, &parent_id); + if created { + // inspect the dependencies of the package let dependencies = version_and_info .info .dependencies_as_entries() - .with_context(|| format!("npm package: {}", id))?; + .with_context(|| { + format!("npm package: {}@{}", name, version_and_info.version) + })?; - pending_dependencies.push_back((node, dependencies)); + self.pending_dependencies.push_back((id, dependencies)); } + Ok(()) + } - let mut pending_peer_dependencies = VecDeque::new(); - while !pending_dependencies.is_empty() - || !pending_peer_dependencies.is_empty() + pub async fn resolve_pending(&mut self) -> Result<(), AnyError> { + while !self.pending_dependencies.is_empty() + || !self.pending_peer_dependencies.is_empty() { // now go down through the dependencies by tree depth - while let Some((parent_node, mut deps)) = pending_dependencies.pop_front() + while let Some((parent_id, mut deps)) = + self.pending_dependencies.pop_front() { // ensure name alphabetical and then version descending deps.sort(); @@ -895,130 +937,178 @@ impl NpmResolution { // resolve the non-peer dependencies for dep in deps { + let package_info = self.api.package_info(&dep.name).await?; + if matches!( dep.kind, NpmDependencyEntryKind::Peer | NpmDependencyEntryKind::OptionalPeer ) { - pending_peer_dependencies.push_back((parent_node.clone(), dep)); - continue; // handle these later - } - - // check if an existing dependency matches this - let package_info = self.api.package_info(&dep.name).await?; - let version_and_info = if let Some(version) = - snapshot.resolve_best_package_version(&dep.name, &dep.version_req) - { - match package_info.versions.get(&version.to_string()) { - Some(version_info) => VersionAndInfo { - version, - info: version_info.clone(), - }, - None => { - bail!("could not find version '{}' for '{}'", version, dep.name) - } - } + self.pending_peer_dependencies.push_back(( + parent_id.clone(), + dep, + package_info, + )); } else { - // get the information - get_resolved_package_version_and_info( + self.analyze_dependency( &dep.name, &dep.version_req, package_info, - None, - )? - }; - - let id = NpmPackageId { - name: dep.name.clone(), - version: version_and_info.version.clone(), - peer_dependencies: Vec::new(), - }; - let (created, node) = graph.get_or_create_for_id(&id); - set_parent_node(&node, &parent_node); - - if !created { - // inspect the dependencies of the package - let dependencies = version_and_info - .info - .dependencies_as_entries() - .with_context(|| { - format!( - "npm package: {}@{}", - dep.name, version_and_info.version - ) - })?; - - pending_dependencies.push_back((node, dependencies)); + &parent_id, + )?; } } } - if let Some((parent_node, peer_dep)) = - pending_peer_dependencies.pop_front() + if let Some((parent_id, peer_dep, peer_package_info)) = + self.pending_peer_dependencies.pop_front() { - graph.resolve_peer_dep(&parent_node, &peer_dep); + self.resolve_peer_dep( + &parent_id, + &peer_dep, + peer_package_info, + vec![], + )?; // peer_dep.version_req.satisfies(version) //parent_node.borrow_mut(). } } - - Ok(snapshot) + Ok(()) } - pub fn resolve_package_from_id( - &self, - id: &NpmPackageId, - ) -> Option { - self.snapshot.read().package_from_id(id).cloned() - } + fn resolve_peer_dep( + &mut self, + child_id: &NpmPackageId, + peer_dep: &NpmDependencyEntry, + peer_package_info: NpmPackageInfo, + mut path: Vec, + ) -> Result<(), AnyError> { + // Peer dependencies are resolved based on its ancestors' siblings. + // If not found, then it resolves based on the version requirement if non-optional + let parents = self.graph.borrow_node(&child_id).parents.clone(); + path.push(child_id.clone()); + for parent in parents { + let children_ids = match &parent { + NodeParent::Node(parent_node_id) => { + self.graph.borrow_node(parent_node_id).children.clone() + } + NodeParent::Req(req) => self + .graph + .package_reqs + .values() + .cloned() + .collect::>(), + }; + for child_id in children_ids { + if child_id.name == peer_dep.name + && peer_dep.version_req.satisfies(&child_id.version) + { + // go down the descendants creating a new path + match &parent { + NodeParent::Node(node_id) => { + let parents = self.graph.borrow_node(node_id).parents.clone(); + self.set_new_peer_dep(parents, node_id, &child_id, path); + return Ok(()); + } + NodeParent::Req(req) => { + let old_id = self.graph.package_reqs.get(&req).unwrap().clone(); + self.set_new_peer_dep( + HashSet::from([NodeParent::Req(req.clone())]), + &old_id, + &child_id, + path, + ); + return Ok(()); + } + } + } + } + } - pub fn resolve_package_from_package( - &self, - name: &str, - referrer: &NpmPackageId, - ) -> Result { - self - .snapshot - .read() - .resolve_package_from_package(name, referrer) - .cloned() + // at this point it means we didn't find anything by searching the ancestor siblings, + // so we need to resolve based on the package info + if !peer_dep.kind.is_optional() { + self.analyze_dependency( + &peer_dep.name, + &peer_dep.version_req, + peer_package_info, + &child_id, + )?; + } + Ok(()) } - /// Resolve a node package from a deno module. - pub fn resolve_package_from_deno_module( - &self, - package: &NpmPackageReq, - ) -> Result { - self - .snapshot - .read() - .resolve_package_from_deno_module(package) - .cloned() - } + fn set_new_peer_dep( + &mut self, + previous_parents: HashSet, + node_id: &NpmPackageId, + peer_dep_id: &NpmPackageId, + path: Vec, + ) { + let old_id = node_id; + let mut new_id = old_id.clone(); + new_id.peer_dependencies.push(peer_dep_id.clone()); + // remove the previous parents from the old node + let old_node_children = { + let old_node = self.graph.borrow_node_mut(old_id); + for previous_parent in &previous_parents { + old_node.parents.remove(previous_parent); + } + old_node.children.clone() + }; - pub fn all_packages(&self) -> Vec { - self.snapshot.read().all_packages() - } + let (created, new_node) = self.graph.get_or_create_for_id(&new_id); - pub fn has_packages(&self) -> bool { - !self.snapshot.read().packages.is_empty() - } + // update the previous parent to point to the new node + // and this node to point at those parents + { + let new_node = (*new_node).borrow_mut(); + for parent in previous_parents { + match &parent { + NodeParent::Node(parent_id) => { + let mut parent = + (**self.graph.packages.get(parent_id).unwrap()).borrow_mut(); + parent.children.remove(&old_id); + parent.children.insert(new_id.clone()); + } + NodeParent::Req(req) => { + self.graph.package_reqs.insert(req.clone(), new_id.clone()); + } + } + new_node.parents.insert(parent); + } - pub fn snapshot(&self) -> NpmResolutionSnapshot { - self.snapshot.read().clone() - } + // now add the previous children to this node + new_node.children.extend(old_node_children); + } - pub fn lock( - &self, - lockfile: &mut Lockfile, - snapshot: &NpmResolutionSnapshot, - ) -> Result<(), AnyError> { - for (package_req, version) in snapshot.package_reqs.iter() { - lockfile.insert_npm_specifier(package_req, version.to_string()); + for child_id in old_node_children { + self + .graph + .borrow_node_mut(&child_id) + .parents + .insert(NodeParent::Node(new_id.clone())); } - for package in self.all_packages() { - lockfile.check_or_insert_npm_package(&package)?; + + if created { + // continue going down the path + let maybe_next_node = path.pop(); + if let Some(next_node_id) = path.pop() { + self.set_new_peer_dep( + HashSet::from([NodeParent::Node(new_id)]), + &next_node_id, + peer_dep_id, + path, + ); + } + } + + // forget the old node at this point if it has no parents + { + let old_node = self.graph.borrow_node_mut(old_id); + if old_node.parents.is_empty() { + self.graph.forget_orphan(&mut old_node); + } } - Ok(()) } } From b7f1424deb59978ec9395f3aa3631ca28b5e3ee8 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 2 Nov 2022 13:23:52 -0400 Subject: [PATCH 04/43] Committing current state. --- cli/npm/registry.rs | 11 +++ cli/npm/resolution.rs | 194 ++++++++++++++++++++++++------------------ 2 files changed, 124 insertions(+), 81 deletions(-) diff --git a/cli/npm/registry.rs b/cli/npm/registry.rs index 2f75d6e95373fe..28591c683f15fb 100644 --- a/cli/npm/registry.rs +++ b/cli/npm/registry.rs @@ -26,6 +26,7 @@ use crate::progress_bar::ProgressBar; use super::cache::NpmCache; use super::resolution::NpmVersionMatcher; +use super::semver::NpmVersion; use super::semver::NpmVersionReq; // npm registry docs: https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md @@ -221,6 +222,16 @@ impl NpmRegistryApi { } } + pub async fn package_version_info( + &self, + name: &str, + version: &NpmVersion, + ) -> Result, AnyError> { + // todo(dsherret): this could be optimized to not clone the entire package info + let mut package_info = self.package_info(name).await?; + Ok(package_info.versions.remove(&version.to_string())) + } + pub async fn maybe_package_info( &self, name: &str, diff --git a/cli/npm/resolution.rs b/cli/npm/resolution.rs index f07261e6219d27..d5b051cb0f2d3d 100644 --- a/cli/npm/resolution.rs +++ b/cli/npm/resolution.rs @@ -278,15 +278,7 @@ impl NpmResolutionSnapshot { req: &NpmPackageReq, ) -> Result<&NpmResolutionPackage, AnyError> { match self.package_reqs.get(req) { - Some(version) => Ok( - self - .packages - .get(&NpmPackageId { - name: req.name.clone(), - version: version.clone(), - }) - .unwrap(), - ), + Some(id) => Ok(self.packages.get(id).unwrap()), None => bail!("could not find npm package directory for '{}'", req), } } @@ -294,11 +286,8 @@ impl NpmResolutionSnapshot { pub fn top_level_packages(&self) -> Vec { self .package_reqs - .iter() - .map(|(req, version)| NpmPackageId { - name: req.name.clone(), - version: version.clone(), - }) + .values() + .cloned() .collect::>() .into_iter() .collect::>() @@ -384,7 +373,7 @@ impl NpmResolutionSnapshot { lockfile: Arc>, api: &NpmRegistryApi, ) -> Result { - let mut package_reqs: HashMap; + let mut package_reqs: HashMap; let mut packages_by_name: HashMap>; let mut packages: HashMap; @@ -405,7 +394,7 @@ impl NpmResolutionSnapshot { let reference = NpmPackageReference::from_str(&format!("npm:{}", key)) .with_context(|| format!("Unable to parse npm specifier: {}", key))?; let package_id = NpmPackageId::deserialize_from_lock_file(value)?; - package_reqs.insert(reference.req, package_id.version.clone()); + package_reqs.insert(reference.req, package_id.clone()); verify_ids.insert(package_id.clone()); } @@ -468,17 +457,16 @@ impl NpmResolutionSnapshot { // ensure the dist is set for each package for package in packages.values_mut() { // this will read from the memory cache now - let package_info = api.package_info(&package.id.name).await?; - let version_info = match package_info - .versions - .get(&package.id.version.to_string()) + let version_info = match api + .package_version_info(&package.id.name, &package.id.version) + .await? { Some(version_info) => version_info, None => { bail!("could not find '{}' specified in the lockfile. Maybe try again with --reload", package.id); } }; - package.dist = version_info.dist.clone(); + package.dist = version_info.dist; } Ok(Self { @@ -565,11 +553,12 @@ impl NpmResolution { async fn add_package_reqs_to_snapshot( &self, mut package_reqs: Vec, - mut snapshot: NpmResolutionSnapshot, + snapshot: NpmResolutionSnapshot, ) -> Result { // convert the snapshot to a traversable graph let mut graph = Graph::default(); graph.fill_with_snapshot(&snapshot); + drop(snapshot); // todo: remove // multiple packages are resolved in alphabetical order package_reqs.sort_by(|a, b| a.name.cmp(&b.name)); @@ -614,7 +603,7 @@ impl NpmResolution { resolver.resolve_pending().await?; - Ok(snapshot) + graph.into_snapshot(&self.api).await } pub fn resolve_package_from_id( @@ -682,8 +671,8 @@ enum NodeParent { struct Node { pub id: NpmPackageId, - pub parents: HashSet, - pub children: HashSet, + pub parents: HashMap, + pub children: HashMap, pub unresolved_peers: Vec, } @@ -721,10 +710,10 @@ impl Graph { pub fn fill_with_snapshot(&mut self, snapshot: &NpmResolutionSnapshot) { for (package_req, id) in &snapshot.package_reqs { let node = self.fill_for_id_with_snapshot(id, snapshot); - (*node) - .borrow_mut() - .parents - .insert(NodeParent::Req(package_req.clone())); + (*node).borrow_mut().parents.insert( + package_req.to_string(), + NodeParent::Req(package_req.clone()), + ); self.package_reqs.insert(package_req.clone(), id.clone()); } } @@ -738,7 +727,7 @@ impl Graph { let node = self.get_or_create_for_id(id).1; for (name, child_id) in resolution.dependencies { let child_node = self.fill_for_id_with_snapshot(&child_id, snapshot); - self.set_child_parent_node(&child_node, &id); + self.set_child_parent_node(&name, &child_node, &id); } node } @@ -755,9 +744,9 @@ impl Graph { assert_eq!(node.parents.len(), 0); self.packages.remove(&node.id); let parent = NodeParent::Node(node.id.clone()); - for child_id in &node.children { + for (specifier, child_id) in &node.children { let mut child = (**self.packages.get(child_id).unwrap()).borrow_mut(); - child.parents.remove(&parent); + child.parents.remove(specifier); if child.parents.is_empty() { self.forget_orphan(&mut child); } @@ -766,14 +755,57 @@ impl Graph { pub fn set_child_parent_node( &mut self, + specifier: &str, child: &Rc>, parent_id: &NpmPackageId, ) { let mut child = (**child).borrow_mut(); let mut parent = (**self.packages.get(parent_id).unwrap()).borrow_mut(); debug_assert_ne!(parent.id, child.id); - parent.children.insert(child.id.clone()); - child.parents.insert(NodeParent::Node(parent.id.clone())); + parent + .children + .insert(specifier.to_string(), child.id.clone()); + child + .parents + .insert(specifier.to_string(), NodeParent::Node(parent.id.clone())); + } + + pub async fn into_snapshot( + self, + api: &NpmRegistryApi, + ) -> Result { + let mut packages = HashMap::with_capacity(self.packages.len()); + for (id, node) in self.packages { + let node = node.borrow(); + assert_eq!(node.unresolved_peers.len(), 0); + packages.insert( + id.clone(), + NpmResolutionPackage { + dist: api + .package_version_info(&id.name, &id.version) + .await? + .unwrap() + .dist, + dependencies: node.children, + id, + }, + ) + } + Ok(NpmResolutionSnapshot { + package_reqs: self.package_reqs, + package_versions_by_name: self + .packages_by_name + .into_iter() + .map(|(name, ids)| { + let mut versions = + ids.into_iter().map(|id| id.version).collect::>(); + versions.sort(); + versions.dedup(); + (name, versions) + }) + .collect::>(), + packages, + }) } } @@ -782,7 +814,7 @@ struct GraphDependencyResolver<'a> { api: &'a NpmRegistryApi, pending_dependencies: VecDeque<(NpmPackageId, Vec)>, pending_peer_dependencies: - VecDeque<(NpmPackageId, NpmDependencyEntry, NpmPackageInfo)>, + VecDeque<((String, NpmPackageId), NpmDependencyEntry, NpmPackageInfo)>, } impl<'a> GraphDependencyResolver<'a> { @@ -855,10 +887,10 @@ impl<'a> GraphDependencyResolver<'a> { peer_dependencies: Vec::new(), }; let node = self.graph.get_or_create_for_id(&id).1; - (*node) - .borrow_mut() - .parents - .insert(NodeParent::Req(package_req.clone())); + (*node).borrow_mut().parents.insert( + package_req.to_string(), + NodeParent::Req(package_req.clone()), + ); let dependencies = version_and_info .info @@ -871,24 +903,25 @@ impl<'a> GraphDependencyResolver<'a> { fn analyze_dependency( &mut self, - name: &str, - version_matcher: &impl NpmVersionMatcher, + entry: &NpmDependencyEntry, package_info: NpmPackageInfo, parent_id: &NpmPackageId, ) -> Result<(), AnyError> { let version_and_info = self.resolve_best_package_version_and_info( - name, - version_matcher, + &entry.name, + &entry.version_req, package_info, )?; let id = NpmPackageId { - name: name.to_string(), + name: entry.name.clone(), version: version_and_info.version.clone(), peer_dependencies: Vec::new(), }; let (created, node) = self.graph.get_or_create_for_id(&id); - self.graph.set_child_parent_node(&node, &parent_id); + self + .graph + .set_child_parent_node(&entry.bare_specifier, &node, &parent_id); if created { // inspect the dependencies of the package @@ -896,7 +929,7 @@ impl<'a> GraphDependencyResolver<'a> { .info .dependencies_as_entries() .with_context(|| { - format!("npm package: {}@{}", name, version_and_info.version) + format!("npm package: {}@{}", &entry.name, version_and_info.version) })?; self.pending_dependencies.push_back((id, dependencies)); @@ -944,25 +977,21 @@ impl<'a> GraphDependencyResolver<'a> { NpmDependencyEntryKind::Peer | NpmDependencyEntryKind::OptionalPeer ) { self.pending_peer_dependencies.push_back(( - parent_id.clone(), + (dep.bare_specifier.clone(), parent_id.clone()), dep, package_info, )); } else { - self.analyze_dependency( - &dep.name, - &dep.version_req, - package_info, - &parent_id, - )?; + self.analyze_dependency(&dep, package_info, &parent_id)?; } } } - if let Some((parent_id, peer_dep, peer_package_info)) = + if let Some(((specifier, parent_id), peer_dep, peer_package_info)) = self.pending_peer_dependencies.pop_front() { self.resolve_peer_dep( + &specifier, &parent_id, &peer_dep, peer_package_info, @@ -977,28 +1006,29 @@ impl<'a> GraphDependencyResolver<'a> { fn resolve_peer_dep( &mut self, + specifier: &str, child_id: &NpmPackageId, peer_dep: &NpmDependencyEntry, peer_package_info: NpmPackageInfo, - mut path: Vec, + mut path: Vec<(String, NpmPackageId)>, ) -> Result<(), AnyError> { // Peer dependencies are resolved based on its ancestors' siblings. // If not found, then it resolves based on the version requirement if non-optional let parents = self.graph.borrow_node(&child_id).parents.clone(); - path.push(child_id.clone()); - for parent in parents { - let children_ids = match &parent { + path.push((specifier.to_string(), child_id.clone())); + for (specifier, parent) in parents { + let children = match &parent { NodeParent::Node(parent_node_id) => { self.graph.borrow_node(parent_node_id).children.clone() } NodeParent::Req(req) => self .graph .package_reqs - .values() - .cloned() - .collect::>(), + .iter() + .map(|(req, id)| (req.to_string(), id.clone())) + .collect::>(), }; - for child_id in children_ids { + for (child_specifier, child_id) in children { if child_id.name == peer_dep.name && peer_dep.version_req.satisfies(&child_id.version) { @@ -1006,13 +1036,19 @@ impl<'a> GraphDependencyResolver<'a> { match &parent { NodeParent::Node(node_id) => { let parents = self.graph.borrow_node(node_id).parents.clone(); - self.set_new_peer_dep(parents, node_id, &child_id, path); + self.set_new_peer_dep( + parents, &specifier, node_id, &child_id, path, + ); return Ok(()); } NodeParent::Req(req) => { let old_id = self.graph.package_reqs.get(&req).unwrap().clone(); self.set_new_peer_dep( - HashSet::from([NodeParent::Req(req.clone())]), + HashMap::from([( + req.to_string(), + NodeParent::Req(req.clone()), + )]), + &specifier, &old_id, &child_id, path, @@ -1027,22 +1063,18 @@ impl<'a> GraphDependencyResolver<'a> { // at this point it means we didn't find anything by searching the ancestor siblings, // so we need to resolve based on the package info if !peer_dep.kind.is_optional() { - self.analyze_dependency( - &peer_dep.name, - &peer_dep.version_req, - peer_package_info, - &child_id, - )?; + self.analyze_dependency(&peer_dep, peer_package_info, &child_id)?; } Ok(()) } fn set_new_peer_dep( &mut self, - previous_parents: HashSet, + previous_parents: HashMap, + specifier: &str, node_id: &NpmPackageId, peer_dep_id: &NpmPackageId, - path: Vec, + path: Vec<(String, NpmPackageId)>, ) { let old_id = node_id; let mut new_id = old_id.clone(); @@ -1050,7 +1082,7 @@ impl<'a> GraphDependencyResolver<'a> { // remove the previous parents from the old node let old_node_children = { let old_node = self.graph.borrow_node_mut(old_id); - for previous_parent in &previous_parents { + for previous_parent in previous_parents.keys() { old_node.parents.remove(previous_parent); } old_node.children.clone() @@ -1062,39 +1094,39 @@ impl<'a> GraphDependencyResolver<'a> { // and this node to point at those parents { let new_node = (*new_node).borrow_mut(); - for parent in previous_parents { + for (specifier, parent) in previous_parents { match &parent { NodeParent::Node(parent_id) => { let mut parent = (**self.graph.packages.get(parent_id).unwrap()).borrow_mut(); - parent.children.remove(&old_id); - parent.children.insert(new_id.clone()); + parent.children.insert(specifier.clone(), new_id.clone()); } NodeParent::Req(req) => { self.graph.package_reqs.insert(req.clone(), new_id.clone()); } } - new_node.parents.insert(parent); + new_node.parents.insert(specifier, parent); } // now add the previous children to this node new_node.children.extend(old_node_children); } - for child_id in old_node_children { + for (specifier, child_id) in old_node_children { self .graph .borrow_node_mut(&child_id) .parents - .insert(NodeParent::Node(new_id.clone())); + .insert(specifier, NodeParent::Node(new_id.clone())); } if created { // continue going down the path let maybe_next_node = path.pop(); - if let Some(next_node_id) = path.pop() { + if let Some((next_specifier, next_node_id)) = path.pop() { self.set_new_peer_dep( - HashSet::from([NodeParent::Node(new_id)]), + HashMap::from([(specifier.to_string(), NodeParent::Node(new_id))]), + &next_specifier, &next_node_id, peer_dep_id, path, From 19dd3d7edda90ae565f756cfdfb4418d61174b05 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 2 Nov 2022 17:05:51 -0400 Subject: [PATCH 05/43] Serialization and deserialization of npm package ids --- cli/lockfile.rs | 18 ++--- cli/npm/cache.rs | 1 + cli/npm/resolution.rs | 166 ++++++++++++++++++++++++++------------- cli/npm/semver/errors.rs | 2 + cli/npm/semver/mod.rs | 2 +- 5 files changed, 125 insertions(+), 64 deletions(-) diff --git a/cli/lockfile.rs b/cli/lockfile.rs index b0cf689177a6e4..3824b006a36e30 100644 --- a/cli/lockfile.rs +++ b/cli/lockfile.rs @@ -267,7 +267,7 @@ impl Lockfile { &mut self, package: &NpmResolutionPackage, ) -> Result<(), LockfileError> { - let specifier = package.id.serialize_for_lock_file(); + let specifier = package.id.as_serializable_name(); if let Some(package_info) = self.content.npm.packages.get(&specifier) { let integrity = package .dist @@ -298,7 +298,7 @@ Use \"--lock-write\" flag to regenerate the lockfile at \"{}\".", let dependencies = package .dependencies .iter() - .map(|(name, id)| (name.to_string(), id.serialize_for_lock_file())) + .map(|(name, id)| (name.to_string(), id.as_serializable_name())) .collect::>(); let integrity = package @@ -307,7 +307,7 @@ Use \"--lock-write\" flag to regenerate the lockfile at \"{}\".", .as_ref() .unwrap_or(&package.dist.shasum); self.content.npm.packages.insert( - package.id.serialize_for_lock_file(), + package.id.as_serializable_name(), NpmPackageInfo { integrity: integrity.to_string(), dependencies, @@ -559,8 +559,8 @@ mod tests { version: NpmVersion::parse("3.3.4").unwrap(), }, dist: NpmPackageVersionDistInfo { - tarball: "foo".to_string(), - shasum: "foo".to_string(), + tarball: "foo".to_string(), + shasum: "foo".to_string(), integrity: Some("sha512-MqBkQh/OHTS2egovRtLk45wEyNXwF+cokD+1YPf9u5VfJiRdAiRwB2froX5Co9Rh20xs4siNPm8naNotSD6RBw==".to_string()) }, dependencies: HashMap::new(), @@ -574,8 +574,8 @@ mod tests { version: NpmVersion::parse("1.0.0").unwrap(), }, dist: NpmPackageVersionDistInfo { - tarball: "foo".to_string(), - shasum: "foo".to_string(), + tarball: "foo".to_string(), + shasum: "foo".to_string(), integrity: Some("sha512-1fygroTLlHu66zi26VoTDv8yRgm0Fccecssto+MhsZ0D/DGW2sm8E8AjW7NU5VVTRt5GxbeZ5qBuJr+HyLYkjQ==".to_string()) }, dependencies: HashMap::new(), @@ -590,8 +590,8 @@ mod tests { version: NpmVersion::parse("1.0.2").unwrap(), }, dist: NpmPackageVersionDistInfo { - tarball: "foo".to_string(), - shasum: "foo".to_string(), + tarball: "foo".to_string(), + shasum: "foo".to_string(), integrity: Some("sha512-R0XvVJ9WusLiqTCEiGCmICCMplcCkIwwR11mOSD9CR5u+IXYdiseeEuXCVAjS54zqwkLcPNnmU4OeJ6tUrWhDw==".to_string()) }, dependencies: HashMap::new(), diff --git a/cli/npm/cache.rs b/cli/npm/cache.rs index 6a0d72b3a527fa..c333daa70ac995 100644 --- a/cli/npm/cache.rs +++ b/cli/npm/cache.rs @@ -153,6 +153,7 @@ impl ReadonlyNpmCache { // examples: // * chalk/5.0.1/ // * @types/chalk/5.0.1/ + // * some-package/5.0.1_peer-dep-name@0.1.0/ let is_scoped_package = relative_url.starts_with('@'); let mut parts = relative_url .split('/') diff --git a/cli/npm/resolution.rs b/cli/npm/resolution.rs index d5b051cb0f2d3d..3125b54e7f3a43 100644 --- a/cli/npm/resolution.rs +++ b/cli/npm/resolution.rs @@ -1,10 +1,8 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. -use std::borrow::BorrowMut; use std::cell::Ref; use std::cell::RefCell; use std::cell::RefMut; -use std::cmp::Ordering; use std::collections::HashMap; use std::collections::HashSet; use std::collections::VecDeque; @@ -185,26 +183,99 @@ impl NpmPackageId { } } - pub fn serialize_for_lock_file(&self) -> String { - if !self.peer_dependencies.is_empty() { - todo!(); + pub fn as_serializable_name(&self) -> String { + self.as_serialize_name_with_level(1) + } + + fn as_serialize_name_with_level(&self, level: usize) -> String { + let mut result = format!("{}@{}", self.name, self.version); + for peer in &self.peer_dependencies { + result.push_str(&"_".repeat(level)); + result.push_str(&peer.as_serialize_name_with_level(level + 1)); } - format!("{}@{}", self.name, self.version) + result } - pub fn deserialize_from_lock_file(id: &str) -> Result { - let reference = NpmPackageReference::from_str(&format!("npm:{}", id)) - .with_context(|| { - format!("Unable to deserialize npm package reference: {}", id) - })?; - let version = - NpmVersion::parse(&reference.req.version_req.unwrap().to_string()) - .unwrap(); - Ok(Self { - name: reference.req.name, - version, - peer_dependencies: Vec::new(), // todo - }) + pub fn deserialize_serializable_name(id: &str) -> Result { + use monch::*; + + fn parse_name(input: &str) -> ParseResult<&str> { + if_not_empty(substring(move |input| { + for (pos, c) in input.char_indices() { + // first character might be a scope, so skip it + if pos > 0 && c == '@' { + return Ok((&input[pos..], ())); + } + } + ParseError::backtrace() + }))(input) + } + + fn parse_version(input: &str) -> ParseResult<&str> { + if_not_empty(substring(skip_while(|c| c != '_')))(input) + } + + fn parse_name_and_version( + input: &str, + ) -> ParseResult<(String, NpmVersion)> { + let (input, name) = parse_name(input)?; + let (input, _) = ch('@')(input)?; + let at_version_input = input; + let (input, version) = parse_version(input)?; + match NpmVersion::parse(version) { + Ok(version) => Ok((input, (name.to_string(), version))), + Err(err) => ParseError::fail(at_version_input, format!("{:#}", err)), + } + } + + fn parse_level_at_level<'a>( + level: usize, + ) -> impl Fn(&'a str) -> ParseResult<'a, ()> { + fn parse_level(input: &str) -> ParseResult { + let parsed_level = input.chars().take_while(|c| *c == '_').count(); + Ok((&input[parsed_level..], parsed_level)) + } + + move |input| { + let (input, parsed_level) = parse_level(input)?; + if parsed_level == level { + Ok((input, ())) + } else { + ParseError::backtrace() + } + } + } + + fn parse_peers_at_level<'a>( + level: usize, + ) -> impl Fn(&'a str) -> ParseResult<'a, Vec> { + // todo(THIS PR): open an issue in monch for 'many_while' + many_till( + parse_id_at_level(level), + check_not(parse_level_at_level(level)), + ) + } + + fn parse_id_at_level<'a>( + level: usize, + ) -> impl Fn(&'a str) -> ParseResult<'a, NpmPackageId> { + move |input| { + let (input, (name, version)) = parse_name_and_version(input)?; + let (input, peer_dependencies) = + parse_peers_at_level(level + 1)(input)?; + Ok(( + input, + NpmPackageId { + name, + version, + peer_dependencies, + }, + )) + } + } + + crate::npm::semver::errors::with_failure_handling(parse_id_at_level(0))(id) + .with_context(|| format!("Invalid npm package id '{}'.", id)) } } @@ -227,7 +298,7 @@ pub struct NpmResolutionPackage { pub struct NpmResolutionSnapshot { #[serde(with = "map_to_vec")] package_reqs: HashMap, - package_versions_by_name: HashMap>, + packages_by_name: HashMap>, #[serde(with = "map_to_vec")] packages: HashMap, } @@ -323,11 +394,7 @@ impl NpmResolutionSnapshot { version_req: None, }; - if let Some(version) = self.resolve_best_package_version(name_, &req) { - let id = NpmPackageId { - name: name_.to_string(), - version, - }; + if let Some(id) = self.resolve_best_package_id(name_, &req) { if let Some(pkg) = self.packages.get(&id) { return Ok(pkg); } @@ -347,26 +414,27 @@ impl NpmResolutionSnapshot { self.packages.values().cloned().collect() } - pub fn resolve_best_package_version( + pub fn resolve_best_package_id( &self, name: &str, version_matcher: &impl NpmVersionMatcher, - ) -> Option { - let mut maybe_best_version: Option<&NpmVersion> = None; - if let Some(versions) = self.package_versions_by_name.get(name) { - for version in versions { - if version_matcher.matches(version) { - let is_best_version = maybe_best_version + ) -> Option { + // todo(THIS PR): this is not correct because some ids will be better than others + let mut maybe_best_id: Option<&NpmPackageId> = None; + if let Some(ids) = self.packages_by_name.get(name) { + for id in ids { + if version_matcher.matches(&id.version) { + let is_best_version = maybe_best_id .as_ref() - .map(|best_version| (*best_version).cmp(version).is_lt()) + .map(|best_id| best_id.version.cmp(&id.version).is_lt()) .unwrap_or(true); if is_best_version { - maybe_best_version = Some(version); + maybe_best_id = Some(id); } } } } - maybe_best_version.cloned() + maybe_best_id.cloned() } pub async fn from_lockfile( @@ -374,7 +442,7 @@ impl NpmResolutionSnapshot { api: &NpmRegistryApi, ) -> Result { let mut package_reqs: HashMap; - let mut packages_by_name: HashMap>; + let mut packages_by_name: HashMap>; let mut packages: HashMap; { @@ -393,23 +461,23 @@ impl NpmResolutionSnapshot { for (key, value) in &lockfile.content.npm.specifiers { let reference = NpmPackageReference::from_str(&format!("npm:{}", key)) .with_context(|| format!("Unable to parse npm specifier: {}", key))?; - let package_id = NpmPackageId::deserialize_from_lock_file(value)?; + let package_id = NpmPackageId::deserialize_serializable_name(value)?; package_reqs.insert(reference.req, package_id.clone()); verify_ids.insert(package_id.clone()); } // then the packages for (key, value) in &lockfile.content.npm.packages { - let package_id = NpmPackageId::deserialize_from_lock_file(key)?; + let package_id = NpmPackageId::deserialize_serializable_name(key)?; let mut dependencies = HashMap::default(); packages_by_name .entry(package_id.name.to_string()) .or_default() - .push(package_id.version.clone()); + .push(package_id.clone()); for (name, specifier) in &value.dependencies { - let dep_id = NpmPackageId::deserialize_from_lock_file(specifier)?; + let dep_id = NpmPackageId::deserialize_serializable_name(specifier)?; dependencies.insert(name.to_string(), dep_id.clone()); verify_ids.insert(dep_id); } @@ -471,7 +539,7 @@ impl NpmResolutionSnapshot { Ok(Self { package_reqs, - package_versions_by_name: packages_by_name, + packages_by_name, packages, }) } @@ -786,24 +854,14 @@ impl Graph { .await? .unwrap() .dist, - dependencies: node.children, + dependencies: node.children.clone(), id, }, - ) + ); } Ok(NpmResolutionSnapshot { package_reqs: self.package_reqs, - package_versions_by_name: self - .packages_by_name - .into_iter() - .map(|(name, ids)| { - let mut versions = - ids.into_iter().map(|id| id.version).collect::>(); - versions.sort(); - versions.dedup(); - (name, versions) - }) - .collect::>(), + packages_by_name: self.packages_by_name, packages, }) } diff --git a/cli/npm/semver/errors.rs b/cli/npm/semver/errors.rs index 530d73c5594ab6..0f2ed1bf0ed058 100644 --- a/cli/npm/semver/errors.rs +++ b/cli/npm/semver/errors.rs @@ -6,6 +6,8 @@ use monch::ParseError; use monch::ParseErrorFailure; use monch::ParseResult; +// todo(THIS PR): open an issue in monch about these + pub fn with_failure_handling<'a, T>( combinator: impl Fn(&'a str) -> ParseResult, ) -> impl Fn(&'a str) -> Result { diff --git a/cli/npm/semver/mod.rs b/cli/npm/semver/mod.rs index 90352817fde592..6e1985adc12075 100644 --- a/cli/npm/semver/mod.rs +++ b/cli/npm/semver/mod.rs @@ -20,7 +20,7 @@ use self::range::VersionRangeSet; use self::range::XRange; pub use self::specifier::SpecifierVersionReq; -mod errors; +pub mod errors; mod range; mod specifier; From 2d9b75ba41cc95e14633380d39a676d94c21e984 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 2 Nov 2022 19:47:52 -0400 Subject: [PATCH 06/43] Compiling... --- cli/lockfile.rs | 4 ++ cli/npm/cache.rs | 70 ++++++------------- cli/npm/registry.rs | 14 ++-- cli/npm/resolution.rs | 153 ++++++++++++++++++++++++++---------------- 4 files changed, 131 insertions(+), 110 deletions(-) diff --git a/cli/lockfile.rs b/cli/lockfile.rs index 3824b006a36e30..3dc420796cd0f6 100644 --- a/cli/lockfile.rs +++ b/cli/lockfile.rs @@ -557,6 +557,7 @@ mod tests { id: NpmPackageId { name: "nanoid".to_string(), version: NpmVersion::parse("3.3.4").unwrap(), + peer_dependencies: Vec::new(), }, dist: NpmPackageVersionDistInfo { tarball: "foo".to_string(), @@ -572,6 +573,7 @@ mod tests { id: NpmPackageId { name: "picocolors".to_string(), version: NpmVersion::parse("1.0.0").unwrap(), + peer_dependencies: Vec::new(), }, dist: NpmPackageVersionDistInfo { tarball: "foo".to_string(), @@ -588,6 +590,7 @@ mod tests { id: NpmPackageId { name: "source-map-js".to_string(), version: NpmVersion::parse("1.0.2").unwrap(), + peer_dependencies: Vec::new(), }, dist: NpmPackageVersionDistInfo { tarball: "foo".to_string(), @@ -604,6 +607,7 @@ mod tests { id: NpmPackageId { name: "source-map-js".to_string(), version: NpmVersion::parse("1.0.2").unwrap(), + peer_dependencies: Vec::new(), }, dist: NpmPackageVersionDistInfo { tarball: "foo".to_string(), diff --git a/cli/npm/cache.rs b/cli/npm/cache.rs index c333daa70ac995..3b06f9b750e8e8 100644 --- a/cli/npm/cache.rs +++ b/cli/npm/cache.rs @@ -19,7 +19,6 @@ use crate::fs_util; use crate::progress_bar::ProgressBar; use super::registry::NpmPackageVersionDistInfo; -use super::semver::NpmVersion; use super::tarball::verify_and_extract_tarball; use super::NpmPackageId; @@ -85,25 +84,21 @@ impl ReadonlyNpmCache { ) -> PathBuf { self .package_name_folder(&id.name, registry_url) - .join(id.version.to_string()) + .join(id.as_serializable_name().strip_prefix(&id.name).unwrap()) } pub fn package_name_folder(&self, name: &str, registry_url: &Url) -> PathBuf { let mut dir = self.registry_folder(registry_url); - let mut parts = name.split('/').map(Cow::Borrowed).collect::>(); - // package names were not always enforced to be lowercase and so we need - // to ensure package names, which are therefore case sensitive, are stored - // on a case insensitive file system to not have conflicts. We do this by - // first putting it in a "_" folder then hashing the package name. + let parts = name.split('/').map(Cow::Borrowed).collect::>(); if name.to_lowercase() != name { - let last_part = parts.last_mut().unwrap(); - *last_part = Cow::Owned(crate::checksum::gen(&[last_part.as_bytes()])); - // We can't just use the hash as part of the directory because it may - // have a collision with an actual package name in case someone wanted - // to name an actual package that. To get around this, put all these - // in a folder called "_" since npm packages can't start with an underscore - // and there is no package currently called just "_". - dir = dir.join("_"); + // Lowercase package names introduce complications. + // When implementing this ensure: + // 1. It works on case insensitive filesystems. ex. JSON should not + // conflict with json... yes you read that right, those are separate + // packages. + // 2. We can figure out the package id from the path. This is used + // in resolve_package_id_from_specifier + todo!("deno currently doesn't support npm package names that are not all lowercase"); } // ensure backslashes are used on windows for part in parts { @@ -164,11 +159,10 @@ impl ReadonlyNpmCache { if parts.len() < 2 { return None; } - let version = parts.pop().unwrap(); + let version_part = parts.pop().unwrap(); // this could also contain the peer dep id info let name = parts.join("/"); - NpmVersion::parse(version) - .ok() - .map(|version| NpmPackageId { name, version }) + let full_name = format!("{}@{}", name, version_part); + NpmPackageId::deserialize_name(&full_name).ok() } pub fn get_cache_location(&self) -> PathBuf { @@ -324,12 +318,12 @@ mod test { let cache = ReadonlyNpmCache::new(root_dir.clone()); let registry_url = Url::parse("https://registry.npmjs.org/").unwrap(); - // all lowercase should be as-is assert_eq!( cache.package_folder( &NpmPackageId { name: "json".to_string(), version: NpmVersion::parse("1.2.5").unwrap(), + peer_dependencies: Vec::new(), }, ®istry_url, ), @@ -338,44 +332,24 @@ mod test { .join("json") .join("1.2.5"), ); - } - #[test] - fn should_handle_non_all_lowercase_package_names() { - // it was possible at one point for npm packages to not just be lowercase - let root_dir = crate::deno_dir::DenoDir::new(None).unwrap().root; - let cache = ReadonlyNpmCache::new(root_dir.clone()); - let registry_url = Url::parse("https://registry.npmjs.org/").unwrap(); - let json_uppercase_hash = - "db1a21a0bc2ef8fbe13ac4cf044e8c9116d29137d5ed8b916ab63dcb2d4290df"; assert_eq!( cache.package_folder( &NpmPackageId { - name: "JSON".to_string(), - version: NpmVersion::parse("1.2.5").unwrap(), - }, - ®istry_url, - ), - root_dir - .join("registry.npmjs.org") - .join("_") - .join(json_uppercase_hash) - .join("1.2.5"), - ); - assert_eq!( - cache.package_folder( - &NpmPackageId { - name: "@types/JSON".to_string(), + name: "json".to_string(), version: NpmVersion::parse("1.2.5").unwrap(), + peer_dependencies: vec![NpmPackageId { + name: "other".to_string(), + version: NpmVersion::parse("3.2.1").unwrap(), + peer_dependencies: Vec::new() + }], }, ®istry_url, ), root_dir .join("registry.npmjs.org") - .join("_") - .join("@types") - .join(json_uppercase_hash) - .join("1.2.5"), + .join("json") + .join("1.2.5_other@3.2.1"), ); } } diff --git a/cli/npm/registry.rs b/cli/npm/registry.rs index 28591c683f15fb..f8354289a23e9c 100644 --- a/cli/npm/registry.rs +++ b/cli/npm/registry.rs @@ -87,6 +87,7 @@ pub struct NpmPeerDependencyMeta { } #[derive(Debug, Default, Deserialize, Serialize, Clone)] +#[serde(rename_all = "camelCase")] pub struct NpmPackageVersionInfo { pub version: String, pub dist: NpmPackageVersionDistInfo, @@ -95,9 +96,9 @@ pub struct NpmPackageVersionInfo { #[serde(default)] pub dependencies: HashMap, #[serde(default)] - pub peerDependencies: HashMap, + pub peer_dependencies: HashMap, #[serde(default)] - pub peerDependenciesMeta: HashMap, + pub peer_dependencies_meta: HashMap, } impl NpmPackageVersionInfo { @@ -134,14 +135,15 @@ impl NpmPackageVersionInfo { }) } - let mut result = - Vec::with_capacity(self.dependencies.len() + self.peerDependencies.len()); + let mut result = Vec::with_capacity( + self.dependencies.len() + self.peer_dependencies.len(), + ); for entry in &self.dependencies { result.push(parse_dep_entry(entry, NpmDependencyEntryKind::Dep)?); } - for entry in &self.peerDependencies { + for entry in &self.peer_dependencies { let is_optional = self - .peerDependenciesMeta + .peer_dependencies_meta .get(entry.0) .map(|d| d.optional) .unwrap_or(false); diff --git a/cli/npm/resolution.rs b/cli/npm/resolution.rs index 3125b54e7f3a43..38d96a94e90ee2 100644 --- a/cli/npm/resolution.rs +++ b/cli/npm/resolution.rs @@ -1,12 +1,8 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. -use std::cell::Ref; -use std::cell::RefCell; -use std::cell::RefMut; use std::collections::HashMap; use std::collections::HashSet; use std::collections::VecDeque; -use std::rc::Rc; use deno_ast::ModuleSpecifier; use deno_core::anyhow::bail; @@ -15,6 +11,7 @@ use deno_core::error::generic_error; use deno_core::error::AnyError; use deno_core::futures; use deno_core::parking_lot::Mutex; +use deno_core::parking_lot::MutexGuard; use deno_core::parking_lot::RwLock; use serde::Deserialize; use serde::Serialize; @@ -184,19 +181,38 @@ impl NpmPackageId { } pub fn as_serializable_name(&self) -> String { - self.as_serialize_name_with_level(1) + self.as_serialize_name_with_level(0) } fn as_serialize_name_with_level(&self, level: usize) -> String { - let mut result = format!("{}@{}", self.name, self.version); + fn encode_level(level: usize) -> String { + // This level will always be max 2 characters. + // npm packages aren't allowed to start with a number + if level <= 1 { + "_".repeat(level) + } else { + // ex. 3 -> _3 + format!("_{}", level) + } + } + + let mut result = format!( + "{}@{}", + if level == 0 { + self.name.to_string() + } else { + self.name.replace("/", "+") + }, + self.version + ); for peer in &self.peer_dependencies { - result.push_str(&"_".repeat(level)); + result.push_str(&encode_level(level + 1)); result.push_str(&peer.as_serialize_name_with_level(level + 1)); } result } - pub fn deserialize_serializable_name(id: &str) -> Result { + pub fn deserialize_name(id: &str) -> Result { use monch::*; fn parse_name(input: &str) -> ParseResult<&str> { @@ -228,12 +244,29 @@ impl NpmPackageId { } } + fn parse_next_char_as_u32<'a>(input: &str) -> ParseResult { + let (input, c) = next_char(input)?; + match c.to_digit(10) { + Some(d) => Ok((input, d)), + None => ParseError::backtrace(), + } + } + fn parse_level_at_level<'a>( level: usize, ) -> impl Fn(&'a str) -> ParseResult<'a, ()> { fn parse_level(input: &str) -> ParseResult { - let parsed_level = input.chars().take_while(|c| *c == '_').count(); - Ok((&input[parsed_level..], parsed_level)) + let (input, _) = ch('_')(input)?; + let (input, maybe_underscore) = maybe(ch('_'))(input)?; + if maybe_underscore.is_some() { + return Ok((input, 2)); + } + let (input, maybe_number) = maybe(parse_next_char_as_u32)(input)?; + if let Some(value) = maybe_number { + Ok((input, value as usize)) + } else { + Ok((input, 1)) + } } move |input| { @@ -261,6 +294,11 @@ impl NpmPackageId { ) -> impl Fn(&'a str) -> ParseResult<'a, NpmPackageId> { move |input| { let (input, (name, version)) = parse_name_and_version(input)?; + let name = if level > 0 { + name.replace("+", "/") + } else { + name + }; let (input, peer_dependencies) = parse_peers_at_level(level + 1)(input)?; Ok(( @@ -274,6 +312,7 @@ impl NpmPackageId { } } + // todo(THIS PR): move this someone re-usable crate::npm::semver::errors::with_failure_handling(parse_id_at_level(0))(id) .with_context(|| format!("Invalid npm package id '{}'.", id)) } @@ -461,14 +500,14 @@ impl NpmResolutionSnapshot { for (key, value) in &lockfile.content.npm.specifiers { let reference = NpmPackageReference::from_str(&format!("npm:{}", key)) .with_context(|| format!("Unable to parse npm specifier: {}", key))?; - let package_id = NpmPackageId::deserialize_serializable_name(value)?; + let package_id = NpmPackageId::deserialize_name(value)?; package_reqs.insert(reference.req, package_id.clone()); verify_ids.insert(package_id.clone()); } // then the packages for (key, value) in &lockfile.content.npm.packages { - let package_id = NpmPackageId::deserialize_serializable_name(key)?; + let package_id = NpmPackageId::deserialize_name(key)?; let mut dependencies = HashMap::default(); packages_by_name @@ -477,7 +516,7 @@ impl NpmResolutionSnapshot { .push(package_id.clone()); for (name, specifier) in &value.dependencies { - let dep_id = NpmPackageId::deserialize_serializable_name(specifier)?; + let dep_id = NpmPackageId::deserialize_name(specifier)?; dependencies.insert(name.to_string(), dep_id.clone()); verify_ids.insert(dep_id); } @@ -748,18 +787,21 @@ struct Node { struct Graph { package_reqs: HashMap, packages_by_name: HashMap>, - packages: HashMap>>, + // Ideally this would be Rc>, but we need to use a Mutex + // because the lsp requires Send and this code is executed in the lsp. + // Would be nice if the lsp wasn't Send. + packages: HashMap>>, } impl Graph { pub fn get_or_create_for_id( &mut self, id: &NpmPackageId, - ) -> (bool, Rc>) { + ) -> (bool, Arc>) { if let Some(node) = self.packages.get(id) { (false, node.clone()) } else { - let node = Rc::new(RefCell::new(Node { + let node = Arc::new(Mutex::new(Node { id: id.clone(), parents: Default::default(), children: Default::default(), @@ -778,7 +820,7 @@ impl Graph { pub fn fill_with_snapshot(&mut self, snapshot: &NpmResolutionSnapshot) { for (package_req, id) in &snapshot.package_reqs { let node = self.fill_for_id_with_snapshot(id, snapshot); - (*node).borrow_mut().parents.insert( + (*node).lock().parents.insert( package_req.to_string(), NodeParent::Req(package_req.clone()), ); @@ -790,33 +832,31 @@ impl Graph { &mut self, id: &NpmPackageId, snapshot: &NpmResolutionSnapshot, - ) -> Rc> { + ) -> Arc> { let resolution = snapshot.packages.get(id).unwrap(); let node = self.get_or_create_for_id(id).1; - for (name, child_id) in resolution.dependencies { + for (name, child_id) in &resolution.dependencies { let child_node = self.fill_for_id_with_snapshot(&child_id, snapshot); self.set_child_parent_node(&name, &child_node, &id); } node } - fn borrow_node(&self, id: &NpmPackageId) -> Ref { - (**self.packages.get(id).unwrap()).borrow() - } - - fn borrow_node_mut(&self, id: &NpmPackageId) -> RefMut { - (**self.packages.get(id).unwrap()).borrow_mut() + fn borrow_node(&self, id: &NpmPackageId) -> MutexGuard { + (**self.packages.get(id).unwrap()).lock() } - fn forget_orphan(&mut self, node: &mut Node) { - assert_eq!(node.parents.len(), 0); - self.packages.remove(&node.id); - let parent = NodeParent::Node(node.id.clone()); - for (specifier, child_id) in &node.children { - let mut child = (**self.packages.get(child_id).unwrap()).borrow_mut(); - child.parents.remove(specifier); - if child.parents.is_empty() { - self.forget_orphan(&mut child); + fn forget_orphan(&mut self, node_id: &NpmPackageId) { + if let Some(node) = self.packages.remove(node_id) { + let node = (*node).lock(); + assert_eq!(node.parents.len(), 0); + for (specifier, child_id) in &node.children { + let mut child = (**self.packages.get(child_id).unwrap()).lock(); + child.parents.remove(specifier); + if child.parents.is_empty() { + drop(child); // stop borrowing from self + self.forget_orphan(&child_id); + } } } } @@ -824,11 +864,11 @@ impl Graph { pub fn set_child_parent_node( &mut self, specifier: &str, - child: &Rc>, + child: &Arc>, parent_id: &NpmPackageId, ) { - let mut child = (**child).borrow_mut(); - let mut parent = (**self.packages.get(parent_id).unwrap()).borrow_mut(); + let mut child = (**child).lock(); + let mut parent = (**self.packages.get(parent_id).unwrap()).lock(); debug_assert_ne!(parent.id, child.id); parent .children @@ -844,16 +884,17 @@ impl Graph { ) -> Result { let mut packages = HashMap::with_capacity(self.packages.len()); for (id, node) in self.packages { - let node = node.borrow(); + let dist = api + .package_version_info(&id.name, &id.version) + .await? + .unwrap() + .dist; + let node = node.lock(); assert_eq!(node.unresolved_peers.len(), 0); packages.insert( id.clone(), NpmResolutionPackage { - dist: api - .package_version_info(&id.name, &id.version) - .await? - .unwrap() - .dist, + dist, dependencies: node.children.clone(), id, }, @@ -945,7 +986,7 @@ impl<'a> GraphDependencyResolver<'a> { peer_dependencies: Vec::new(), }; let node = self.graph.get_or_create_for_id(&id).1; - (*node).borrow_mut().parents.insert( + (*node).lock().parents.insert( package_req.to_string(), NodeParent::Req(package_req.clone()), ); @@ -1055,8 +1096,6 @@ impl<'a> GraphDependencyResolver<'a> { peer_package_info, vec![], )?; - // peer_dep.version_req.satisfies(version) - //parent_node.borrow_mut(). } } Ok(()) @@ -1079,13 +1118,15 @@ impl<'a> GraphDependencyResolver<'a> { NodeParent::Node(parent_node_id) => { self.graph.borrow_node(parent_node_id).children.clone() } - NodeParent::Req(req) => self + NodeParent::Req(parent_req) => self .graph .package_reqs .iter() + .filter(|(req, _)| *req == parent_req) .map(|(req, id)| (req.to_string(), id.clone())) .collect::>(), }; + // todo(THIS PR): don't we need to use the specifier here? for (child_specifier, child_id) in children { if child_id.name == peer_dep.name && peer_dep.version_req.satisfies(&child_id.version) @@ -1132,14 +1173,14 @@ impl<'a> GraphDependencyResolver<'a> { specifier: &str, node_id: &NpmPackageId, peer_dep_id: &NpmPackageId, - path: Vec<(String, NpmPackageId)>, + mut path: Vec<(String, NpmPackageId)>, ) { let old_id = node_id; let mut new_id = old_id.clone(); new_id.peer_dependencies.push(peer_dep_id.clone()); // remove the previous parents from the old node let old_node_children = { - let old_node = self.graph.borrow_node_mut(old_id); + let mut old_node = self.graph.borrow_node(old_id); for previous_parent in previous_parents.keys() { old_node.parents.remove(previous_parent); } @@ -1151,12 +1192,12 @@ impl<'a> GraphDependencyResolver<'a> { // update the previous parent to point to the new node // and this node to point at those parents { - let new_node = (*new_node).borrow_mut(); + let mut new_node = (*new_node).lock(); for (specifier, parent) in previous_parents { match &parent { NodeParent::Node(parent_id) => { let mut parent = - (**self.graph.packages.get(parent_id).unwrap()).borrow_mut(); + (**self.graph.packages.get(parent_id).unwrap()).lock(); parent.children.insert(specifier.clone(), new_id.clone()); } NodeParent::Req(req) => { @@ -1167,20 +1208,19 @@ impl<'a> GraphDependencyResolver<'a> { } // now add the previous children to this node - new_node.children.extend(old_node_children); + new_node.children.extend(old_node_children.clone()); } for (specifier, child_id) in old_node_children { self .graph - .borrow_node_mut(&child_id) + .borrow_node(&child_id) .parents .insert(specifier, NodeParent::Node(new_id.clone())); } if created { // continue going down the path - let maybe_next_node = path.pop(); if let Some((next_specifier, next_node_id)) = path.pop() { self.set_new_peer_dep( HashMap::from([(specifier.to_string(), NodeParent::Node(new_id))]), @@ -1194,9 +1234,10 @@ impl<'a> GraphDependencyResolver<'a> { // forget the old node at this point if it has no parents { - let old_node = self.graph.borrow_node_mut(old_id); + let old_node = self.graph.borrow_node(old_id); if old_node.parents.is_empty() { - self.graph.forget_orphan(&mut old_node); + drop(old_node); // stop borrowing + self.graph.forget_orphan(old_id); } } } From 53f11d7d706e519e1c9c6322bb63fcb9a4d774fe Mon Sep 17 00:00:00 2001 From: David Sherret Date: Wed, 2 Nov 2022 20:46:56 -0400 Subject: [PATCH 07/43] Passing existing tests... now to write new ones --- cli/npm/cache.rs | 10 +++- cli/npm/registry.rs | 4 +- cli/npm/resolution.rs | 124 +++++++++++++++++++++++++----------------- 3 files changed, 83 insertions(+), 55 deletions(-) diff --git a/cli/npm/cache.rs b/cli/npm/cache.rs index 3b06f9b750e8e8..c4e0d148e15453 100644 --- a/cli/npm/cache.rs +++ b/cli/npm/cache.rs @@ -82,9 +82,13 @@ impl ReadonlyNpmCache { id: &NpmPackageId, registry_url: &Url, ) -> PathBuf { - self - .package_name_folder(&id.name, registry_url) - .join(id.as_serializable_name().strip_prefix(&id.name).unwrap()) + self.package_name_folder(&id.name, registry_url).join( + id.as_serializable_name() + .strip_prefix(&id.name) + .unwrap() + .strip_prefix("@") + .unwrap(), + ) } pub fn package_name_folder(&self, name: &str, registry_url: &Url) -> PathBuf { diff --git a/cli/npm/registry.rs b/cli/npm/registry.rs index f8354289a23e9c..014d07d9b35533 100644 --- a/cli/npm/registry.rs +++ b/cli/npm/registry.rs @@ -39,7 +39,7 @@ pub struct NpmPackageInfo { pub dist_tags: HashMap, } -#[derive(Eq, PartialEq)] +#[derive(Debug, Eq, PartialEq)] pub enum NpmDependencyEntryKind { Dep, Peer, @@ -52,7 +52,7 @@ impl NpmDependencyEntryKind { } } -#[derive(Eq, PartialEq)] +#[derive(Debug, Eq, PartialEq)] pub struct NpmDependencyEntry { pub bare_specifier: String, pub name: String, diff --git a/cli/npm/resolution.rs b/cli/npm/resolution.rs index 38d96a94e90ee2..1ed8344ee6cda6 100644 --- a/cli/npm/resolution.rs +++ b/cli/npm/resolution.rs @@ -185,17 +185,6 @@ impl NpmPackageId { } fn as_serialize_name_with_level(&self, level: usize) -> String { - fn encode_level(level: usize) -> String { - // This level will always be max 2 characters. - // npm packages aren't allowed to start with a number - if level <= 1 { - "_".repeat(level) - } else { - // ex. 3 -> _3 - format!("_{}", level) - } - } - let mut result = format!( "{}@{}", if level == 0 { @@ -206,7 +195,10 @@ impl NpmPackageId { self.version ); for peer in &self.peer_dependencies { - result.push_str(&encode_level(level + 1)); + // unfortunately we can't do something like `_3` when + // this gets deep because npm package names can start + // with a number + result.push_str(&"_".repeat(level + 1)); result.push_str(&peer.as_serialize_name_with_level(level + 1)); } result @@ -244,29 +236,12 @@ impl NpmPackageId { } } - fn parse_next_char_as_u32<'a>(input: &str) -> ParseResult { - let (input, c) = next_char(input)?; - match c.to_digit(10) { - Some(d) => Ok((input, d)), - None => ParseError::backtrace(), - } - } - fn parse_level_at_level<'a>( level: usize, ) -> impl Fn(&'a str) -> ParseResult<'a, ()> { fn parse_level(input: &str) -> ParseResult { - let (input, _) = ch('_')(input)?; - let (input, maybe_underscore) = maybe(ch('_'))(input)?; - if maybe_underscore.is_some() { - return Ok((input, 2)); - } - let (input, maybe_number) = maybe(parse_next_char_as_u32)(input)?; - if let Some(value) = maybe_number { - Ok((input, value as usize)) - } else { - Ok((input, 1)) - } + let level = input.chars().take_while(|c| *c == '_').count(); + Ok((&input[level..], level)) } move |input| { @@ -282,11 +257,16 @@ impl NpmPackageId { fn parse_peers_at_level<'a>( level: usize, ) -> impl Fn(&'a str) -> ParseResult<'a, Vec> { - // todo(THIS PR): open an issue in monch for 'many_while' - many_till( - parse_id_at_level(level), - check_not(parse_level_at_level(level)), - ) + move |mut input| { + let mut peers = Vec::new(); + while let Ok((level_input, _)) = parse_level_at_level(level)(input) { + input = level_input; + let peer_result = parse_id_at_level(level)(input)?; + input = peer_result.0; + peers.push(peer_result.1); + } + Ok((input, peers)) + } } fn parse_id_at_level<'a>( @@ -770,12 +750,13 @@ impl NpmResolution { Ok(()) } } -#[derive(Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash)] enum NodeParent { Req(NpmPackageReq), Node(NpmPackageId), } +#[derive(Debug)] struct Node { pub id: NpmPackageId, pub parents: HashMap, @@ -783,7 +764,7 @@ struct Node { pub unresolved_peers: Vec, } -#[derive(Default)] +#[derive(Debug, Default)] struct Graph { package_reqs: HashMap, packages_by_name: HashMap>, @@ -887,7 +868,7 @@ impl Graph { let dist = api .package_version_info(&id.name, &id.version) .await? - .unwrap() + .unwrap() // todo(THIS PR): don't unwrap here .dist; let node = node.lock(); assert_eq!(node.unresolved_peers.len(), 0); @@ -990,6 +971,10 @@ impl<'a> GraphDependencyResolver<'a> { package_req.to_string(), NodeParent::Req(package_req.clone()), ); + self + .graph + .package_reqs + .insert(package_req.clone(), id.clone()); let dependencies = version_and_info .info @@ -1071,17 +1056,18 @@ impl<'a> GraphDependencyResolver<'a> { for dep in deps { let package_info = self.api.package_info(&dep.name).await?; - if matches!( - dep.kind, - NpmDependencyEntryKind::Peer | NpmDependencyEntryKind::OptionalPeer - ) { - self.pending_peer_dependencies.push_back(( - (dep.bare_specifier.clone(), parent_id.clone()), - dep, - package_info, - )); - } else { - self.analyze_dependency(&dep, package_info, &parent_id)?; + match dep.kind { + NpmDependencyEntryKind::Dep => { + self.analyze_dependency(&dep, package_info, &parent_id)?; + } + NpmDependencyEntryKind::Peer + | NpmDependencyEntryKind::OptionalPeer => { + self.pending_peer_dependencies.push_back(( + (dep.bare_specifier.clone(), parent_id.clone()), + dep, + package_info, + )); + } } } } @@ -1510,4 +1496,42 @@ mod tests { ); assert_eq!(result.unwrap().version.to_string(), "1.0.0-alpha"); } + + #[test] + fn serialize_npm_package_id() { + let id = NpmPackageId { + name: "pkg-a".to_string(), + version: NpmVersion::parse("1.2.3").unwrap(), + peer_dependencies: vec![ + NpmPackageId { + name: "pkg-b".to_string(), + version: NpmVersion::parse("3.2.1").unwrap(), + peer_dependencies: vec![ + NpmPackageId { + name: "pkg-c".to_string(), + version: NpmVersion::parse("1.3.2").unwrap(), + peer_dependencies: vec![], + }, + NpmPackageId { + name: "pkg-d".to_string(), + version: NpmVersion::parse("2.3.4").unwrap(), + peer_dependencies: vec![], + }, + ], + }, + NpmPackageId { + name: "pkg-e".to_string(), + version: NpmVersion::parse("2.3.1").unwrap(), + peer_dependencies: vec![NpmPackageId { + name: "pkg-f".to_string(), + version: NpmVersion::parse("2.3.1").unwrap(), + peer_dependencies: vec![], + }], + }, + ], + }; + let serialized = id.as_serializable_name(); + assert_eq!(serialized, "pkg-a@1.2.3_pkg-b@3.2.1__pkg-c@1.3.2__pkg-d@2.3.4_pkg-e@2.3.1__pkg-f@2.3.1"); + assert_eq!(NpmPackageId::deserialize_name(&serialized).unwrap(), id); + } } From b4c0d71ca635373ee09891a8c7d9a0145f9f5ec7 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 3 Nov 2022 09:53:36 -0400 Subject: [PATCH 08/43] Split up resolution.rs into multiple files --- cli/npm/resolution.rs | 1537 -------------------------------- cli/npm/resolution/graph.rs | 668 ++++++++++++++ cli/npm/resolution/mod.rs | 637 +++++++++++++ cli/npm/resolution/snapshot.rs | 302 +++++++ 4 files changed, 1607 insertions(+), 1537 deletions(-) delete mode 100644 cli/npm/resolution.rs create mode 100644 cli/npm/resolution/graph.rs create mode 100644 cli/npm/resolution/mod.rs create mode 100644 cli/npm/resolution/snapshot.rs diff --git a/cli/npm/resolution.rs b/cli/npm/resolution.rs deleted file mode 100644 index 1ed8344ee6cda6..00000000000000 --- a/cli/npm/resolution.rs +++ /dev/null @@ -1,1537 +0,0 @@ -// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. - -use std::collections::HashMap; -use std::collections::HashSet; -use std::collections::VecDeque; - -use deno_ast::ModuleSpecifier; -use deno_core::anyhow::bail; -use deno_core::anyhow::Context; -use deno_core::error::generic_error; -use deno_core::error::AnyError; -use deno_core::futures; -use deno_core::parking_lot::Mutex; -use deno_core::parking_lot::MutexGuard; -use deno_core::parking_lot::RwLock; -use serde::Deserialize; -use serde::Serialize; -use std::sync::Arc; - -use crate::lockfile::Lockfile; -use crate::npm::registry::NpmDependencyEntry; -use crate::npm::registry::NpmDependencyEntryKind; - -use super::cache::should_sync_download; -use super::registry::NpmPackageInfo; -use super::registry::NpmPackageVersionDistInfo; -use super::registry::NpmPackageVersionInfo; -use super::registry::NpmRegistryApi; -use super::semver::NpmVersion; -use super::semver::NpmVersionReq; -use super::semver::SpecifierVersionReq; - -/// The version matcher used for npm schemed urls is more strict than -/// the one used by npm packages and so we represent either via a trait. -pub trait NpmVersionMatcher { - fn tag(&self) -> Option<&str>; - fn matches(&self, version: &NpmVersion) -> bool; - fn version_text(&self) -> String; -} - -#[derive(Clone, Debug, Default, PartialEq, Eq)] -pub struct NpmPackageReference { - pub req: NpmPackageReq, - pub sub_path: Option, -} - -impl NpmPackageReference { - pub fn from_specifier( - specifier: &ModuleSpecifier, - ) -> Result { - Self::from_str(specifier.as_str()) - } - - pub fn from_str(specifier: &str) -> Result { - let specifier = match specifier.strip_prefix("npm:") { - Some(s) => s, - None => { - bail!("Not an npm specifier: {}", specifier); - } - }; - let parts = specifier.split('/').collect::>(); - let name_part_len = if specifier.starts_with('@') { 2 } else { 1 }; - if parts.len() < name_part_len { - return Err(generic_error(format!("Not a valid package: {}", specifier))); - } - let name_parts = &parts[0..name_part_len]; - let last_name_part = &name_parts[name_part_len - 1]; - let (name, version_req) = if let Some(at_index) = last_name_part.rfind('@') - { - let version = &last_name_part[at_index + 1..]; - let last_name_part = &last_name_part[..at_index]; - let version_req = SpecifierVersionReq::parse(version) - .with_context(|| "Invalid version requirement.")?; - let name = if name_part_len == 1 { - last_name_part.to_string() - } else { - format!("{}/{}", name_parts[0], last_name_part) - }; - (name, Some(version_req)) - } else { - (name_parts.join("/"), None) - }; - let sub_path = if parts.len() == name_parts.len() { - None - } else { - Some(parts[name_part_len..].join("/")) - }; - - if let Some(sub_path) = &sub_path { - if let Some(at_index) = sub_path.rfind('@') { - let (new_sub_path, version) = sub_path.split_at(at_index); - let msg = format!( - "Invalid package specifier 'npm:{}/{}'. Did you mean to write 'npm:{}{}/{}'?", - name, sub_path, name, version, new_sub_path - ); - return Err(generic_error(msg)); - } - } - - Ok(NpmPackageReference { - req: NpmPackageReq { name, version_req }, - sub_path, - }) - } -} - -impl std::fmt::Display for NpmPackageReference { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - if let Some(sub_path) = &self.sub_path { - write!(f, "{}/{}", self.req, sub_path) - } else { - write!(f, "{}", self.req) - } - } -} - -#[derive( - Clone, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize, -)] -pub struct NpmPackageReq { - pub name: String, - pub version_req: Option, -} - -impl std::fmt::Display for NpmPackageReq { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - match &self.version_req { - Some(req) => write!(f, "{}@{}", self.name, req), - None => write!(f, "{}", self.name), - } - } -} - -impl NpmVersionMatcher for NpmPackageReq { - fn tag(&self) -> Option<&str> { - match &self.version_req { - Some(version_req) => version_req.tag(), - None => Some("latest"), - } - } - - fn matches(&self, version: &NpmVersion) -> bool { - match self.version_req.as_ref() { - Some(req) => { - assert_eq!(self.tag(), None); - match req.range() { - Some(range) => range.satisfies(version), - None => false, - } - } - None => version.pre.is_empty(), - } - } - - fn version_text(&self) -> String { - self - .version_req - .as_ref() - .map(|v| format!("{}", v)) - .unwrap_or_else(|| "non-prerelease".to_string()) - } -} - -#[derive( - Debug, Clone, PartialOrd, Ord, PartialEq, Eq, Hash, Serialize, Deserialize, -)] -pub struct NpmPackageId { - pub name: String, - pub version: NpmVersion, - pub peer_dependencies: Vec, -} - -impl NpmPackageId { - #[allow(unused)] - pub fn scope(&self) -> Option<&str> { - if self.name.starts_with('@') && self.name.contains('/') { - self.name.split('/').next() - } else { - None - } - } - - pub fn as_serializable_name(&self) -> String { - self.as_serialize_name_with_level(0) - } - - fn as_serialize_name_with_level(&self, level: usize) -> String { - let mut result = format!( - "{}@{}", - if level == 0 { - self.name.to_string() - } else { - self.name.replace("/", "+") - }, - self.version - ); - for peer in &self.peer_dependencies { - // unfortunately we can't do something like `_3` when - // this gets deep because npm package names can start - // with a number - result.push_str(&"_".repeat(level + 1)); - result.push_str(&peer.as_serialize_name_with_level(level + 1)); - } - result - } - - pub fn deserialize_name(id: &str) -> Result { - use monch::*; - - fn parse_name(input: &str) -> ParseResult<&str> { - if_not_empty(substring(move |input| { - for (pos, c) in input.char_indices() { - // first character might be a scope, so skip it - if pos > 0 && c == '@' { - return Ok((&input[pos..], ())); - } - } - ParseError::backtrace() - }))(input) - } - - fn parse_version(input: &str) -> ParseResult<&str> { - if_not_empty(substring(skip_while(|c| c != '_')))(input) - } - - fn parse_name_and_version( - input: &str, - ) -> ParseResult<(String, NpmVersion)> { - let (input, name) = parse_name(input)?; - let (input, _) = ch('@')(input)?; - let at_version_input = input; - let (input, version) = parse_version(input)?; - match NpmVersion::parse(version) { - Ok(version) => Ok((input, (name.to_string(), version))), - Err(err) => ParseError::fail(at_version_input, format!("{:#}", err)), - } - } - - fn parse_level_at_level<'a>( - level: usize, - ) -> impl Fn(&'a str) -> ParseResult<'a, ()> { - fn parse_level(input: &str) -> ParseResult { - let level = input.chars().take_while(|c| *c == '_').count(); - Ok((&input[level..], level)) - } - - move |input| { - let (input, parsed_level) = parse_level(input)?; - if parsed_level == level { - Ok((input, ())) - } else { - ParseError::backtrace() - } - } - } - - fn parse_peers_at_level<'a>( - level: usize, - ) -> impl Fn(&'a str) -> ParseResult<'a, Vec> { - move |mut input| { - let mut peers = Vec::new(); - while let Ok((level_input, _)) = parse_level_at_level(level)(input) { - input = level_input; - let peer_result = parse_id_at_level(level)(input)?; - input = peer_result.0; - peers.push(peer_result.1); - } - Ok((input, peers)) - } - } - - fn parse_id_at_level<'a>( - level: usize, - ) -> impl Fn(&'a str) -> ParseResult<'a, NpmPackageId> { - move |input| { - let (input, (name, version)) = parse_name_and_version(input)?; - let name = if level > 0 { - name.replace("+", "/") - } else { - name - }; - let (input, peer_dependencies) = - parse_peers_at_level(level + 1)(input)?; - Ok(( - input, - NpmPackageId { - name, - version, - peer_dependencies, - }, - )) - } - } - - // todo(THIS PR): move this someone re-usable - crate::npm::semver::errors::with_failure_handling(parse_id_at_level(0))(id) - .with_context(|| format!("Invalid npm package id '{}'.", id)) - } -} - -impl std::fmt::Display for NpmPackageId { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}@{}", self.name, self.version) - } -} - -#[derive(Debug, Clone, Serialize, Deserialize)] -pub struct NpmResolutionPackage { - pub id: NpmPackageId, - pub dist: NpmPackageVersionDistInfo, - /// Key is what the package refers to the other package as, - /// which could be different from the package name. - pub dependencies: HashMap, -} - -#[derive(Debug, Clone, Default, Serialize, Deserialize)] -pub struct NpmResolutionSnapshot { - #[serde(with = "map_to_vec")] - package_reqs: HashMap, - packages_by_name: HashMap>, - #[serde(with = "map_to_vec")] - packages: HashMap, -} - -// This is done so the maps with non-string keys get serialized and deserialized as vectors. -// Adapted from: https://github.com/serde-rs/serde/issues/936#issuecomment-302281792 -mod map_to_vec { - use std::collections::HashMap; - - use serde::de::Deserialize; - use serde::de::Deserializer; - use serde::ser::Serializer; - use serde::Serialize; - - pub fn serialize( - map: &HashMap, - serializer: S, - ) -> Result - where - S: Serializer, - { - serializer.collect_seq(map.iter()) - } - - pub fn deserialize< - 'de, - D, - K: Deserialize<'de> + Eq + std::hash::Hash, - V: Deserialize<'de>, - >( - deserializer: D, - ) -> Result, D::Error> - where - D: Deserializer<'de>, - { - let mut map = HashMap::new(); - for (key, value) in Vec::<(K, V)>::deserialize(deserializer)? { - map.insert(key, value); - } - Ok(map) - } -} - -impl NpmResolutionSnapshot { - /// Resolve a node package from a deno module. - pub fn resolve_package_from_deno_module( - &self, - req: &NpmPackageReq, - ) -> Result<&NpmResolutionPackage, AnyError> { - match self.package_reqs.get(req) { - Some(id) => Ok(self.packages.get(id).unwrap()), - None => bail!("could not find npm package directory for '{}'", req), - } - } - - pub fn top_level_packages(&self) -> Vec { - self - .package_reqs - .values() - .cloned() - .collect::>() - .into_iter() - .collect::>() - } - - pub fn package_from_id( - &self, - id: &NpmPackageId, - ) -> Option<&NpmResolutionPackage> { - self.packages.get(id) - } - - pub fn resolve_package_from_package( - &self, - name: &str, - referrer: &NpmPackageId, - ) -> Result<&NpmResolutionPackage, AnyError> { - match self.packages.get(referrer) { - Some(referrer_package) => { - let name_ = name_without_path(name); - if let Some(id) = referrer_package.dependencies.get(name_) { - return Ok(self.packages.get(id).unwrap()); - } - - if referrer_package.id.name == name_ { - return Ok(referrer_package); - } - - // TODO(bartlomieju): this should use a reverse lookup table in the - // snapshot instead of resolving best version again. - let req = NpmPackageReq { - name: name_.to_string(), - version_req: None, - }; - - if let Some(id) = self.resolve_best_package_id(name_, &req) { - if let Some(pkg) = self.packages.get(&id) { - return Ok(pkg); - } - } - - bail!( - "could not find npm package '{}' referenced by '{}'", - name, - referrer - ) - } - None => bail!("could not find referrer npm package '{}'", referrer), - } - } - - pub fn all_packages(&self) -> Vec { - self.packages.values().cloned().collect() - } - - pub fn resolve_best_package_id( - &self, - name: &str, - version_matcher: &impl NpmVersionMatcher, - ) -> Option { - // todo(THIS PR): this is not correct because some ids will be better than others - let mut maybe_best_id: Option<&NpmPackageId> = None; - if let Some(ids) = self.packages_by_name.get(name) { - for id in ids { - if version_matcher.matches(&id.version) { - let is_best_version = maybe_best_id - .as_ref() - .map(|best_id| best_id.version.cmp(&id.version).is_lt()) - .unwrap_or(true); - if is_best_version { - maybe_best_id = Some(id); - } - } - } - } - maybe_best_id.cloned() - } - - pub async fn from_lockfile( - lockfile: Arc>, - api: &NpmRegistryApi, - ) -> Result { - let mut package_reqs: HashMap; - let mut packages_by_name: HashMap>; - let mut packages: HashMap; - - { - let lockfile = lockfile.lock(); - - // pre-allocate collections - package_reqs = - HashMap::with_capacity(lockfile.content.npm.specifiers.len()); - packages = HashMap::with_capacity(lockfile.content.npm.packages.len()); - packages_by_name = - HashMap::with_capacity(lockfile.content.npm.packages.len()); // close enough - let mut verify_ids = - HashSet::with_capacity(lockfile.content.npm.packages.len()); - - // collect the specifiers to version mappings - for (key, value) in &lockfile.content.npm.specifiers { - let reference = NpmPackageReference::from_str(&format!("npm:{}", key)) - .with_context(|| format!("Unable to parse npm specifier: {}", key))?; - let package_id = NpmPackageId::deserialize_name(value)?; - package_reqs.insert(reference.req, package_id.clone()); - verify_ids.insert(package_id.clone()); - } - - // then the packages - for (key, value) in &lockfile.content.npm.packages { - let package_id = NpmPackageId::deserialize_name(key)?; - let mut dependencies = HashMap::default(); - - packages_by_name - .entry(package_id.name.to_string()) - .or_default() - .push(package_id.clone()); - - for (name, specifier) in &value.dependencies { - let dep_id = NpmPackageId::deserialize_name(specifier)?; - dependencies.insert(name.to_string(), dep_id.clone()); - verify_ids.insert(dep_id); - } - - let package = NpmResolutionPackage { - id: package_id.clone(), - // temporary dummy value - dist: NpmPackageVersionDistInfo { - tarball: "foobar".to_string(), - shasum: "foobar".to_string(), - integrity: Some("foobar".to_string()), - }, - dependencies, - }; - - packages.insert(package_id, package); - } - - // verify that all these ids exist in packages - for id in &verify_ids { - if !packages.contains_key(id) { - bail!( - "the lockfile ({}) is corrupt. You can recreate it with --lock-write", - lockfile.filename.display(), - ); - } - } - } - - let mut unresolved_tasks = Vec::with_capacity(packages_by_name.len()); - - // cache the package names in parallel in the registry api - for package_name in packages_by_name.keys() { - let package_name = package_name.clone(); - let api = api.clone(); - unresolved_tasks.push(tokio::task::spawn(async move { - api.package_info(&package_name).await?; - Result::<_, AnyError>::Ok(()) - })); - } - for result in futures::future::join_all(unresolved_tasks).await { - result??; - } - - // ensure the dist is set for each package - for package in packages.values_mut() { - // this will read from the memory cache now - let version_info = match api - .package_version_info(&package.id.name, &package.id.version) - .await? - { - Some(version_info) => version_info, - None => { - bail!("could not find '{}' specified in the lockfile. Maybe try again with --reload", package.id); - } - }; - package.dist = version_info.dist; - } - - Ok(Self { - package_reqs, - packages_by_name, - packages, - }) - } -} - -pub struct NpmResolution { - api: NpmRegistryApi, - snapshot: RwLock, - update_sempahore: tokio::sync::Semaphore, -} - -impl std::fmt::Debug for NpmResolution { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - let snapshot = self.snapshot.read(); - f.debug_struct("NpmResolution") - .field("snapshot", &snapshot) - .finish() - } -} - -impl NpmResolution { - pub fn new( - api: NpmRegistryApi, - initial_snapshot: Option, - ) -> Self { - Self { - api, - snapshot: RwLock::new(initial_snapshot.unwrap_or_default()), - update_sempahore: tokio::sync::Semaphore::new(1), - } - } - - pub async fn add_package_reqs( - &self, - package_reqs: Vec, - ) -> Result<(), AnyError> { - // only allow one thread in here at a time - let _permit = self.update_sempahore.acquire().await.unwrap(); - let snapshot = self.snapshot.read().clone(); - - let snapshot = self - .add_package_reqs_to_snapshot(package_reqs, snapshot) - .await?; - - *self.snapshot.write() = snapshot; - Ok(()) - } - - pub async fn set_package_reqs( - &self, - package_reqs: HashSet, - ) -> Result<(), AnyError> { - // only allow one thread in here at a time - let _permit = self.update_sempahore.acquire().await.unwrap(); - let snapshot = self.snapshot.read().clone(); - - let has_removed_package = !snapshot - .package_reqs - .keys() - .all(|req| package_reqs.contains(req)); - // if any packages were removed, we need to completely recreate the npm resolution snapshot - let snapshot = if has_removed_package { - NpmResolutionSnapshot::default() - } else { - snapshot - }; - let snapshot = self - .add_package_reqs_to_snapshot( - package_reqs.into_iter().collect(), - snapshot, - ) - .await?; - - *self.snapshot.write() = snapshot; - - Ok(()) - } - - async fn add_package_reqs_to_snapshot( - &self, - mut package_reqs: Vec, - snapshot: NpmResolutionSnapshot, - ) -> Result { - // convert the snapshot to a traversable graph - let mut graph = Graph::default(); - graph.fill_with_snapshot(&snapshot); - drop(snapshot); // todo: remove - - // multiple packages are resolved in alphabetical order - package_reqs.sort_by(|a, b| a.name.cmp(&b.name)); - - // go over the top level packages first, then down the - // tree one level at a time through all the branches - let mut unresolved_tasks = Vec::with_capacity(package_reqs.len()); - for package_req in package_reqs { - if graph.package_reqs.contains_key(&package_req) { - // skip analyzing this package, as there's already a matching top level package - continue; - } - - // no existing best version, so resolve the current packages - let api = self.api.clone(); - let maybe_info = if should_sync_download() { - // for deterministic test output - Some(api.package_info(&package_req.name).await) - } else { - None - }; - unresolved_tasks.push(tokio::task::spawn(async move { - let info = match maybe_info { - Some(info) => info?, - None => api.package_info(&package_req.name).await?, - }; - Result::<_, AnyError>::Ok((package_req, info)) - })); - } - - let mut resolver = GraphDependencyResolver { - graph: &mut graph, - api: &self.api, - pending_dependencies: Default::default(), - pending_peer_dependencies: Default::default(), - }; - - for result in futures::future::join_all(unresolved_tasks).await { - let (package_req, info) = result??; - resolver.resolve_npm_package_req(&package_req, info)?; - } - - resolver.resolve_pending().await?; - - graph.into_snapshot(&self.api).await - } - - pub fn resolve_package_from_id( - &self, - id: &NpmPackageId, - ) -> Option { - self.snapshot.read().package_from_id(id).cloned() - } - - pub fn resolve_package_from_package( - &self, - name: &str, - referrer: &NpmPackageId, - ) -> Result { - self - .snapshot - .read() - .resolve_package_from_package(name, referrer) - .cloned() - } - - /// Resolve a node package from a deno module. - pub fn resolve_package_from_deno_module( - &self, - package: &NpmPackageReq, - ) -> Result { - self - .snapshot - .read() - .resolve_package_from_deno_module(package) - .cloned() - } - - pub fn all_packages(&self) -> Vec { - self.snapshot.read().all_packages() - } - - pub fn has_packages(&self) -> bool { - !self.snapshot.read().packages.is_empty() - } - - pub fn snapshot(&self) -> NpmResolutionSnapshot { - self.snapshot.read().clone() - } - - pub fn lock( - &self, - lockfile: &mut Lockfile, - snapshot: &NpmResolutionSnapshot, - ) -> Result<(), AnyError> { - for (package_req, version) in snapshot.package_reqs.iter() { - lockfile.insert_npm_specifier(package_req, version.to_string()); - } - for package in self.all_packages() { - lockfile.check_or_insert_npm_package(&package)?; - } - Ok(()) - } -} -#[derive(Debug, Clone, PartialEq, Eq, Hash)] -enum NodeParent { - Req(NpmPackageReq), - Node(NpmPackageId), -} - -#[derive(Debug)] -struct Node { - pub id: NpmPackageId, - pub parents: HashMap, - pub children: HashMap, - pub unresolved_peers: Vec, -} - -#[derive(Debug, Default)] -struct Graph { - package_reqs: HashMap, - packages_by_name: HashMap>, - // Ideally this would be Rc>, but we need to use a Mutex - // because the lsp requires Send and this code is executed in the lsp. - // Would be nice if the lsp wasn't Send. - packages: HashMap>>, -} - -impl Graph { - pub fn get_or_create_for_id( - &mut self, - id: &NpmPackageId, - ) -> (bool, Arc>) { - if let Some(node) = self.packages.get(id) { - (false, node.clone()) - } else { - let node = Arc::new(Mutex::new(Node { - id: id.clone(), - parents: Default::default(), - children: Default::default(), - unresolved_peers: Default::default(), - })); - self - .packages_by_name - .entry(id.name.clone()) - .or_default() - .push(id.clone()); - self.packages.insert(id.clone(), node.clone()); - (true, node) - } - } - - pub fn fill_with_snapshot(&mut self, snapshot: &NpmResolutionSnapshot) { - for (package_req, id) in &snapshot.package_reqs { - let node = self.fill_for_id_with_snapshot(id, snapshot); - (*node).lock().parents.insert( - package_req.to_string(), - NodeParent::Req(package_req.clone()), - ); - self.package_reqs.insert(package_req.clone(), id.clone()); - } - } - - fn fill_for_id_with_snapshot( - &mut self, - id: &NpmPackageId, - snapshot: &NpmResolutionSnapshot, - ) -> Arc> { - let resolution = snapshot.packages.get(id).unwrap(); - let node = self.get_or_create_for_id(id).1; - for (name, child_id) in &resolution.dependencies { - let child_node = self.fill_for_id_with_snapshot(&child_id, snapshot); - self.set_child_parent_node(&name, &child_node, &id); - } - node - } - - fn borrow_node(&self, id: &NpmPackageId) -> MutexGuard { - (**self.packages.get(id).unwrap()).lock() - } - - fn forget_orphan(&mut self, node_id: &NpmPackageId) { - if let Some(node) = self.packages.remove(node_id) { - let node = (*node).lock(); - assert_eq!(node.parents.len(), 0); - for (specifier, child_id) in &node.children { - let mut child = (**self.packages.get(child_id).unwrap()).lock(); - child.parents.remove(specifier); - if child.parents.is_empty() { - drop(child); // stop borrowing from self - self.forget_orphan(&child_id); - } - } - } - } - - pub fn set_child_parent_node( - &mut self, - specifier: &str, - child: &Arc>, - parent_id: &NpmPackageId, - ) { - let mut child = (**child).lock(); - let mut parent = (**self.packages.get(parent_id).unwrap()).lock(); - debug_assert_ne!(parent.id, child.id); - parent - .children - .insert(specifier.to_string(), child.id.clone()); - child - .parents - .insert(specifier.to_string(), NodeParent::Node(parent.id.clone())); - } - - pub async fn into_snapshot( - self, - api: &NpmRegistryApi, - ) -> Result { - let mut packages = HashMap::with_capacity(self.packages.len()); - for (id, node) in self.packages { - let dist = api - .package_version_info(&id.name, &id.version) - .await? - .unwrap() // todo(THIS PR): don't unwrap here - .dist; - let node = node.lock(); - assert_eq!(node.unresolved_peers.len(), 0); - packages.insert( - id.clone(), - NpmResolutionPackage { - dist, - dependencies: node.children.clone(), - id, - }, - ); - } - Ok(NpmResolutionSnapshot { - package_reqs: self.package_reqs, - packages_by_name: self.packages_by_name, - packages, - }) - } -} - -struct GraphDependencyResolver<'a> { - graph: &'a mut Graph, - api: &'a NpmRegistryApi, - pending_dependencies: VecDeque<(NpmPackageId, Vec)>, - pending_peer_dependencies: - VecDeque<((String, NpmPackageId), NpmDependencyEntry, NpmPackageInfo)>, -} - -impl<'a> GraphDependencyResolver<'a> { - pub fn resolve_best_package_version_and_info( - &self, - name: &str, - version_matcher: &impl NpmVersionMatcher, - package_info: NpmPackageInfo, - ) -> Result { - if let Some(version) = - self.resolve_best_package_version(name, version_matcher) - { - match package_info.versions.get(&version.to_string()) { - Some(version_info) => Ok(VersionAndInfo { - version, - info: version_info.clone(), - }), - None => { - bail!("could not find version '{}' for '{}'", version, name) - } - } - } else { - // get the information - get_resolved_package_version_and_info( - name, - version_matcher, - package_info, - None, - ) - } - } - - pub fn resolve_best_package_version( - &self, - name: &str, - version_matcher: &impl NpmVersionMatcher, - ) -> Option { - let mut maybe_best_version: Option<&NpmVersion> = None; - if let Some(ids) = self.graph.packages_by_name.get(name) { - for version in ids.iter().map(|id| &id.version) { - if version_matcher.matches(version) { - let is_best_version = maybe_best_version - .as_ref() - .map(|best_version| (*best_version).cmp(version).is_lt()) - .unwrap_or(true); - if is_best_version { - maybe_best_version = Some(version); - } - } - } - } - maybe_best_version.cloned() - } - - pub fn resolve_npm_package_req( - &mut self, - package_req: &NpmPackageReq, - info: NpmPackageInfo, - ) -> Result<(), AnyError> { - // inspect if there's a match in the list of current packages and otherwise - // fall back to looking at the registry - let version_and_info = self.resolve_best_package_version_and_info( - &package_req.name, - package_req, - info, - )?; - let id = NpmPackageId { - name: package_req.name.clone(), - version: version_and_info.version.clone(), - peer_dependencies: Vec::new(), - }; - let node = self.graph.get_or_create_for_id(&id).1; - (*node).lock().parents.insert( - package_req.to_string(), - NodeParent::Req(package_req.clone()), - ); - self - .graph - .package_reqs - .insert(package_req.clone(), id.clone()); - - let dependencies = version_and_info - .info - .dependencies_as_entries() - .with_context(|| format!("npm package: {}", id))?; - - self.pending_dependencies.push_back((id, dependencies)); - Ok(()) - } - - fn analyze_dependency( - &mut self, - entry: &NpmDependencyEntry, - package_info: NpmPackageInfo, - parent_id: &NpmPackageId, - ) -> Result<(), AnyError> { - let version_and_info = self.resolve_best_package_version_and_info( - &entry.name, - &entry.version_req, - package_info, - )?; - - let id = NpmPackageId { - name: entry.name.clone(), - version: version_and_info.version.clone(), - peer_dependencies: Vec::new(), - }; - let (created, node) = self.graph.get_or_create_for_id(&id); - self - .graph - .set_child_parent_node(&entry.bare_specifier, &node, &parent_id); - - if created { - // inspect the dependencies of the package - let dependencies = version_and_info - .info - .dependencies_as_entries() - .with_context(|| { - format!("npm package: {}@{}", &entry.name, version_and_info.version) - })?; - - self.pending_dependencies.push_back((id, dependencies)); - } - Ok(()) - } - - pub async fn resolve_pending(&mut self) -> Result<(), AnyError> { - while !self.pending_dependencies.is_empty() - || !self.pending_peer_dependencies.is_empty() - { - // now go down through the dependencies by tree depth - while let Some((parent_id, mut deps)) = - self.pending_dependencies.pop_front() - { - // ensure name alphabetical and then version descending - deps.sort(); - - // cache all the dependencies' registry infos in parallel if should - if !should_sync_download() { - let handles = deps - .iter() - .map(|dep| { - let name = dep.name.clone(); - let api = self.api.clone(); - tokio::task::spawn(async move { - // it's ok to call this without storing the result, because - // NpmRegistryApi will cache the package info in memory - api.package_info(&name).await - }) - }) - .collect::>(); - let results = futures::future::join_all(handles).await; - for result in results { - result??; // surface the first error - } - } - - // resolve the non-peer dependencies - for dep in deps { - let package_info = self.api.package_info(&dep.name).await?; - - match dep.kind { - NpmDependencyEntryKind::Dep => { - self.analyze_dependency(&dep, package_info, &parent_id)?; - } - NpmDependencyEntryKind::Peer - | NpmDependencyEntryKind::OptionalPeer => { - self.pending_peer_dependencies.push_back(( - (dep.bare_specifier.clone(), parent_id.clone()), - dep, - package_info, - )); - } - } - } - } - - if let Some(((specifier, parent_id), peer_dep, peer_package_info)) = - self.pending_peer_dependencies.pop_front() - { - self.resolve_peer_dep( - &specifier, - &parent_id, - &peer_dep, - peer_package_info, - vec![], - )?; - } - } - Ok(()) - } - - fn resolve_peer_dep( - &mut self, - specifier: &str, - child_id: &NpmPackageId, - peer_dep: &NpmDependencyEntry, - peer_package_info: NpmPackageInfo, - mut path: Vec<(String, NpmPackageId)>, - ) -> Result<(), AnyError> { - // Peer dependencies are resolved based on its ancestors' siblings. - // If not found, then it resolves based on the version requirement if non-optional - let parents = self.graph.borrow_node(&child_id).parents.clone(); - path.push((specifier.to_string(), child_id.clone())); - for (specifier, parent) in parents { - let children = match &parent { - NodeParent::Node(parent_node_id) => { - self.graph.borrow_node(parent_node_id).children.clone() - } - NodeParent::Req(parent_req) => self - .graph - .package_reqs - .iter() - .filter(|(req, _)| *req == parent_req) - .map(|(req, id)| (req.to_string(), id.clone())) - .collect::>(), - }; - // todo(THIS PR): don't we need to use the specifier here? - for (child_specifier, child_id) in children { - if child_id.name == peer_dep.name - && peer_dep.version_req.satisfies(&child_id.version) - { - // go down the descendants creating a new path - match &parent { - NodeParent::Node(node_id) => { - let parents = self.graph.borrow_node(node_id).parents.clone(); - self.set_new_peer_dep( - parents, &specifier, node_id, &child_id, path, - ); - return Ok(()); - } - NodeParent::Req(req) => { - let old_id = self.graph.package_reqs.get(&req).unwrap().clone(); - self.set_new_peer_dep( - HashMap::from([( - req.to_string(), - NodeParent::Req(req.clone()), - )]), - &specifier, - &old_id, - &child_id, - path, - ); - return Ok(()); - } - } - } - } - } - - // at this point it means we didn't find anything by searching the ancestor siblings, - // so we need to resolve based on the package info - if !peer_dep.kind.is_optional() { - self.analyze_dependency(&peer_dep, peer_package_info, &child_id)?; - } - Ok(()) - } - - fn set_new_peer_dep( - &mut self, - previous_parents: HashMap, - specifier: &str, - node_id: &NpmPackageId, - peer_dep_id: &NpmPackageId, - mut path: Vec<(String, NpmPackageId)>, - ) { - let old_id = node_id; - let mut new_id = old_id.clone(); - new_id.peer_dependencies.push(peer_dep_id.clone()); - // remove the previous parents from the old node - let old_node_children = { - let mut old_node = self.graph.borrow_node(old_id); - for previous_parent in previous_parents.keys() { - old_node.parents.remove(previous_parent); - } - old_node.children.clone() - }; - - let (created, new_node) = self.graph.get_or_create_for_id(&new_id); - - // update the previous parent to point to the new node - // and this node to point at those parents - { - let mut new_node = (*new_node).lock(); - for (specifier, parent) in previous_parents { - match &parent { - NodeParent::Node(parent_id) => { - let mut parent = - (**self.graph.packages.get(parent_id).unwrap()).lock(); - parent.children.insert(specifier.clone(), new_id.clone()); - } - NodeParent::Req(req) => { - self.graph.package_reqs.insert(req.clone(), new_id.clone()); - } - } - new_node.parents.insert(specifier, parent); - } - - // now add the previous children to this node - new_node.children.extend(old_node_children.clone()); - } - - for (specifier, child_id) in old_node_children { - self - .graph - .borrow_node(&child_id) - .parents - .insert(specifier, NodeParent::Node(new_id.clone())); - } - - if created { - // continue going down the path - if let Some((next_specifier, next_node_id)) = path.pop() { - self.set_new_peer_dep( - HashMap::from([(specifier.to_string(), NodeParent::Node(new_id))]), - &next_specifier, - &next_node_id, - peer_dep_id, - path, - ); - } - } - - // forget the old node at this point if it has no parents - { - let old_node = self.graph.borrow_node(old_id); - if old_node.parents.is_empty() { - drop(old_node); // stop borrowing - self.graph.forget_orphan(old_id); - } - } - } -} - -#[derive(Clone)] -struct VersionAndInfo { - version: NpmVersion, - info: NpmPackageVersionInfo, -} - -fn get_resolved_package_version_and_info( - pkg_name: &str, - version_matcher: &impl NpmVersionMatcher, - info: NpmPackageInfo, - parent: Option<&NpmPackageId>, -) -> Result { - let mut maybe_best_version: Option = None; - if let Some(tag) = version_matcher.tag() { - // For when someone just specifies @types/node, we want to pull in a - // "known good" version of @types/node that works well with Deno and - // not necessarily the latest version. For example, we might only be - // compatible with Node vX, but then Node vY is published so we wouldn't - // want to pull that in. - // Note: If the user doesn't want this behavior, then they can specify an - // explicit version. - if tag == "latest" && pkg_name == "@types/node" { - return get_resolved_package_version_and_info( - pkg_name, - &NpmVersionReq::parse("18.0.0 - 18.8.2").unwrap(), - info, - parent, - ); - } - - if let Some(version) = info.dist_tags.get(tag) { - match info.versions.get(version) { - Some(info) => { - return Ok(VersionAndInfo { - version: NpmVersion::parse(version)?, - info: info.clone(), - }); - } - None => { - bail!( - "Could not find version '{}' referenced in dist-tag '{}'.", - version, - tag, - ) - } - } - } else { - bail!("Could not find dist-tag '{}'.", tag,) - } - } else { - for (_, version_info) in info.versions.into_iter() { - let version = NpmVersion::parse(&version_info.version)?; - if version_matcher.matches(&version) { - let is_best_version = maybe_best_version - .as_ref() - .map(|best_version| best_version.version.cmp(&version).is_lt()) - .unwrap_or(true); - if is_best_version { - maybe_best_version = Some(VersionAndInfo { - version, - info: version_info, - }); - } - } - } - } - - match maybe_best_version { - Some(v) => Ok(v), - // If the package isn't found, it likely means that the user needs to use - // `--reload` to get the latest npm package information. Although it seems - // like we could make this smart by fetching the latest information for - // this package here, we really need a full restart. There could be very - // interesting bugs that occur if this package's version was resolved by - // something previous using the old information, then now being smart here - // causes a new fetch of the package information, meaning this time the - // previous resolution of this package's version resolved to an older - // version, but next time to a different version because it has new information. - None => bail!( - concat!( - "Could not find npm package '{}' matching {}{}. ", - "Try retrieving the latest npm package information by running with --reload", - ), - pkg_name, - version_matcher.version_text(), - match parent { - Some(id) => format!(" as specified in {}", id), - None => String::new(), - } - ), - } -} - -fn name_without_path(name: &str) -> &str { - let mut search_start_index = 0; - if name.starts_with('@') { - if let Some(slash_index) = name.find('/') { - search_start_index = slash_index + 1; - } - } - if let Some(slash_index) = &name[search_start_index..].find('/') { - // get the name up until the path slash - &name[0..search_start_index + slash_index] - } else { - name - } -} - -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn parse_npm_package_ref() { - assert_eq!( - NpmPackageReference::from_str("npm:@package/test").unwrap(), - NpmPackageReference { - req: NpmPackageReq { - name: "@package/test".to_string(), - version_req: None, - }, - sub_path: None, - } - ); - - assert_eq!( - NpmPackageReference::from_str("npm:@package/test@1").unwrap(), - NpmPackageReference { - req: NpmPackageReq { - name: "@package/test".to_string(), - version_req: Some(SpecifierVersionReq::parse("1").unwrap()), - }, - sub_path: None, - } - ); - - assert_eq!( - NpmPackageReference::from_str("npm:@package/test@~1.1/sub_path").unwrap(), - NpmPackageReference { - req: NpmPackageReq { - name: "@package/test".to_string(), - version_req: Some(SpecifierVersionReq::parse("~1.1").unwrap()), - }, - sub_path: Some("sub_path".to_string()), - } - ); - - assert_eq!( - NpmPackageReference::from_str("npm:@package/test/sub_path").unwrap(), - NpmPackageReference { - req: NpmPackageReq { - name: "@package/test".to_string(), - version_req: None, - }, - sub_path: Some("sub_path".to_string()), - } - ); - - assert_eq!( - NpmPackageReference::from_str("npm:test").unwrap(), - NpmPackageReference { - req: NpmPackageReq { - name: "test".to_string(), - version_req: None, - }, - sub_path: None, - } - ); - - assert_eq!( - NpmPackageReference::from_str("npm:test@^1.2").unwrap(), - NpmPackageReference { - req: NpmPackageReq { - name: "test".to_string(), - version_req: Some(SpecifierVersionReq::parse("^1.2").unwrap()), - }, - sub_path: None, - } - ); - - assert_eq!( - NpmPackageReference::from_str("npm:test@~1.1/sub_path").unwrap(), - NpmPackageReference { - req: NpmPackageReq { - name: "test".to_string(), - version_req: Some(SpecifierVersionReq::parse("~1.1").unwrap()), - }, - sub_path: Some("sub_path".to_string()), - } - ); - - assert_eq!( - NpmPackageReference::from_str("npm:@package/test/sub_path").unwrap(), - NpmPackageReference { - req: NpmPackageReq { - name: "@package/test".to_string(), - version_req: None, - }, - sub_path: Some("sub_path".to_string()), - } - ); - - assert_eq!( - NpmPackageReference::from_str("npm:@package") - .err() - .unwrap() - .to_string(), - "Not a valid package: @package" - ); - } - - #[test] - fn test_name_without_path() { - assert_eq!(name_without_path("foo"), "foo"); - assert_eq!(name_without_path("@foo/bar"), "@foo/bar"); - assert_eq!(name_without_path("@foo/bar/baz"), "@foo/bar"); - assert_eq!(name_without_path("@hello"), "@hello"); - } - - #[test] - fn test_get_resolved_package_version_and_info() { - // dist tag where version doesn't exist - let package_ref = NpmPackageReference::from_str("npm:test").unwrap(); - let result = get_resolved_package_version_and_info( - "test", - &package_ref.req, - NpmPackageInfo { - name: "test".to_string(), - versions: HashMap::new(), - dist_tags: HashMap::from([( - "latest".to_string(), - "1.0.0-alpha".to_string(), - )]), - }, - None, - ); - assert_eq!( - result.err().unwrap().to_string(), - "Could not find version '1.0.0-alpha' referenced in dist-tag 'latest'." - ); - - // dist tag where version is a pre-release - let package_ref = NpmPackageReference::from_str("npm:test").unwrap(); - let result = get_resolved_package_version_and_info( - "test", - &package_ref.req, - NpmPackageInfo { - name: "test".to_string(), - versions: HashMap::from([ - ("0.1.0".to_string(), NpmPackageVersionInfo::default()), - ( - "1.0.0-alpha".to_string(), - NpmPackageVersionInfo { - version: "0.1.0-alpha".to_string(), - ..Default::default() - }, - ), - ]), - dist_tags: HashMap::from([( - "latest".to_string(), - "1.0.0-alpha".to_string(), - )]), - }, - None, - ); - assert_eq!(result.unwrap().version.to_string(), "1.0.0-alpha"); - } - - #[test] - fn serialize_npm_package_id() { - let id = NpmPackageId { - name: "pkg-a".to_string(), - version: NpmVersion::parse("1.2.3").unwrap(), - peer_dependencies: vec![ - NpmPackageId { - name: "pkg-b".to_string(), - version: NpmVersion::parse("3.2.1").unwrap(), - peer_dependencies: vec![ - NpmPackageId { - name: "pkg-c".to_string(), - version: NpmVersion::parse("1.3.2").unwrap(), - peer_dependencies: vec![], - }, - NpmPackageId { - name: "pkg-d".to_string(), - version: NpmVersion::parse("2.3.4").unwrap(), - peer_dependencies: vec![], - }, - ], - }, - NpmPackageId { - name: "pkg-e".to_string(), - version: NpmVersion::parse("2.3.1").unwrap(), - peer_dependencies: vec![NpmPackageId { - name: "pkg-f".to_string(), - version: NpmVersion::parse("2.3.1").unwrap(), - peer_dependencies: vec![], - }], - }, - ], - }; - let serialized = id.as_serializable_name(); - assert_eq!(serialized, "pkg-a@1.2.3_pkg-b@3.2.1__pkg-c@1.3.2__pkg-d@2.3.4_pkg-e@2.3.1__pkg-f@2.3.1"); - assert_eq!(NpmPackageId::deserialize_name(&serialized).unwrap(), id); - } -} diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs new file mode 100644 index 00000000000000..84988d7b73c95c --- /dev/null +++ b/cli/npm/resolution/graph.rs @@ -0,0 +1,668 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +use std::collections::HashMap; +use std::collections::VecDeque; +use std::sync::Arc; + +use deno_core::anyhow::bail; +use deno_core::anyhow::Context; +use deno_core::error::AnyError; +use deno_core::futures; +use deno_core::parking_lot::Mutex; +use deno_core::parking_lot::MutexGuard; + +use crate::npm::cache::should_sync_download; +use crate::npm::registry::NpmDependencyEntry; +use crate::npm::registry::NpmDependencyEntryKind; +use crate::npm::registry::NpmPackageInfo; +use crate::npm::registry::NpmPackageVersionInfo; +use crate::npm::registry::NpmRegistryApi; +use crate::npm::semver::NpmVersion; +use crate::npm::semver::NpmVersionReq; + +use super::snapshot::NpmResolutionSnapshot; +use super::NpmPackageId; +use super::NpmPackageReq; +use super::NpmResolutionPackage; +use super::NpmVersionMatcher; + +#[derive(Debug, Clone, PartialEq, Eq, Hash)] +enum NodeParent { + Req(NpmPackageReq), + Node(NpmPackageId), +} + +#[derive(Debug)] +struct Node { + pub id: NpmPackageId, + pub parents: HashMap, + pub children: HashMap, + pub unresolved_peers: Vec, +} + +#[derive(Debug, Default)] +pub struct Graph { + package_reqs: HashMap, + packages_by_name: HashMap>, + // Ideally this would be Rc>, but we need to use a Mutex + // because the lsp requires Send and this code is executed in the lsp. + // Would be nice if the lsp wasn't Send. + packages: HashMap>>, +} + +impl Graph { + pub fn has_package_req(&self, req: &NpmPackageReq) -> bool { + self.package_reqs.contains_key(req) + } + + pub fn fill_with_snapshot(&mut self, snapshot: &NpmResolutionSnapshot) { + for (package_req, id) in &snapshot.package_reqs { + let node = self.fill_for_id_with_snapshot(id, snapshot); + (*node).lock().parents.insert( + package_req.to_string(), + NodeParent::Req(package_req.clone()), + ); + self.package_reqs.insert(package_req.clone(), id.clone()); + } + } + + fn get_or_create_for_id( + &mut self, + id: &NpmPackageId, + ) -> (bool, Arc>) { + if let Some(node) = self.packages.get(id) { + (false, node.clone()) + } else { + let node = Arc::new(Mutex::new(Node { + id: id.clone(), + parents: Default::default(), + children: Default::default(), + unresolved_peers: Default::default(), + })); + self + .packages_by_name + .entry(id.name.clone()) + .or_default() + .push(id.clone()); + self.packages.insert(id.clone(), node.clone()); + (true, node) + } + } + + fn fill_for_id_with_snapshot( + &mut self, + id: &NpmPackageId, + snapshot: &NpmResolutionSnapshot, + ) -> Arc> { + let resolution = snapshot.packages.get(id).unwrap(); + let node = self.get_or_create_for_id(id).1; + for (name, child_id) in &resolution.dependencies { + let child_node = self.fill_for_id_with_snapshot(&child_id, snapshot); + self.set_child_parent_node(&name, &child_node, &id); + } + node + } + + fn borrow_node(&self, id: &NpmPackageId) -> MutexGuard { + (**self.packages.get(id).unwrap()).lock() + } + + fn forget_orphan(&mut self, node_id: &NpmPackageId) { + if let Some(node) = self.packages.remove(node_id) { + let node = (*node).lock(); + assert_eq!(node.parents.len(), 0); + for (specifier, child_id) in &node.children { + let mut child = (**self.packages.get(child_id).unwrap()).lock(); + child.parents.remove(specifier); + if child.parents.is_empty() { + drop(child); // stop borrowing from self + self.forget_orphan(&child_id); + } + } + } + } + + fn set_child_parent_node( + &mut self, + specifier: &str, + child: &Arc>, + parent_id: &NpmPackageId, + ) { + let mut child = (**child).lock(); + let mut parent = (**self.packages.get(parent_id).unwrap()).lock(); + debug_assert_ne!(parent.id, child.id); + parent + .children + .insert(specifier.to_string(), child.id.clone()); + child + .parents + .insert(specifier.to_string(), NodeParent::Node(parent.id.clone())); + } + + pub async fn into_snapshot( + self, + api: &NpmRegistryApi, + ) -> Result { + let mut packages = HashMap::with_capacity(self.packages.len()); + for (id, node) in self.packages { + let dist = api + .package_version_info(&id.name, &id.version) + .await? + .unwrap() // todo(THIS PR): don't unwrap here + .dist; + let node = node.lock(); + assert_eq!(node.unresolved_peers.len(), 0); + packages.insert( + id.clone(), + NpmResolutionPackage { + dist, + dependencies: node.children.clone(), + id, + }, + ); + } + Ok(NpmResolutionSnapshot { + package_reqs: self.package_reqs, + packages_by_name: self.packages_by_name, + packages, + }) + } +} + +pub struct GraphDependencyResolver<'a> { + graph: &'a mut Graph, + api: &'a NpmRegistryApi, + pending_dependencies: VecDeque<(NpmPackageId, Vec)>, + pending_peer_dependencies: + VecDeque<((String, NpmPackageId), NpmDependencyEntry, NpmPackageInfo)>, +} + +impl<'a> GraphDependencyResolver<'a> { + pub fn new(graph: &'a mut Graph, api: &'a NpmRegistryApi) -> Self { + Self { + graph, + api, + pending_dependencies: Default::default(), + pending_peer_dependencies: Default::default(), + } + } + + fn resolve_best_package_version_and_info( + &self, + name: &str, + version_matcher: &impl NpmVersionMatcher, + package_info: NpmPackageInfo, + ) -> Result { + if let Some(version) = + self.resolve_best_package_version(name, version_matcher) + { + match package_info.versions.get(&version.to_string()) { + Some(version_info) => Ok(VersionAndInfo { + version, + info: version_info.clone(), + }), + None => { + bail!("could not find version '{}' for '{}'", version, name) + } + } + } else { + // get the information + get_resolved_package_version_and_info( + name, + version_matcher, + package_info, + None, + ) + } + } + + fn resolve_best_package_version( + &self, + name: &str, + version_matcher: &impl NpmVersionMatcher, + ) -> Option { + let mut maybe_best_version: Option<&NpmVersion> = None; + if let Some(ids) = self.graph.packages_by_name.get(name) { + for version in ids.iter().map(|id| &id.version) { + if version_matcher.matches(version) { + let is_best_version = maybe_best_version + .as_ref() + .map(|best_version| (*best_version).cmp(version).is_lt()) + .unwrap_or(true); + if is_best_version { + maybe_best_version = Some(version); + } + } + } + } + maybe_best_version.cloned() + } + + pub fn resolve_npm_package_req( + &mut self, + package_req: &NpmPackageReq, + info: NpmPackageInfo, + ) -> Result<(), AnyError> { + // inspect if there's a match in the list of current packages and otherwise + // fall back to looking at the registry + let version_and_info = self.resolve_best_package_version_and_info( + &package_req.name, + package_req, + info, + )?; + let id = NpmPackageId { + name: package_req.name.clone(), + version: version_and_info.version.clone(), + peer_dependencies: Vec::new(), + }; + let node = self.graph.get_or_create_for_id(&id).1; + (*node).lock().parents.insert( + package_req.to_string(), + NodeParent::Req(package_req.clone()), + ); + self + .graph + .package_reqs + .insert(package_req.clone(), id.clone()); + + let dependencies = version_and_info + .info + .dependencies_as_entries() + .with_context(|| format!("npm package: {}", id))?; + + self.pending_dependencies.push_back((id, dependencies)); + Ok(()) + } + + fn analyze_dependency( + &mut self, + entry: &NpmDependencyEntry, + package_info: NpmPackageInfo, + parent_id: &NpmPackageId, + ) -> Result<(), AnyError> { + let version_and_info = self.resolve_best_package_version_and_info( + &entry.name, + &entry.version_req, + package_info, + )?; + + let id = NpmPackageId { + name: entry.name.clone(), + version: version_and_info.version.clone(), + peer_dependencies: Vec::new(), + }; + let (created, node) = self.graph.get_or_create_for_id(&id); + self + .graph + .set_child_parent_node(&entry.bare_specifier, &node, &parent_id); + + if created { + // inspect the dependencies of the package + let dependencies = version_and_info + .info + .dependencies_as_entries() + .with_context(|| { + format!("npm package: {}@{}", &entry.name, version_and_info.version) + })?; + + self.pending_dependencies.push_back((id, dependencies)); + } + Ok(()) + } + + pub async fn resolve_pending(&mut self) -> Result<(), AnyError> { + while !self.pending_dependencies.is_empty() + || !self.pending_peer_dependencies.is_empty() + { + // now go down through the dependencies by tree depth + while let Some((parent_id, mut deps)) = + self.pending_dependencies.pop_front() + { + // ensure name alphabetical and then version descending + deps.sort(); + + // cache all the dependencies' registry infos in parallel if should + if !should_sync_download() { + let handles = deps + .iter() + .map(|dep| { + let name = dep.name.clone(); + let api = self.api.clone(); + tokio::task::spawn(async move { + // it's ok to call this without storing the result, because + // NpmRegistryApi will cache the package info in memory + api.package_info(&name).await + }) + }) + .collect::>(); + let results = futures::future::join_all(handles).await; + for result in results { + result??; // surface the first error + } + } + + // resolve the non-peer dependencies + for dep in deps { + let package_info = self.api.package_info(&dep.name).await?; + + match dep.kind { + NpmDependencyEntryKind::Dep => { + self.analyze_dependency(&dep, package_info, &parent_id)?; + } + NpmDependencyEntryKind::Peer + | NpmDependencyEntryKind::OptionalPeer => { + self.pending_peer_dependencies.push_back(( + (dep.bare_specifier.clone(), parent_id.clone()), + dep, + package_info, + )); + } + } + } + } + + if let Some(((specifier, parent_id), peer_dep, peer_package_info)) = + self.pending_peer_dependencies.pop_front() + { + self.resolve_peer_dep( + &specifier, + &parent_id, + &peer_dep, + peer_package_info, + vec![], + )?; + } + } + Ok(()) + } + + fn resolve_peer_dep( + &mut self, + specifier: &str, + child_id: &NpmPackageId, + peer_dep: &NpmDependencyEntry, + peer_package_info: NpmPackageInfo, + mut path: Vec<(String, NpmPackageId)>, + ) -> Result<(), AnyError> { + // Peer dependencies are resolved based on its ancestors' siblings. + // If not found, then it resolves based on the version requirement if non-optional + let parents = self.graph.borrow_node(&child_id).parents.clone(); + path.push((specifier.to_string(), child_id.clone())); + for (specifier, parent) in parents { + let children = match &parent { + NodeParent::Node(parent_node_id) => { + self.graph.borrow_node(parent_node_id).children.clone() + } + NodeParent::Req(parent_req) => self + .graph + .package_reqs + .iter() + .filter(|(req, _)| *req == parent_req) + .map(|(req, id)| (req.to_string(), id.clone())) + .collect::>(), + }; + // todo(THIS PR): don't we need to use the specifier here? + for (child_specifier, child_id) in children { + if child_id.name == peer_dep.name + && peer_dep.version_req.satisfies(&child_id.version) + { + // go down the descendants creating a new path + match &parent { + NodeParent::Node(node_id) => { + let parents = self.graph.borrow_node(node_id).parents.clone(); + self.set_new_peer_dep( + parents, &specifier, node_id, &child_id, path, + ); + return Ok(()); + } + NodeParent::Req(req) => { + let old_id = self.graph.package_reqs.get(&req).unwrap().clone(); + self.set_new_peer_dep( + HashMap::from([( + req.to_string(), + NodeParent::Req(req.clone()), + )]), + &specifier, + &old_id, + &child_id, + path, + ); + return Ok(()); + } + } + } + } + } + + // at this point it means we didn't find anything by searching the ancestor siblings, + // so we need to resolve based on the package info + if !peer_dep.kind.is_optional() { + self.analyze_dependency(&peer_dep, peer_package_info, &child_id)?; + } + Ok(()) + } + + fn set_new_peer_dep( + &mut self, + previous_parents: HashMap, + specifier: &str, + node_id: &NpmPackageId, + peer_dep_id: &NpmPackageId, + mut path: Vec<(String, NpmPackageId)>, + ) { + let old_id = node_id; + let mut new_id = old_id.clone(); + new_id.peer_dependencies.push(peer_dep_id.clone()); + // remove the previous parents from the old node + let old_node_children = { + let mut old_node = self.graph.borrow_node(old_id); + for previous_parent in previous_parents.keys() { + old_node.parents.remove(previous_parent); + } + old_node.children.clone() + }; + + let (created, new_node) = self.graph.get_or_create_for_id(&new_id); + + // update the previous parent to point to the new node + // and this node to point at those parents + { + let mut new_node = (*new_node).lock(); + for (specifier, parent) in previous_parents { + match &parent { + NodeParent::Node(parent_id) => { + let mut parent = + (**self.graph.packages.get(parent_id).unwrap()).lock(); + parent.children.insert(specifier.clone(), new_id.clone()); + } + NodeParent::Req(req) => { + self.graph.package_reqs.insert(req.clone(), new_id.clone()); + } + } + new_node.parents.insert(specifier, parent); + } + + // now add the previous children to this node + new_node.children.extend(old_node_children.clone()); + } + + for (specifier, child_id) in old_node_children { + self + .graph + .borrow_node(&child_id) + .parents + .insert(specifier, NodeParent::Node(new_id.clone())); + } + + if created { + // continue going down the path + if let Some((next_specifier, next_node_id)) = path.pop() { + self.set_new_peer_dep( + HashMap::from([(specifier.to_string(), NodeParent::Node(new_id))]), + &next_specifier, + &next_node_id, + peer_dep_id, + path, + ); + } + } + + // forget the old node at this point if it has no parents + { + let old_node = self.graph.borrow_node(old_id); + if old_node.parents.is_empty() { + drop(old_node); // stop borrowing + self.graph.forget_orphan(old_id); + } + } + } +} + +#[derive(Clone)] +struct VersionAndInfo { + version: NpmVersion, + info: NpmPackageVersionInfo, +} + +fn get_resolved_package_version_and_info( + pkg_name: &str, + version_matcher: &impl NpmVersionMatcher, + info: NpmPackageInfo, + parent: Option<&NpmPackageId>, +) -> Result { + let mut maybe_best_version: Option = None; + if let Some(tag) = version_matcher.tag() { + // For when someone just specifies @types/node, we want to pull in a + // "known good" version of @types/node that works well with Deno and + // not necessarily the latest version. For example, we might only be + // compatible with Node vX, but then Node vY is published so we wouldn't + // want to pull that in. + // Note: If the user doesn't want this behavior, then they can specify an + // explicit version. + if tag == "latest" && pkg_name == "@types/node" { + return get_resolved_package_version_and_info( + pkg_name, + &NpmVersionReq::parse("18.0.0 - 18.8.2").unwrap(), + info, + parent, + ); + } + + if let Some(version) = info.dist_tags.get(tag) { + match info.versions.get(version) { + Some(info) => { + return Ok(VersionAndInfo { + version: NpmVersion::parse(version)?, + info: info.clone(), + }); + } + None => { + bail!( + "Could not find version '{}' referenced in dist-tag '{}'.", + version, + tag, + ) + } + } + } else { + bail!("Could not find dist-tag '{}'.", tag,) + } + } else { + for (_, version_info) in info.versions.into_iter() { + let version = NpmVersion::parse(&version_info.version)?; + if version_matcher.matches(&version) { + let is_best_version = maybe_best_version + .as_ref() + .map(|best_version| best_version.version.cmp(&version).is_lt()) + .unwrap_or(true); + if is_best_version { + maybe_best_version = Some(VersionAndInfo { + version, + info: version_info, + }); + } + } + } + } + + match maybe_best_version { + Some(v) => Ok(v), + // If the package isn't found, it likely means that the user needs to use + // `--reload` to get the latest npm package information. Although it seems + // like we could make this smart by fetching the latest information for + // this package here, we really need a full restart. There could be very + // interesting bugs that occur if this package's version was resolved by + // something previous using the old information, then now being smart here + // causes a new fetch of the package information, meaning this time the + // previous resolution of this package's version resolved to an older + // version, but next time to a different version because it has new information. + None => bail!( + concat!( + "Could not find npm package '{}' matching {}{}. ", + "Try retrieving the latest npm package information by running with --reload", + ), + pkg_name, + version_matcher.version_text(), + match parent { + Some(id) => format!(" as specified in {}", id), + None => String::new(), + } + ), + } +} + +#[cfg(test)] +mod test { + use crate::npm::NpmPackageReference; + + use super::*; + + #[test] + fn test_get_resolved_package_version_and_info() { + // dist tag where version doesn't exist + let package_ref = NpmPackageReference::from_str("npm:test").unwrap(); + let result = get_resolved_package_version_and_info( + "test", + &package_ref.req, + NpmPackageInfo { + name: "test".to_string(), + versions: HashMap::new(), + dist_tags: HashMap::from([( + "latest".to_string(), + "1.0.0-alpha".to_string(), + )]), + }, + None, + ); + assert_eq!( + result.err().unwrap().to_string(), + "Could not find version '1.0.0-alpha' referenced in dist-tag 'latest'." + ); + + // dist tag where version is a pre-release + let package_ref = NpmPackageReference::from_str("npm:test").unwrap(); + let result = get_resolved_package_version_and_info( + "test", + &package_ref.req, + NpmPackageInfo { + name: "test".to_string(), + versions: HashMap::from([ + ("0.1.0".to_string(), NpmPackageVersionInfo::default()), + ( + "1.0.0-alpha".to_string(), + NpmPackageVersionInfo { + version: "0.1.0-alpha".to_string(), + ..Default::default() + }, + ), + ]), + dist_tags: HashMap::from([( + "latest".to_string(), + "1.0.0-alpha".to_string(), + )]), + }, + None, + ); + assert_eq!(result.unwrap().version.to_string(), "1.0.0-alpha"); + } +} diff --git a/cli/npm/resolution/mod.rs b/cli/npm/resolution/mod.rs new file mode 100644 index 00000000000000..33b8fa3455bd76 --- /dev/null +++ b/cli/npm/resolution/mod.rs @@ -0,0 +1,637 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +use std::collections::HashMap; +use std::collections::HashSet; + +use deno_ast::ModuleSpecifier; +use deno_core::anyhow::bail; +use deno_core::anyhow::Context; +use deno_core::error::generic_error; +use deno_core::error::AnyError; +use deno_core::futures; +use deno_core::parking_lot::RwLock; +use serde::Deserialize; +use serde::Serialize; + +use crate::lockfile::Lockfile; + +use self::graph::GraphDependencyResolver; + +use super::cache::should_sync_download; +use super::registry::NpmPackageVersionDistInfo; +use super::registry::NpmRegistryApi; +use super::semver::NpmVersion; +use super::semver::SpecifierVersionReq; + +mod graph; +mod snapshot; + +use graph::Graph; +pub use snapshot::NpmResolutionSnapshot; + +/// The version matcher used for npm schemed urls is more strict than +/// the one used by npm packages and so we represent either via a trait. +pub trait NpmVersionMatcher { + fn tag(&self) -> Option<&str>; + fn matches(&self, version: &NpmVersion) -> bool; + fn version_text(&self) -> String; +} + +#[derive(Clone, Debug, Default, PartialEq, Eq)] +pub struct NpmPackageReference { + pub req: NpmPackageReq, + pub sub_path: Option, +} + +impl NpmPackageReference { + pub fn from_specifier( + specifier: &ModuleSpecifier, + ) -> Result { + Self::from_str(specifier.as_str()) + } + + pub fn from_str(specifier: &str) -> Result { + let specifier = match specifier.strip_prefix("npm:") { + Some(s) => s, + None => { + bail!("Not an npm specifier: {}", specifier); + } + }; + let parts = specifier.split('/').collect::>(); + let name_part_len = if specifier.starts_with('@') { 2 } else { 1 }; + if parts.len() < name_part_len { + return Err(generic_error(format!("Not a valid package: {}", specifier))); + } + let name_parts = &parts[0..name_part_len]; + let last_name_part = &name_parts[name_part_len - 1]; + let (name, version_req) = if let Some(at_index) = last_name_part.rfind('@') + { + let version = &last_name_part[at_index + 1..]; + let last_name_part = &last_name_part[..at_index]; + let version_req = SpecifierVersionReq::parse(version) + .with_context(|| "Invalid version requirement.")?; + let name = if name_part_len == 1 { + last_name_part.to_string() + } else { + format!("{}/{}", name_parts[0], last_name_part) + }; + (name, Some(version_req)) + } else { + (name_parts.join("/"), None) + }; + let sub_path = if parts.len() == name_parts.len() { + None + } else { + Some(parts[name_part_len..].join("/")) + }; + + if let Some(sub_path) = &sub_path { + if let Some(at_index) = sub_path.rfind('@') { + let (new_sub_path, version) = sub_path.split_at(at_index); + let msg = format!( + "Invalid package specifier 'npm:{}/{}'. Did you mean to write 'npm:{}{}/{}'?", + name, sub_path, name, version, new_sub_path + ); + return Err(generic_error(msg)); + } + } + + Ok(NpmPackageReference { + req: NpmPackageReq { name, version_req }, + sub_path, + }) + } +} + +impl std::fmt::Display for NpmPackageReference { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + if let Some(sub_path) = &self.sub_path { + write!(f, "{}/{}", self.req, sub_path) + } else { + write!(f, "{}", self.req) + } + } +} + +#[derive( + Clone, Debug, Default, PartialEq, Eq, Hash, Serialize, Deserialize, +)] +pub struct NpmPackageReq { + pub name: String, + pub version_req: Option, +} + +impl std::fmt::Display for NpmPackageReq { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + match &self.version_req { + Some(req) => write!(f, "{}@{}", self.name, req), + None => write!(f, "{}", self.name), + } + } +} + +impl NpmVersionMatcher for NpmPackageReq { + fn tag(&self) -> Option<&str> { + match &self.version_req { + Some(version_req) => version_req.tag(), + None => Some("latest"), + } + } + + fn matches(&self, version: &NpmVersion) -> bool { + match self.version_req.as_ref() { + Some(req) => { + assert_eq!(self.tag(), None); + match req.range() { + Some(range) => range.satisfies(version), + None => false, + } + } + None => version.pre.is_empty(), + } + } + + fn version_text(&self) -> String { + self + .version_req + .as_ref() + .map(|v| format!("{}", v)) + .unwrap_or_else(|| "non-prerelease".to_string()) + } +} + +#[derive( + Debug, Clone, PartialOrd, Ord, PartialEq, Eq, Hash, Serialize, Deserialize, +)] +pub struct NpmPackageId { + pub name: String, + pub version: NpmVersion, + pub peer_dependencies: Vec, +} + +impl NpmPackageId { + #[allow(unused)] + pub fn scope(&self) -> Option<&str> { + if self.name.starts_with('@') && self.name.contains('/') { + self.name.split('/').next() + } else { + None + } + } + + pub fn as_serializable_name(&self) -> String { + self.as_serialize_name_with_level(0) + } + + fn as_serialize_name_with_level(&self, level: usize) -> String { + let mut result = format!( + "{}@{}", + if level == 0 { + self.name.to_string() + } else { + self.name.replace("/", "+") + }, + self.version + ); + for peer in &self.peer_dependencies { + // unfortunately we can't do something like `_3` when + // this gets deep because npm package names can start + // with a number + result.push_str(&"_".repeat(level + 1)); + result.push_str(&peer.as_serialize_name_with_level(level + 1)); + } + result + } + + pub fn deserialize_name(id: &str) -> Result { + use monch::*; + + fn parse_name(input: &str) -> ParseResult<&str> { + if_not_empty(substring(move |input| { + for (pos, c) in input.char_indices() { + // first character might be a scope, so skip it + if pos > 0 && c == '@' { + return Ok((&input[pos..], ())); + } + } + ParseError::backtrace() + }))(input) + } + + fn parse_version(input: &str) -> ParseResult<&str> { + if_not_empty(substring(skip_while(|c| c != '_')))(input) + } + + fn parse_name_and_version( + input: &str, + ) -> ParseResult<(String, NpmVersion)> { + let (input, name) = parse_name(input)?; + let (input, _) = ch('@')(input)?; + let at_version_input = input; + let (input, version) = parse_version(input)?; + match NpmVersion::parse(version) { + Ok(version) => Ok((input, (name.to_string(), version))), + Err(err) => ParseError::fail(at_version_input, format!("{:#}", err)), + } + } + + fn parse_level_at_level<'a>( + level: usize, + ) -> impl Fn(&'a str) -> ParseResult<'a, ()> { + fn parse_level(input: &str) -> ParseResult { + let level = input.chars().take_while(|c| *c == '_').count(); + Ok((&input[level..], level)) + } + + move |input| { + let (input, parsed_level) = parse_level(input)?; + if parsed_level == level { + Ok((input, ())) + } else { + ParseError::backtrace() + } + } + } + + fn parse_peers_at_level<'a>( + level: usize, + ) -> impl Fn(&'a str) -> ParseResult<'a, Vec> { + move |mut input| { + let mut peers = Vec::new(); + while let Ok((level_input, _)) = parse_level_at_level(level)(input) { + input = level_input; + let peer_result = parse_id_at_level(level)(input)?; + input = peer_result.0; + peers.push(peer_result.1); + } + Ok((input, peers)) + } + } + + fn parse_id_at_level<'a>( + level: usize, + ) -> impl Fn(&'a str) -> ParseResult<'a, NpmPackageId> { + move |input| { + let (input, (name, version)) = parse_name_and_version(input)?; + let name = if level > 0 { + name.replace("+", "/") + } else { + name + }; + let (input, peer_dependencies) = + parse_peers_at_level(level + 1)(input)?; + Ok(( + input, + NpmPackageId { + name, + version, + peer_dependencies, + }, + )) + } + } + + // todo(THIS PR): move this someone re-usable + crate::npm::semver::errors::with_failure_handling(parse_id_at_level(0))(id) + .with_context(|| format!("Invalid npm package id '{}'.", id)) + } +} + +impl std::fmt::Display for NpmPackageId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}@{}", self.name, self.version) + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct NpmResolutionPackage { + pub id: NpmPackageId, + pub dist: NpmPackageVersionDistInfo, + /// Key is what the package refers to the other package as, + /// which could be different from the package name. + pub dependencies: HashMap, +} + +pub struct NpmResolution { + api: NpmRegistryApi, + snapshot: RwLock, + update_sempahore: tokio::sync::Semaphore, +} + +impl std::fmt::Debug for NpmResolution { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + let snapshot = self.snapshot.read(); + f.debug_struct("NpmResolution") + .field("snapshot", &snapshot) + .finish() + } +} + +impl NpmResolution { + pub fn new( + api: NpmRegistryApi, + initial_snapshot: Option, + ) -> Self { + Self { + api, + snapshot: RwLock::new(initial_snapshot.unwrap_or_default()), + update_sempahore: tokio::sync::Semaphore::new(1), + } + } + + pub async fn add_package_reqs( + &self, + package_reqs: Vec, + ) -> Result<(), AnyError> { + // only allow one thread in here at a time + let _permit = self.update_sempahore.acquire().await.unwrap(); + let snapshot = self.snapshot.read().clone(); + + let snapshot = self + .add_package_reqs_to_snapshot(package_reqs, snapshot) + .await?; + + *self.snapshot.write() = snapshot; + Ok(()) + } + + pub async fn set_package_reqs( + &self, + package_reqs: HashSet, + ) -> Result<(), AnyError> { + // only allow one thread in here at a time + let _permit = self.update_sempahore.acquire().await.unwrap(); + let snapshot = self.snapshot.read().clone(); + + let has_removed_package = !snapshot + .package_reqs + .keys() + .all(|req| package_reqs.contains(req)); + // if any packages were removed, we need to completely recreate the npm resolution snapshot + let snapshot = if has_removed_package { + NpmResolutionSnapshot::default() + } else { + snapshot + }; + let snapshot = self + .add_package_reqs_to_snapshot( + package_reqs.into_iter().collect(), + snapshot, + ) + .await?; + + *self.snapshot.write() = snapshot; + + Ok(()) + } + + async fn add_package_reqs_to_snapshot( + &self, + mut package_reqs: Vec, + snapshot: NpmResolutionSnapshot, + ) -> Result { + // convert the snapshot to a traversable graph + let mut graph = Graph::default(); + graph.fill_with_snapshot(&snapshot); + drop(snapshot); // todo: remove + + // multiple packages are resolved in alphabetical order + package_reqs.sort_by(|a, b| a.name.cmp(&b.name)); + + // go over the top level packages first, then down the + // tree one level at a time through all the branches + let mut unresolved_tasks = Vec::with_capacity(package_reqs.len()); + for package_req in package_reqs { + if graph.has_package_req(&package_req) { + // skip analyzing this package, as there's already a matching top level package + continue; + } + + // no existing best version, so resolve the current packages + let api = self.api.clone(); + let maybe_info = if should_sync_download() { + // for deterministic test output + Some(api.package_info(&package_req.name).await) + } else { + None + }; + unresolved_tasks.push(tokio::task::spawn(async move { + let info = match maybe_info { + Some(info) => info?, + None => api.package_info(&package_req.name).await?, + }; + Result::<_, AnyError>::Ok((package_req, info)) + })); + } + + let mut resolver = GraphDependencyResolver::new(&mut graph, &self.api); + + for result in futures::future::join_all(unresolved_tasks).await { + let (package_req, info) = result??; + resolver.resolve_npm_package_req(&package_req, info)?; + } + + resolver.resolve_pending().await?; + + graph.into_snapshot(&self.api).await + } + + pub fn resolve_package_from_id( + &self, + id: &NpmPackageId, + ) -> Option { + self.snapshot.read().package_from_id(id).cloned() + } + + pub fn resolve_package_from_package( + &self, + name: &str, + referrer: &NpmPackageId, + ) -> Result { + self + .snapshot + .read() + .resolve_package_from_package(name, referrer) + .cloned() + } + + /// Resolve a node package from a deno module. + pub fn resolve_package_from_deno_module( + &self, + package: &NpmPackageReq, + ) -> Result { + self + .snapshot + .read() + .resolve_package_from_deno_module(package) + .cloned() + } + + pub fn all_packages(&self) -> Vec { + self.snapshot.read().all_packages() + } + + pub fn has_packages(&self) -> bool { + !self.snapshot.read().packages.is_empty() + } + + pub fn snapshot(&self) -> NpmResolutionSnapshot { + self.snapshot.read().clone() + } + + pub fn lock( + &self, + lockfile: &mut Lockfile, + snapshot: &NpmResolutionSnapshot, + ) -> Result<(), AnyError> { + for (package_req, version) in snapshot.package_reqs.iter() { + lockfile.insert_npm_specifier(package_req, version.to_string()); + } + for package in self.all_packages() { + lockfile.check_or_insert_npm_package(&package)?; + } + Ok(()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parse_npm_package_ref() { + assert_eq!( + NpmPackageReference::from_str("npm:@package/test").unwrap(), + NpmPackageReference { + req: NpmPackageReq { + name: "@package/test".to_string(), + version_req: None, + }, + sub_path: None, + } + ); + + assert_eq!( + NpmPackageReference::from_str("npm:@package/test@1").unwrap(), + NpmPackageReference { + req: NpmPackageReq { + name: "@package/test".to_string(), + version_req: Some(SpecifierVersionReq::parse("1").unwrap()), + }, + sub_path: None, + } + ); + + assert_eq!( + NpmPackageReference::from_str("npm:@package/test@~1.1/sub_path").unwrap(), + NpmPackageReference { + req: NpmPackageReq { + name: "@package/test".to_string(), + version_req: Some(SpecifierVersionReq::parse("~1.1").unwrap()), + }, + sub_path: Some("sub_path".to_string()), + } + ); + + assert_eq!( + NpmPackageReference::from_str("npm:@package/test/sub_path").unwrap(), + NpmPackageReference { + req: NpmPackageReq { + name: "@package/test".to_string(), + version_req: None, + }, + sub_path: Some("sub_path".to_string()), + } + ); + + assert_eq!( + NpmPackageReference::from_str("npm:test").unwrap(), + NpmPackageReference { + req: NpmPackageReq { + name: "test".to_string(), + version_req: None, + }, + sub_path: None, + } + ); + + assert_eq!( + NpmPackageReference::from_str("npm:test@^1.2").unwrap(), + NpmPackageReference { + req: NpmPackageReq { + name: "test".to_string(), + version_req: Some(SpecifierVersionReq::parse("^1.2").unwrap()), + }, + sub_path: None, + } + ); + + assert_eq!( + NpmPackageReference::from_str("npm:test@~1.1/sub_path").unwrap(), + NpmPackageReference { + req: NpmPackageReq { + name: "test".to_string(), + version_req: Some(SpecifierVersionReq::parse("~1.1").unwrap()), + }, + sub_path: Some("sub_path".to_string()), + } + ); + + assert_eq!( + NpmPackageReference::from_str("npm:@package/test/sub_path").unwrap(), + NpmPackageReference { + req: NpmPackageReq { + name: "@package/test".to_string(), + version_req: None, + }, + sub_path: Some("sub_path".to_string()), + } + ); + + assert_eq!( + NpmPackageReference::from_str("npm:@package") + .err() + .unwrap() + .to_string(), + "Not a valid package: @package" + ); + } + + #[test] + fn serialize_npm_package_id() { + let id = NpmPackageId { + name: "pkg-a".to_string(), + version: NpmVersion::parse("1.2.3").unwrap(), + peer_dependencies: vec![ + NpmPackageId { + name: "pkg-b".to_string(), + version: NpmVersion::parse("3.2.1").unwrap(), + peer_dependencies: vec![ + NpmPackageId { + name: "pkg-c".to_string(), + version: NpmVersion::parse("1.3.2").unwrap(), + peer_dependencies: vec![], + }, + NpmPackageId { + name: "pkg-d".to_string(), + version: NpmVersion::parse("2.3.4").unwrap(), + peer_dependencies: vec![], + }, + ], + }, + NpmPackageId { + name: "pkg-e".to_string(), + version: NpmVersion::parse("2.3.1").unwrap(), + peer_dependencies: vec![NpmPackageId { + name: "pkg-f".to_string(), + version: NpmVersion::parse("2.3.1").unwrap(), + peer_dependencies: vec![], + }], + }, + ], + }; + let serialized = id.as_serializable_name(); + assert_eq!(serialized, "pkg-a@1.2.3_pkg-b@3.2.1__pkg-c@1.3.2__pkg-d@2.3.4_pkg-e@2.3.1__pkg-f@2.3.1"); + assert_eq!(NpmPackageId::deserialize_name(&serialized).unwrap(), id); + } +} diff --git a/cli/npm/resolution/snapshot.rs b/cli/npm/resolution/snapshot.rs new file mode 100644 index 00000000000000..a30168651a4ac7 --- /dev/null +++ b/cli/npm/resolution/snapshot.rs @@ -0,0 +1,302 @@ +// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. + +use std::collections::HashMap; +use std::collections::HashSet; +use std::sync::Arc; + +use deno_core::anyhow::bail; +use deno_core::anyhow::Context; +use deno_core::error::AnyError; +use deno_core::futures; +use deno_core::parking_lot::Mutex; +use serde::Deserialize; +use serde::Serialize; + +use crate::lockfile::Lockfile; +use crate::npm::registry::NpmPackageVersionDistInfo; +use crate::npm::registry::NpmRegistryApi; + +use super::NpmPackageId; +use super::NpmPackageReference; +use super::NpmPackageReq; +use super::NpmResolutionPackage; +use super::NpmVersionMatcher; + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct NpmResolutionSnapshot { + #[serde(with = "map_to_vec")] + pub(super) package_reqs: HashMap, + pub(super) packages_by_name: HashMap>, + #[serde(with = "map_to_vec")] + pub(super) packages: HashMap, +} + +// This is done so the maps with non-string keys get serialized and deserialized as vectors. +// Adapted from: https://github.com/serde-rs/serde/issues/936#issuecomment-302281792 +mod map_to_vec { + use std::collections::HashMap; + + use serde::de::Deserialize; + use serde::de::Deserializer; + use serde::ser::Serializer; + use serde::Serialize; + + pub fn serialize( + map: &HashMap, + serializer: S, + ) -> Result + where + S: Serializer, + { + serializer.collect_seq(map.iter()) + } + + pub fn deserialize< + 'de, + D, + K: Deserialize<'de> + Eq + std::hash::Hash, + V: Deserialize<'de>, + >( + deserializer: D, + ) -> Result, D::Error> + where + D: Deserializer<'de>, + { + let mut map = HashMap::new(); + for (key, value) in Vec::<(K, V)>::deserialize(deserializer)? { + map.insert(key, value); + } + Ok(map) + } +} + +impl NpmResolutionSnapshot { + /// Resolve a node package from a deno module. + pub fn resolve_package_from_deno_module( + &self, + req: &NpmPackageReq, + ) -> Result<&NpmResolutionPackage, AnyError> { + match self.package_reqs.get(req) { + Some(id) => Ok(self.packages.get(id).unwrap()), + None => bail!("could not find npm package directory for '{}'", req), + } + } + + pub fn top_level_packages(&self) -> Vec { + self + .package_reqs + .values() + .cloned() + .collect::>() + .into_iter() + .collect::>() + } + + pub fn package_from_id( + &self, + id: &NpmPackageId, + ) -> Option<&NpmResolutionPackage> { + self.packages.get(id) + } + + pub fn resolve_package_from_package( + &self, + name: &str, + referrer: &NpmPackageId, + ) -> Result<&NpmResolutionPackage, AnyError> { + match self.packages.get(referrer) { + Some(referrer_package) => { + let name_ = name_without_path(name); + if let Some(id) = referrer_package.dependencies.get(name_) { + return Ok(self.packages.get(id).unwrap()); + } + + if referrer_package.id.name == name_ { + return Ok(referrer_package); + } + + // TODO(bartlomieju): this should use a reverse lookup table in the + // snapshot instead of resolving best version again. + let req = NpmPackageReq { + name: name_.to_string(), + version_req: None, + }; + + if let Some(id) = self.resolve_best_package_id(name_, &req) { + if let Some(pkg) = self.packages.get(&id) { + return Ok(pkg); + } + } + + bail!( + "could not find npm package '{}' referenced by '{}'", + name, + referrer + ) + } + None => bail!("could not find referrer npm package '{}'", referrer), + } + } + + pub fn all_packages(&self) -> Vec { + self.packages.values().cloned().collect() + } + + pub fn resolve_best_package_id( + &self, + name: &str, + version_matcher: &impl NpmVersionMatcher, + ) -> Option { + // todo(THIS PR): this is not correct because some ids will be better than others + let mut maybe_best_id: Option<&NpmPackageId> = None; + if let Some(ids) = self.packages_by_name.get(name) { + for id in ids { + if version_matcher.matches(&id.version) { + let is_best_version = maybe_best_id + .as_ref() + .map(|best_id| best_id.version.cmp(&id.version).is_lt()) + .unwrap_or(true); + if is_best_version { + maybe_best_id = Some(id); + } + } + } + } + maybe_best_id.cloned() + } + + pub async fn from_lockfile( + lockfile: Arc>, + api: &NpmRegistryApi, + ) -> Result { + let mut package_reqs: HashMap; + let mut packages_by_name: HashMap>; + let mut packages: HashMap; + + { + let lockfile = lockfile.lock(); + + // pre-allocate collections + package_reqs = + HashMap::with_capacity(lockfile.content.npm.specifiers.len()); + packages = HashMap::with_capacity(lockfile.content.npm.packages.len()); + packages_by_name = + HashMap::with_capacity(lockfile.content.npm.packages.len()); // close enough + let mut verify_ids = + HashSet::with_capacity(lockfile.content.npm.packages.len()); + + // collect the specifiers to version mappings + for (key, value) in &lockfile.content.npm.specifiers { + let reference = NpmPackageReference::from_str(&format!("npm:{}", key)) + .with_context(|| format!("Unable to parse npm specifier: {}", key))?; + let package_id = NpmPackageId::deserialize_name(value)?; + package_reqs.insert(reference.req, package_id.clone()); + verify_ids.insert(package_id.clone()); + } + + // then the packages + for (key, value) in &lockfile.content.npm.packages { + let package_id = NpmPackageId::deserialize_name(key)?; + let mut dependencies = HashMap::default(); + + packages_by_name + .entry(package_id.name.to_string()) + .or_default() + .push(package_id.clone()); + + for (name, specifier) in &value.dependencies { + let dep_id = NpmPackageId::deserialize_name(specifier)?; + dependencies.insert(name.to_string(), dep_id.clone()); + verify_ids.insert(dep_id); + } + + let package = NpmResolutionPackage { + id: package_id.clone(), + // temporary dummy value + dist: NpmPackageVersionDistInfo { + tarball: "foobar".to_string(), + shasum: "foobar".to_string(), + integrity: Some("foobar".to_string()), + }, + dependencies, + }; + + packages.insert(package_id, package); + } + + // verify that all these ids exist in packages + for id in &verify_ids { + if !packages.contains_key(id) { + bail!( + "the lockfile ({}) is corrupt. You can recreate it with --lock-write", + lockfile.filename.display(), + ); + } + } + } + + let mut unresolved_tasks = Vec::with_capacity(packages_by_name.len()); + + // cache the package names in parallel in the registry api + for package_name in packages_by_name.keys() { + let package_name = package_name.clone(); + let api = api.clone(); + unresolved_tasks.push(tokio::task::spawn(async move { + api.package_info(&package_name).await?; + Result::<_, AnyError>::Ok(()) + })); + } + for result in futures::future::join_all(unresolved_tasks).await { + result??; + } + + // ensure the dist is set for each package + for package in packages.values_mut() { + // this will read from the memory cache now + let version_info = match api + .package_version_info(&package.id.name, &package.id.version) + .await? + { + Some(version_info) => version_info, + None => { + bail!("could not find '{}' specified in the lockfile. Maybe try again with --reload", package.id); + } + }; + package.dist = version_info.dist; + } + + Ok(Self { + package_reqs, + packages_by_name, + packages, + }) + } +} + +fn name_without_path(name: &str) -> &str { + let mut search_start_index = 0; + if name.starts_with('@') { + if let Some(slash_index) = name.find('/') { + search_start_index = slash_index + 1; + } + } + if let Some(slash_index) = &name[search_start_index..].find('/') { + // get the name up until the path slash + &name[0..search_start_index + slash_index] + } else { + name + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_name_without_path() { + assert_eq!(name_without_path("foo"), "foo"); + assert_eq!(name_without_path("@foo/bar"), "@foo/bar"); + assert_eq!(name_without_path("@foo/bar/baz"), "@foo/bar"); + assert_eq!(name_without_path("@hello"), "@hello"); + } +} From 96a8975efacda1acb2e7a041542098233aa82709 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 3 Nov 2022 11:17:18 -0400 Subject: [PATCH 09/43] Ability to inject a custom npm registry api for testing purposes --- cli/lsp/language_server.rs | 6 +-- cli/npm/cache.rs | 1 + cli/npm/mod.rs | 1 + cli/npm/registry.rs | 89 +++++++++++++++++++++++----------- cli/npm/resolution/graph.rs | 14 +++--- cli/npm/resolution/mod.rs | 7 +-- cli/npm/resolution/snapshot.rs | 3 +- cli/npm/resolvers/global.rs | 4 +- cli/npm/resolvers/local.rs | 4 +- cli/npm/resolvers/mod.rs | 8 +-- cli/proc_state.rs | 6 +-- 11 files changed, 92 insertions(+), 51 deletions(-) diff --git a/cli/lsp/language_server.rs b/cli/lsp/language_server.rs index 3a0906636dc4bb..aa4e98b1d7c293 100644 --- a/cli/lsp/language_server.rs +++ b/cli/lsp/language_server.rs @@ -71,7 +71,7 @@ use crate::fs_util; use crate::graph_util::graph_valid; use crate::npm::NpmCache; use crate::npm::NpmPackageResolver; -use crate::npm::NpmRegistryApi; +use crate::npm::RealNpmRegistryApi; use crate::proc_state::import_map_from_text; use crate::proc_state::ProcState; use crate::progress_bar::ProgressBar; @@ -258,7 +258,7 @@ impl Inner { ts_server.clone(), ); let assets = Assets::new(ts_server.clone()); - let registry_url = NpmRegistryApi::default_url(); + let registry_url = RealNpmRegistryApi::default_url(); // Use an "only" cache setting in order to make the // user do an explicit "cache" command and prevent // the cache from being filled with lots of packages while @@ -270,7 +270,7 @@ impl Inner { cache_setting.clone(), progress_bar.clone(), ); - let api = NpmRegistryApi::new( + let api = RealNpmRegistryApi::new( registry_url, npm_cache.clone(), cache_setting, diff --git a/cli/npm/cache.rs b/cli/npm/cache.rs index c4e0d148e15453..75d75d3b44a98b 100644 --- a/cli/npm/cache.rs +++ b/cli/npm/cache.rs @@ -102,6 +102,7 @@ impl ReadonlyNpmCache { // packages. // 2. We can figure out the package id from the path. This is used // in resolve_package_id_from_specifier + // Maybe consider only supporting this if people use --node-modules-dir todo!("deno currently doesn't support npm package names that are not all lowercase"); } // ensure backslashes are used on windows diff --git a/cli/npm/mod.rs b/cli/npm/mod.rs index 1c37276db87355..86ed8572c2b248 100644 --- a/cli/npm/mod.rs +++ b/cli/npm/mod.rs @@ -13,6 +13,7 @@ pub use cache::NpmCache; #[cfg(test)] pub use registry::NpmPackageVersionDistInfo; pub use registry::NpmRegistryApi; +pub use registry::RealNpmRegistryApi; pub use resolution::NpmPackageId; pub use resolution::NpmPackageReference; pub use resolution::NpmPackageReq; diff --git a/cli/npm/registry.rs b/cli/npm/registry.rs index 014d07d9b35533..bd12973d5bd435 100644 --- a/cli/npm/registry.rs +++ b/cli/npm/registry.rs @@ -11,6 +11,8 @@ use deno_core::anyhow::bail; use deno_core::anyhow::Context; use deno_core::error::custom_error; use deno_core::error::AnyError; +use deno_core::futures::future::BoxFuture; +use deno_core::futures::FutureExt; use deno_core::parking_lot::Mutex; use deno_core::serde::Deserialize; use deno_core::serde_json; @@ -165,16 +167,49 @@ pub struct NpmPackageVersionDistInfo { pub integrity: Option, } -#[derive(Clone)] -pub struct NpmRegistryApi { - base_url: Url, - cache: NpmCache, - mem_cache: Arc>>>, - cache_setting: CacheSetting, - progress_bar: ProgressBar, +pub trait NpmRegistryApi: Clone + Sync + Send + 'static { + fn maybe_package_info( + &self, + name: &str, + ) -> BoxFuture<'static, Result, AnyError>>; + + fn package_info( + &self, + name: &str, + ) -> BoxFuture<'static, Result> { + let api = self.clone(); + let name = name.to_string(); + async move { + let maybe_package_info = api.maybe_package_info(&name).await?; + match maybe_package_info { + Some(package_info) => Ok(package_info), + None => bail!("npm package '{}' does not exist", name), + } + } + .boxed() + } + + fn package_version_info( + &self, + name: &str, + version: &NpmVersion, + ) -> BoxFuture<'static, Result, AnyError>> { + let api = self.clone(); + let name = name.to_string(); + let version = version.to_string(); + async move { + // todo(dsherret): this could be optimized to not clone the entire package info + let mut package_info = api.package_info(&name).await?; + Ok(package_info.versions.remove(&version)) + } + .boxed() + } } -impl NpmRegistryApi { +#[derive(Clone)] +pub struct RealNpmRegistryApi(Arc); + +impl RealNpmRegistryApi { pub fn default_url() -> Url { let env_var_name = "DENO_NPM_REGISTRY"; if let Ok(registry_url) = std::env::var(env_var_name) { @@ -200,40 +235,40 @@ impl NpmRegistryApi { cache_setting: CacheSetting, progress_bar: ProgressBar, ) -> Self { - Self { + Self(Arc::new(RealNpmRegistryApiInner { base_url, cache, mem_cache: Default::default(), cache_setting, progress_bar, - } + })) } pub fn base_url(&self) -> &Url { - &self.base_url + &self.0.base_url } +} - pub async fn package_info( +impl NpmRegistryApi for RealNpmRegistryApi { + fn maybe_package_info( &self, name: &str, - ) -> Result { - let maybe_package_info = self.maybe_package_info(name).await?; - match maybe_package_info { - Some(package_info) => Ok(package_info), - None => bail!("npm package '{}' does not exist", name), - } + ) -> BoxFuture<'static, Result, AnyError>> { + let api = self.clone(); + let name = name.to_string(); + async move { api.0.maybe_package_info(&name).await }.boxed() } +} - pub async fn package_version_info( - &self, - name: &str, - version: &NpmVersion, - ) -> Result, AnyError> { - // todo(dsherret): this could be optimized to not clone the entire package info - let mut package_info = self.package_info(name).await?; - Ok(package_info.versions.remove(&version.to_string())) - } +struct RealNpmRegistryApiInner { + base_url: Url, + cache: NpmCache, + mem_cache: Mutex>>, + cache_setting: CacheSetting, + progress_bar: ProgressBar, +} +impl RealNpmRegistryApiInner { pub async fn maybe_package_info( &self, name: &str, diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index 84988d7b73c95c..53d875fad6ca3f 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -16,9 +16,9 @@ use crate::npm::registry::NpmDependencyEntry; use crate::npm::registry::NpmDependencyEntryKind; use crate::npm::registry::NpmPackageInfo; use crate::npm::registry::NpmPackageVersionInfo; -use crate::npm::registry::NpmRegistryApi; use crate::npm::semver::NpmVersion; use crate::npm::semver::NpmVersionReq; +use crate::npm::NpmRegistryApi; use super::snapshot::NpmResolutionSnapshot; use super::NpmPackageId; @@ -141,7 +141,7 @@ impl Graph { pub async fn into_snapshot( self, - api: &NpmRegistryApi, + api: &impl NpmRegistryApi, ) -> Result { let mut packages = HashMap::with_capacity(self.packages.len()); for (id, node) in self.packages { @@ -169,16 +169,18 @@ impl Graph { } } -pub struct GraphDependencyResolver<'a> { +pub struct GraphDependencyResolver<'a, TNpmRegistryApi: NpmRegistryApi> { graph: &'a mut Graph, - api: &'a NpmRegistryApi, + api: &'a TNpmRegistryApi, pending_dependencies: VecDeque<(NpmPackageId, Vec)>, pending_peer_dependencies: VecDeque<((String, NpmPackageId), NpmDependencyEntry, NpmPackageInfo)>, } -impl<'a> GraphDependencyResolver<'a> { - pub fn new(graph: &'a mut Graph, api: &'a NpmRegistryApi) -> Self { +impl<'a, TNpmRegistryApi: NpmRegistryApi> + GraphDependencyResolver<'a, TNpmRegistryApi> +{ + pub fn new(graph: &'a mut Graph, api: &'a TNpmRegistryApi) -> Self { Self { graph, api, diff --git a/cli/npm/resolution/mod.rs b/cli/npm/resolution/mod.rs index 33b8fa3455bd76..8f4ac0766d7764 100644 --- a/cli/npm/resolution/mod.rs +++ b/cli/npm/resolution/mod.rs @@ -19,9 +19,10 @@ use self::graph::GraphDependencyResolver; use super::cache::should_sync_download; use super::registry::NpmPackageVersionDistInfo; -use super::registry::NpmRegistryApi; +use super::registry::RealNpmRegistryApi; use super::semver::NpmVersion; use super::semver::SpecifierVersionReq; +use super::NpmRegistryApi; mod graph; mod snapshot; @@ -313,7 +314,7 @@ pub struct NpmResolutionPackage { } pub struct NpmResolution { - api: NpmRegistryApi, + api: RealNpmRegistryApi, snapshot: RwLock, update_sempahore: tokio::sync::Semaphore, } @@ -329,7 +330,7 @@ impl std::fmt::Debug for NpmResolution { impl NpmResolution { pub fn new( - api: NpmRegistryApi, + api: RealNpmRegistryApi, initial_snapshot: Option, ) -> Self { Self { diff --git a/cli/npm/resolution/snapshot.rs b/cli/npm/resolution/snapshot.rs index a30168651a4ac7..5ed22848dbbe83 100644 --- a/cli/npm/resolution/snapshot.rs +++ b/cli/npm/resolution/snapshot.rs @@ -15,6 +15,7 @@ use serde::Serialize; use crate::lockfile::Lockfile; use crate::npm::registry::NpmPackageVersionDistInfo; use crate::npm::registry::NpmRegistryApi; +use crate::npm::registry::RealNpmRegistryApi; use super::NpmPackageId; use super::NpmPackageReference; @@ -167,7 +168,7 @@ impl NpmResolutionSnapshot { pub async fn from_lockfile( lockfile: Arc>, - api: &NpmRegistryApi, + api: &RealNpmRegistryApi, ) -> Result { let mut package_reqs: HashMap; let mut packages_by_name: HashMap>; diff --git a/cli/npm/resolvers/global.rs b/cli/npm/resolvers/global.rs index 42090415ae24eb..4fb3195d06e2bc 100644 --- a/cli/npm/resolvers/global.rs +++ b/cli/npm/resolvers/global.rs @@ -23,7 +23,7 @@ use crate::npm::resolvers::common::cache_packages; use crate::npm::NpmCache; use crate::npm::NpmPackageId; use crate::npm::NpmPackageReq; -use crate::npm::NpmRegistryApi; +use crate::npm::RealNpmRegistryApi; use super::common::ensure_registry_read_permission; use super::common::InnerNpmPackageResolver; @@ -39,7 +39,7 @@ pub struct GlobalNpmPackageResolver { impl GlobalNpmPackageResolver { pub fn new( cache: NpmCache, - api: NpmRegistryApi, + api: RealNpmRegistryApi, initial_snapshot: Option, ) -> Self { let registry_url = api.base_url().to_owned(); diff --git a/cli/npm/resolvers/local.rs b/cli/npm/resolvers/local.rs index cad940d5638d3e..47842627ad0cef 100644 --- a/cli/npm/resolvers/local.rs +++ b/cli/npm/resolvers/local.rs @@ -29,7 +29,7 @@ use crate::npm::resolution::NpmResolutionSnapshot; use crate::npm::NpmCache; use crate::npm::NpmPackageId; use crate::npm::NpmPackageReq; -use crate::npm::NpmRegistryApi; +use crate::npm::RealNpmRegistryApi; use super::common::ensure_registry_read_permission; use super::common::InnerNpmPackageResolver; @@ -48,7 +48,7 @@ pub struct LocalNpmPackageResolver { impl LocalNpmPackageResolver { pub fn new( cache: NpmCache, - api: NpmRegistryApi, + api: RealNpmRegistryApi, node_modules_folder: PathBuf, initial_snapshot: Option, ) -> Self { diff --git a/cli/npm/resolvers/mod.rs b/cli/npm/resolvers/mod.rs index 71c2abc00da1f3..a0b500823537f1 100644 --- a/cli/npm/resolvers/mod.rs +++ b/cli/npm/resolvers/mod.rs @@ -29,8 +29,8 @@ use self::local::LocalNpmPackageResolver; use super::NpmCache; use super::NpmPackageId; use super::NpmPackageReq; -use super::NpmRegistryApi; use super::NpmResolutionSnapshot; +use super::RealNpmRegistryApi; const RESOLUTION_STATE_ENV_VAR_NAME: &str = "DENO_DONT_USE_INTERNAL_NODE_COMPAT_STATE"; @@ -71,7 +71,7 @@ pub struct NpmPackageResolver { no_npm: bool, inner: Arc, local_node_modules_path: Option, - api: NpmRegistryApi, + api: RealNpmRegistryApi, cache: NpmCache, maybe_lockfile: Option>>, } @@ -90,7 +90,7 @@ impl std::fmt::Debug for NpmPackageResolver { impl NpmPackageResolver { pub fn new( cache: NpmCache, - api: NpmRegistryApi, + api: RealNpmRegistryApi, unstable: bool, no_npm: bool, local_node_modules_path: Option, @@ -133,7 +133,7 @@ impl NpmPackageResolver { fn new_with_maybe_snapshot( cache: NpmCache, - api: NpmRegistryApi, + api: RealNpmRegistryApi, unstable: bool, no_npm: bool, local_node_modules_path: Option, diff --git a/cli/proc_state.rs b/cli/proc_state.rs index 148f44923d4997..ae3a54a2017b10 100644 --- a/cli/proc_state.rs +++ b/cli/proc_state.rs @@ -26,7 +26,7 @@ use crate::node::NodeResolution; use crate::npm::NpmCache; use crate::npm::NpmPackageReference; use crate::npm::NpmPackageResolver; -use crate::npm::NpmRegistryApi; +use crate::npm::RealNpmRegistryApi; use crate::progress_bar::ProgressBar; use crate::resolver::CliResolver; use crate::tools::check; @@ -211,13 +211,13 @@ impl ProcState { let emit_cache = EmitCache::new(dir.gen_cache.clone()); let parsed_source_cache = ParsedSourceCache::new(Some(dir.dep_analysis_db_file_path())); - let registry_url = NpmRegistryApi::default_url(); + let registry_url = RealNpmRegistryApi::default_url(); let npm_cache = NpmCache::from_deno_dir( &dir, cli_options.cache_setting(), progress_bar.clone(), ); - let api = NpmRegistryApi::new( + let api = RealNpmRegistryApi::new( registry_url, npm_cache.clone(), cli_options.cache_setting(), From 8e8c4389f797455bb1dfb30829850afbb52b820d Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 3 Nov 2022 12:29:36 -0400 Subject: [PATCH 10/43] Start of adding unit tests for graph resolver code. --- cli/npm/registry.rs | 104 ++++++++++++++++++++++++++++++++++-- cli/npm/resolution/graph.rs | 83 ++++++++++++++++++++++++++++ cli/npm/resolution/mod.rs | 2 +- 3 files changed, 185 insertions(+), 4 deletions(-) diff --git a/cli/npm/registry.rs b/cli/npm/registry.rs index bd12973d5bd435..bfd7d5b81ddb03 100644 --- a/cli/npm/registry.rs +++ b/cli/npm/registry.rs @@ -33,7 +33,7 @@ use super::semver::NpmVersionReq; // npm registry docs: https://github.com/npm/registry/blob/master/docs/REGISTRY-API.md -#[derive(Debug, Deserialize, Serialize, Clone)] +#[derive(Debug, Default, Deserialize, Serialize, Clone)] pub struct NpmPackageInfo { pub name: String, pub versions: HashMap, @@ -159,7 +159,7 @@ impl NpmPackageVersionInfo { } } -#[derive(Debug, Default, Clone, Serialize, Deserialize)] +#[derive(Debug, Default, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct NpmPackageVersionDistInfo { /// URL to the tarball. pub tarball: String, @@ -198,7 +198,8 @@ pub trait NpmRegistryApi: Clone + Sync + Send + 'static { let name = name.to_string(); let version = version.to_string(); async move { - // todo(dsherret): this could be optimized to not clone the entire package info + // todo(dsherret): this could be optimized to not clone the + // entire package info in the case of the RealNpmRegistryApi let mut package_info = api.package_info(&name).await?; Ok(package_info.versions.remove(&version)) } @@ -441,3 +442,100 @@ impl RealNpmRegistryApiInner { name_folder_path.join("registry.json") } } + +/// Note: This test struct is not thread safe for setup +/// purposes. Construct everything on the same thread. +#[cfg(test)] +#[derive(Clone, Default)] +pub struct TestNpmRegistryApi { + package_infos: Arc>>, +} + +#[cfg(test)] +impl TestNpmRegistryApi { + pub fn add_package_info(&self, name: &str, info: NpmPackageInfo) { + let previous = self.package_infos.lock().insert(name.to_string(), info); + assert!(previous.is_none()); + } + + pub fn ensure_package(&self, name: &str) { + if !self.package_infos.lock().contains_key(name) { + self.add_package_info( + name, + NpmPackageInfo { + name: name.to_string(), + ..Default::default() + }, + ); + } + } + + pub fn ensure_package_version(&self, name: &str, version: &str) { + self.ensure_package(name); + let mut infos = self.package_infos.lock(); + let info = infos.get_mut(name).unwrap(); + if !info.versions.contains_key(version) { + info.versions.insert( + version.to_string(), + NpmPackageVersionInfo { + version: version.to_string(), + ..Default::default() + }, + ); + } + } + + pub fn add_dependency( + &self, + package_from: (&str, &str), + package_to: (&str, &str), + ) { + let mut infos = self.package_infos.lock(); + let info = infos.get_mut(package_from.0).unwrap(); + let version = info.versions.get_mut(package_from.1).unwrap(); + version + .dependencies + .insert(package_to.0.to_string(), package_to.1.to_string()); + } + + pub fn add_peer_dependency( + &self, + package_from: (&str, &str), + package_to: (&str, &str), + ) { + let mut infos = self.package_infos.lock(); + let info = infos.get_mut(package_from.0).unwrap(); + let version = info.versions.get_mut(package_from.1).unwrap(); + version + .peer_dependencies + .insert(package_to.0.to_string(), package_to.1.to_string()); + } + + pub fn add_optional_peer_dependency( + &self, + package_from: (&str, &str), + package_to: (&str, &str), + ) { + let mut infos = self.package_infos.lock(); + let info = infos.get_mut(package_from.0).unwrap(); + let version = info.versions.get_mut(package_from.1).unwrap(); + version + .peer_dependencies + .insert(package_to.0.to_string(), package_to.1.to_string()); + version.peer_dependencies_meta.insert( + package_to.0.to_string(), + NpmPeerDependencyMeta { optional: true }, + ); + } +} + +#[cfg(test)] +impl NpmRegistryApi for TestNpmRegistryApi { + fn maybe_package_info( + &self, + name: &str, + ) -> BoxFuture<'static, Result, AnyError>> { + let result = self.package_infos.lock().get(name).cloned(); + Box::pin(deno_core::futures::future::ready(Ok(result))) + } +} diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index 53d875fad6ca3f..6c8b5676e92c57 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -363,6 +363,9 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> } } + // if a peer dependency was found, resolve one of them then go back + // to resolving dependenices above before moving on to the next + // peer dependency if let Some(((specifier, parent_id), peer_dep, peer_package_info)) = self.pending_peer_dependencies.pop_front() { @@ -615,6 +618,9 @@ fn get_resolved_package_version_and_info( #[cfg(test)] mod test { + use pretty_assertions::assert_eq; + + use crate::npm::registry::TestNpmRegistryApi; use crate::npm::NpmPackageReference; use super::*; @@ -667,4 +673,81 @@ mod test { ); assert_eq!(result.unwrap().version.to_string(), "1.0.0-alpha"); } + + #[tokio::test] + async fn resolve_no_peer_deps() { + let api = TestNpmRegistryApi::default(); + api.ensure_package_version("package-a", "1.0.0"); + api.ensure_package_version("package-b", "2.0.0"); + api.ensure_package_version("package-c", "0.1.0"); + api.ensure_package_version("package-c", "0.0.10"); + api.ensure_package_version("package-d", "3.2.1"); + api.ensure_package_version("package-d", "3.2.0"); + api.add_dependency(("package-a", "1.0.0"), ("package-b", "^2")); + api.add_dependency(("package-a", "1.0.0"), ("package-c", "^0.1")); + api.add_dependency(("package-c", "0.1.0"), ("package-d", "*")); + + let packages = + run_resolver_and_get_output(api, vec!["npm:package-a@1"]).await; + assert_eq!( + packages, + vec![ + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-a@1.0.0").unwrap(), + dependencies: HashMap::from([ + ( + "package-b".to_string(), + NpmPackageId::deserialize_name("package-b@2.0.0").unwrap(), + ), + ( + "package-c".to_string(), + NpmPackageId::deserialize_name("package-c@0.1.0").unwrap(), + ), + ]), + dist: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-b@2.0.0").unwrap(), + dist: Default::default(), + dependencies: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-c@0.1.0").unwrap(), + dist: Default::default(), + dependencies: HashMap::from([( + "package-d".to_string(), + NpmPackageId::deserialize_name("package-d@3.2.1").unwrap(), + ),]) + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-d@3.2.1").unwrap(), + dist: Default::default(), + dependencies: Default::default(), + }, + ] + ); + } + + async fn run_resolver_and_get_output( + api: TestNpmRegistryApi, + reqs: Vec<&str>, + ) -> Vec { + let mut graph = Graph::default(); + let mut resolver = GraphDependencyResolver::new(&mut graph, &api); + + for req in reqs { + let req = NpmPackageReference::from_str(req).unwrap().req; + resolver + .resolve_npm_package_req( + &req, + api.package_info(&req.name).await.unwrap(), + ) + .unwrap(); + } + + resolver.resolve_pending().await.unwrap(); + let mut packages = graph.into_snapshot(&api).await.unwrap().all_packages(); + packages.sort_by(|a, b| a.id.cmp(&b.id)); + packages + } } diff --git a/cli/npm/resolution/mod.rs b/cli/npm/resolution/mod.rs index 8f4ac0766d7764..f977d08cf252d7 100644 --- a/cli/npm/resolution/mod.rs +++ b/cli/npm/resolution/mod.rs @@ -304,7 +304,7 @@ impl std::fmt::Display for NpmPackageId { } } -#[derive(Debug, Clone, Serialize, Deserialize)] +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct NpmResolutionPackage { pub id: NpmPackageId, pub dist: NpmPackageVersionDistInfo, From c2899a6231b0ae3d2932b86e898c2695cc0bc154 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Thu, 3 Nov 2022 18:06:44 -0400 Subject: [PATCH 11/43] More progress, but test is still failing --- cli/npm/resolution/graph.rs | 537 ++++++++++++++++++++++++++---------- 1 file changed, 388 insertions(+), 149 deletions(-) diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index 6c8b5676e92c57..11354ef64e10f6 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -1,6 +1,7 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. use std::collections::HashMap; +use std::collections::HashSet; use std::collections::VecDeque; use std::sync::Arc; @@ -35,9 +36,45 @@ enum NodeParent { #[derive(Debug)] struct Node { pub id: NpmPackageId, - pub parents: HashMap, + pub parents: HashMap>, pub children: HashMap, - pub unresolved_peers: Vec, + pub unresolved_deps: Vec, +} + +impl Node { + pub fn add_parent(&mut self, specifier: String, parent: NodeParent) { + eprintln!( + "ADDING parent {}: {} {} {}", + self.id.as_serializable_name(), + specifier, + match &parent { + NodeParent::Node(n) => n.as_serializable_name(), + NodeParent::Req(req) => req.to_string(), + }, + self.parents.entry(specifier.clone()).or_default().len(), + ); + self.parents.entry(specifier).or_default().insert(parent); + } + + pub fn remove_parent(&mut self, specifier: &String, parent: &NodeParent) { + eprintln!( + "REMOVING parent {}: {} {} {}", + self.id.as_serializable_name(), + specifier, + match parent { + NodeParent::Node(n) => n.as_serializable_name(), + NodeParent::Req(req) => req.to_string(), + }, + self.parents.entry(specifier.clone()).or_default().len(), + ); + if let Some(parents) = self.parents.get_mut(specifier) { + parents.remove(parent); + if parents.is_empty() { + drop(parents); + self.parents.remove(specifier); + } + } + } } #[derive(Debug, Default)] @@ -58,7 +95,7 @@ impl Graph { pub fn fill_with_snapshot(&mut self, snapshot: &NpmResolutionSnapshot) { for (package_req, id) in &snapshot.package_reqs { let node = self.fill_for_id_with_snapshot(id, snapshot); - (*node).lock().parents.insert( + (*node).lock().add_parent( package_req.to_string(), NodeParent::Req(package_req.clone()), ); @@ -77,7 +114,7 @@ impl Graph { id: id.clone(), parents: Default::default(), children: Default::default(), - unresolved_peers: Default::default(), + unresolved_deps: Default::default(), })); self .packages_by_name @@ -104,16 +141,32 @@ impl Graph { } fn borrow_node(&self, id: &NpmPackageId) -> MutexGuard { - (**self.packages.get(id).unwrap()).lock() + (**self.packages.get(id).unwrap_or_else(|| { + panic!( + "could not find id {} in the tree", + id.as_serializable_name() + ) + })) + .lock() } fn forget_orphan(&mut self, node_id: &NpmPackageId) { if let Some(node) = self.packages.remove(node_id) { + eprintln!( + "REMAINING: {:?}", + self + .packages + .values() + .map(|n| n.lock().id.as_serializable_name()) + .collect::>() + ); let node = (*node).lock(); + eprintln!("FORGOT: {}", node.id.as_serializable_name()); assert_eq!(node.parents.len(), 0); + let parent = NodeParent::Node(node.id.clone()); for (specifier, child_id) in &node.children { - let mut child = (**self.packages.get(child_id).unwrap()).lock(); - child.parents.remove(specifier); + let mut child = self.borrow_node(child_id); + child.remove_parent(specifier, &parent); if child.parents.is_empty() { drop(child); // stop borrowing from self self.forget_orphan(&child_id); @@ -122,21 +175,40 @@ impl Graph { } } + fn set_child_parent( + &mut self, + specifier: &str, + child: &Mutex, + parent: &NodeParent, + ) { + match parent { + NodeParent::Node(parent_id) => { + self.set_child_parent_node(&specifier, &child, &parent_id); + } + NodeParent::Req(package_req) => { + let mut node = (*child).lock(); + node.add_parent(specifier.to_string(), parent.clone()); + self + .package_reqs + .insert(package_req.clone(), node.id.clone()); + } + } + } + fn set_child_parent_node( &mut self, specifier: &str, - child: &Arc>, + child: &Mutex, parent_id: &NpmPackageId, ) { - let mut child = (**child).lock(); + let mut child = (*child).lock(); let mut parent = (**self.packages.get(parent_id).unwrap()).lock(); debug_assert_ne!(parent.id, child.id); parent .children .insert(specifier.to_string(), child.id.clone()); child - .parents - .insert(specifier.to_string(), NodeParent::Node(parent.id.clone())); + .add_parent(specifier.to_string(), NodeParent::Node(parent.id.clone())); } pub async fn into_snapshot( @@ -151,7 +223,7 @@ impl Graph { .unwrap() // todo(THIS PR): don't unwrap here .dist; let node = node.lock(); - assert_eq!(node.unresolved_peers.len(), 0); + assert_eq!(node.unresolved_deps.len(), 0); packages.insert( id.clone(), NpmResolutionPackage { @@ -172,9 +244,7 @@ impl Graph { pub struct GraphDependencyResolver<'a, TNpmRegistryApi: NpmRegistryApi> { graph: &'a mut Graph, api: &'a TNpmRegistryApi, - pending_dependencies: VecDeque<(NpmPackageId, Vec)>, - pending_peer_dependencies: - VecDeque<((String, NpmPackageId), NpmDependencyEntry, NpmPackageInfo)>, + pending_unresolved_nodes: VecDeque>>, } impl<'a, TNpmRegistryApi: NpmRegistryApi> @@ -184,8 +254,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> Self { graph, api, - pending_dependencies: Default::default(), - pending_peer_dependencies: Default::default(), + pending_unresolved_nodes: Default::default(), } } @@ -243,36 +312,15 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> pub fn resolve_npm_package_req( &mut self, package_req: &NpmPackageReq, - info: NpmPackageInfo, + package_info: NpmPackageInfo, ) -> Result<(), AnyError> { - // inspect if there's a match in the list of current packages and otherwise - // fall back to looking at the registry - let version_and_info = self.resolve_best_package_version_and_info( + let _ = self.resolve_node_from_info( + &package_req.to_string(), &package_req.name, package_req, - info, + package_info, + &NodeParent::Req(package_req.clone()), )?; - let id = NpmPackageId { - name: package_req.name.clone(), - version: version_and_info.version.clone(), - peer_dependencies: Vec::new(), - }; - let node = self.graph.get_or_create_for_id(&id).1; - (*node).lock().parents.insert( - package_req.to_string(), - NodeParent::Req(package_req.clone()), - ); - self - .graph - .package_reqs - .insert(package_req.clone(), id.clone()); - - let dependencies = version_and_info - .info - .dependencies_as_entries() - .with_context(|| format!("npm package: {}", id))?; - - self.pending_dependencies.push_back((id, dependencies)); Ok(()) } @@ -282,45 +330,62 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> package_info: NpmPackageInfo, parent_id: &NpmPackageId, ) -> Result<(), AnyError> { - let version_and_info = self.resolve_best_package_version_and_info( + let _ = self.resolve_node_from_info( + &entry.bare_specifier, &entry.name, &entry.version_req, package_info, + &NodeParent::Node(parent_id.clone()), )?; + Ok(()) + } + fn resolve_node_from_info( + &mut self, + specifier: &str, + name: &str, + version_matcher: &impl NpmVersionMatcher, + package_info: NpmPackageInfo, + parent: &NodeParent, + ) -> Result>, AnyError> { + let version_and_info = self.resolve_best_package_version_and_info( + name, + version_matcher, + package_info, + )?; let id = NpmPackageId { - name: entry.name.clone(), + name: name.to_string(), version: version_and_info.version.clone(), peer_dependencies: Vec::new(), }; let (created, node) = self.graph.get_or_create_for_id(&id); - self - .graph - .set_child_parent_node(&entry.bare_specifier, &node, &parent_id); - if created { - // inspect the dependencies of the package - let dependencies = version_and_info + eprintln!("RESOLVED: {}", id.as_serializable_name()); + self.pending_unresolved_nodes.push_back(node.clone()); + let mut node = (*node).lock(); + node.unresolved_deps = version_and_info .info .dependencies_as_entries() - .with_context(|| { - format!("npm package: {}@{}", &entry.name, version_and_info.version) - })?; - - self.pending_dependencies.push_back((id, dependencies)); + .with_context(|| format!("npm package: {}", id))?; } - Ok(()) + + self.graph.set_child_parent(&specifier, &node, &parent); + Ok(node) } pub async fn resolve_pending(&mut self) -> Result<(), AnyError> { - while !self.pending_dependencies.is_empty() - || !self.pending_peer_dependencies.is_empty() - { + while !self.pending_unresolved_nodes.is_empty() { // now go down through the dependencies by tree depth - while let Some((parent_id, mut deps)) = - self.pending_dependencies.pop_front() - { - // ensure name alphabetical and then version descending + while let Some(parent_node) = self.pending_unresolved_nodes.pop_front() { + let (mut parent_id, mut deps) = { + let mut parent_node = parent_node.lock(); + ( + parent_node.id.clone(), + parent_node.unresolved_deps.drain(..).collect::>(), + ) + }; + // Ensure name alphabetical and then version descending + // so these are resolved in that order deps.sort(); // cache all the dependencies' registry infos in parallel if should @@ -353,30 +418,20 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> } NpmDependencyEntryKind::Peer | NpmDependencyEntryKind::OptionalPeer => { - self.pending_peer_dependencies.push_back(( - (dep.bare_specifier.clone(), parent_id.clone()), - dep, + let maybe_new_parent_id = self.resolve_peer_dep( + &dep.bare_specifier, + &parent_id, + &dep, package_info, - )); + vec![], + )?; + if let Some(new_parent_id) = maybe_new_parent_id { + parent_id = new_parent_id; + } } } } } - - // if a peer dependency was found, resolve one of them then go back - // to resolving dependenices above before moving on to the next - // peer dependency - if let Some(((specifier, parent_id), peer_dep, peer_package_info)) = - self.pending_peer_dependencies.pop_front() - { - self.resolve_peer_dep( - &specifier, - &parent_id, - &peer_dep, - peer_package_info, - vec![], - )?; - } } Ok(()) } @@ -384,86 +439,132 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> fn resolve_peer_dep( &mut self, specifier: &str, - child_id: &NpmPackageId, + parent_id: &NpmPackageId, peer_dep: &NpmDependencyEntry, peer_package_info: NpmPackageInfo, - mut path: Vec<(String, NpmPackageId)>, - ) -> Result<(), AnyError> { + mut path: Vec, + ) -> Result, AnyError> { + eprintln!("[resolve_peer_dep]: Specifier: {}", specifier); // Peer dependencies are resolved based on its ancestors' siblings. - // If not found, then it resolves based on the version requirement if non-optional - let parents = self.graph.borrow_node(&child_id).parents.clone(); - path.push((specifier.to_string(), child_id.clone())); - for (specifier, parent) in parents { - let children = match &parent { - NodeParent::Node(parent_node_id) => { - self.graph.borrow_node(parent_node_id).children.clone() + // If not found, then it resolves based on the version requirement if non-optional. + let mut pending_ancestors = VecDeque::new(); // go up the tree by depth + path.push(specifier.to_string()); + eprintln!("[resolve_peer_dep]: Path: {:?}", path); + for (specifier, grand_parents) in + self.graph.borrow_node(&parent_id).parents.clone() + { + for grand_parent in grand_parents { + pending_ancestors.push_back(( + specifier.clone(), + grand_parent, + path.clone(), + )); + } + } + + while let Some((specifier, ancestor, path)) = pending_ancestors.pop_front() + { + let children = match &ancestor { + NodeParent::Node(ancestor_node_id) => { + let ancestor = self.graph.borrow_node(ancestor_node_id); + let mut new_path = path.clone(); + new_path.push(specifier.to_string()); + for (specifier, parents) in &ancestor.parents { + for parent in parents { + pending_ancestors.push_back(( + specifier.clone(), + parent.clone(), + new_path.clone(), + )); + } + } + ancestor.children.clone() + } + NodeParent::Req(_) => { + // in this case, the parent is the root so the children are all the package requirements + self + .graph + .package_reqs + .iter() + .map(|(req, id)| (req.to_string(), id.clone())) + .collect::>() } - NodeParent::Req(parent_req) => self - .graph - .package_reqs - .iter() - .filter(|(req, _)| *req == parent_req) - .map(|(req, id)| (req.to_string(), id.clone())) - .collect::>(), }; - // todo(THIS PR): don't we need to use the specifier here? - for (child_specifier, child_id) in children { + for child_id in children.values() { if child_id.name == peer_dep.name && peer_dep.version_req.satisfies(&child_id.version) { + eprintln!("MATCHED: {}", child_id.as_serializable_name()); + eprintln!("PATH: {:?}", path); // go down the descendants creating a new path - match &parent { + match &ancestor { NodeParent::Node(node_id) => { let parents = self.graph.borrow_node(node_id).parents.clone(); - self.set_new_peer_dep( + return Ok(Some(self.set_new_peer_dep( parents, &specifier, node_id, &child_id, path, - ); - return Ok(()); + ))); } NodeParent::Req(req) => { let old_id = self.graph.package_reqs.get(&req).unwrap().clone(); - self.set_new_peer_dep( + return Ok(Some(self.set_new_peer_dep( HashMap::from([( req.to_string(), - NodeParent::Req(req.clone()), + HashSet::from([NodeParent::Req(req.clone())]), )]), &specifier, &old_id, &child_id, path, - ); - return Ok(()); + ))); } } } } } - // at this point it means we didn't find anything by searching the ancestor siblings, - // so we need to resolve based on the package info + // We didn't find anything by searching the ancestor siblings, so we need + // to resolve based on the package info and will treat this just like any + // other dependency when not optional if !peer_dep.kind.is_optional() { - self.analyze_dependency(&peer_dep, peer_package_info, &child_id)?; + self.analyze_dependency(&peer_dep, peer_package_info, &parent_id)?; } - Ok(()) + + Ok(None) } fn set_new_peer_dep( &mut self, - previous_parents: HashMap, + previous_parents: HashMap>, specifier: &str, node_id: &NpmPackageId, peer_dep_id: &NpmPackageId, - mut path: Vec<(String, NpmPackageId)>, - ) { + mut path: Vec, + ) -> NpmPackageId { let old_id = node_id; + if old_id.peer_dependencies.contains(peer_dep_id) { + // the parent has already resolved to using this peer dependency + // via some other path, so we don't need to update its ids, + // but instead only make a link to it + let node = self.graph.get_or_create_for_id(peer_dep_id).1; + self.graph.set_child_parent_node(&specifier, &node, &old_id); + return old_id.clone(); + } + let mut new_id = old_id.clone(); new_id.peer_dependencies.push(peer_dep_id.clone()); + eprintln!("NEW ID: {}", new_id.as_serializable_name()); + eprintln!("PATH: {:?}", path); // remove the previous parents from the old node let old_node_children = { let mut old_node = self.graph.borrow_node(old_id); - for previous_parent in previous_parents.keys() { - old_node.parents.remove(previous_parent); + for (specifier, parents) in &previous_parents { + for parent in parents { + old_node.remove_parent(specifier, parent); + } } + // This should never have elements because we should always + // be at the bottom of the tree when evaluating dependencies + assert!(old_node.unresolved_deps.is_empty()); old_node.children.clone() }; @@ -471,44 +572,44 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> // update the previous parent to point to the new node // and this node to point at those parents - { - let mut new_node = (*new_node).lock(); - for (specifier, parent) in previous_parents { - match &parent { - NodeParent::Node(parent_id) => { - let mut parent = - (**self.graph.packages.get(parent_id).unwrap()).lock(); - parent.children.insert(specifier.clone(), new_id.clone()); - } - NodeParent::Req(req) => { - self.graph.package_reqs.insert(req.clone(), new_id.clone()); - } - } - new_node.parents.insert(specifier, parent); + for (specifier, parents) in previous_parents { + for parent in parents { + self.graph.set_child_parent(&specifier, &new_node, &parent); } - - // now add the previous children to this node - new_node.children.extend(old_node_children.clone()); } - for (specifier, child_id) in old_node_children { + // now add the previous children to this node + let new_id_as_parent = NodeParent::Node(new_id.clone()); + for (specifier, child_id) in &old_node_children { + let child = self.graph.packages.get(child_id).unwrap().clone(); self .graph - .borrow_node(&child_id) - .parents - .insert(specifier, NodeParent::Node(new_id.clone())); + .set_child_parent(&specifier, &child, &new_id_as_parent); } if created { // continue going down the path - if let Some((next_specifier, next_node_id)) = path.pop() { - self.set_new_peer_dep( - HashMap::from([(specifier.to_string(), NodeParent::Node(new_id))]), - &next_specifier, - &next_node_id, - peer_dep_id, - path, - ); + if let Some(next_specifier) = path.pop() { + if path.is_empty() { + // this means we're at the peer dependency now + assert!(!old_node_children.contains_key(&next_specifier)); + let node = self.graph.get_or_create_for_id(&peer_dep_id).1; + self + .graph + .set_child_parent_node(&next_specifier, &node, &new_id); + } else { + let next_node_id = old_node_children.get(&next_specifier).unwrap(); + self.set_new_peer_dep( + HashMap::from([( + specifier.to_string(), + HashSet::from([NodeParent::Node(new_id.clone())]), + )]), + &next_specifier, + &next_node_id, + peer_dep_id, + path, + ); + } } } @@ -520,6 +621,8 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> self.graph.forget_orphan(old_id); } } + + return new_id; } } @@ -570,7 +673,7 @@ fn get_resolved_package_version_and_info( } } } else { - bail!("Could not find dist-tag '{}'.", tag,) + bail!("Could not find dist-tag '{}'.", tag) } } else { for (_, version_info) in info.versions.into_iter() { @@ -728,6 +831,142 @@ mod test { ); } + #[tokio::test] + async fn resolve_with_peer_deps_top_tree() { + let api = TestNpmRegistryApi::default(); + api.ensure_package_version("package-a", "1.0.0"); + api.ensure_package_version("package-b", "2.0.0"); + api.ensure_package_version("package-c", "3.0.0"); + api.ensure_package_version("package-peer", "4.0.0"); + api.ensure_package_version("package-peer", "4.1.0"); + api.add_dependency(("package-a", "1.0.0"), ("package-b", "^2")); + api.add_dependency(("package-a", "1.0.0"), ("package-c", "^3")); + api.add_peer_dependency(("package-b", "2.0.0"), ("package-peer", "4")); + api.add_peer_dependency(("package-c", "3.0.0"), ("package-peer", "*")); + + let packages = run_resolver_and_get_output( + api, + // the peer dependency is specified here at the top of the tree + // so it should resolve to 4.0.0 instead of 4.1.0 + vec!["npm:package-a@1", "npm:package-peer@4.0.0"], + ) + .await; + assert_eq!( + packages, + vec![ + NpmResolutionPackage { + id: NpmPackageId::deserialize_name( + "package-a@1.0.0_package-peer@4.0.0" + ) + .unwrap(), + dependencies: HashMap::from([ + ( + "package-b".to_string(), + NpmPackageId::deserialize_name( + "package-b@2.0.0_package-peer@4.0.0" + ) + .unwrap(), + ), + ( + "package-c".to_string(), + NpmPackageId::deserialize_name( + "package-c@3.0.0_package-peer@4.0.0" + ) + .unwrap(), + ), + ]), + dist: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name( + "package-b@2.0.0_package-peer@4.0.0" + ) + .unwrap(), + dist: Default::default(), + dependencies: HashMap::from([( + "package-peer".to_string(), + NpmPackageId::deserialize_name("package-peer@4.0.0").unwrap(), + )]) + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name( + "package-c@3.0.0_package-peer@4.0.0" + ) + .unwrap(), + dist: Default::default(), + dependencies: HashMap::from([( + "package-peer".to_string(), + NpmPackageId::deserialize_name("package-peer@4.0.0").unwrap(), + )]) + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-peer@4.0.0").unwrap(), + dist: Default::default(), + dependencies: Default::default(), + }, + ] + ); + } + + #[tokio::test] + async fn resolve_with_peer_deps_auto_resolved() { + // in this case, the peer dependency is not found in the tree + // so it's auto-resolved based on the registry + let api = TestNpmRegistryApi::default(); + api.ensure_package_version("package-a", "1.0.0"); + api.ensure_package_version("package-b", "2.0.0"); + api.ensure_package_version("package-c", "3.0.0"); + api.ensure_package_version("package-peer", "4.0.0"); + api.ensure_package_version("package-peer", "4.1.0"); + api.add_dependency(("package-a", "1.0.0"), ("package-b", "^2")); + api.add_dependency(("package-a", "1.0.0"), ("package-c", "^3")); + api.add_peer_dependency(("package-b", "2.0.0"), ("package-peer", "4")); + api.add_peer_dependency(("package-c", "3.0.0"), ("package-peer", "*")); + + let packages = + run_resolver_and_get_output(api, vec!["npm:package-a@1"]).await; + assert_eq!( + packages, + vec![ + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-a@1.0.0").unwrap(), + dependencies: HashMap::from([ + ( + "package-b".to_string(), + NpmPackageId::deserialize_name("package-b@2.0.0").unwrap(), + ), + ( + "package-c".to_string(), + NpmPackageId::deserialize_name("package-c@3.0.0").unwrap(), + ), + ]), + dist: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-b@2.0.0").unwrap(), + dist: Default::default(), + dependencies: HashMap::from([( + "package-peer".to_string(), + NpmPackageId::deserialize_name("package-peer@4.1.0").unwrap(), + )]) + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-c@3.0.0").unwrap(), + dist: Default::default(), + dependencies: HashMap::from([( + "package-peer".to_string(), + NpmPackageId::deserialize_name("package-peer@4.1.0").unwrap(), + )]) + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-peer@4.1.0").unwrap(), + dist: Default::default(), + dependencies: Default::default(), + }, + ] + ); + } + async fn run_resolver_and_get_output( api: TestNpmRegistryApi, reqs: Vec<&str>, From 53385234e64007bb8f3a812bd3f7f46a73c8adb1 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Fri, 4 Nov 2022 11:40:09 -0400 Subject: [PATCH 12/43] Working basic resolution... now to add more complex tests. --- cli/npm/resolution/graph.rs | 178 +++++++++++++++++++++--------------- 1 file changed, 105 insertions(+), 73 deletions(-) diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index 11354ef64e10f6..5651729c4a6821 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -44,7 +44,7 @@ struct Node { impl Node { pub fn add_parent(&mut self, specifier: String, parent: NodeParent) { eprintln!( - "ADDING parent {}: {} {} {}", + "ADDING parent to {}: {} {} {}", self.id.as_serializable_name(), specifier, match &parent { @@ -56,16 +56,16 @@ impl Node { self.parents.entry(specifier).or_default().insert(parent); } - pub fn remove_parent(&mut self, specifier: &String, parent: &NodeParent) { + pub fn remove_parent(&mut self, specifier: &str, parent: &NodeParent) { eprintln!( - "REMOVING parent {}: {} {} {}", + "REMOVING parent from {}: {} {} {}", self.id.as_serializable_name(), specifier, match parent { NodeParent::Node(n) => n.as_serializable_name(), NodeParent::Req(req) => req.to_string(), }, - self.parents.entry(specifier.clone()).or_default().len(), + self.parents.entry(specifier.to_string()).or_default().len(), ); if let Some(parents) = self.parents.get_mut(specifier) { parents.remove(parent); @@ -167,6 +167,7 @@ impl Graph { for (specifier, child_id) in &node.children { let mut child = self.borrow_node(child_id); child.remove_parent(specifier, &parent); + eprintln!("CHILD PARENTS: {:?}", child.parents); if child.parents.is_empty() { drop(child); // stop borrowing from self self.forget_orphan(&child_id); @@ -211,6 +212,31 @@ impl Graph { .add_parent(specifier.to_string(), NodeParent::Node(parent.id.clone())); } + fn remove_child_parent( + &mut self, + specifier: &str, + child_id: &NpmPackageId, + parent: &NodeParent, + ) { + match parent { + NodeParent::Node(parent_id) => { + eprintln!("PARENT: {}", parent_id.as_serializable_name()); + eprintln!("SPECIFIER: {}", specifier); + let mut node = self.borrow_node(parent_id); + if let Some(removed_child_id) = node.children.remove(specifier) { + assert_eq!(removed_child_id, *child_id); + } + } + NodeParent::Req(req) => { + assert_eq!(req.to_string(), specifier); + if let Some(removed_child_id) = self.package_reqs.remove(req) { + assert_eq!(removed_child_id, *child_id); + } + } + } + self.borrow_node(child_id).remove_parent(specifier, parent); + } + pub async fn into_snapshot( self, api: &impl NpmRegistryApi, @@ -500,9 +526,9 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> match &ancestor { NodeParent::Node(node_id) => { let parents = self.graph.borrow_node(node_id).parents.clone(); - return Ok(Some(self.set_new_peer_dep( - parents, &specifier, node_id, &child_id, path, - ))); + return Ok(Some( + self.set_new_peer_dep(parents, node_id, &child_id, path), + )); } NodeParent::Req(req) => { let old_id = self.graph.package_reqs.get(&req).unwrap().clone(); @@ -511,7 +537,6 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> req.to_string(), HashSet::from([NodeParent::Req(req.clone())]), )]), - &specifier, &old_id, &child_id, path, @@ -535,86 +560,93 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> fn set_new_peer_dep( &mut self, previous_parents: HashMap>, - specifier: &str, node_id: &NpmPackageId, peer_dep_id: &NpmPackageId, mut path: Vec, ) -> NpmPackageId { + eprintln!("PREVIOUS PARENTS: {:?}", previous_parents); let old_id = node_id; - if old_id.peer_dependencies.contains(peer_dep_id) { - // the parent has already resolved to using this peer dependency - // via some other path, so we don't need to update its ids, - // but instead only make a link to it - let node = self.graph.get_or_create_for_id(peer_dep_id).1; - self.graph.set_child_parent_node(&specifier, &node, &old_id); - return old_id.clone(); - } - - let mut new_id = old_id.clone(); - new_id.peer_dependencies.push(peer_dep_id.clone()); - eprintln!("NEW ID: {}", new_id.as_serializable_name()); - eprintln!("PATH: {:?}", path); - // remove the previous parents from the old node - let old_node_children = { - let mut old_node = self.graph.borrow_node(old_id); - for (specifier, parents) in &previous_parents { - for parent in parents { - old_node.remove_parent(specifier, parent); - } - } - // This should never have elements because we should always - // be at the bottom of the tree when evaluating dependencies - assert!(old_node.unresolved_deps.is_empty()); - old_node.children.clone() - }; - - let (created, new_node) = self.graph.get_or_create_for_id(&new_id); + let (new_id, old_node_children) = + if old_id.peer_dependencies.contains(peer_dep_id) { + // the parent has already resolved to using this peer dependency + // via some other path, so we don't need to update its ids, + // but instead only make a link to it + ( + old_id.clone(), + self.graph.borrow_node(old_id).children.clone(), + ) + } else { + let mut new_id = old_id.clone(); + new_id.peer_dependencies.push(peer_dep_id.clone()); + eprintln!("NEW ID: {}", new_id.as_serializable_name()); + eprintln!("PATH: {:?}", path); + // remove the previous parents from the old node + let old_node_children = { + for (specifier, parents) in &previous_parents { + for parent in parents { + self.graph.remove_child_parent(&specifier, old_id, parent); + } + } + // This should never have elements because the bottom of the + // tree should be the only place that has any + let old_node = self.graph.borrow_node(old_id); + assert!(old_node.unresolved_deps.is_empty()); + old_node.children.clone() + }; - // update the previous parent to point to the new node - // and this node to point at those parents - for (specifier, parents) in previous_parents { - for parent in parents { - self.graph.set_child_parent(&specifier, &new_node, &parent); - } - } + let (_, new_node) = self.graph.get_or_create_for_id(&new_id); - // now add the previous children to this node - let new_id_as_parent = NodeParent::Node(new_id.clone()); - for (specifier, child_id) in &old_node_children { - let child = self.graph.packages.get(child_id).unwrap().clone(); - self - .graph - .set_child_parent(&specifier, &child, &new_id_as_parent); - } + // update the previous parent to point to the new node + // and this node to point at those parents + for (specifier, parents) in previous_parents { + for parent in parents { + self.graph.set_child_parent(&specifier, &new_node, &parent); + } + } - if created { - // continue going down the path - if let Some(next_specifier) = path.pop() { - if path.is_empty() { - // this means we're at the peer dependency now - assert!(!old_node_children.contains_key(&next_specifier)); - let node = self.graph.get_or_create_for_id(&peer_dep_id).1; + // now add the previous children to this node + let new_id_as_parent = NodeParent::Node(new_id.clone()); + for (specifier, child_id) in &old_node_children { + let child = self.graph.packages.get(child_id).unwrap().clone(); self .graph - .set_child_parent_node(&next_specifier, &node, &new_id); - } else { - let next_node_id = old_node_children.get(&next_specifier).unwrap(); - self.set_new_peer_dep( - HashMap::from([( - specifier.to_string(), - HashSet::from([NodeParent::Node(new_id.clone())]), - )]), - &next_specifier, - &next_node_id, - peer_dep_id, - path, - ); + .set_child_parent(&specifier, &child, &new_id_as_parent); } + (new_id, old_node_children) + }; + + // continue going down the path + if let Some(next_specifier) = path.pop() { + if path.is_empty() { + // this means we're at the peer dependency now + assert!(!old_node_children.contains_key(&next_specifier)); + let node = self.graph.get_or_create_for_id(&peer_dep_id).1; + self + .graph + .set_child_parent_node(&next_specifier, &node, &new_id); + } else { + let next_node_id = old_node_children.get(&next_specifier).unwrap(); + // if new_id != *old_id { + // self.graph.remove_child_parent_node( + // &next_specifier, + // next_node_id, + // old_id, + // ); + // } + self.set_new_peer_dep( + HashMap::from([( + next_specifier.to_string(), + HashSet::from([NodeParent::Node(new_id.clone())]), + )]), + &next_node_id, + peer_dep_id, + path, + ); } } // forget the old node at this point if it has no parents - { + if new_id != *old_id { let old_node = self.graph.borrow_node(old_id); if old_node.parents.is_empty() { drop(old_node); // stop borrowing From 13d4f01aa1fcc88d00eaf21427a6f28bd6d95e54 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Fri, 4 Nov 2022 12:53:29 -0400 Subject: [PATCH 13/43] Tests for resolving ancestor sibling not at the top of the tree. --- cli/npm/resolution/graph.rs | 122 +++++++++++++++++++++++++++++++----- 1 file changed, 108 insertions(+), 14 deletions(-) diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index 5651729c4a6821..f7d8a5f106d87a 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -479,29 +479,22 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> for (specifier, grand_parents) in self.graph.borrow_node(&parent_id).parents.clone() { + let mut path = path.clone(); + path.push(specifier.clone()); for grand_parent in grand_parents { - pending_ancestors.push_back(( - specifier.clone(), - grand_parent, - path.clone(), - )); + pending_ancestors.push_back((grand_parent, path.clone())); } } - while let Some((specifier, ancestor, path)) = pending_ancestors.pop_front() - { + while let Some((ancestor, path)) = pending_ancestors.pop_front() { let children = match &ancestor { NodeParent::Node(ancestor_node_id) => { let ancestor = self.graph.borrow_node(ancestor_node_id); - let mut new_path = path.clone(); - new_path.push(specifier.to_string()); for (specifier, parents) in &ancestor.parents { + let mut new_path = path.clone(); + new_path.push(specifier.to_string()); for parent in parents { - pending_ancestors.push_back(( - specifier.clone(), - parent.clone(), - new_path.clone(), - )); + pending_ancestors.push_back((parent.clone(), new_path.clone())); } } ancestor.children.clone() @@ -532,6 +525,10 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> } NodeParent::Req(req) => { let old_id = self.graph.package_reqs.get(&req).unwrap().clone(); + // todo: should refactor all this code to have a node for everything, but an id + // that could be `Root` or `NpmPackageId` maybe? + let mut path = path; + path.pop(); // this path will be at the root, but there's no "node" for that return Ok(Some(self.set_new_peer_dep( HashMap::from([( req.to_string(), @@ -617,6 +614,11 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> // continue going down the path if let Some(next_specifier) = path.pop() { + eprintln!( + "Next specifier: {}, peer dep id: {}", + next_specifier, + peer_dep_id.as_serializable_name() + ); if path.is_empty() { // this means we're at the peer dependency now assert!(!old_node_children.contains_key(&next_specifier)); @@ -940,6 +942,98 @@ mod test { ); } + #[tokio::test] + async fn resolve_with_peer_deps_ancestor_sibling_not_top_tree() { + let api = TestNpmRegistryApi::default(); + api.ensure_package_version("package-0", "1.1.1"); + api.ensure_package_version("package-a", "1.0.0"); + api.ensure_package_version("package-b", "2.0.0"); + api.ensure_package_version("package-c", "3.0.0"); + api.ensure_package_version("package-peer", "4.0.0"); + api.ensure_package_version("package-peer", "4.1.0"); + api.add_dependency(("package-0", "1.1.1"), ("package-a", "1")); + api.add_dependency(("package-a", "1.0.0"), ("package-b", "^2")); + api.add_dependency(("package-a", "1.0.0"), ("package-c", "^3")); + // the peer dependency is specified here as a sibling of "a" and "b" + // so it should resolve to 4.0.0 instead of 4.1.0 + api.add_dependency(("package-a", "1.0.0"), ("package-peer", "4.0.0")); + api.add_peer_dependency(("package-b", "2.0.0"), ("package-peer", "4")); + api.add_peer_dependency(("package-c", "3.0.0"), ("package-peer", "*")); + + let packages = + run_resolver_and_get_output(api, vec!["npm:package-0@1.1.1"]).await; + assert_eq!( + packages, + vec![ + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-0@1.1.1").unwrap(), + dependencies: HashMap::from([( + "package-a".to_string(), + NpmPackageId::deserialize_name( + "package-a@1.0.0_package-peer@4.0.0" + ) + .unwrap(), + ),]), + dist: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name( + "package-a@1.0.0_package-peer@4.0.0" + ) + .unwrap(), + dependencies: HashMap::from([ + ( + "package-b".to_string(), + NpmPackageId::deserialize_name( + "package-b@2.0.0_package-peer@4.0.0" + ) + .unwrap(), + ), + ( + "package-c".to_string(), + NpmPackageId::deserialize_name( + "package-c@3.0.0_package-peer@4.0.0" + ) + .unwrap(), + ), + ( + "package-peer".to_string(), + NpmPackageId::deserialize_name("package-peer@4.0.0").unwrap(), + ), + ]), + dist: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name( + "package-b@2.0.0_package-peer@4.0.0" + ) + .unwrap(), + dist: Default::default(), + dependencies: HashMap::from([( + "package-peer".to_string(), + NpmPackageId::deserialize_name("package-peer@4.0.0").unwrap(), + )]) + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name( + "package-c@3.0.0_package-peer@4.0.0" + ) + .unwrap(), + dist: Default::default(), + dependencies: HashMap::from([( + "package-peer".to_string(), + NpmPackageId::deserialize_name("package-peer@4.0.0").unwrap(), + )]) + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-peer@4.0.0").unwrap(), + dist: Default::default(), + dependencies: Default::default(), + }, + ] + ); + } + #[tokio::test] async fn resolve_with_peer_deps_auto_resolved() { // in this case, the peer dependency is not found in the tree From d1867f45ab63391197ed0260f86bf40f8924b35a Mon Sep 17 00:00:00 2001 From: David Sherret Date: Fri, 4 Nov 2022 15:17:08 -0400 Subject: [PATCH 14/43] More tests. --- cli/npm/resolution/graph.rs | 143 +++++++++++++++++++++++++++++++++++- 1 file changed, 140 insertions(+), 3 deletions(-) diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index f7d8a5f106d87a..9ce5b35e698080 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -29,10 +29,14 @@ use super::NpmVersionMatcher; #[derive(Debug, Clone, PartialEq, Eq, Hash)] enum NodeParent { + /// These are top of the graph npm package requirements + /// as specified in Deno code. Req(NpmPackageReq), + /// A reference to another node, which is a resolved package. Node(NpmPackageId), } +/// A resolved package in the resolution graph. #[derive(Debug)] struct Node { pub id: NpmPackageId, @@ -525,10 +529,8 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> } NodeParent::Req(req) => { let old_id = self.graph.package_reqs.get(&req).unwrap().clone(); - // todo: should refactor all this code to have a node for everything, but an id - // that could be `Root` or `NpmPackageId` maybe? let mut path = path; - path.pop(); // this path will be at the root, but there's no "node" for that + path.pop(); // go back down one level return Ok(Some(self.set_new_peer_dep( HashMap::from([( req.to_string(), @@ -1093,6 +1095,141 @@ mod test { ); } + #[tokio::test] + async fn resolve_with_optional_peer_dep_not_resolved() { + // in this case, the peer dependency is not found in the tree + // so it's auto-resolved based on the registry + let api = TestNpmRegistryApi::default(); + api.ensure_package_version("package-a", "1.0.0"); + api.ensure_package_version("package-b", "2.0.0"); + api.ensure_package_version("package-c", "3.0.0"); + api.ensure_package_version("package-peer", "4.0.0"); + api.ensure_package_version("package-peer", "4.1.0"); + api.add_dependency(("package-a", "1.0.0"), ("package-b", "^2")); + api.add_dependency(("package-a", "1.0.0"), ("package-c", "^3")); + api.add_optional_peer_dependency( + ("package-b", "2.0.0"), + ("package-peer", "4"), + ); + api.add_optional_peer_dependency( + ("package-c", "3.0.0"), + ("package-peer", "*"), + ); + + let packages = + run_resolver_and_get_output(api, vec!["npm:package-a@1"]).await; + assert_eq!( + packages, + vec![ + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-a@1.0.0").unwrap(), + dependencies: HashMap::from([ + ( + "package-b".to_string(), + NpmPackageId::deserialize_name("package-b@2.0.0").unwrap(), + ), + ( + "package-c".to_string(), + NpmPackageId::deserialize_name("package-c@3.0.0").unwrap(), + ), + ]), + dist: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-b@2.0.0").unwrap(), + dist: Default::default(), + dependencies: HashMap::new(), + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-c@3.0.0").unwrap(), + dist: Default::default(), + dependencies: HashMap::new(), + }, + ] + ); + } + + #[tokio::test] + async fn resolve_with_optional_peer_found() { + let api = TestNpmRegistryApi::default(); + api.ensure_package_version("package-a", "1.0.0"); + api.ensure_package_version("package-b", "2.0.0"); + api.ensure_package_version("package-c", "3.0.0"); + api.ensure_package_version("package-peer", "4.0.0"); + api.ensure_package_version("package-peer", "4.1.0"); + api.add_dependency(("package-a", "1.0.0"), ("package-b", "^2")); + api.add_dependency(("package-a", "1.0.0"), ("package-c", "^3")); + api.add_optional_peer_dependency( + ("package-b", "2.0.0"), + ("package-peer", "4"), + ); + api.add_optional_peer_dependency( + ("package-c", "3.0.0"), + ("package-peer", "*"), + ); + + let packages = run_resolver_and_get_output( + api, + vec!["npm:package-a@1", "npm:package-peer@4.0.0"], + ) + .await; + assert_eq!( + packages, + vec![ + NpmResolutionPackage { + id: NpmPackageId::deserialize_name( + "package-a@1.0.0_package-peer@4.0.0" + ) + .unwrap(), + dependencies: HashMap::from([ + ( + "package-b".to_string(), + NpmPackageId::deserialize_name( + "package-b@2.0.0_package-peer@4.0.0" + ) + .unwrap(), + ), + ( + "package-c".to_string(), + NpmPackageId::deserialize_name( + "package-c@3.0.0_package-peer@4.0.0" + ) + .unwrap(), + ), + ]), + dist: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name( + "package-b@2.0.0_package-peer@4.0.0" + ) + .unwrap(), + dist: Default::default(), + dependencies: HashMap::from([( + "package-peer".to_string(), + NpmPackageId::deserialize_name("package-peer@4.0.0").unwrap(), + )]) + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name( + "package-c@3.0.0_package-peer@4.0.0" + ) + .unwrap(), + dist: Default::default(), + dependencies: HashMap::from([( + "package-peer".to_string(), + NpmPackageId::deserialize_name("package-peer@4.0.0").unwrap(), + )]) + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-peer@4.0.0").unwrap(), + dist: Default::default(), + dependencies: Default::default(), + }, + ] + ); + } + async fn run_resolver_and_get_output( api: TestNpmRegistryApi, reqs: Vec<&str>, From 6f02b54ec1b78dd4eba5cd4243902e919c795bd0 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 5 Nov 2022 11:29:00 -0400 Subject: [PATCH 15/43] Fix another failing test. --- cli/npm/resolution/graph.rs | 302 ++++++++++++++++++++++++++++++++---- cli/npm/resolution/mod.rs | 2 +- cli/npm/resolvers/local.rs | 2 +- 3 files changed, 271 insertions(+), 35 deletions(-) diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index 9ce5b35e698080..e0f6c878fae1d3 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -42,7 +42,7 @@ struct Node { pub id: NpmPackageId, pub parents: HashMap>, pub children: HashMap, - pub unresolved_deps: Vec, + pub deps: Arc>, } impl Node { @@ -118,7 +118,7 @@ impl Graph { id: id.clone(), parents: Default::default(), children: Default::default(), - unresolved_deps: Default::default(), + deps: Default::default(), })); self .packages_by_name @@ -253,7 +253,6 @@ impl Graph { .unwrap() // todo(THIS PR): don't unwrap here .dist; let node = node.lock(); - assert_eq!(node.unresolved_deps.len(), 0); packages.insert( id.clone(), NpmResolutionPackage { @@ -339,18 +338,19 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> maybe_best_version.cloned() } - pub fn resolve_npm_package_req( + pub fn add_npm_package_req( &mut self, package_req: &NpmPackageReq, package_info: NpmPackageInfo, ) -> Result<(), AnyError> { - let _ = self.resolve_node_from_info( + let node = self.resolve_node_from_info( &package_req.to_string(), &package_req.name, package_req, package_info, &NodeParent::Req(package_req.clone()), )?; + self.pending_unresolved_nodes.push_back(node); Ok(()) } @@ -360,13 +360,14 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> package_info: NpmPackageInfo, parent_id: &NpmPackageId, ) -> Result<(), AnyError> { - let _ = self.resolve_node_from_info( + let node = self.resolve_node_from_info( &entry.bare_specifier, &entry.name, &entry.version_req, package_info, &NodeParent::Node(parent_id.clone()), )?; + self.pending_unresolved_nodes.push_back(node); Ok(()) } @@ -391,12 +392,15 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> let (created, node) = self.graph.get_or_create_for_id(&id); if created { eprintln!("RESOLVED: {}", id.as_serializable_name()); - self.pending_unresolved_nodes.push_back(node.clone()); let mut node = (*node).lock(); - node.unresolved_deps = version_and_info + let mut deps = version_and_info .info .dependencies_as_entries() .with_context(|| format!("npm package: {}", id))?; + // Ensure name alphabetical and then version descending + // so these are resolved in that order + deps.sort(); + node.deps = Arc::new(deps); } self.graph.set_child_parent(&specifier, &node, &parent); @@ -407,16 +411,10 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> while !self.pending_unresolved_nodes.is_empty() { // now go down through the dependencies by tree depth while let Some(parent_node) = self.pending_unresolved_nodes.pop_front() { - let (mut parent_id, mut deps) = { - let mut parent_node = parent_node.lock(); - ( - parent_node.id.clone(), - parent_node.unresolved_deps.drain(..).collect::>(), - ) + let (mut parent_id, deps) = { + let parent_node = parent_node.lock(); + (parent_node.id.clone(), parent_node.deps.clone()) }; - // Ensure name alphabetical and then version descending - // so these are resolved in that order - deps.sort(); // cache all the dependencies' registry infos in parallel if should if !should_sync_download() { @@ -438,16 +436,22 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> } } - // resolve the non-peer dependencies - for dep in deps { + // resolve the dependencies + for dep in deps.iter() { let package_info = self.api.package_info(&dep.name).await?; + eprintln!( + "-- DEPENDENCY: {} ({})", + dep.name, + parent_id.as_serializable_name() + ); match dep.kind { NpmDependencyEntryKind::Dep => { self.analyze_dependency(&dep, package_info, &parent_id)?; } NpmDependencyEntryKind::Peer | NpmDependencyEntryKind::OptionalPeer => { + eprintln!("ANALYZING PEER DEP: {}", dep.name); let maybe_new_parent_id = self.resolve_peer_dep( &dep.bare_specifier, &parent_id, @@ -586,10 +590,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> self.graph.remove_child_parent(&specifier, old_id, parent); } } - // This should never have elements because the bottom of the - // tree should be the only place that has any let old_node = self.graph.borrow_node(old_id); - assert!(old_node.unresolved_deps.is_empty()); old_node.children.clone() }; @@ -625,18 +626,12 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> // this means we're at the peer dependency now assert!(!old_node_children.contains_key(&next_specifier)); let node = self.graph.get_or_create_for_id(&peer_dep_id).1; + self.pending_unresolved_nodes.push_back(node.clone()); self .graph .set_child_parent_node(&next_specifier, &node, &new_id); } else { let next_node_id = old_node_children.get(&next_specifier).unwrap(); - // if new_id != *old_id { - // self.graph.remove_child_parent_node( - // &next_specifier, - // next_node_id, - // old_id, - // ); - // } self.set_new_peer_dep( HashMap::from([( next_specifier.to_string(), @@ -651,7 +646,13 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> // forget the old node at this point if it has no parents if new_id != *old_id { + eprintln!( + "CHANGING ID: {} -> {}", + old_id.as_serializable_name(), + new_id.as_serializable_name() + ); let old_node = self.graph.borrow_node(old_id); + eprintln!("OLD PARENTS: {:?}", old_node.parents); if old_node.parents.is_empty() { drop(old_node); // stop borrowing self.graph.forget_orphan(old_id); @@ -1230,6 +1231,244 @@ mod test { ); } + #[tokio::test] + async fn resolve_nested_peer_deps_auto_resolved() { + let api = TestNpmRegistryApi::default(); + api.ensure_package_version("package-0", "1.0.0"); + api.ensure_package_version("package-peer-a", "2.0.0"); + api.ensure_package_version("package-peer-b", "3.0.0"); + api.add_peer_dependency(("package-0", "1.0.0"), ("package-peer-a", "2")); + api.add_peer_dependency( + ("package-peer-a", "2.0.0"), + ("package-peer-b", "3"), + ); + + let packages = + run_resolver_and_get_output(api, vec!["npm:package-0@1.0"]).await; + assert_eq!( + packages, + vec![ + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-0@1.0.0").unwrap(), + dependencies: HashMap::from([( + "package-peer-a".to_string(), + NpmPackageId::deserialize_name("package-peer-a@2.0.0").unwrap(), + )]), + dist: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-peer-a@2.0.0").unwrap(), + dependencies: HashMap::from([( + "package-peer-b".to_string(), + NpmPackageId::deserialize_name("package-peer-b@3.0.0").unwrap(), + )]), + dist: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-peer-b@3.0.0").unwrap(), + dependencies: HashMap::new(), + dist: Default::default(), + }, + ] + ); + } + + #[tokio::test] + async fn resolve_nested_peer_deps_ancestor_sibling_deps() { + let api = TestNpmRegistryApi::default(); + api.ensure_package_version("package-0", "1.0.0"); + api.ensure_package_version("package-peer-a", "2.0.0"); + api.ensure_package_version("package-peer-b", "3.0.0"); + api.add_dependency(("package-0", "1.0.0"), ("package-peer-b", "*")); + api.add_peer_dependency(("package-0", "1.0.0"), ("package-peer-a", "2")); + api.add_peer_dependency( + ("package-peer-a", "2.0.0"), + ("package-peer-b", "3"), + ); + + let packages = run_resolver_and_get_output( + api, + vec![ + "npm:package-0@1.0", + "npm:package-peer-a@2", + "npm:package-peer-b@3", + ], + ) + .await; + assert_eq!( + packages, + vec![ + NpmResolutionPackage { + id: NpmPackageId::deserialize_name( + "package-0@1.0.0_package-peer-a@2.0.0_package-peer-b@3.0.0" + ) + .unwrap(), + dependencies: HashMap::from([ + ( + "package-peer-a".to_string(), + NpmPackageId::deserialize_name( + "package-peer-a@2.0.0_package-peer-b@3.0.0" + ) + .unwrap(), + ), + ( + "package-peer-b".to_string(), + NpmPackageId::deserialize_name("package-peer-b@3.0.0").unwrap(), + ) + ]), + dist: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name( + "package-peer-a@2.0.0_package-peer-b@3.0.0" + ) + .unwrap(), + dependencies: HashMap::from([( + "package-peer-b".to_string(), + NpmPackageId::deserialize_name("package-peer-b@3.0.0").unwrap(), + )]), + dist: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-peer-b@3.0.0").unwrap(), + dependencies: HashMap::new(), + dist: Default::default(), + }, + ] + ); + } + + #[tokio::test] + async fn resolve_with_peer_deps_multiple() { + let api = TestNpmRegistryApi::default(); + api.ensure_package_version("package-0", "1.1.1"); + api.ensure_package_version("package-a", "1.0.0"); + api.ensure_package_version("package-b", "2.0.0"); + api.ensure_package_version("package-c", "3.0.0"); + api.ensure_package_version("package-d", "3.5.0"); + api.ensure_package_version("package-e", "3.6.0"); + api.ensure_package_version("package-peer-a", "4.0.0"); + api.ensure_package_version("package-peer-a", "4.1.0"); + api.ensure_package_version("package-peer-b", "5.3.0"); + api.ensure_package_version("package-peer-b", "5.4.1"); + api.ensure_package_version("package-peer-c", "6.2.0"); + api.add_dependency(("package-0", "1.1.1"), ("package-a", "1")); + api.add_dependency(("package-a", "1.0.0"), ("package-b", "^2")); + api.add_dependency(("package-a", "1.0.0"), ("package-c", "^3")); + api.add_dependency(("package-a", "1.0.0"), ("package-d", "^3")); + api.add_dependency(("package-a", "1.0.0"), ("package-peer-a", "4.0.0")); + api.add_peer_dependency(("package-b", "2.0.0"), ("package-peer-a", "4")); + api.add_peer_dependency( + ("package-b", "2.0.0"), + ("package-peer-c", "=6.2.0"), + ); + api.add_peer_dependency(("package-c", "3.0.0"), ("package-peer-a", "*")); + api.add_peer_dependency( + ("package-peer-a", "4.0.0"), + ("package-peer-b", "^5.4"), // will be auto-resolved + ); + + let packages = run_resolver_and_get_output( + api, + vec!["npm:package-0@1.1.1", "npm:package-e@3"], + ) + .await; + assert_eq!( + packages, + vec![ + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-0@1.1.1").unwrap(), + dependencies: HashMap::from([( + "package-a".to_string(), + NpmPackageId::deserialize_name( + "package-a@1.0.0_package-peer-a@4.0.0" + ) + .unwrap(), + ),]), + dist: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name( + "package-a@1.0.0_package-peer-a@4.0.0" + ) + .unwrap(), + dependencies: HashMap::from([ + ( + "package-b".to_string(), + NpmPackageId::deserialize_name( + "package-b@2.0.0_package-peer-a@4.0.0" + ) + .unwrap(), + ), + ( + "package-c".to_string(), + NpmPackageId::deserialize_name( + "package-c@3.0.0_package-peer-a@4.0.0" + ) + .unwrap(), + ), + ( + "package-d".to_string(), + NpmPackageId::deserialize_name("package-d@3.5.0").unwrap(), + ), + ( + "package-peer-a".to_string(), + NpmPackageId::deserialize_name("package-peer-a@4.0.0").unwrap(), + ), + ]), + dist: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name( + "package-b@2.0.0_package-peer@4.0.0" + ) + .unwrap(), + dist: Default::default(), + dependencies: HashMap::from([( + "package-peer".to_string(), + NpmPackageId::deserialize_name("package-peer@4.0.0").unwrap(), + )]) + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name( + "package-c@3.0.0_package-peer@4.0.0" + ) + .unwrap(), + dist: Default::default(), + dependencies: HashMap::from([( + "package-peer".to_string(), + NpmPackageId::deserialize_name("package-peer@4.0.0").unwrap(), + )]) + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-d@3.5.0").unwrap(), + dependencies: HashMap::from([]), + dist: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-e@3.6.0").unwrap(), + dependencies: HashMap::from([]), + dist: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-peer-a@4.0.0").unwrap(), + dist: Default::default(), + dependencies: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-peer-b@5.4.1").unwrap(), + dist: Default::default(), + dependencies: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-peer-c@6.2.0").unwrap(), + dist: Default::default(), + dependencies: Default::default(), + }, + ] + ); + } + async fn run_resolver_and_get_output( api: TestNpmRegistryApi, reqs: Vec<&str>, @@ -1240,10 +1479,7 @@ mod test { for req in reqs { let req = NpmPackageReference::from_str(req).unwrap().req; resolver - .resolve_npm_package_req( - &req, - api.package_info(&req.name).await.unwrap(), - ) + .add_npm_package_req(&req, api.package_info(&req.name).await.unwrap()) .unwrap(); } diff --git a/cli/npm/resolution/mod.rs b/cli/npm/resolution/mod.rs index f977d08cf252d7..9433066b792186 100644 --- a/cli/npm/resolution/mod.rs +++ b/cli/npm/resolution/mod.rs @@ -429,7 +429,7 @@ impl NpmResolution { for result in futures::future::join_all(unresolved_tasks).await { let (package_req, info) = result??; - resolver.resolve_npm_package_req(&package_req, info)?; + resolver.add_npm_package_req(&package_req, info)?; } resolver.resolve_pending().await?; diff --git a/cli/npm/resolvers/local.rs b/cli/npm/resolvers/local.rs index 47842627ad0cef..5895dadafde198 100644 --- a/cli/npm/resolvers/local.rs +++ b/cli/npm/resolvers/local.rs @@ -280,7 +280,7 @@ async fn sync_resolution_with_fs( for package in &all_packages { let folder_name = get_package_folder_name(&package.id); let folder_path = deno_local_registry_dir.join(&folder_name); - let initialized_file = folder_path.join("deno_initialized"); + let initialized_file = folder_path.join(".initialized"); if !initialized_file.exists() { let cache = cache.clone(); let registry_url = registry_url.clone(); From db8465f24b5de22a75481b7c2b3283ead1748141 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 5 Nov 2022 12:18:35 -0400 Subject: [PATCH 16/43] Fix tests. --- cli/npm/resolution/graph.rs | 60 ++++++++++++++++++++++++++----------- 1 file changed, 42 insertions(+), 18 deletions(-) diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index e0f6c878fae1d3..d2c4aec804d3ef 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -344,12 +344,15 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> package_info: NpmPackageInfo, ) -> Result<(), AnyError> { let node = self.resolve_node_from_info( - &package_req.to_string(), &package_req.name, package_req, package_info, - &NodeParent::Req(package_req.clone()), )?; + self.graph.set_child_parent( + &package_req.to_string(), + &node, + &NodeParent::Req(package_req.clone()), + ); self.pending_unresolved_nodes.push_back(node); Ok(()) } @@ -361,23 +364,24 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> parent_id: &NpmPackageId, ) -> Result<(), AnyError> { let node = self.resolve_node_from_info( - &entry.bare_specifier, &entry.name, &entry.version_req, package_info, - &NodeParent::Node(parent_id.clone()), )?; + self.graph.set_child_parent( + &entry.bare_specifier, + &node, + &NodeParent::Node(parent_id.clone()), + ); self.pending_unresolved_nodes.push_back(node); Ok(()) } fn resolve_node_from_info( &mut self, - specifier: &str, name: &str, version_matcher: &impl NpmVersionMatcher, package_info: NpmPackageInfo, - parent: &NodeParent, ) -> Result>, AnyError> { let version_and_info = self.resolve_best_package_version_and_info( name, @@ -403,7 +407,6 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> node.deps = Arc::new(deps); } - self.graph.set_child_parent(&specifier, &node, &parent); Ok(node) } @@ -460,6 +463,15 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> vec![], )?; if let Some(new_parent_id) = maybe_new_parent_id { + eprintln!( + "NEW PARENT ID: {} -> {}", + parent_id.as_serializable_name(), + new_parent_id.as_serializable_name() + ); + assert_eq!( + (&new_parent_id.name, &new_parent_id.version), + (&parent_id.name, &parent_id.version) + ); parent_id = new_parent_id; } } @@ -615,6 +627,9 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> (new_id, old_node_children) }; + // this is the parent id found at the bottom of the path + let mut bottom_parent_id = new_id.clone(); + // continue going down the path if let Some(next_specifier) = path.pop() { eprintln!( @@ -632,7 +647,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> .set_child_parent_node(&next_specifier, &node, &new_id); } else { let next_node_id = old_node_children.get(&next_specifier).unwrap(); - self.set_new_peer_dep( + bottom_parent_id = self.set_new_peer_dep( HashMap::from([( next_specifier.to_string(), HashSet::from([NodeParent::Node(new_id.clone())]), @@ -659,7 +674,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> } } - return new_id; + return bottom_parent_id; } } @@ -1420,24 +1435,30 @@ mod test { }, NpmResolutionPackage { id: NpmPackageId::deserialize_name( - "package-b@2.0.0_package-peer@4.0.0" + "package-b@2.0.0_package-peer-a@4.0.0" ) .unwrap(), dist: Default::default(), - dependencies: HashMap::from([( - "package-peer".to_string(), - NpmPackageId::deserialize_name("package-peer@4.0.0").unwrap(), - )]) + dependencies: HashMap::from([ + ( + "package-peer-a".to_string(), + NpmPackageId::deserialize_name("package-peer-a@4.0.0").unwrap(), + ), + ( + "package-peer-c".to_string(), + NpmPackageId::deserialize_name("package-peer-c@6.2.0").unwrap(), + ) + ]) }, NpmResolutionPackage { id: NpmPackageId::deserialize_name( - "package-c@3.0.0_package-peer@4.0.0" + "package-c@3.0.0_package-peer-a@4.0.0" ) .unwrap(), dist: Default::default(), dependencies: HashMap::from([( - "package-peer".to_string(), - NpmPackageId::deserialize_name("package-peer@4.0.0").unwrap(), + "package-peer-a".to_string(), + NpmPackageId::deserialize_name("package-peer-a@4.0.0").unwrap(), )]) }, NpmResolutionPackage { @@ -1453,7 +1474,10 @@ mod test { NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-peer-a@4.0.0").unwrap(), dist: Default::default(), - dependencies: Default::default(), + dependencies: HashMap::from([( + "package-peer-b".to_string(), + NpmPackageId::deserialize_name("package-peer-b@5.4.1").unwrap(), + ),]) }, NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-peer-b@5.4.1").unwrap(), From b2dac597d2f54e5fb57e5e1e2805c1b85a322abb Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 5 Nov 2022 12:48:07 -0400 Subject: [PATCH 17/43] Handle dependency being specified as dep and peer dep --- cli/npm/registry.rs | 30 ++++++++++++++++++++++-------- cli/npm/resolution/graph.rs | 14 ++++++++++++-- 2 files changed, 34 insertions(+), 10 deletions(-) diff --git a/cli/npm/registry.rs b/cli/npm/registry.rs index bfd7d5b81ddb03..2a89d4463b3b4c 100644 --- a/cli/npm/registry.rs +++ b/cli/npm/registry.rs @@ -56,10 +56,14 @@ impl NpmDependencyEntryKind { #[derive(Debug, Eq, PartialEq)] pub struct NpmDependencyEntry { + pub kind: NpmDependencyEntryKind, pub bare_specifier: String, pub name: String, pub version_req: NpmVersionReq, - pub kind: NpmDependencyEntryKind, + /// When the dependency is also marked as a peer dependency, + /// use this entry to resolve the dependency when it can't + /// be resolved as a peer dependency. + pub peer_dep_version_req: Option, } impl PartialOrd for NpmDependencyEntry { @@ -130,19 +134,17 @@ impl NpmPackageVersionInfo { ) })?; Ok(NpmDependencyEntry { + kind, bare_specifier, name, version_req, - kind, + peer_dep_version_req: None, }) } - let mut result = Vec::with_capacity( + let mut result = HashMap::with_capacity( self.dependencies.len() + self.peer_dependencies.len(), ); - for entry in &self.dependencies { - result.push(parse_dep_entry(entry, NpmDependencyEntryKind::Dep)?); - } for entry in &self.peer_dependencies { let is_optional = self .peer_dependencies_meta @@ -153,9 +155,21 @@ impl NpmPackageVersionInfo { true => NpmDependencyEntryKind::OptionalPeer, false => NpmDependencyEntryKind::Peer, }; - result.push(parse_dep_entry(entry, kind)?); + let entry = parse_dep_entry(entry, kind)?; + result.insert(entry.bare_specifier.clone(), entry); + } + for entry in &self.dependencies { + let entry = parse_dep_entry(entry, NpmDependencyEntryKind::Dep)?; + // people may define a dependency as a peer dependency as well, + // so in those cases, attempt to resolve as a peer dependency, + // but then use this dependency version requirement otherwise + if let Some(peer_dep_entry) = result.get_mut(&entry.bare_specifier) { + peer_dep_entry.peer_dep_version_req = Some(entry.version_req); + } else { + result.insert(entry.bare_specifier.clone(), entry); + } } - Ok(result) + Ok(result.into_values().collect()) } } diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index d2c4aec804d3ef..9a27542a55d6dc 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -365,7 +365,17 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> ) -> Result<(), AnyError> { let node = self.resolve_node_from_info( &entry.name, - &entry.version_req, + match entry.kind { + NpmDependencyEntryKind::Dep => &entry.version_req, + // when resolving a peer dependency as a dependency, it should + // use the "dependencies" entry version requirement if it exists + NpmDependencyEntryKind::Peer | NpmDependencyEntryKind::OptionalPeer => { + &entry + .peer_dep_version_req + .as_ref() + .unwrap_or(&entry.version_req) + } + }, package_info, )?; self.graph.set_child_parent( @@ -1477,7 +1487,7 @@ mod test { dependencies: HashMap::from([( "package-peer-b".to_string(), NpmPackageId::deserialize_name("package-peer-b@5.4.1").unwrap(), - ),]) + )]) }, NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-peer-b@5.4.1").unwrap(), From 01c05f225e1cf0268bfdae8ec542a1ae403a5462 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 5 Nov 2022 16:52:31 -0400 Subject: [PATCH 18/43] Starting to add support for handling circular stuff. --- cli/npm/resolution/graph.rs | 343 ++++++++++++++++++++++++++++-------- 1 file changed, 272 insertions(+), 71 deletions(-) diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index 9a27542a55d6dc..7b9eac81774731 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -27,6 +27,50 @@ use super::NpmPackageReq; use super::NpmResolutionPackage; use super::NpmVersionMatcher; +#[derive(Default, Clone)] +pub struct VisitedVersions(HashSet); + +impl VisitedVersions { + pub fn add(&mut self, id: &NpmPackageId) -> bool { + self.0.insert(Self::id_as_key(id)) + } + + pub fn has_visited(&self, id: &NpmPackageId) -> bool { + self.0.contains(&Self::id_as_key(id)) + } + + fn id_as_key(id: &NpmPackageId) -> String { + // we only key on name and version, and ignore the peer dependency + // information because the peer dependency data could change above and below us, but the names and versions won't + format!("{}@{}", id.name, id.version) + } +} + +#[derive(Default, Clone)] +pub struct GraphPath { + visited_versions: VisitedVersions, + specifiers: Vec, +} + +impl GraphPath { + pub fn with_step(&self, specifier: &str, id: &NpmPackageId) -> GraphPath { + let mut copy = self.clone(); + assert!(copy.visited_versions.add(id)); + copy.specifiers.push(specifier.to_string()); + copy + } + + pub fn with_specifier(&self, specifier: String) -> GraphPath { + let mut copy = self.clone(); + copy.specifiers.push(specifier); + copy + } + + pub fn has_visited_version(&self, id: &NpmPackageId) -> bool { + self.visited_versions.has_visited(id) + } +} + #[derive(Debug, Clone, PartialEq, Eq, Hash)] enum NodeParent { /// These are top of the graph npm package requirements @@ -208,7 +252,7 @@ impl Graph { ) { let mut child = (*child).lock(); let mut parent = (**self.packages.get(parent_id).unwrap()).lock(); - debug_assert_ne!(parent.id, child.id); + assert_ne!(parent.id, child.id); parent .children .insert(specifier.to_string(), child.id.clone()); @@ -273,7 +317,7 @@ impl Graph { pub struct GraphDependencyResolver<'a, TNpmRegistryApi: NpmRegistryApi> { graph: &'a mut Graph, api: &'a TNpmRegistryApi, - pending_unresolved_nodes: VecDeque>>, + pending_unresolved_nodes: VecDeque<(VisitedVersions, Arc>)>, } impl<'a, TNpmRegistryApi: NpmRegistryApi> @@ -353,7 +397,9 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> &node, &NodeParent::Req(package_req.clone()), ); - self.pending_unresolved_nodes.push_back(node); + self + .pending_unresolved_nodes + .push_back((VisitedVersions::default(), node)); Ok(()) } @@ -362,6 +408,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> entry: &NpmDependencyEntry, package_info: NpmPackageInfo, parent_id: &NpmPackageId, + visited_versions: &VisitedVersions, ) -> Result<(), AnyError> { let node = self.resolve_node_from_info( &entry.name, @@ -383,7 +430,9 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> &node, &NodeParent::Node(parent_id.clone()), ); - self.pending_unresolved_nodes.push_back(node); + self + .pending_unresolved_nodes + .push_back((visited_versions.clone(), node)); Ok(()) } @@ -423,12 +472,18 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> pub async fn resolve_pending(&mut self) -> Result<(), AnyError> { while !self.pending_unresolved_nodes.is_empty() { // now go down through the dependencies by tree depth - while let Some(parent_node) = self.pending_unresolved_nodes.pop_front() { + while let Some((mut visited_versions, parent_node)) = + self.pending_unresolved_nodes.pop_front() + { let (mut parent_id, deps) = { let parent_node = parent_node.lock(); (parent_node.id.clone(), parent_node.deps.clone()) }; + if !visited_versions.add(&parent_id) { + continue; // circular + } + // cache all the dependencies' registry infos in parallel if should if !should_sync_download() { let handles = deps @@ -460,7 +515,12 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> ); match dep.kind { NpmDependencyEntryKind::Dep => { - self.analyze_dependency(&dep, package_info, &parent_id)?; + self.analyze_dependency( + &dep, + package_info, + &parent_id, + &visited_versions, + )?; } NpmDependencyEntryKind::Peer | NpmDependencyEntryKind::OptionalPeer => { @@ -470,7 +530,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> &parent_id, &dep, package_info, - vec![], + &visited_versions, )?; if let Some(new_parent_id) = maybe_new_parent_id { eprintln!( @@ -498,75 +558,91 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> parent_id: &NpmPackageId, peer_dep: &NpmDependencyEntry, peer_package_info: NpmPackageInfo, - mut path: Vec, + visited_ancestor_versions: &VisitedVersions, ) -> Result, AnyError> { + fn find_matching_child<'a>( + peer_dep: &NpmDependencyEntry, + children: impl Iterator, + ) -> Option { + for child_id in children { + if child_id.name == peer_dep.name + && peer_dep.version_req.satisfies(&child_id.version) + { + return Some(child_id.clone()); + } + } + None + } + eprintln!("[resolve_peer_dep]: Specifier: {}", specifier); // Peer dependencies are resolved based on its ancestors' siblings. // If not found, then it resolves based on the version requirement if non-optional. let mut pending_ancestors = VecDeque::new(); // go up the tree by depth - path.push(specifier.to_string()); - eprintln!("[resolve_peer_dep]: Path: {:?}", path); + let path = GraphPath::default().with_step(specifier, parent_id); + eprintln!("[resolve_peer_dep]: Path: {:?}", path.specifiers); + + // skip over the current node for (specifier, grand_parents) in self.graph.borrow_node(&parent_id).parents.clone() { - let mut path = path.clone(); - path.push(specifier.clone()); + let path = path.with_specifier(specifier); for grand_parent in grand_parents { pending_ancestors.push_back((grand_parent, path.clone())); } } while let Some((ancestor, path)) = pending_ancestors.pop_front() { - let children = match &ancestor { + match &ancestor { NodeParent::Node(ancestor_node_id) => { - let ancestor = self.graph.borrow_node(ancestor_node_id); - for (specifier, parents) in &ancestor.parents { - let mut new_path = path.clone(); - new_path.push(specifier.to_string()); - for parent in parents { - pending_ancestors.push_back((parent.clone(), new_path.clone())); + // we've gone in a full circle, so don't keep looking + if path.has_visited_version(ancestor_node_id) { + continue; + } + + let maybe_peer_dep_id = if ancestor_node_id.name == peer_dep.name + && peer_dep.version_req.satisfies(&ancestor_node_id.version) + { + Some(ancestor_node_id.clone()) + } else { + let ancestor = self.graph.borrow_node(ancestor_node_id); + for (specifier, parents) in &ancestor.parents { + let new_path = path.with_step(specifier, ancestor_node_id); + for parent in parents { + pending_ancestors.push_back((parent.clone(), new_path.clone())); + } } + find_matching_child(peer_dep, ancestor.children.values()) + }; + if let Some(peer_dep_id) = maybe_peer_dep_id { + let parents = + self.graph.borrow_node(ancestor_node_id).parents.clone(); + return Ok(Some(self.set_new_peer_dep( + parents, + ancestor_node_id, + &peer_dep_id, + path.specifiers, + visited_ancestor_versions, + ))); } - ancestor.children.clone() } - NodeParent::Req(_) => { + NodeParent::Req(req) => { // in this case, the parent is the root so the children are all the package requirements - self - .graph - .package_reqs - .iter() - .map(|(req, id)| (req.to_string(), id.clone())) - .collect::>() - } - }; - for child_id in children.values() { - if child_id.name == peer_dep.name - && peer_dep.version_req.satisfies(&child_id.version) - { - eprintln!("MATCHED: {}", child_id.as_serializable_name()); - eprintln!("PATH: {:?}", path); - // go down the descendants creating a new path - match &ancestor { - NodeParent::Node(node_id) => { - let parents = self.graph.borrow_node(node_id).parents.clone(); - return Ok(Some( - self.set_new_peer_dep(parents, node_id, &child_id, path), - )); - } - NodeParent::Req(req) => { - let old_id = self.graph.package_reqs.get(&req).unwrap().clone(); - let mut path = path; - path.pop(); // go back down one level - return Ok(Some(self.set_new_peer_dep( - HashMap::from([( - req.to_string(), - HashSet::from([NodeParent::Req(req.clone())]), - )]), - &old_id, - &child_id, - path, - ))); - } + if let Some(child_id) = + find_matching_child(peer_dep, self.graph.package_reqs.values()) + { + let old_id = self.graph.package_reqs.get(&req).unwrap().clone(); + let mut path = path.specifiers; + path.pop(); // go back down one level + return Ok(Some(self.set_new_peer_dep( + HashMap::from([( + req.to_string(), + HashSet::from([NodeParent::Req(req.clone())]), + )]), + &old_id, + &child_id, + path, + visited_ancestor_versions, + ))); } } } @@ -576,7 +652,12 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> // to resolve based on the package info and will treat this just like any // other dependency when not optional if !peer_dep.kind.is_optional() { - self.analyze_dependency(&peer_dep, peer_package_info, &parent_id)?; + self.analyze_dependency( + &peer_dep, + peer_package_info, + &parent_id, + &visited_ancestor_versions, + )?; } Ok(None) @@ -588,6 +669,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> node_id: &NpmPackageId, peer_dep_id: &NpmPackageId, mut path: Vec, + visited_ancestor_versions: &VisitedVersions, ) -> NpmPackageId { eprintln!("PREVIOUS PARENTS: {:?}", previous_parents); let old_id = node_id; @@ -651,7 +733,9 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> // this means we're at the peer dependency now assert!(!old_node_children.contains_key(&next_specifier)); let node = self.graph.get_or_create_for_id(&peer_dep_id).1; - self.pending_unresolved_nodes.push_back(node.clone()); + self + .pending_unresolved_nodes + .push_back((visited_ancestor_versions.clone(), node.clone())); self .graph .set_child_parent_node(&next_specifier, &node, &new_id); @@ -665,6 +749,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> &next_node_id, peer_dep_id, path, + visited_ancestor_versions, ); } } @@ -852,7 +937,7 @@ mod test { api.add_dependency(("package-a", "1.0.0"), ("package-c", "^0.1")); api.add_dependency(("package-c", "0.1.0"), ("package-d", "*")); - let packages = + let (packages, package_reqs) = run_resolver_and_get_output(api, vec!["npm:package-a@1"]).await; assert_eq!( packages, @@ -891,6 +976,10 @@ mod test { }, ] ); + assert_eq!( + package_reqs, + vec![("package-a@1".to_string(), "package-a@1.0.0".to_string())] + ); } #[tokio::test] @@ -906,7 +995,7 @@ mod test { api.add_peer_dependency(("package-b", "2.0.0"), ("package-peer", "4")); api.add_peer_dependency(("package-c", "3.0.0"), ("package-peer", "*")); - let packages = run_resolver_and_get_output( + let (packages, package_reqs) = run_resolver_and_get_output( api, // the peer dependency is specified here at the top of the tree // so it should resolve to 4.0.0 instead of 4.1.0 @@ -968,6 +1057,19 @@ mod test { }, ] ); + assert_eq!( + package_reqs, + vec![ + ( + "package-a@1".to_string(), + "package-a@1.0.0_package-peer@4.0.0".to_string() + ), + ( + "package-peer@4.0.0".to_string(), + "package-peer@4.0.0".to_string() + ) + ] + ); } #[tokio::test] @@ -988,7 +1090,7 @@ mod test { api.add_peer_dependency(("package-b", "2.0.0"), ("package-peer", "4")); api.add_peer_dependency(("package-c", "3.0.0"), ("package-peer", "*")); - let packages = + let (packages, package_reqs) = run_resolver_and_get_output(api, vec!["npm:package-0@1.1.1"]).await; assert_eq!( packages, @@ -1060,6 +1162,10 @@ mod test { }, ] ); + assert_eq!( + package_reqs, + vec![("package-0@1.1.1".to_string(), "package-0@1.1.1".to_string())] + ); } #[tokio::test] @@ -1077,7 +1183,7 @@ mod test { api.add_peer_dependency(("package-b", "2.0.0"), ("package-peer", "4")); api.add_peer_dependency(("package-c", "3.0.0"), ("package-peer", "*")); - let packages = + let (packages, package_reqs) = run_resolver_and_get_output(api, vec!["npm:package-a@1"]).await; assert_eq!( packages, @@ -1119,6 +1225,10 @@ mod test { }, ] ); + assert_eq!( + package_reqs, + vec![("package-a@1".to_string(), "package-a@1.0.0".to_string())] + ); } #[tokio::test] @@ -1142,7 +1252,7 @@ mod test { ("package-peer", "*"), ); - let packages = + let (packages, package_reqs) = run_resolver_and_get_output(api, vec!["npm:package-a@1"]).await; assert_eq!( packages, @@ -1173,6 +1283,10 @@ mod test { }, ] ); + assert_eq!( + package_reqs, + vec![("package-a@1".to_string(), "package-a@1.0.0".to_string())] + ); } #[tokio::test] @@ -1194,7 +1308,7 @@ mod test { ("package-peer", "*"), ); - let packages = run_resolver_and_get_output( + let (packages, package_reqs) = run_resolver_and_get_output( api, vec!["npm:package-a@1", "npm:package-peer@4.0.0"], ) @@ -1254,6 +1368,19 @@ mod test { }, ] ); + assert_eq!( + package_reqs, + vec![ + ( + "package-a@1".to_string(), + "package-a@1.0.0_package-peer@4.0.0".to_string() + ), + ( + "package-peer@4.0.0".to_string(), + "package-peer@4.0.0".to_string() + ) + ] + ); } #[tokio::test] @@ -1268,7 +1395,7 @@ mod test { ("package-peer-b", "3"), ); - let packages = + let (packages, package_reqs) = run_resolver_and_get_output(api, vec!["npm:package-0@1.0"]).await; assert_eq!( packages, @@ -1296,6 +1423,10 @@ mod test { }, ] ); + assert_eq!( + package_reqs, + vec![("package-0@1.0".to_string(), "package-a@1.0.0".to_string())] + ); } #[tokio::test] @@ -1311,7 +1442,7 @@ mod test { ("package-peer-b", "3"), ); - let packages = run_resolver_and_get_output( + let (packages, package_reqs) = run_resolver_and_get_output( api, vec![ "npm:package-0@1.0", @@ -1361,6 +1492,24 @@ mod test { }, ] ); + assert_eq!( + package_reqs, + vec![ + ( + "package-0@1.0".to_string(), + "package-0@1.0.0_package-peer-a@2.0.0_package-peer-b@3.0.0" + .to_string() + ), + ( + "package-peer-a@2".to_string(), + "package-peer-a@2.0.0_package-peer-b@3.0.0".to_string() + ), + ( + "package-peer-b@3".to_string(), + "package-peer-b@3.0.0".to_string() + ) + ] + ); } #[tokio::test] @@ -1393,7 +1542,7 @@ mod test { ("package-peer-b", "^5.4"), // will be auto-resolved ); - let packages = run_resolver_and_get_output( + let (packages, package_reqs) = run_resolver_and_get_output( api, vec!["npm:package-0@1.1.1", "npm:package-e@3"], ) @@ -1501,12 +1650,57 @@ mod test { }, ] ); + assert_eq!( + package_reqs, + vec![ + ("package-0@1.1.1".to_string(), "package-0@1.1.1".to_string()), + ("package-e@3".to_string(), "package-e@3.6.0".to_string()), + ] + ); + } + + #[tokio::test] + async fn resolve_peer_deps_circular() { + let api = TestNpmRegistryApi::default(); + api.ensure_package_version("package-a", "1.0.0"); + api.ensure_package_version("package-b", "2.0.0"); + api.add_dependency(("package-a", "1.0.0"), ("package-b", "*")); + api.add_peer_dependency(("package-b", "2.0.0"), ("package-a", "1")); + + let (packages, package_reqs) = + run_resolver_and_get_output(api, vec!["npm:package-a@1.0"]).await; + assert_eq!( + packages, + vec![ + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-a@1.0.0_package-a@1.0.0") + .unwrap(), + dependencies: HashMap::from([( + "package-b".to_string(), + NpmPackageId::deserialize_name("package-b@2.0.0_package-a@1.0.0") + .unwrap(), + )]), + dist: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-b@2.0.0_package-a@1.0.0") + .unwrap(), + dependencies: HashMap::from([( + "package-a".to_string(), + NpmPackageId::deserialize_name("package-a@1.0.0_package-a@1.0.0") + .unwrap(), + )]), + dist: Default::default(), + }, + ] + ); + assert_eq!(package_reqs, vec![]); } async fn run_resolver_and_get_output( api: TestNpmRegistryApi, reqs: Vec<&str>, - ) -> Vec { + ) -> (Vec, Vec<(String, String)>) { let mut graph = Graph::default(); let mut resolver = GraphDependencyResolver::new(&mut graph, &api); @@ -1518,8 +1712,15 @@ mod test { } resolver.resolve_pending().await.unwrap(); - let mut packages = graph.into_snapshot(&api).await.unwrap().all_packages(); + let snapshot = graph.into_snapshot(&api).await.unwrap(); + let mut packages = snapshot.all_packages(); packages.sort_by(|a, b| a.id.cmp(&b.id)); - packages + let mut package_reqs = snapshot + .package_reqs + .into_iter() + .map(|(a, b)| (a.to_string(), b.as_serializable_name())) + .collect::>(); + package_reqs.sort_by(|a, b| a.0.to_string().cmp(&b.0.to_string())); + (packages, package_reqs) } } From de0e1bcfea10c2b85bd37e07798d4eb060d97394 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 5 Nov 2022 17:53:56 -0400 Subject: [PATCH 19/43] Working resolution::graph tests --- cli/npm/resolution/graph.rs | 49 ++++++++++++++++++++++++++++--------- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index 7b9eac81774731..1c70c507f8b145 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -1,5 +1,6 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. +use std::borrow::Cow; use std::collections::HashMap; use std::collections::HashSet; use std::collections::VecDeque; @@ -40,8 +41,9 @@ impl VisitedVersions { } fn id_as_key(id: &NpmPackageId) -> String { - // we only key on name and version, and ignore the peer dependency - // information because the peer dependency data could change above and below us, but the names and versions won't + // we only key on name and version in the id and not peer dependencies + // because the peer dependencies could change above and below us, + // but the names and versions won't format!("{}@{}", id.name, id.version) } } @@ -101,6 +103,14 @@ impl Node { }, self.parents.entry(specifier.clone()).or_default().len(), ); + if self.id.as_serializable_name() == "package-a@1.0.0" + && match &parent { + NodeParent::Node(n) => n.as_serializable_name(), + NodeParent::Req(req) => req.to_string(), + } == "package-b@2.0.0_package-a@1.0.0" + { + panic!("STOP"); + } self.parents.entry(specifier).or_default().insert(parent); } @@ -672,9 +682,10 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> visited_ancestor_versions: &VisitedVersions, ) -> NpmPackageId { eprintln!("PREVIOUS PARENTS: {:?}", previous_parents); + let mut peer_dep_id = Cow::Borrowed(peer_dep_id); let old_id = node_id; let (new_id, old_node_children) = - if old_id.peer_dependencies.contains(peer_dep_id) { + if old_id.peer_dependencies.contains(&peer_dep_id) { // the parent has already resolved to using this peer dependency // via some other path, so we don't need to update its ids, // but instead only make a link to it @@ -684,7 +695,13 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> ) } else { let mut new_id = old_id.clone(); - new_id.peer_dependencies.push(peer_dep_id.clone()); + new_id.peer_dependencies.push(peer_dep_id.as_ref().clone()); + + // this will happen for circular dependencies + if *old_id == *peer_dep_id { + peer_dep_id = Cow::Owned(new_id.clone()); + } + eprintln!("NEW ID: {}", new_id.as_serializable_name()); eprintln!("PATH: {:?}", path); // remove the previous parents from the old node @@ -747,7 +764,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> HashSet::from([NodeParent::Node(new_id.clone())]), )]), &next_node_id, - peer_dep_id, + &peer_dep_id, path, visited_ancestor_versions, ); @@ -1425,7 +1442,7 @@ mod test { ); assert_eq!( package_reqs, - vec![("package-0@1.0".to_string(), "package-a@1.0.0".to_string())] + vec![("package-0@1.0".to_string(), "package-0@1.0.0".to_string())] ); } @@ -1677,14 +1694,18 @@ mod test { .unwrap(), dependencies: HashMap::from([( "package-b".to_string(), - NpmPackageId::deserialize_name("package-b@2.0.0_package-a@1.0.0") - .unwrap(), + NpmPackageId::deserialize_name( + "package-b@2.0.0_package-a@1.0.0__package-a@1.0.0" + ) + .unwrap(), )]), dist: Default::default(), }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-b@2.0.0_package-a@1.0.0") - .unwrap(), + id: NpmPackageId::deserialize_name( + "package-b@2.0.0_package-a@1.0.0__package-a@1.0.0" + ) + .unwrap(), dependencies: HashMap::from([( "package-a".to_string(), NpmPackageId::deserialize_name("package-a@1.0.0_package-a@1.0.0") @@ -1694,7 +1715,13 @@ mod test { }, ] ); - assert_eq!(package_reqs, vec![]); + assert_eq!( + package_reqs, + vec![( + "package-a@1.0".to_string(), + "package-a@1.0.0_package-a@1.0.0".to_string() + )] + ); } async fn run_resolver_and_get_output( From f71961cb1fd083e22a6a96767167c758576e0ae2 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 5 Nov 2022 18:30:51 -0400 Subject: [PATCH 20/43] Fix clippy. --- cli/npm/cache.rs | 2 +- cli/npm/resolution/graph.rs | 116 +++++++++--------------------------- cli/npm/resolution/mod.rs | 7 +-- 3 files changed, 33 insertions(+), 92 deletions(-) diff --git a/cli/npm/cache.rs b/cli/npm/cache.rs index 75d75d3b44a98b..e2010ada63202c 100644 --- a/cli/npm/cache.rs +++ b/cli/npm/cache.rs @@ -86,7 +86,7 @@ impl ReadonlyNpmCache { id.as_serializable_name() .strip_prefix(&id.name) .unwrap() - .strip_prefix("@") + .strip_prefix('@') .unwrap(), ) } diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index 1c70c507f8b145..de49d5b7fb2bfd 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -12,6 +12,7 @@ use deno_core::error::AnyError; use deno_core::futures; use deno_core::parking_lot::Mutex; use deno_core::parking_lot::MutexGuard; +use log::debug; use crate::npm::cache::should_sync_download; use crate::npm::registry::NpmDependencyEntry; @@ -93,42 +94,13 @@ struct Node { impl Node { pub fn add_parent(&mut self, specifier: String, parent: NodeParent) { - eprintln!( - "ADDING parent to {}: {} {} {}", - self.id.as_serializable_name(), - specifier, - match &parent { - NodeParent::Node(n) => n.as_serializable_name(), - NodeParent::Req(req) => req.to_string(), - }, - self.parents.entry(specifier.clone()).or_default().len(), - ); - if self.id.as_serializable_name() == "package-a@1.0.0" - && match &parent { - NodeParent::Node(n) => n.as_serializable_name(), - NodeParent::Req(req) => req.to_string(), - } == "package-b@2.0.0_package-a@1.0.0" - { - panic!("STOP"); - } self.parents.entry(specifier).or_default().insert(parent); } pub fn remove_parent(&mut self, specifier: &str, parent: &NodeParent) { - eprintln!( - "REMOVING parent from {}: {} {} {}", - self.id.as_serializable_name(), - specifier, - match parent { - NodeParent::Node(n) => n.as_serializable_name(), - NodeParent::Req(req) => req.to_string(), - }, - self.parents.entry(specifier.to_string()).or_default().len(), - ); if let Some(parents) = self.parents.get_mut(specifier) { parents.remove(parent); if parents.is_empty() { - drop(parents); self.parents.remove(specifier); } } @@ -192,8 +164,8 @@ impl Graph { let resolution = snapshot.packages.get(id).unwrap(); let node = self.get_or_create_for_id(id).1; for (name, child_id) in &resolution.dependencies { - let child_node = self.fill_for_id_with_snapshot(&child_id, snapshot); - self.set_child_parent_node(&name, &child_node, &id); + let child_node = self.fill_for_id_with_snapshot(child_id, snapshot); + self.set_child_parent_node(name, &child_node, id); } node } @@ -210,25 +182,15 @@ impl Graph { fn forget_orphan(&mut self, node_id: &NpmPackageId) { if let Some(node) = self.packages.remove(node_id) { - eprintln!( - "REMAINING: {:?}", - self - .packages - .values() - .map(|n| n.lock().id.as_serializable_name()) - .collect::>() - ); let node = (*node).lock(); - eprintln!("FORGOT: {}", node.id.as_serializable_name()); assert_eq!(node.parents.len(), 0); let parent = NodeParent::Node(node.id.clone()); for (specifier, child_id) in &node.children { let mut child = self.borrow_node(child_id); child.remove_parent(specifier, &parent); - eprintln!("CHILD PARENTS: {:?}", child.parents); if child.parents.is_empty() { drop(child); // stop borrowing from self - self.forget_orphan(&child_id); + self.forget_orphan(child_id); } } } @@ -242,7 +204,7 @@ impl Graph { ) { match parent { NodeParent::Node(parent_id) => { - self.set_child_parent_node(&specifier, &child, &parent_id); + self.set_child_parent_node(specifier, child, parent_id); } NodeParent::Req(package_req) => { let mut node = (*child).lock(); @@ -278,8 +240,6 @@ impl Graph { ) { match parent { NodeParent::Node(parent_id) => { - eprintln!("PARENT: {}", parent_id.as_serializable_name()); - eprintln!("SPECIFIER: {}", specifier); let mut node = self.borrow_node(parent_id); if let Some(removed_child_id) = node.children.remove(specifier) { assert_eq!(removed_child_id, *child_id); @@ -392,7 +352,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> maybe_best_version.cloned() } - pub fn add_npm_package_req( + pub fn add_package_req( &mut self, package_req: &NpmPackageReq, package_info: NpmPackageInfo, @@ -427,7 +387,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> // when resolving a peer dependency as a dependency, it should // use the "dependencies" entry version requirement if it exists NpmDependencyEntryKind::Peer | NpmDependencyEntryKind::OptionalPeer => { - &entry + entry .peer_dep_version_req .as_ref() .unwrap_or(&entry.version_req) @@ -462,9 +422,14 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> version: version_and_info.version.clone(), peer_dependencies: Vec::new(), }; + debug!( + "Resolved {}@{} to {}", + name, + version_matcher.version_text(), + id + ); let (created, node) = self.graph.get_or_create_for_id(&id); if created { - eprintln!("RESOLVED: {}", id.as_serializable_name()); let mut node = (*node).lock(); let mut deps = version_and_info .info @@ -518,15 +483,10 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> for dep in deps.iter() { let package_info = self.api.package_info(&dep.name).await?; - eprintln!( - "-- DEPENDENCY: {} ({})", - dep.name, - parent_id.as_serializable_name() - ); match dep.kind { NpmDependencyEntryKind::Dep => { self.analyze_dependency( - &dep, + dep, package_info, &parent_id, &visited_versions, @@ -534,20 +494,14 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> } NpmDependencyEntryKind::Peer | NpmDependencyEntryKind::OptionalPeer => { - eprintln!("ANALYZING PEER DEP: {}", dep.name); let maybe_new_parent_id = self.resolve_peer_dep( &dep.bare_specifier, &parent_id, - &dep, + dep, package_info, &visited_versions, )?; if let Some(new_parent_id) = maybe_new_parent_id { - eprintln!( - "NEW PARENT ID: {} -> {}", - parent_id.as_serializable_name(), - new_parent_id.as_serializable_name() - ); assert_eq!( (&new_parent_id.name, &new_parent_id.version), (&parent_id.name, &parent_id.version) @@ -584,16 +538,14 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> None } - eprintln!("[resolve_peer_dep]: Specifier: {}", specifier); // Peer dependencies are resolved based on its ancestors' siblings. // If not found, then it resolves based on the version requirement if non-optional. let mut pending_ancestors = VecDeque::new(); // go up the tree by depth let path = GraphPath::default().with_step(specifier, parent_id); - eprintln!("[resolve_peer_dep]: Path: {:?}", path.specifiers); // skip over the current node for (specifier, grand_parents) in - self.graph.borrow_node(&parent_id).parents.clone() + self.graph.borrow_node(parent_id).parents.clone() { let path = path.with_specifier(specifier); for grand_parent in grand_parents { @@ -640,7 +592,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> if let Some(child_id) = find_matching_child(peer_dep, self.graph.package_reqs.values()) { - let old_id = self.graph.package_reqs.get(&req).unwrap().clone(); + let old_id = self.graph.package_reqs.get(req).unwrap().clone(); let mut path = path.specifiers; path.pop(); // go back down one level return Ok(Some(self.set_new_peer_dep( @@ -663,10 +615,10 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> // other dependency when not optional if !peer_dep.kind.is_optional() { self.analyze_dependency( - &peer_dep, + peer_dep, peer_package_info, - &parent_id, - &visited_ancestor_versions, + parent_id, + visited_ancestor_versions, )?; } @@ -681,7 +633,6 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> mut path: Vec, visited_ancestor_versions: &VisitedVersions, ) -> NpmPackageId { - eprintln!("PREVIOUS PARENTS: {:?}", previous_parents); let mut peer_dep_id = Cow::Borrowed(peer_dep_id); let old_id = node_id; let (new_id, old_node_children) = @@ -702,13 +653,11 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> peer_dep_id = Cow::Owned(new_id.clone()); } - eprintln!("NEW ID: {}", new_id.as_serializable_name()); - eprintln!("PATH: {:?}", path); // remove the previous parents from the old node let old_node_children = { for (specifier, parents) in &previous_parents { for parent in parents { - self.graph.remove_child_parent(&specifier, old_id, parent); + self.graph.remove_child_parent(specifier, old_id, parent); } } let old_node = self.graph.borrow_node(old_id); @@ -731,7 +680,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> let child = self.graph.packages.get(child_id).unwrap().clone(); self .graph - .set_child_parent(&specifier, &child, &new_id_as_parent); + .set_child_parent(specifier, &child, &new_id_as_parent); } (new_id, old_node_children) }; @@ -741,13 +690,12 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> // continue going down the path if let Some(next_specifier) = path.pop() { - eprintln!( - "Next specifier: {}, peer dep id: {}", - next_specifier, - peer_dep_id.as_serializable_name() - ); if path.is_empty() { // this means we're at the peer dependency now + debug!( + "Resolved peer dependency for {} in {} to {}", + &next_specifier, &new_id, &peer_dep_id, + ); assert!(!old_node_children.contains_key(&next_specifier)); let node = self.graph.get_or_create_for_id(&peer_dep_id).1; self @@ -763,7 +711,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> next_specifier.to_string(), HashSet::from([NodeParent::Node(new_id.clone())]), )]), - &next_node_id, + next_node_id, &peer_dep_id, path, visited_ancestor_versions, @@ -773,20 +721,14 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> // forget the old node at this point if it has no parents if new_id != *old_id { - eprintln!( - "CHANGING ID: {} -> {}", - old_id.as_serializable_name(), - new_id.as_serializable_name() - ); let old_node = self.graph.borrow_node(old_id); - eprintln!("OLD PARENTS: {:?}", old_node.parents); if old_node.parents.is_empty() { drop(old_node); // stop borrowing self.graph.forget_orphan(old_id); } } - return bottom_parent_id; + bottom_parent_id } } @@ -1734,7 +1676,7 @@ mod test { for req in reqs { let req = NpmPackageReference::from_str(req).unwrap().req; resolver - .add_npm_package_req(&req, api.package_info(&req.name).await.unwrap()) + .add_package_req(&req, api.package_info(&req.name).await.unwrap()) .unwrap(); } diff --git a/cli/npm/resolution/mod.rs b/cli/npm/resolution/mod.rs index 9433066b792186..b6ed58ea5abf4f 100644 --- a/cli/npm/resolution/mod.rs +++ b/cli/npm/resolution/mod.rs @@ -190,7 +190,7 @@ impl NpmPackageId { if level == 0 { self.name.to_string() } else { - self.name.replace("/", "+") + self.name.replace('/', "+") }, self.version ); @@ -275,7 +275,7 @@ impl NpmPackageId { move |input| { let (input, (name, version)) = parse_name_and_version(input)?; let name = if level > 0 { - name.replace("+", "/") + name.replace('+', "/") } else { name }; @@ -394,7 +394,6 @@ impl NpmResolution { // convert the snapshot to a traversable graph let mut graph = Graph::default(); graph.fill_with_snapshot(&snapshot); - drop(snapshot); // todo: remove // multiple packages are resolved in alphabetical order package_reqs.sort_by(|a, b| a.name.cmp(&b.name)); @@ -429,7 +428,7 @@ impl NpmResolution { for result in futures::future::join_all(unresolved_tasks).await { let (package_req, info) = result??; - resolver.add_npm_package_req(&package_req, info)?; + resolver.add_package_req(&package_req, info)?; } resolver.resolve_pending().await?; From 647e391f2ecdca67b84c65eb452d677e37ca6a1c Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 5 Nov 2022 19:04:19 -0400 Subject: [PATCH 21/43] Add test for circular dependencies --- cli/npm/resolution/graph.rs | 44 ++++++++++++++++++++++++++++++++++--- 1 file changed, 41 insertions(+), 3 deletions(-) diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index de49d5b7fb2bfd..134fd9a9b11326 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -30,7 +30,7 @@ use super::NpmResolutionPackage; use super::NpmVersionMatcher; #[derive(Default, Clone)] -pub struct VisitedVersions(HashSet); +struct VisitedVersions(HashSet); impl VisitedVersions { pub fn add(&mut self, id: &NpmPackageId) -> bool { @@ -50,7 +50,7 @@ impl VisitedVersions { } #[derive(Default, Clone)] -pub struct GraphPath { +struct GraphPath { visited_versions: VisitedVersions, specifiers: Vec, } @@ -264,7 +264,7 @@ impl Graph { let dist = api .package_version_info(&id.name, &id.version) .await? - .unwrap() // todo(THIS PR): don't unwrap here + .unwrap() .dist; let node = node.lock(); packages.insert( @@ -276,6 +276,7 @@ impl Graph { }, ); } + Ok(NpmResolutionSnapshot { package_reqs: self.package_reqs, packages_by_name: self.packages_by_name, @@ -1666,6 +1667,43 @@ mod test { ); } + #[tokio::test] + async fn resolve_deps_circular() { + let api = TestNpmRegistryApi::default(); + api.ensure_package_version("package-a", "1.0.0"); + api.ensure_package_version("package-b", "2.0.0"); + api.add_dependency(("package-a", "1.0.0"), ("package-b", "*")); + api.add_dependency(("package-b", "2.0.0"), ("package-a", "1")); + + let (packages, package_reqs) = + run_resolver_and_get_output(api, vec!["npm:package-a@1.0"]).await; + assert_eq!( + packages, + vec![ + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-a@1.0.0").unwrap(), + dependencies: HashMap::from([( + "package-b".to_string(), + NpmPackageId::deserialize_name("package-b@2.0.0").unwrap(), + )]), + dist: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::deserialize_name("package-b@2.0.0").unwrap(), + dependencies: HashMap::from([( + "package-a".to_string(), + NpmPackageId::deserialize_name("package-a@1.0.0").unwrap(), + )]), + dist: Default::default(), + }, + ] + ); + assert_eq!( + package_reqs, + vec![("package-a@1.0".to_string(), "package-a@1.0.0".to_string())] + ); + } + async fn run_resolver_and_get_output( api: TestNpmRegistryApi, reqs: Vec<&str>, From 86227ca6f0e1472db90cc57c5bc8a701986213be Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sat, 5 Nov 2022 20:08:43 -0400 Subject: [PATCH 22/43] Add comment. --- cli/npm/resolution/graph.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index 134fd9a9b11326..85566e1226c996 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -52,6 +52,7 @@ impl VisitedVersions { #[derive(Default, Clone)] struct GraphPath { visited_versions: VisitedVersions, + // todo(THIS PR): switch to a singly linked list here specifiers: Vec, } From e316c67cd9676d870c5a2a7284b612e6dfe2aff8 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sun, 6 Nov 2022 09:53:00 -0500 Subject: [PATCH 23/43] Basic idea for linking this up to the cache folder. Need to map what's in the snapshot to a NpmPackageFolderId --- cli/npm/cache.rs | 217 ++++++++++++++++++++++++++---------- cli/npm/resolution/graph.rs | 1 + cli/npm/resolution/mod.rs | 1 + cli/npm/resolvers/global.rs | 9 +- cli/npm/tarball.rs | 83 +++++--------- 5 files changed, 194 insertions(+), 117 deletions(-) diff --git a/cli/npm/cache.rs b/cli/npm/cache.rs index e2010ada63202c..c50a5268e85ec8 100644 --- a/cli/npm/cache.rs +++ b/cli/npm/cache.rs @@ -19,8 +19,8 @@ use crate::fs_util; use crate::progress_bar::ProgressBar; use super::registry::NpmPackageVersionDistInfo; +use super::semver::NpmVersion; use super::tarball::verify_and_extract_tarball; -use super::NpmPackageId; /// For some of the tests, we want downloading of packages /// to be deterministic so that the output is always the same @@ -28,7 +28,107 @@ pub fn should_sync_download() -> bool { std::env::var("DENO_UNSTABLE_NPM_SYNC_DOWNLOAD") == Ok("1".to_string()) } -pub const NPM_PACKAGE_SYNC_LOCK_FILENAME: &str = ".deno_sync_lock"; +const NPM_PACKAGE_SYNC_LOCK_FILENAME: &str = ".deno_sync_lock"; + +pub fn with_folder_sync_lock( + package: (&str, &NpmVersion), + output_folder: &Path, + action: impl FnOnce() -> Result<(), AnyError>, +) -> Result<(), AnyError> { + fn inner( + output_folder: &Path, + action: impl FnOnce() -> Result<(), AnyError>, + ) -> Result<(), AnyError> { + fs::create_dir_all(output_folder).with_context(|| { + format!("Error creating '{}'.", output_folder.display()) + })?; + + // This sync lock file is a way to ensure that partially created + // npm package directories aren't considered valid. This could maybe + // be a bit smarter in the future to not bother extracting here + // if another process has taken the lock in the past X seconds and + // wait for the other process to finish (it could try to create the + // file with `create_new(true)` then if it exists, check the metadata + // then wait until the other process finishes with a timeout), but + // for now this is good enough. + let sync_lock_path = output_folder.join(NPM_PACKAGE_SYNC_LOCK_FILENAME); + match fs::OpenOptions::new() + .write(true) + .create(true) + .open(&sync_lock_path) + { + Ok(_) => { + action()?; + // extraction succeeded, so only now delete this file + let _ignore = std::fs::remove_file(&sync_lock_path); + Ok(()) + } + Err(err) => { + bail!( + concat!( + "Error creating package sync lock file at '{}'. ", + "Maybe try manually deleting this folder.\n\n{:#}", + ), + output_folder.display(), + err + ); + } + } + } + + match inner(output_folder, action) { + Ok(()) => Ok(()), + Err(err) => { + if let Err(remove_err) = fs::remove_dir_all(&output_folder) { + if remove_err.kind() != std::io::ErrorKind::NotFound { + bail!( + concat!( + "Failed setting up package cache directory for {}@{}, then ", + "failed cleaning it up.\n\nOriginal error:\n\n{}\n\n", + "Remove error:\n\n{}\n\nPlease manually ", + "delete this folder or you will run into issues using this ", + "package in the future:\n\n{}" + ), + package.0, + package.1, + err, + remove_err, + output_folder.display(), + ); + } + } + Err(err) + } + } +} + +pub struct NpmPackageCacheFolderId { + pub name: String, + pub version: NpmVersion, + /// Peer dependency resolution may require us to have duplicate copies + /// of the same package. + pub copy_count: usize, +} + +impl NpmPackageCacheFolderId { + pub fn with_no_count(&self) -> Self { + Self { + name: self.name.clone(), + version: self.version.clone(), + copy_count: 0, + } + } +} + +impl std::fmt::Display for NpmPackageCacheFolderId { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{}@{}", self.name, self.version)?; + if self.copy_count > 0 { + write!(f, "_{}", self.copy_count)?; + } + Ok(()) + } +} #[derive(Clone, Debug)] pub struct ReadonlyNpmCache { @@ -79,15 +179,15 @@ impl ReadonlyNpmCache { pub fn package_folder( &self, - id: &NpmPackageId, + id: &NpmPackageCacheFolderId, registry_url: &Url, ) -> PathBuf { self.package_name_folder(&id.name, registry_url).join( - id.as_serializable_name() - .strip_prefix(&id.name) - .unwrap() - .strip_prefix('@') - .unwrap(), + if id.copy_count == 0 { + id.version.to_string() + } else { + format!("{}_{}", id.version.to_string(), id.copy_count) + }, ) } @@ -118,23 +218,24 @@ impl ReadonlyNpmCache { .join(fs_util::root_url_to_safe_local_dirname(registry_url)) } - pub fn resolve_package_id_from_specifier( + pub fn resolve_package_folder_id_from_specifier( &self, specifier: &ModuleSpecifier, registry_url: &Url, - ) -> Result { - match self.maybe_resolve_package_id_from_specifier(specifier, registry_url) + ) -> Result { + match self + .maybe_resolve_package_folder_id_from_specifier(specifier, registry_url) { Some(id) => Ok(id), None => bail!("could not find npm package for '{}'", specifier), } } - fn maybe_resolve_package_id_from_specifier( + fn maybe_resolve_package_folder_id_from_specifier( &self, specifier: &ModuleSpecifier, registry_url: &Url, - ) -> Option { + ) -> Option { let registry_root_dir = self .root_dir_url .join(&format!( @@ -153,7 +254,7 @@ impl ReadonlyNpmCache { // examples: // * chalk/5.0.1/ // * @types/chalk/5.0.1/ - // * some-package/5.0.1_peer-dep-name@0.1.0/ + // * some-package/5.0.1_1/ -- where the `_1` (/_\d+/) is a copy of the folder for peer deps let is_scoped_package = relative_url.starts_with('@'); let mut parts = relative_url .split('/') @@ -164,10 +265,19 @@ impl ReadonlyNpmCache { if parts.len() < 2 { return None; } - let version_part = parts.pop().unwrap(); // this could also contain the peer dep id info + let version_part = parts.pop().unwrap(); let name = parts.join("/"); - let full_name = format!("{}@{}", name, version_part); - NpmPackageId::deserialize_name(&full_name).ok() + let (version, copy_count) = + if let Some((version, copy_count)) = version_part.split_once('_') { + (version, copy_count.parse::().ok()?) + } else { + (version_part, 0) + }; + Some(NpmPackageCacheFolderId { + name: name.to_string(), + version: NpmVersion::parse(version).ok()?, + copy_count, + }) } pub fn get_cache_location(&self) -> PathBuf { @@ -202,7 +312,7 @@ impl NpmCache { pub async fn ensure_package( &self, - id: &NpmPackageId, + id: &NpmPackageCacheFolderId, dist: &NpmPackageVersionDistInfo, registry_url: &Url, ) -> Result<(), AnyError> { @@ -214,10 +324,11 @@ impl NpmCache { async fn ensure_package_inner( &self, - id: &NpmPackageId, + id: &NpmPackageCacheFolderId, dist: &NpmPackageVersionDistInfo, registry_url: &Url, ) -> Result<(), AnyError> { + // todo(THIS PR): callers should cache the non count version first let package_folder = self.readonly.package_folder(id, registry_url); if package_folder.exists() // if this file exists, then the package didn't successfully extract @@ -237,6 +348,21 @@ impl NpmCache { ); } + // This package is a copy of the original package for peer dependency purposes. + if id.copy_count > 0 { + // it's assumed that the main package folder will exist here + let main_package_folder = self + .readonly + .package_folder(&id.with_no_count(), registry_url); + // todo(THIS PR): use hard linking instead of copying + with_folder_sync_lock( + (id.name.as_str(), &id.version), + &package_folder, + || fs_util::copy_dir_recursive(&main_package_folder, &package_folder), + )?; + return Ok(()); + } + let _guard = self.progress_bar.update(&dist.tarball); let response = reqwest::get(&dist.tarball).await?; @@ -256,35 +382,18 @@ impl NpmCache { } else { let bytes = response.bytes().await?; - match verify_and_extract_tarball(id, &bytes, dist, &package_folder) { - Ok(()) => Ok(()), - Err(err) => { - if let Err(remove_err) = fs::remove_dir_all(&package_folder) { - if remove_err.kind() != std::io::ErrorKind::NotFound { - bail!( - concat!( - "Failed verifying and extracting npm tarball for {}, then ", - "failed cleaning up package cache folder.\n\nOriginal ", - "error:\n\n{}\n\nRemove error:\n\n{}\n\nPlease manually ", - "delete this folder or you will run into issues using this ", - "package in the future:\n\n{}" - ), - id, - err, - remove_err, - package_folder.display(), - ); - } - } - Err(err) - } - } + verify_and_extract_tarball( + (id.name.as_str(), &id.version), + &bytes, + dist, + &package_folder, + ) } } pub fn package_folder( &self, - id: &NpmPackageId, + id: &NpmPackageCacheFolderId, registry_url: &Url, ) -> PathBuf { self.readonly.package_folder(id, registry_url) @@ -298,14 +407,14 @@ impl NpmCache { self.readonly.registry_folder(registry_url) } - pub fn resolve_package_id_from_specifier( + pub fn resolve_package_folder_id_from_specifier( &self, specifier: &ModuleSpecifier, registry_url: &Url, - ) -> Result { + ) -> Result { self .readonly - .resolve_package_id_from_specifier(specifier, registry_url) + .resolve_package_folder_id_from_specifier(specifier, registry_url) } } @@ -314,8 +423,8 @@ mod test { use deno_core::url::Url; use super::ReadonlyNpmCache; + use crate::npm::cache::NpmPackageCacheFolderId; use crate::npm::semver::NpmVersion; - use crate::npm::NpmPackageId; #[test] fn should_get_lowercase_package_folder() { @@ -325,10 +434,10 @@ mod test { assert_eq!( cache.package_folder( - &NpmPackageId { + &NpmPackageCacheFolderId { name: "json".to_string(), version: NpmVersion::parse("1.2.5").unwrap(), - peer_dependencies: Vec::new(), + copy_count: 0, }, ®istry_url, ), @@ -340,21 +449,17 @@ mod test { assert_eq!( cache.package_folder( - &NpmPackageId { + &NpmPackageCacheFolderId { name: "json".to_string(), version: NpmVersion::parse("1.2.5").unwrap(), - peer_dependencies: vec![NpmPackageId { - name: "other".to_string(), - version: NpmVersion::parse("3.2.1").unwrap(), - peer_dependencies: Vec::new() - }], + copy_count: 1, }, ®istry_url, ), root_dir .join("registry.npmjs.org") .join("json") - .join("1.2.5_other@3.2.1"), + .join("1.2.5_1"), ); } } diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index 85566e1226c996..6c115db34681c0 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -51,6 +51,7 @@ impl VisitedVersions { #[derive(Default, Clone)] struct GraphPath { + // todo(THIS PR): investigate if this should use a singly linked list too visited_versions: VisitedVersions, // todo(THIS PR): switch to a singly linked list here specifiers: Vec, diff --git a/cli/npm/resolution/mod.rs b/cli/npm/resolution/mod.rs index b6ed58ea5abf4f..239b3963afc768 100644 --- a/cli/npm/resolution/mod.rs +++ b/cli/npm/resolution/mod.rs @@ -185,6 +185,7 @@ impl NpmPackageId { } fn as_serialize_name_with_level(&self, level: usize) -> String { + // WARNING: This should not change because it's used in the lockfile let mut result = format!( "{}@{}", if level == 0 { diff --git a/cli/npm/resolvers/global.rs b/cli/npm/resolvers/global.rs index 4fb3195d06e2bc..936704ee413a83 100644 --- a/cli/npm/resolvers/global.rs +++ b/cli/npm/resolvers/global.rs @@ -74,7 +74,7 @@ impl InnerNpmPackageResolver for GlobalNpmPackageResolver { ) -> Result { let referrer_pkg_id = self .cache - .resolve_package_id_from_specifier(referrer, &self.registry_url)?; + .resolve_package_folder_id_from_specifier(referrer, &self.registry_url)?; let pkg_result = self .resolution .resolve_package_from_package(name, &referrer_pkg_id); @@ -105,9 +105,10 @@ impl InnerNpmPackageResolver for GlobalNpmPackageResolver { &self, specifier: &ModuleSpecifier, ) -> Result { - let pkg_id = self - .cache - .resolve_package_id_from_specifier(specifier, &self.registry_url)?; + let pkg_id = self.cache.resolve_package_folder_id_from_specifier( + specifier, + &self.registry_url, + )?; Ok(self.package_folder(&pkg_id)) } diff --git a/cli/npm/tarball.rs b/cli/npm/tarball.rs index 928331a71a9bb3..12503244be4bdd 100644 --- a/cli/npm/tarball.rs +++ b/cli/npm/tarball.rs @@ -6,18 +6,17 @@ use std::path::Path; use std::path::PathBuf; use deno_core::anyhow::bail; -use deno_core::anyhow::Context; use deno_core::error::AnyError; use flate2::read::GzDecoder; use tar::Archive; use tar::EntryType; -use super::cache::NPM_PACKAGE_SYNC_LOCK_FILENAME; +use super::cache::with_folder_sync_lock; use super::registry::NpmPackageVersionDistInfo; -use super::NpmPackageId; +use super::semver::NpmVersion; pub fn verify_and_extract_tarball( - package: &NpmPackageId, + package: (&str, &NpmVersion), data: &[u8], dist_info: &NpmPackageVersionDistInfo, output_folder: &Path, @@ -27,50 +26,19 @@ pub fn verify_and_extract_tarball( } else { // todo(dsherret): check shasum here bail!( - "Errored on '{}': npm packages with no integrity are not implemented.", - package + "Errored on '{}@{}': npm packages with no integrity are not implemented.", + package.0, + package.1, ); } - fs::create_dir_all(output_folder).with_context(|| { - format!("Error creating '{}'.", output_folder.display()) - })?; - - // This sync lock file is a way to ensure that partially created - // npm package directories aren't considered valid. This could maybe - // be a bit smarter in the future to not bother extracting here - // if another process has taken the lock in the past X seconds and - // wait for the other process to finish (it could try to create the - // file with `create_new(true)` then if it exists, check the metadata - // then wait until the other process finishes with a timeout), but - // for now this is good enough. - let sync_lock_path = output_folder.join(NPM_PACKAGE_SYNC_LOCK_FILENAME); - match fs::OpenOptions::new() - .write(true) - .create(true) - .open(&sync_lock_path) - { - Ok(_) => { - extract_tarball(data, output_folder)?; - // extraction succeeded, so only now delete this file - let _ignore = std::fs::remove_file(&sync_lock_path); - Ok(()) - } - Err(err) => { - bail!( - concat!( - "Error creating package sync lock file at '{}'. ", - "Maybe try manually deleting this folder.\n\n{:#}", - ), - output_folder.display(), - err - ); - } - } + with_folder_sync_lock(package, output_folder, || { + extract_tarball(data, output_folder) + }) } fn verify_tarball_integrity( - package: &NpmPackageId, + package: (&str, &NpmVersion), data: &[u8], npm_integrity: &str, ) -> Result<(), AnyError> { @@ -81,16 +49,18 @@ fn verify_tarball_integrity( let algo = match hash_kind { "sha512" => &SHA512, hash_kind => bail!( - "Not implemented hash function for {}: {}", - package, + "Not implemented hash function for {}@{}: {}", + package.0, + package.1, hash_kind ), }; (algo, checksum.to_lowercase()) } None => bail!( - "Not implemented integrity kind for {}: {}", - package, + "Not implemented integrity kind for {}@{}: {}", + package.0, + package.1, npm_integrity ), }; @@ -101,8 +71,9 @@ fn verify_tarball_integrity( let tarball_checksum = base64::encode(digest.as_ref()).to_lowercase(); if tarball_checksum != expected_checksum { bail!( - "Tarball checksum did not match what was provided by npm registry for {}.\n\nExpected: {}\nActual: {}", - package, + "Tarball checksum did not match what was provided by npm registry for {}@{}.\n\nExpected: {}\nActual: {}", + package.0, + package.1, expected_checksum, tarball_checksum, ) @@ -162,33 +133,31 @@ mod test { #[test] pub fn test_verify_tarball() { - let package_id = NpmPackageId { - name: "package".to_string(), - version: NpmVersion::parse("1.0.0").unwrap(), - peer_dependencies: Vec::new(), - }; + let package_name = "package".to_string(); + let package_version = NpmVersion::parse("1.0.0").unwrap(); + let package = (package_name.as_str(), &package_version); let actual_checksum = "z4phnx7vul3xvchq1m2ab9yg5aulvxxcg/spidns6c5h0ne8xyxysp+dgnkhfuwvy7kxvudbeoglodj6+sfapg=="; assert_eq!( - verify_tarball_integrity(&package_id, &Vec::new(), "test") + verify_tarball_integrity(&package, &Vec::new(), "test") .unwrap_err() .to_string(), "Not implemented integrity kind for package@1.0.0: test", ); assert_eq!( - verify_tarball_integrity(&package_id, &Vec::new(), "sha1-test") + verify_tarball_integrity(&package, &Vec::new(), "sha1-test") .unwrap_err() .to_string(), "Not implemented hash function for package@1.0.0: sha1", ); assert_eq!( - verify_tarball_integrity(&package_id, &Vec::new(), "sha512-test") + verify_tarball_integrity(&package, &Vec::new(), "sha512-test") .unwrap_err() .to_string(), format!("Tarball checksum did not match what was provided by npm registry for package@1.0.0.\n\nExpected: test\nActual: {}", actual_checksum), ); assert!(verify_tarball_integrity( - &package_id, + &package, &Vec::new(), &format!("sha512-{}", actual_checksum) ) From b481234b6c6a63c58b812effb56d147ea8a73b51 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sun, 6 Nov 2022 14:50:22 -0500 Subject: [PATCH 24/43] Working on hooking up the cache with the snapshot. --- cli/npm/cache.rs | 28 +++++++------ cli/npm/resolution/graph.rs | 59 ++++++++++++++++---------- cli/npm/resolution/mod.rs | 3 +- cli/npm/resolution/snapshot.rs | 75 +++++++++++++++++++++++++++++++--- cli/npm/tarball.rs | 8 ++-- 5 files changed, 128 insertions(+), 45 deletions(-) diff --git a/cli/npm/cache.rs b/cli/npm/cache.rs index c50a5268e85ec8..2926003385dbad 100644 --- a/cli/npm/cache.rs +++ b/cli/npm/cache.rs @@ -107,7 +107,7 @@ pub struct NpmPackageCacheFolderId { pub version: NpmVersion, /// Peer dependency resolution may require us to have duplicate copies /// of the same package. - pub copy_count: usize, + pub copy_index: usize, } impl NpmPackageCacheFolderId { @@ -115,7 +115,7 @@ impl NpmPackageCacheFolderId { Self { name: self.name.clone(), version: self.version.clone(), - copy_count: 0, + copy_index: 0, } } } @@ -123,8 +123,8 @@ impl NpmPackageCacheFolderId { impl std::fmt::Display for NpmPackageCacheFolderId { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!(f, "{}@{}", self.name, self.version)?; - if self.copy_count > 0 { - write!(f, "_{}", self.copy_count)?; + if self.copy_index > 0 { + write!(f, "_{}", self.copy_index)?; } Ok(()) } @@ -183,10 +183,10 @@ impl ReadonlyNpmCache { registry_url: &Url, ) -> PathBuf { self.package_name_folder(&id.name, registry_url).join( - if id.copy_count == 0 { + if id.copy_index == 0 { id.version.to_string() } else { - format!("{}_{}", id.version.to_string(), id.copy_count) + format!("{}_{}", id.version.to_string(), id.copy_index) }, ) } @@ -267,7 +267,7 @@ impl ReadonlyNpmCache { } let version_part = parts.pop().unwrap(); let name = parts.join("/"); - let (version, copy_count) = + let (version, copy_index) = if let Some((version, copy_count)) = version_part.split_once('_') { (version, copy_count.parse::().ok()?) } else { @@ -276,7 +276,7 @@ impl ReadonlyNpmCache { Some(NpmPackageCacheFolderId { name: name.to_string(), version: NpmVersion::parse(version).ok()?, - copy_count, + copy_index, }) } @@ -328,7 +328,7 @@ impl NpmCache { dist: &NpmPackageVersionDistInfo, registry_url: &Url, ) -> Result<(), AnyError> { - // todo(THIS PR): callers should cache the non count version first + // todo(THIS PR): callers should cache the 0-indexed version first let package_folder = self.readonly.package_folder(id, registry_url); if package_folder.exists() // if this file exists, then the package didn't successfully extract @@ -349,8 +349,10 @@ impl NpmCache { } // This package is a copy of the original package for peer dependency purposes. - if id.copy_count > 0 { - // it's assumed that the main package folder will exist here + if id.copy_index > 0 { + // This code assumes that the main package folder will exist here and that + // should be enforced at a higher level to prevent contention between multiple + // threads on the cache folder. let main_package_folder = self .readonly .package_folder(&id.with_no_count(), registry_url); @@ -437,7 +439,7 @@ mod test { &NpmPackageCacheFolderId { name: "json".to_string(), version: NpmVersion::parse("1.2.5").unwrap(), - copy_count: 0, + copy_index: 0, }, ®istry_url, ), @@ -452,7 +454,7 @@ mod test { &NpmPackageCacheFolderId { name: "json".to_string(), version: NpmVersion::parse("1.2.5").unwrap(), - copy_count: 1, + copy_index: 1, }, ®istry_url, ), diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index 6c115db34681c0..37410a9c223ea9 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -24,6 +24,7 @@ use crate::npm::semver::NpmVersionReq; use crate::npm::NpmRegistryApi; use super::snapshot::NpmResolutionSnapshot; +use super::snapshot::SnapshotPackageCopyIndexesBuilder; use super::NpmPackageId; use super::NpmPackageReq; use super::NpmResolutionPackage; @@ -113,26 +114,48 @@ impl Node { pub struct Graph { package_reqs: HashMap, packages_by_name: HashMap>, - // Ideally this would be Rc>, but we need to use a Mutex + // Ideally this value would be Rc>, but we need to use a Mutex // because the lsp requires Send and this code is executed in the lsp. // Would be nice if the lsp wasn't Send. packages: HashMap>>, + // This will be set when creating from a snapshot, then + // inform the final snapshot creation. + packages_to_copy_index: HashMap, } impl Graph { - pub fn has_package_req(&self, req: &NpmPackageReq) -> bool { - self.package_reqs.contains_key(req) - } + pub fn from_snapshot(snapshot: NpmResolutionSnapshot) -> Self { + fn fill_for_id( + graph: &mut Graph, + id: &NpmPackageId, + packages: &HashMap, + ) -> Arc> { + let resolution = packages.get(id).unwrap(); + let node = graph.get_or_create_for_id(id).1; + for (name, child_id) in &resolution.dependencies { + let child_node = fill_for_id(graph, child_id, packages); + graph.set_child_parent_node(name, &child_node, id); + } + node + } - pub fn fill_with_snapshot(&mut self, snapshot: &NpmResolutionSnapshot) { + let mut graph = Self { + packages_to_copy_index: snapshot.packages_to_copy_index, + ..Default::default() + }; for (package_req, id) in &snapshot.package_reqs { - let node = self.fill_for_id_with_snapshot(id, snapshot); + let node = fill_for_id(&mut graph, id, &snapshot.packages); (*node).lock().add_parent( package_req.to_string(), NodeParent::Req(package_req.clone()), ); - self.package_reqs.insert(package_req.clone(), id.clone()); + graph.package_reqs.insert(package_req.clone(), id.clone()); } + graph + } + + pub fn has_package_req(&self, req: &NpmPackageReq) -> bool { + self.package_reqs.contains_key(req) } fn get_or_create_for_id( @@ -158,20 +181,6 @@ impl Graph { } } - fn fill_for_id_with_snapshot( - &mut self, - id: &NpmPackageId, - snapshot: &NpmResolutionSnapshot, - ) -> Arc> { - let resolution = snapshot.packages.get(id).unwrap(); - let node = self.get_or_create_for_id(id).1; - for (name, child_id) in &resolution.dependencies { - let child_node = self.fill_for_id_with_snapshot(child_id, snapshot); - self.set_child_parent_node(name, &child_node, id); - } - node - } - fn borrow_node(&self, id: &NpmPackageId) -> MutexGuard { (**self.packages.get(id).unwrap_or_else(|| { panic!( @@ -261,8 +270,15 @@ impl Graph { self, api: &impl NpmRegistryApi, ) -> Result { + let mut copy_index_builder = + SnapshotPackageCopyIndexesBuilder::from_map_with_capacity( + self.packages_to_copy_index, + self.packages.len(), + ); + let mut packages = HashMap::with_capacity(self.packages.len()); for (id, node) in self.packages { + copy_index_builder.add_package(&id); let dist = api .package_version_info(&id.name, &id.version) .await? @@ -283,6 +299,7 @@ impl Graph { package_reqs: self.package_reqs, packages_by_name: self.packages_by_name, packages, + packages_to_copy_index: copy_index_builder.into_map(), }) } } diff --git a/cli/npm/resolution/mod.rs b/cli/npm/resolution/mod.rs index 239b3963afc768..3937589b32bcd8 100644 --- a/cli/npm/resolution/mod.rs +++ b/cli/npm/resolution/mod.rs @@ -393,8 +393,7 @@ impl NpmResolution { snapshot: NpmResolutionSnapshot, ) -> Result { // convert the snapshot to a traversable graph - let mut graph = Graph::default(); - graph.fill_with_snapshot(&snapshot); + let mut graph = Graph::from_snapshot(snapshot); // multiple packages are resolved in alphabetical order package_reqs.sort_by(|a, b| a.name.cmp(&b.name)); diff --git a/cli/npm/resolution/snapshot.rs b/cli/npm/resolution/snapshot.rs index 5ed22848dbbe83..a12afcf5a58c9c 100644 --- a/cli/npm/resolution/snapshot.rs +++ b/cli/npm/resolution/snapshot.rs @@ -30,6 +30,8 @@ pub struct NpmResolutionSnapshot { pub(super) packages_by_name: HashMap>, #[serde(with = "map_to_vec")] pub(super) packages: HashMap, + #[serde(with = "map_to_vec")] + pub(super) packages_to_copy_index: HashMap, } // This is done so the maps with non-string keys get serialized and deserialized as vectors. @@ -173,6 +175,7 @@ impl NpmResolutionSnapshot { let mut package_reqs: HashMap; let mut packages_by_name: HashMap>; let mut packages: HashMap; + let mut copy_indexes_builder: SnapshotPackageCopyIndexesBuilder; { let lockfile = lockfile.lock(); @@ -180,11 +183,12 @@ impl NpmResolutionSnapshot { // pre-allocate collections package_reqs = HashMap::with_capacity(lockfile.content.npm.specifiers.len()); - packages = HashMap::with_capacity(lockfile.content.npm.packages.len()); - packages_by_name = - HashMap::with_capacity(lockfile.content.npm.packages.len()); // close enough - let mut verify_ids = - HashSet::with_capacity(lockfile.content.npm.packages.len()); + let packages_len = lockfile.content.npm.packages.len(); + packages = HashMap::with_capacity(packages_len); + packages_by_name = HashMap::with_capacity(packages_len); // close enough + copy_indexes_builder = + SnapshotPackageCopyIndexesBuilder::with_capacity(packages_len); + let mut verify_ids = HashSet::with_capacity(packages_len); // collect the specifiers to version mappings for (key, value) in &lockfile.content.npm.specifiers { @@ -198,6 +202,11 @@ impl NpmResolutionSnapshot { // then the packages for (key, value) in &lockfile.content.npm.packages { let package_id = NpmPackageId::deserialize_name(key)?; + + // Update the package copy index + copy_indexes_builder.add_package(&package_id); + + // collect the dependencies let mut dependencies = HashMap::default(); packages_by_name @@ -270,10 +279,66 @@ impl NpmResolutionSnapshot { package_reqs, packages_by_name, packages, + packages_to_copy_index: copy_indexes_builder.into_map(), }) } } +pub struct SnapshotPackageCopyIndexesBuilder { + packages_to_copy_index: HashMap, + package_name_version_to_copy_count: HashMap<(String, String), usize>, +} + +impl SnapshotPackageCopyIndexesBuilder { + pub fn with_capacity(capacity: usize) -> Self { + Self { + packages_to_copy_index: HashMap::with_capacity(capacity), + package_name_version_to_copy_count: HashMap::with_capacity(capacity), // close enough + } + } + + pub fn from_map_with_capacity( + packages_to_copy_index: HashMap, + capacity: usize, + ) -> Self { + let mut package_name_version_to_copy_count = + HashMap::with_capacity(capacity); // close enough + if capacity > packages_to_copy_index.len() { + packages_to_copy_index.reserve(capacity - packages_to_copy_index.len()); + } + + for (id, index) in &packages_to_copy_index { + let entry = package_name_version_to_copy_count + .entry((id.name.to_string(), id.version.to_string())) + .or_insert(0); + if *entry < *index { + *entry = *index; + } + } + Self { + packages_to_copy_index, + package_name_version_to_copy_count, + } + } + + pub fn into_map(self) -> HashMap { + self.packages_to_copy_index + } + + pub fn add_package(&mut self, id: &NpmPackageId) { + if !self.packages_to_copy_index.contains_key(id) { + let index = *self + .package_name_version_to_copy_count + .entry((id.name.to_string(), id.version.to_string())) + .and_modify(|count| { + *count += 1; + }) + .or_insert(0); + self.packages_to_copy_index.insert(id.clone(), index); + } + } +} + fn name_without_path(name: &str) -> &str { let mut search_start_index = 0; if name.starts_with('@') { diff --git a/cli/npm/tarball.rs b/cli/npm/tarball.rs index 12503244be4bdd..751e093f5a5ee3 100644 --- a/cli/npm/tarball.rs +++ b/cli/npm/tarball.rs @@ -139,25 +139,25 @@ mod test { let actual_checksum = "z4phnx7vul3xvchq1m2ab9yg5aulvxxcg/spidns6c5h0ne8xyxysp+dgnkhfuwvy7kxvudbeoglodj6+sfapg=="; assert_eq!( - verify_tarball_integrity(&package, &Vec::new(), "test") + verify_tarball_integrity(package, &Vec::new(), "test") .unwrap_err() .to_string(), "Not implemented integrity kind for package@1.0.0: test", ); assert_eq!( - verify_tarball_integrity(&package, &Vec::new(), "sha1-test") + verify_tarball_integrity(package, &Vec::new(), "sha1-test") .unwrap_err() .to_string(), "Not implemented hash function for package@1.0.0: sha1", ); assert_eq!( - verify_tarball_integrity(&package, &Vec::new(), "sha512-test") + verify_tarball_integrity(package, &Vec::new(), "sha512-test") .unwrap_err() .to_string(), format!("Tarball checksum did not match what was provided by npm registry for package@1.0.0.\n\nExpected: test\nActual: {}", actual_checksum), ); assert!(verify_tarball_integrity( - &package, + package, &Vec::new(), &format!("sha512-{}", actual_checksum) ) From 406c923a1c80606b10da5e19a9babeca696c6384 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sun, 6 Nov 2022 19:09:34 -0500 Subject: [PATCH 25/43] Hooked up resolution to execution. Untested. --- cli/lockfile.rs | 4 ++ cli/npm/cache.rs | 119 +++++++++++++++++++----------- cli/npm/resolution/graph.rs | 58 +++++++++++++-- cli/npm/resolution/mod.rs | 34 ++++++++- cli/npm/resolution/snapshot.rs | 127 ++++++++++++++++++++++----------- cli/npm/resolvers/common.rs | 8 ++- cli/npm/resolvers/global.rs | 32 +++++++-- cli/npm/resolvers/local.rs | 95 ++++++++++++++++++------ 8 files changed, 361 insertions(+), 116 deletions(-) diff --git a/cli/lockfile.rs b/cli/lockfile.rs index 3dc420796cd0f6..94611ac889aff4 100644 --- a/cli/lockfile.rs +++ b/cli/lockfile.rs @@ -559,6 +559,7 @@ mod tests { version: NpmVersion::parse("3.3.4").unwrap(), peer_dependencies: Vec::new(), }, + copy_index: 0, dist: NpmPackageVersionDistInfo { tarball: "foo".to_string(), shasum: "foo".to_string(), @@ -575,6 +576,7 @@ mod tests { version: NpmVersion::parse("1.0.0").unwrap(), peer_dependencies: Vec::new(), }, + copy_index: 0, dist: NpmPackageVersionDistInfo { tarball: "foo".to_string(), shasum: "foo".to_string(), @@ -592,6 +594,7 @@ mod tests { version: NpmVersion::parse("1.0.2").unwrap(), peer_dependencies: Vec::new(), }, + copy_index: 0, dist: NpmPackageVersionDistInfo { tarball: "foo".to_string(), shasum: "foo".to_string(), @@ -609,6 +612,7 @@ mod tests { version: NpmVersion::parse("1.0.2").unwrap(), peer_dependencies: Vec::new(), }, + copy_index: 0, dist: NpmPackageVersionDistInfo { tarball: "foo".to_string(), shasum: "foo".to_string(), diff --git a/cli/npm/cache.rs b/cli/npm/cache.rs index 2926003385dbad..97119944e0716b 100644 --- a/cli/npm/cache.rs +++ b/cli/npm/cache.rs @@ -177,18 +177,33 @@ impl ReadonlyNpmCache { Self::new(dir.root.join("npm")) } - pub fn package_folder( + pub fn package_folder_for_id( &self, id: &NpmPackageCacheFolderId, registry_url: &Url, ) -> PathBuf { - self.package_name_folder(&id.name, registry_url).join( - if id.copy_index == 0 { - id.version.to_string() - } else { - format!("{}_{}", id.version.to_string(), id.copy_index) - }, - ) + if id.copy_index == 0 { + self.package_folder_for_name_and_version( + &id.name, + &id.version, + registry_url, + ) + } else { + self + .package_name_folder(&id.name, registry_url) + .join(format!("{}_{}", id.version.to_string(), id.copy_index)) + } + } + + pub fn package_folder_for_name_and_version( + &self, + name: &str, + version: &NpmVersion, + registry_url: &Url, + ) -> PathBuf { + self + .package_name_folder(name, registry_url) + .join(version.to_string()) } pub fn package_name_folder(&self, name: &str, registry_url: &Url) -> PathBuf { @@ -312,29 +327,35 @@ impl NpmCache { pub async fn ensure_package( &self, - id: &NpmPackageCacheFolderId, + package: (&str, &NpmVersion), dist: &NpmPackageVersionDistInfo, registry_url: &Url, ) -> Result<(), AnyError> { self - .ensure_package_inner(id, dist, registry_url) + .ensure_package_inner(package, dist, registry_url) .await - .with_context(|| format!("Failed caching npm package '{}'.", id)) + .with_context(|| { + format!("Failed caching npm package '{}@{}'.", package.0, package.1) + }) } async fn ensure_package_inner( &self, - id: &NpmPackageCacheFolderId, + package: (&str, &NpmVersion), dist: &NpmPackageVersionDistInfo, registry_url: &Url, ) -> Result<(), AnyError> { // todo(THIS PR): callers should cache the 0-indexed version first - let package_folder = self.readonly.package_folder(id, registry_url); + let package_folder = self.readonly.package_folder_for_name_and_version( + package.0, + package.1, + registry_url, + ); if package_folder.exists() // if this file exists, then the package didn't successfully extract // the first time, or another process is currently extracting the zip file && !package_folder.join(NPM_PACKAGE_SYNC_LOCK_FILENAME).exists() - && self.cache_setting.should_use_for_npm_package(&id.name) + && self.cache_setting.should_use_for_npm_package(&package.0) { return Ok(()); } else if self.cache_setting == CacheSetting::Only { @@ -342,29 +363,12 @@ impl NpmCache { "NotCached", format!( "An npm specifier not found in cache: \"{}\", --cached-only is specified.", - id.name + &package.0 ) ) ); } - // This package is a copy of the original package for peer dependency purposes. - if id.copy_index > 0 { - // This code assumes that the main package folder will exist here and that - // should be enforced at a higher level to prevent contention between multiple - // threads on the cache folder. - let main_package_folder = self - .readonly - .package_folder(&id.with_no_count(), registry_url); - // todo(THIS PR): use hard linking instead of copying - with_folder_sync_lock( - (id.name.as_str(), &id.version), - &package_folder, - || fs_util::copy_dir_recursive(&main_package_folder, &package_folder), - )?; - return Ok(()); - } - let _guard = self.progress_bar.update(&dist.tarball); let response = reqwest::get(&dist.tarball).await?; @@ -384,21 +388,52 @@ impl NpmCache { } else { let bytes = response.bytes().await?; - verify_and_extract_tarball( - (id.name.as_str(), &id.version), - &bytes, - dist, - &package_folder, - ) + verify_and_extract_tarball(package, &bytes, dist, &package_folder) } } - pub fn package_folder( + /// Ensures a copy of the package exists in the global cache. + /// + /// This assumes that the original package folder being hard linked + /// from exists before this is called. + pub fn ensure_copy_package( + &self, + id: &NpmPackageCacheFolderId, + registry_url: &Url, + ) -> Result<(), AnyError> { + assert_ne!(id.copy_index, 0); + let package_folder = self.readonly.package_folder_for_id(id, registry_url); + let original_package_folder = self + .readonly + .package_folder_for_name_and_version(&id.name, &id.version, registry_url); + // todo(THIS PR): use hard linking instead of copying + with_folder_sync_lock( + (&id.name.as_str(), &id.version), + &package_folder, + || fs_util::copy_dir_recursive(&original_package_folder, &package_folder), + )?; + return Ok(()); + } + + pub fn package_folder_for_id( &self, id: &NpmPackageCacheFolderId, registry_url: &Url, ) -> PathBuf { - self.readonly.package_folder(id, registry_url) + self.readonly.package_folder_for_id(id, registry_url) + } + + pub fn package_folder_for_name_and_version( + &self, + name: &str, + version: &NpmVersion, + registry_url: &Url, + ) -> PathBuf { + self.readonly.package_folder_for_name_and_version( + name, + version, + registry_url, + ) } pub fn package_name_folder(&self, name: &str, registry_url: &Url) -> PathBuf { @@ -435,7 +470,7 @@ mod test { let registry_url = Url::parse("https://registry.npmjs.org/").unwrap(); assert_eq!( - cache.package_folder( + cache.package_folder_for_id( &NpmPackageCacheFolderId { name: "json".to_string(), version: NpmVersion::parse("1.2.5").unwrap(), @@ -450,7 +485,7 @@ mod test { ); assert_eq!( - cache.package_folder( + cache.package_folder_for_id( &NpmPackageCacheFolderId { name: "json".to_string(), version: NpmVersion::parse("1.2.5").unwrap(), diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index 37410a9c223ea9..5faee06c6e39c7 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -140,7 +140,13 @@ impl Graph { } let mut graph = Self { - packages_to_copy_index: snapshot.packages_to_copy_index, + // Note: It might be more correct to store the copy index + // from past resolutions with the node somehow, but maybe not. + packages_to_copy_index: snapshot + .packages + .iter() + .map(|(id, p)| (id.clone(), p.copy_index)) + .collect(), ..Default::default() }; for (package_req, id) in &snapshot.package_reqs { @@ -278,7 +284,6 @@ impl Graph { let mut packages = HashMap::with_capacity(self.packages.len()); for (id, node) in self.packages { - copy_index_builder.add_package(&id); let dist = api .package_version_info(&id.name, &id.version) .await? @@ -288,9 +293,10 @@ impl Graph { packages.insert( id.clone(), NpmResolutionPackage { + copy_index: copy_index_builder.add_package(&id), + id, dist, dependencies: node.children.clone(), - id, }, ); } @@ -299,7 +305,6 @@ impl Graph { package_reqs: self.package_reqs, packages_by_name: self.packages_by_name, packages, - packages_to_copy_index: copy_index_builder.into_map(), }) } } @@ -923,6 +928,7 @@ mod test { vec![ NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-a@1.0.0").unwrap(), + copy_index: 0, dependencies: HashMap::from([ ( "package-b".to_string(), @@ -937,19 +943,22 @@ mod test { }, NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-b@2.0.0").unwrap(), + copy_index: 0, dist: Default::default(), dependencies: Default::default(), }, NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-c@0.1.0").unwrap(), + copy_index: 0, dist: Default::default(), dependencies: HashMap::from([( "package-d".to_string(), NpmPackageId::deserialize_name("package-d@3.2.1").unwrap(), - ),]) + )]) }, NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-d@3.2.1").unwrap(), + copy_index: 0, dist: Default::default(), dependencies: Default::default(), }, @@ -989,6 +998,7 @@ mod test { "package-a@1.0.0_package-peer@4.0.0" ) .unwrap(), + copy_index: 0, dependencies: HashMap::from([ ( "package-b".to_string(), @@ -1012,6 +1022,7 @@ mod test { "package-b@2.0.0_package-peer@4.0.0" ) .unwrap(), + copy_index: 0, dist: Default::default(), dependencies: HashMap::from([( "package-peer".to_string(), @@ -1023,6 +1034,7 @@ mod test { "package-c@3.0.0_package-peer@4.0.0" ) .unwrap(), + copy_index: 0, dist: Default::default(), dependencies: HashMap::from([( "package-peer".to_string(), @@ -1031,6 +1043,7 @@ mod test { }, NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-peer@4.0.0").unwrap(), + copy_index: 0, dist: Default::default(), dependencies: Default::default(), }, @@ -1076,6 +1089,7 @@ mod test { vec![ NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-0@1.1.1").unwrap(), + copy_index: 0, dependencies: HashMap::from([( "package-a".to_string(), NpmPackageId::deserialize_name( @@ -1090,6 +1104,7 @@ mod test { "package-a@1.0.0_package-peer@4.0.0" ) .unwrap(), + copy_index: 0, dependencies: HashMap::from([ ( "package-b".to_string(), @@ -1117,6 +1132,7 @@ mod test { "package-b@2.0.0_package-peer@4.0.0" ) .unwrap(), + copy_index: 0, dist: Default::default(), dependencies: HashMap::from([( "package-peer".to_string(), @@ -1128,6 +1144,7 @@ mod test { "package-c@3.0.0_package-peer@4.0.0" ) .unwrap(), + copy_index: 0, dist: Default::default(), dependencies: HashMap::from([( "package-peer".to_string(), @@ -1136,6 +1153,7 @@ mod test { }, NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-peer@4.0.0").unwrap(), + copy_index: 0, dist: Default::default(), dependencies: Default::default(), }, @@ -1169,6 +1187,7 @@ mod test { vec![ NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-a@1.0.0").unwrap(), + copy_index: 0, dependencies: HashMap::from([ ( "package-b".to_string(), @@ -1183,6 +1202,7 @@ mod test { }, NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-b@2.0.0").unwrap(), + copy_index: 0, dist: Default::default(), dependencies: HashMap::from([( "package-peer".to_string(), @@ -1191,6 +1211,7 @@ mod test { }, NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-c@3.0.0").unwrap(), + copy_index: 0, dist: Default::default(), dependencies: HashMap::from([( "package-peer".to_string(), @@ -1199,6 +1220,7 @@ mod test { }, NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-peer@4.1.0").unwrap(), + copy_index: 0, dist: Default::default(), dependencies: Default::default(), }, @@ -1238,6 +1260,7 @@ mod test { vec![ NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-a@1.0.0").unwrap(), + copy_index: 0, dependencies: HashMap::from([ ( "package-b".to_string(), @@ -1252,11 +1275,13 @@ mod test { }, NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-b@2.0.0").unwrap(), + copy_index: 0, dist: Default::default(), dependencies: HashMap::new(), }, NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-c@3.0.0").unwrap(), + copy_index: 0, dist: Default::default(), dependencies: HashMap::new(), }, @@ -1300,6 +1325,7 @@ mod test { "package-a@1.0.0_package-peer@4.0.0" ) .unwrap(), + copy_index: 0, dependencies: HashMap::from([ ( "package-b".to_string(), @@ -1323,6 +1349,7 @@ mod test { "package-b@2.0.0_package-peer@4.0.0" ) .unwrap(), + copy_index: 0, dist: Default::default(), dependencies: HashMap::from([( "package-peer".to_string(), @@ -1334,6 +1361,7 @@ mod test { "package-c@3.0.0_package-peer@4.0.0" ) .unwrap(), + copy_index: 0, dist: Default::default(), dependencies: HashMap::from([( "package-peer".to_string(), @@ -1342,6 +1370,7 @@ mod test { }, NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-peer@4.0.0").unwrap(), + copy_index: 0, dist: Default::default(), dependencies: Default::default(), }, @@ -1381,6 +1410,7 @@ mod test { vec![ NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-0@1.0.0").unwrap(), + copy_index: 0, dependencies: HashMap::from([( "package-peer-a".to_string(), NpmPackageId::deserialize_name("package-peer-a@2.0.0").unwrap(), @@ -1389,6 +1419,7 @@ mod test { }, NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-peer-a@2.0.0").unwrap(), + copy_index: 0, dependencies: HashMap::from([( "package-peer-b".to_string(), NpmPackageId::deserialize_name("package-peer-b@3.0.0").unwrap(), @@ -1397,6 +1428,7 @@ mod test { }, NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-peer-b@3.0.0").unwrap(), + copy_index: 0, dependencies: HashMap::new(), dist: Default::default(), }, @@ -1438,6 +1470,7 @@ mod test { "package-0@1.0.0_package-peer-a@2.0.0_package-peer-b@3.0.0" ) .unwrap(), + copy_index: 0, dependencies: HashMap::from([ ( "package-peer-a".to_string(), @@ -1458,6 +1491,7 @@ mod test { "package-peer-a@2.0.0_package-peer-b@3.0.0" ) .unwrap(), + copy_index: 0, dependencies: HashMap::from([( "package-peer-b".to_string(), NpmPackageId::deserialize_name("package-peer-b@3.0.0").unwrap(), @@ -1466,6 +1500,7 @@ mod test { }, NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-peer-b@3.0.0").unwrap(), + copy_index: 0, dependencies: HashMap::new(), dist: Default::default(), }, @@ -1531,6 +1566,7 @@ mod test { vec![ NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-0@1.1.1").unwrap(), + copy_index: 0, dependencies: HashMap::from([( "package-a".to_string(), NpmPackageId::deserialize_name( @@ -1545,6 +1581,7 @@ mod test { "package-a@1.0.0_package-peer-a@4.0.0" ) .unwrap(), + copy_index: 0, dependencies: HashMap::from([ ( "package-b".to_string(), @@ -1576,6 +1613,7 @@ mod test { "package-b@2.0.0_package-peer-a@4.0.0" ) .unwrap(), + copy_index: 0, dist: Default::default(), dependencies: HashMap::from([ ( @@ -1593,6 +1631,7 @@ mod test { "package-c@3.0.0_package-peer-a@4.0.0" ) .unwrap(), + copy_index: 0, dist: Default::default(), dependencies: HashMap::from([( "package-peer-a".to_string(), @@ -1601,16 +1640,19 @@ mod test { }, NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-d@3.5.0").unwrap(), + copy_index: 0, dependencies: HashMap::from([]), dist: Default::default(), }, NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-e@3.6.0").unwrap(), + copy_index: 0, dependencies: HashMap::from([]), dist: Default::default(), }, NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-peer-a@4.0.0").unwrap(), + copy_index: 0, dist: Default::default(), dependencies: HashMap::from([( "package-peer-b".to_string(), @@ -1619,11 +1661,13 @@ mod test { }, NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-peer-b@5.4.1").unwrap(), + copy_index: 0, dist: Default::default(), dependencies: Default::default(), }, NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-peer-c@6.2.0").unwrap(), + copy_index: 0, dist: Default::default(), dependencies: Default::default(), }, @@ -1654,6 +1698,7 @@ mod test { NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-a@1.0.0_package-a@1.0.0") .unwrap(), + copy_index: 0, dependencies: HashMap::from([( "package-b".to_string(), NpmPackageId::deserialize_name( @@ -1668,6 +1713,7 @@ mod test { "package-b@2.0.0_package-a@1.0.0__package-a@1.0.0" ) .unwrap(), + copy_index: 0, dependencies: HashMap::from([( "package-a".to_string(), NpmPackageId::deserialize_name("package-a@1.0.0_package-a@1.0.0") @@ -1701,6 +1747,7 @@ mod test { vec![ NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-a@1.0.0").unwrap(), + copy_index: 0, dependencies: HashMap::from([( "package-b".to_string(), NpmPackageId::deserialize_name("package-b@2.0.0").unwrap(), @@ -1709,6 +1756,7 @@ mod test { }, NpmResolutionPackage { id: NpmPackageId::deserialize_name("package-b@2.0.0").unwrap(), + copy_index: 0, dependencies: HashMap::from([( "package-a".to_string(), NpmPackageId::deserialize_name("package-a@1.0.0").unwrap(), diff --git a/cli/npm/resolution/mod.rs b/cli/npm/resolution/mod.rs index 3937589b32bcd8..cb17a3c9a933f6 100644 --- a/cli/npm/resolution/mod.rs +++ b/cli/npm/resolution/mod.rs @@ -16,8 +16,10 @@ use serde::Serialize; use crate::lockfile::Lockfile; use self::graph::GraphDependencyResolver; +use self::snapshot::NpmPackagesPartitioned; use super::cache::should_sync_download; +use super::cache::NpmPackageCacheFolderId; use super::registry::NpmPackageVersionDistInfo; use super::registry::RealNpmRegistryApi; use super::semver::NpmVersion; @@ -308,12 +310,27 @@ impl std::fmt::Display for NpmPackageId { #[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] pub struct NpmResolutionPackage { pub id: NpmPackageId, + /// The peer dependency resolution can differ for the same + /// package (name and version) depending on where it is in + /// the resolution tree. This copy index indicates which + /// copy of the package this is. + pub copy_index: usize, pub dist: NpmPackageVersionDistInfo, /// Key is what the package refers to the other package as, /// which could be different from the package name. pub dependencies: HashMap, } +impl NpmResolutionPackage { + pub fn get_package_cache_folder_id(&self) -> NpmPackageCacheFolderId { + NpmPackageCacheFolderId { + name: self.id.name.clone(), + version: self.id.version.clone(), + copy_index: self.copy_index, + } + } +} + pub struct NpmResolution { api: RealNpmRegistryApi, snapshot: RwLock, @@ -443,10 +460,21 @@ impl NpmResolution { self.snapshot.read().package_from_id(id).cloned() } + pub fn resolve_package_cache_folder_id_from_id( + &self, + id: &NpmPackageId, + ) -> Option { + self + .snapshot + .read() + .package_from_id(id) + .map(|p| p.get_package_cache_folder_id()) + } + pub fn resolve_package_from_package( &self, name: &str, - referrer: &NpmPackageId, + referrer: &NpmPackageCacheFolderId, ) -> Result { self .snapshot @@ -471,6 +499,10 @@ impl NpmResolution { self.snapshot.read().all_packages() } + pub fn all_packages_partitioned(&self) -> NpmPackagesPartitioned { + self.snapshot.read().all_packages_partitioned() + } + pub fn has_packages(&self) -> bool { !self.snapshot.read().packages.is_empty() } diff --git a/cli/npm/resolution/snapshot.rs b/cli/npm/resolution/snapshot.rs index a12afcf5a58c9c..4d887a2bf731ca 100644 --- a/cli/npm/resolution/snapshot.rs +++ b/cli/npm/resolution/snapshot.rs @@ -4,6 +4,7 @@ use std::collections::HashMap; use std::collections::HashSet; use std::sync::Arc; +use deno_core::anyhow::anyhow; use deno_core::anyhow::bail; use deno_core::anyhow::Context; use deno_core::error::AnyError; @@ -13,6 +14,7 @@ use serde::Deserialize; use serde::Serialize; use crate::lockfile::Lockfile; +use crate::npm::cache::NpmPackageCacheFolderId; use crate::npm::registry::NpmPackageVersionDistInfo; use crate::npm::registry::NpmRegistryApi; use crate::npm::registry::RealNpmRegistryApi; @@ -23,6 +25,21 @@ use super::NpmPackageReq; use super::NpmResolutionPackage; use super::NpmVersionMatcher; +/// Packages partitioned by if they are "copy" packages or not. Copy +/// packages are used for peer dependencies. +pub struct NpmPackagesPartitioned { + pub packages: Vec, + pub copy_packages: Vec, +} + +impl NpmPackagesPartitioned { + pub fn into_all(self) -> Vec { + let mut packages = self.packages; + packages.extend(self.copy_packages); + packages + } +} + #[derive(Debug, Clone, Default, Serialize, Deserialize)] pub struct NpmResolutionSnapshot { #[serde(with = "map_to_vec")] @@ -30,8 +47,6 @@ pub struct NpmResolutionSnapshot { pub(super) packages_by_name: HashMap>, #[serde(with = "map_to_vec")] pub(super) packages: HashMap, - #[serde(with = "map_to_vec")] - pub(super) packages_to_copy_index: HashMap, } // This is done so the maps with non-string keys get serialized and deserialized as vectors. @@ -105,46 +120,80 @@ impl NpmResolutionSnapshot { pub fn resolve_package_from_package( &self, name: &str, - referrer: &NpmPackageId, + referrer: &NpmPackageCacheFolderId, ) -> Result<&NpmResolutionPackage, AnyError> { - match self.packages.get(referrer) { - Some(referrer_package) => { - let name_ = name_without_path(name); - if let Some(id) = referrer_package.dependencies.get(name_) { - return Ok(self.packages.get(id).unwrap()); - } - - if referrer_package.id.name == name_ { - return Ok(referrer_package); - } + // todo(dsherret): do we need an additional hashmap to get this quickly? + let referrer_package = self + .packages_by_name + .get(&referrer.name) + .and_then(|packages| { + packages + .iter() + .filter(|p| p.version == referrer.version) + .filter_map(|id| { + let package = self.packages.get(id)?; + if package.copy_index == referrer.copy_index { + Some(package) + } else { + None + } + }) + .next() + }) + .ok_or_else(|| { + anyhow!("could not find referrer npm package '{}'", referrer) + })?; + + let name = name_without_path(&name); + if let Some(id) = referrer_package.dependencies.get(name) { + return Ok(self.packages.get(id).unwrap()); + } - // TODO(bartlomieju): this should use a reverse lookup table in the - // snapshot instead of resolving best version again. - let req = NpmPackageReq { - name: name_.to_string(), - version_req: None, - }; + if referrer_package.id.name == name { + return Ok(referrer_package); + } - if let Some(id) = self.resolve_best_package_id(name_, &req) { - if let Some(pkg) = self.packages.get(&id) { - return Ok(pkg); - } - } + // TODO(bartlomieju): this should use a reverse lookup table in the + // snapshot instead of resolving best version again. + let req = NpmPackageReq { + name: name.to_string(), + version_req: None, + }; - bail!( - "could not find npm package '{}' referenced by '{}'", - name, - referrer - ) + if let Some(id) = self.resolve_best_package_id(name, &req) { + if let Some(pkg) = self.packages.get(&id) { + return Ok(pkg); } - None => bail!("could not find referrer npm package '{}'", referrer), } + + bail!( + "could not find npm package '{}' referenced by '{}'", + name, + referrer + ) } pub fn all_packages(&self) -> Vec { self.packages.values().cloned().collect() } + pub fn all_packages_partitioned(&self) -> NpmPackagesPartitioned { + let mut packages = self.all_packages(); + let mut copy_packages = Vec::with_capacity(packages.len() / 2); // at most 1 copy for every package + + // partition out any packages that are "copy" packages + for i in (0..packages.len()).rev() { + if packages[i].copy_index > 0 { + copy_packages.push(packages.swap_remove(i)); + } + } + + NpmPackagesPartitioned { + packages, + copy_packages, + } + } + pub fn resolve_best_package_id( &self, name: &str, @@ -203,9 +252,6 @@ impl NpmResolutionSnapshot { for (key, value) in &lockfile.content.npm.packages { let package_id = NpmPackageId::deserialize_name(key)?; - // Update the package copy index - copy_indexes_builder.add_package(&package_id); - // collect the dependencies let mut dependencies = HashMap::default(); @@ -222,6 +268,7 @@ impl NpmResolutionSnapshot { let package = NpmResolutionPackage { id: package_id.clone(), + copy_index: copy_indexes_builder.add_package(&package_id), // temporary dummy value dist: NpmPackageVersionDistInfo { tarball: "foobar".to_string(), @@ -279,7 +326,6 @@ impl NpmResolutionSnapshot { package_reqs, packages_by_name, packages, - packages_to_copy_index: copy_indexes_builder.into_map(), }) } } @@ -298,7 +344,7 @@ impl SnapshotPackageCopyIndexesBuilder { } pub fn from_map_with_capacity( - packages_to_copy_index: HashMap, + mut packages_to_copy_index: HashMap, capacity: usize, ) -> Self { let mut package_name_version_to_copy_count = @@ -321,12 +367,10 @@ impl SnapshotPackageCopyIndexesBuilder { } } - pub fn into_map(self) -> HashMap { - self.packages_to_copy_index - } - - pub fn add_package(&mut self, id: &NpmPackageId) { - if !self.packages_to_copy_index.contains_key(id) { + pub fn add_package(&mut self, id: &NpmPackageId) -> usize { + if let Some(index) = self.packages_to_copy_index.get(id) { + *index + } else { let index = *self .package_name_version_to_copy_count .entry((id.name.to_string(), id.version.to_string())) @@ -335,6 +379,7 @@ impl SnapshotPackageCopyIndexesBuilder { }) .or_insert(0); self.packages_to_copy_index.insert(id.clone(), index); + index } } } diff --git a/cli/npm/resolvers/common.rs b/cli/npm/resolvers/common.rs index 07996c4e108b37..32b8293cd01179 100644 --- a/cli/npm/resolvers/common.rs +++ b/cli/npm/resolvers/common.rs @@ -70,13 +70,19 @@ pub async fn cache_packages( // and we want the output to be deterministic packages.sort_by(|a, b| a.id.cmp(&b.id)); } + let mut handles = Vec::with_capacity(packages.len()); for package in packages { + assert_eq!(package.copy_index, 0); // the caller should not provide any of these let cache = cache.clone(); let registry_url = registry_url.clone(); let handle = tokio::task::spawn(async move { cache - .ensure_package(&package.id, &package.dist, ®istry_url) + .ensure_package( + (package.id.name.as_str(), &package.id.version), + &package.dist, + ®istry_url, + ) .await }); if sync_download { diff --git a/cli/npm/resolvers/global.rs b/cli/npm/resolvers/global.rs index 936704ee413a83..474cb55d6aedbf 100644 --- a/cli/npm/resolvers/global.rs +++ b/cli/npm/resolvers/global.rs @@ -53,7 +53,13 @@ impl GlobalNpmPackageResolver { } fn package_folder(&self, id: &NpmPackageId) -> PathBuf { - self.cache.package_folder(id, &self.registry_url) + let folder_id = self + .resolution + .resolve_package_cache_folder_id_from_id(id) + .unwrap(); + self + .cache + .package_folder_for_id(&folder_id, &self.registry_url) } } @@ -105,11 +111,15 @@ impl InnerNpmPackageResolver for GlobalNpmPackageResolver { &self, specifier: &ModuleSpecifier, ) -> Result { - let pkg_id = self.cache.resolve_package_folder_id_from_specifier( + let pkg_folder_id = self.cache.resolve_package_folder_id_from_specifier( specifier, &self.registry_url, )?; - Ok(self.package_folder(&pkg_id)) + Ok( + self + .cache + .package_folder_for_id(&pkg_folder_id, &self.registry_url), + ) } fn package_size(&self, package_id: &NpmPackageId) -> Result { @@ -163,10 +173,22 @@ impl InnerNpmPackageResolver for GlobalNpmPackageResolver { async fn cache_packages_in_resolver( resolver: &GlobalNpmPackageResolver, ) -> Result<(), AnyError> { + let package_partitions = resolver.resolution.all_packages_partitioned(); + cache_packages( - resolver.resolution.all_packages(), + package_partitions.packages, &resolver.cache, &resolver.registry_url, ) - .await + .await?; + + // create the copy package folders + for copy in package_partitions.copy_packages { + resolver.cache.ensure_copy_package( + ©.get_package_cache_folder_id(), + &resolver.registry_url, + )?; + } + + Ok(()) } diff --git a/cli/npm/resolvers/local.rs b/cli/npm/resolvers/local.rs index 5895dadafde198..c9c9d7e4a417a4 100644 --- a/cli/npm/resolvers/local.rs +++ b/cli/npm/resolvers/local.rs @@ -24,6 +24,7 @@ use tokio::task::JoinHandle; use crate::fs_util; use crate::lockfile::Lockfile; use crate::npm::cache::should_sync_download; +use crate::npm::cache::NpmPackageCacheFolderId; use crate::npm::resolution::NpmResolution; use crate::npm::resolution::NpmResolutionSnapshot; use crate::npm::NpmCache; @@ -255,8 +256,13 @@ async fn sync_resolution_with_fs( registry_url: &Url, root_node_modules_dir_path: &Path, ) -> Result<(), AnyError> { - fn get_package_folder_name(package_id: &NpmPackageId) -> String { - package_id.to_string().replace('/', "+") + fn get_package_folder_name(id: &NpmPackageCacheFolderId) -> String { + let copy_str = if id.copy_index == 0 { + "".to_string() + } else { + format!("_{}", id.copy_index) + }; + format!("{}@{}{}", id.name, id.version, copy_str).replace('/', "+") } let deno_local_registry_dir = root_node_modules_dir_path.join(".deno"); @@ -269,16 +275,17 @@ async fn sync_resolution_with_fs( // Copy (hardlink in future) // to // node_modules/.deno//node_modules/ let sync_download = should_sync_download(); - let mut all_packages = snapshot.all_packages(); + let mut package_partitions = snapshot.all_packages_partitioned(); if sync_download { // we're running the tests not with --quiet // and we want the output to be deterministic - all_packages.sort_by(|a, b| a.id.cmp(&b.id)); + package_partitions.packages.sort_by(|a, b| a.id.cmp(&b.id)); } let mut handles: Vec>> = - Vec::with_capacity(all_packages.len()); - for package in &all_packages { - let folder_name = get_package_folder_name(&package.id); + Vec::with_capacity(package_partitions.packages.len()); + for package in &package_partitions.packages { + let folder_name = + get_package_folder_name(&package.get_package_cache_folder_id()); let folder_path = deno_local_registry_dir.join(&folder_name); let initialized_file = folder_path.join(".initialized"); if !initialized_file.exists() { @@ -287,14 +294,22 @@ async fn sync_resolution_with_fs( let package = package.clone(); let handle = tokio::task::spawn(async move { cache - .ensure_package(&package.id, &package.dist, ®istry_url) + .ensure_package( + (&package.id.name, &package.id.version), + &package.dist, + ®istry_url, + ) .await?; let sub_node_modules = folder_path.join("node_modules"); let package_path = join_package_name(&sub_node_modules, &package.id.name); fs::create_dir_all(&package_path) .with_context(|| format!("Creating '{}'", folder_path.display()))?; - let cache_folder = cache.package_folder(&package.id, ®istry_url); + let cache_folder = cache.package_folder_for_name_and_version( + &package.id.name, + &package.id.version, + ®istry_url, + ); // for now copy, but in the future consider hard linking fs_util::copy_dir_recursive(&cache_folder, &package_path)?; // write out a file that indicates this folder has been initialized @@ -314,16 +329,51 @@ async fn sync_resolution_with_fs( result??; // surface the first error } - // 2. Symlink all the dependencies into the .deno directory. + // 2. Create any "copy" packages, which are used for peer dependencies + for package in &package_partitions.copy_packages { + let package_cache_folder_id = package.get_package_cache_folder_id(); + let destination_path = deno_local_registry_dir + .join(&get_package_folder_name(&package_cache_folder_id)); + let initialized_file = destination_path.join(".initialized"); + if !initialized_file.exists() { + let sub_node_modules = destination_path.join("node_modules"); + let package_path = join_package_name(&sub_node_modules, &package.id.name); + fs::create_dir_all(&package_path).with_context(|| { + format!("Creating '{}'", destination_path.display()) + })?; + let source_path = join_package_name( + &deno_local_registry_dir + .join(&get_package_folder_name( + &package_cache_folder_id.with_no_count(), + )) + .join("node_modules"), + &package.id.name, + ); + // todo(THIS PR): hard link instead + fs_util::copy_dir_recursive(&source_path, &package_path)?; + // write out a file that indicates this folder has been initialized + fs::write(initialized_file, "")?; + } + } + + let all_packages = package_partitions.into_all(); + + // 3. Symlink all the dependencies into the .deno directory. // // Symlink node_modules/.deno//node_modules/ to // node_modules/.deno//node_modules/ for package in &all_packages { let sub_node_modules = deno_local_registry_dir - .join(&get_package_folder_name(&package.id)) + .join(&get_package_folder_name( + &package.get_package_cache_folder_id(), + )) .join("node_modules"); for (name, dep_id) in &package.dependencies { - let dep_folder_name = get_package_folder_name(dep_id); + let dep_cache_folder_id = snapshot + .package_from_id(dep_id) + .unwrap() + .get_package_cache_folder_id(); + let dep_folder_name = get_package_folder_name(&dep_cache_folder_id); let dep_folder_path = join_package_name( &deno_local_registry_dir .join(dep_folder_name) @@ -337,7 +387,7 @@ async fn sync_resolution_with_fs( } } - // 3. Create all the packages in the node_modules folder, which are symlinks. + // 4. Create all the packages in the node_modules folder, which are symlinks. // // Symlink node_modules/ to // node_modules/.deno//node_modules/ @@ -357,19 +407,22 @@ async fn sync_resolution_with_fs( } else { continue; // skip, already handled }; - let local_registry_package_path = deno_local_registry_dir - .join(&get_package_folder_name(&package_id)) - .join("node_modules") - .join(&package_id.name); + let package = snapshot.package_from_id(&package_id).unwrap(); + let local_registry_package_path = join_package_name( + &deno_local_registry_dir + .join(&get_package_folder_name( + &package.get_package_cache_folder_id(), + )) + .join("node_modules"), + &package_id.name, + ); symlink_package_dir( &local_registry_package_path, &join_package_name(root_node_modules_dir_path, &root_folder_name), )?; - if let Some(package) = snapshot.package_from_id(&package_id) { - for id in package.dependencies.values() { - pending_packages.push_back((id.clone(), false)); - } + for id in package.dependencies.values() { + pending_packages.push_back((id.clone(), false)); } } From b5342a7665a6151fcce0cffaecd0599dbf4ec510 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Sun, 6 Nov 2022 19:11:33 -0500 Subject: [PATCH 26/43] Clippy. --- cli/npm/cache.rs | 10 +++++----- cli/npm/resolution/snapshot.rs | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cli/npm/cache.rs b/cli/npm/cache.rs index 97119944e0716b..2f2a1fe0864a78 100644 --- a/cli/npm/cache.rs +++ b/cli/npm/cache.rs @@ -191,7 +191,7 @@ impl ReadonlyNpmCache { } else { self .package_name_folder(&id.name, registry_url) - .join(format!("{}_{}", id.version.to_string(), id.copy_index)) + .join(format!("{}_{}", id.version, id.copy_index)) } } @@ -289,7 +289,7 @@ impl ReadonlyNpmCache { (version_part, 0) }; Some(NpmPackageCacheFolderId { - name: name.to_string(), + name, version: NpmVersion::parse(version).ok()?, copy_index, }) @@ -355,7 +355,7 @@ impl NpmCache { // if this file exists, then the package didn't successfully extract // the first time, or another process is currently extracting the zip file && !package_folder.join(NPM_PACKAGE_SYNC_LOCK_FILENAME).exists() - && self.cache_setting.should_use_for_npm_package(&package.0) + && self.cache_setting.should_use_for_npm_package(package.0) { return Ok(()); } else if self.cache_setting == CacheSetting::Only { @@ -408,11 +408,11 @@ impl NpmCache { .package_folder_for_name_and_version(&id.name, &id.version, registry_url); // todo(THIS PR): use hard linking instead of copying with_folder_sync_lock( - (&id.name.as_str(), &id.version), + (id.name.as_str(), &id.version), &package_folder, || fs_util::copy_dir_recursive(&original_package_folder, &package_folder), )?; - return Ok(()); + Ok(()) } pub fn package_folder_for_id( diff --git a/cli/npm/resolution/snapshot.rs b/cli/npm/resolution/snapshot.rs index 4d887a2bf731ca..7e29f9efb17069 100644 --- a/cli/npm/resolution/snapshot.rs +++ b/cli/npm/resolution/snapshot.rs @@ -144,7 +144,7 @@ impl NpmResolutionSnapshot { anyhow!("could not find referrer npm package '{}'", referrer) })?; - let name = name_without_path(&name); + let name = name_without_path(name); if let Some(id) = referrer_package.dependencies.get(name) { return Ok(self.packages.get(id).unwrap()); } From 7dee311bae807ac6a0dd0ba5fc56771b9a4b8523 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 7 Nov 2022 10:11:02 -0500 Subject: [PATCH 27/43] Add test and fix issues for "copy_index" --- cli/lockfile.rs | 6 +- cli/npm/resolution/graph.rs | 345 +++++++++++++++++++++++---------- cli/npm/resolution/mod.rs | 14 +- cli/npm/resolution/snapshot.rs | 64 +++++- 4 files changed, 303 insertions(+), 126 deletions(-) diff --git a/cli/lockfile.rs b/cli/lockfile.rs index 94611ac889aff4..8ee20b6fc314c9 100644 --- a/cli/lockfile.rs +++ b/cli/lockfile.rs @@ -267,7 +267,7 @@ impl Lockfile { &mut self, package: &NpmResolutionPackage, ) -> Result<(), LockfileError> { - let specifier = package.id.as_serializable_name(); + let specifier = package.id.as_serialized(); if let Some(package_info) = self.content.npm.packages.get(&specifier) { let integrity = package .dist @@ -298,7 +298,7 @@ Use \"--lock-write\" flag to regenerate the lockfile at \"{}\".", let dependencies = package .dependencies .iter() - .map(|(name, id)| (name.to_string(), id.as_serializable_name())) + .map(|(name, id)| (name.to_string(), id.as_serialized())) .collect::>(); let integrity = package @@ -307,7 +307,7 @@ Use \"--lock-write\" flag to regenerate the lockfile at \"{}\".", .as_ref() .unwrap_or(&package.dist.shasum); self.content.npm.packages.insert( - package.id.as_serializable_name(), + package.id.as_serialized(), NpmPackageInfo { integrity: integrity.to_string(), dependencies, diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index 5faee06c6e39c7..19671623f345b2 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -24,7 +24,7 @@ use crate::npm::semver::NpmVersionReq; use crate::npm::NpmRegistryApi; use super::snapshot::NpmResolutionSnapshot; -use super::snapshot::SnapshotPackageCopyIndexesBuilder; +use super::snapshot::SnapshotPackageCopyIndexResolver; use super::NpmPackageId; use super::NpmPackageReq; use super::NpmResolutionPackage; @@ -189,10 +189,7 @@ impl Graph { fn borrow_node(&self, id: &NpmPackageId) -> MutexGuard { (**self.packages.get(id).unwrap_or_else(|| { - panic!( - "could not find id {} in the tree", - id.as_serializable_name() - ) + panic!("could not find id {} in the tree", id.as_serialized()) })) .lock() } @@ -201,6 +198,16 @@ impl Graph { if let Some(node) = self.packages.remove(node_id) { let node = (*node).lock(); assert_eq!(node.parents.len(), 0); + + // Remove the id from the list of packages by name. + let packages_with_name = + self.packages_by_name.get_mut(&node.id.name).unwrap(); + let remove_index = packages_with_name + .iter() + .position(|id| id == &node.id) + .unwrap(); + packages_with_name.remove(remove_index); + let parent = NodeParent::Node(node.id.clone()); for (specifier, child_id) in &node.children { let mut child = self.borrow_node(child_id); @@ -276,12 +283,23 @@ impl Graph { self, api: &impl NpmRegistryApi, ) -> Result { - let mut copy_index_builder = - SnapshotPackageCopyIndexesBuilder::from_map_with_capacity( + let mut copy_index_resolver = + SnapshotPackageCopyIndexResolver::from_map_with_capacity( self.packages_to_copy_index, self.packages.len(), ); + // Iterate through the packages vector in each packages_by_name in order + // to set the copy index as this will be deterministic rather than + // iterating over the hashmap below. + for packages in self.packages_by_name.values() { + if packages.len() > 1 { + for id in packages { + copy_index_resolver.resolve(id); + } + } + } + let mut packages = HashMap::with_capacity(self.packages.len()); for (id, node) in self.packages { let dist = api @@ -293,7 +311,7 @@ impl Graph { packages.insert( id.clone(), NpmResolutionPackage { - copy_index: copy_index_builder.add_package(&id), + copy_index: copy_index_resolver.resolve(&id), id, dist, dependencies: node.children.clone(), @@ -909,7 +927,7 @@ mod test { } #[tokio::test] - async fn resolve_no_peer_deps() { + async fn resolve_deps_no_peer() { let api = TestNpmRegistryApi::default(); api.ensure_package_version("package-a", "1.0.0"); api.ensure_package_version("package-b", "2.0.0"); @@ -927,37 +945,37 @@ mod test { packages, vec![ NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-a@1.0.0").unwrap(), + id: NpmPackageId::from_serialized("package-a@1.0.0").unwrap(), copy_index: 0, dependencies: HashMap::from([ ( "package-b".to_string(), - NpmPackageId::deserialize_name("package-b@2.0.0").unwrap(), + NpmPackageId::from_serialized("package-b@2.0.0").unwrap(), ), ( "package-c".to_string(), - NpmPackageId::deserialize_name("package-c@0.1.0").unwrap(), + NpmPackageId::from_serialized("package-c@0.1.0").unwrap(), ), ]), dist: Default::default(), }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-b@2.0.0").unwrap(), + id: NpmPackageId::from_serialized("package-b@2.0.0").unwrap(), copy_index: 0, dist: Default::default(), dependencies: Default::default(), }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-c@0.1.0").unwrap(), + id: NpmPackageId::from_serialized("package-c@0.1.0").unwrap(), copy_index: 0, dist: Default::default(), dependencies: HashMap::from([( "package-d".to_string(), - NpmPackageId::deserialize_name("package-d@3.2.1").unwrap(), + NpmPackageId::from_serialized("package-d@3.2.1").unwrap(), )]) }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-d@3.2.1").unwrap(), + id: NpmPackageId::from_serialized("package-d@3.2.1").unwrap(), copy_index: 0, dist: Default::default(), dependencies: Default::default(), @@ -970,6 +988,45 @@ mod test { ); } + #[tokio::test] + async fn resolve_deps_circular() { + let api = TestNpmRegistryApi::default(); + api.ensure_package_version("package-a", "1.0.0"); + api.ensure_package_version("package-b", "2.0.0"); + api.add_dependency(("package-a", "1.0.0"), ("package-b", "*")); + api.add_dependency(("package-b", "2.0.0"), ("package-a", "1")); + + let (packages, package_reqs) = + run_resolver_and_get_output(api, vec!["npm:package-a@1.0"]).await; + assert_eq!( + packages, + vec![ + NpmResolutionPackage { + id: NpmPackageId::from_serialized("package-a@1.0.0").unwrap(), + copy_index: 0, + dependencies: HashMap::from([( + "package-b".to_string(), + NpmPackageId::from_serialized("package-b@2.0.0").unwrap(), + )]), + dist: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::from_serialized("package-b@2.0.0").unwrap(), + copy_index: 0, + dependencies: HashMap::from([( + "package-a".to_string(), + NpmPackageId::from_serialized("package-a@1.0.0").unwrap(), + )]), + dist: Default::default(), + }, + ] + ); + assert_eq!( + package_reqs, + vec![("package-a@1.0".to_string(), "package-a@1.0.0".to_string())] + ); + } + #[tokio::test] async fn resolve_with_peer_deps_top_tree() { let api = TestNpmRegistryApi::default(); @@ -994,7 +1051,7 @@ mod test { packages, vec![ NpmResolutionPackage { - id: NpmPackageId::deserialize_name( + id: NpmPackageId::from_serialized( "package-a@1.0.0_package-peer@4.0.0" ) .unwrap(), @@ -1002,14 +1059,14 @@ mod test { dependencies: HashMap::from([ ( "package-b".to_string(), - NpmPackageId::deserialize_name( + NpmPackageId::from_serialized( "package-b@2.0.0_package-peer@4.0.0" ) .unwrap(), ), ( "package-c".to_string(), - NpmPackageId::deserialize_name( + NpmPackageId::from_serialized( "package-c@3.0.0_package-peer@4.0.0" ) .unwrap(), @@ -1018,7 +1075,7 @@ mod test { dist: Default::default(), }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name( + id: NpmPackageId::from_serialized( "package-b@2.0.0_package-peer@4.0.0" ) .unwrap(), @@ -1026,11 +1083,11 @@ mod test { dist: Default::default(), dependencies: HashMap::from([( "package-peer".to_string(), - NpmPackageId::deserialize_name("package-peer@4.0.0").unwrap(), + NpmPackageId::from_serialized("package-peer@4.0.0").unwrap(), )]) }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name( + id: NpmPackageId::from_serialized( "package-c@3.0.0_package-peer@4.0.0" ) .unwrap(), @@ -1038,11 +1095,11 @@ mod test { dist: Default::default(), dependencies: HashMap::from([( "package-peer".to_string(), - NpmPackageId::deserialize_name("package-peer@4.0.0").unwrap(), + NpmPackageId::from_serialized("package-peer@4.0.0").unwrap(), )]) }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-peer@4.0.0").unwrap(), + id: NpmPackageId::from_serialized("package-peer@4.0.0").unwrap(), copy_index: 0, dist: Default::default(), dependencies: Default::default(), @@ -1088,19 +1145,17 @@ mod test { packages, vec![ NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-0@1.1.1").unwrap(), + id: NpmPackageId::from_serialized("package-0@1.1.1").unwrap(), copy_index: 0, dependencies: HashMap::from([( "package-a".to_string(), - NpmPackageId::deserialize_name( - "package-a@1.0.0_package-peer@4.0.0" - ) - .unwrap(), + NpmPackageId::from_serialized("package-a@1.0.0_package-peer@4.0.0") + .unwrap(), ),]), dist: Default::default(), }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name( + id: NpmPackageId::from_serialized( "package-a@1.0.0_package-peer@4.0.0" ) .unwrap(), @@ -1108,27 +1163,27 @@ mod test { dependencies: HashMap::from([ ( "package-b".to_string(), - NpmPackageId::deserialize_name( + NpmPackageId::from_serialized( "package-b@2.0.0_package-peer@4.0.0" ) .unwrap(), ), ( "package-c".to_string(), - NpmPackageId::deserialize_name( + NpmPackageId::from_serialized( "package-c@3.0.0_package-peer@4.0.0" ) .unwrap(), ), ( "package-peer".to_string(), - NpmPackageId::deserialize_name("package-peer@4.0.0").unwrap(), + NpmPackageId::from_serialized("package-peer@4.0.0").unwrap(), ), ]), dist: Default::default(), }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name( + id: NpmPackageId::from_serialized( "package-b@2.0.0_package-peer@4.0.0" ) .unwrap(), @@ -1136,11 +1191,11 @@ mod test { dist: Default::default(), dependencies: HashMap::from([( "package-peer".to_string(), - NpmPackageId::deserialize_name("package-peer@4.0.0").unwrap(), + NpmPackageId::from_serialized("package-peer@4.0.0").unwrap(), )]) }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name( + id: NpmPackageId::from_serialized( "package-c@3.0.0_package-peer@4.0.0" ) .unwrap(), @@ -1148,11 +1203,11 @@ mod test { dist: Default::default(), dependencies: HashMap::from([( "package-peer".to_string(), - NpmPackageId::deserialize_name("package-peer@4.0.0").unwrap(), + NpmPackageId::from_serialized("package-peer@4.0.0").unwrap(), )]) }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-peer@4.0.0").unwrap(), + id: NpmPackageId::from_serialized("package-peer@4.0.0").unwrap(), copy_index: 0, dist: Default::default(), dependencies: Default::default(), @@ -1186,40 +1241,40 @@ mod test { packages, vec![ NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-a@1.0.0").unwrap(), + id: NpmPackageId::from_serialized("package-a@1.0.0").unwrap(), copy_index: 0, dependencies: HashMap::from([ ( "package-b".to_string(), - NpmPackageId::deserialize_name("package-b@2.0.0").unwrap(), + NpmPackageId::from_serialized("package-b@2.0.0").unwrap(), ), ( "package-c".to_string(), - NpmPackageId::deserialize_name("package-c@3.0.0").unwrap(), + NpmPackageId::from_serialized("package-c@3.0.0").unwrap(), ), ]), dist: Default::default(), }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-b@2.0.0").unwrap(), + id: NpmPackageId::from_serialized("package-b@2.0.0").unwrap(), copy_index: 0, dist: Default::default(), dependencies: HashMap::from([( "package-peer".to_string(), - NpmPackageId::deserialize_name("package-peer@4.1.0").unwrap(), + NpmPackageId::from_serialized("package-peer@4.1.0").unwrap(), )]) }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-c@3.0.0").unwrap(), + id: NpmPackageId::from_serialized("package-c@3.0.0").unwrap(), copy_index: 0, dist: Default::default(), dependencies: HashMap::from([( "package-peer".to_string(), - NpmPackageId::deserialize_name("package-peer@4.1.0").unwrap(), + NpmPackageId::from_serialized("package-peer@4.1.0").unwrap(), )]) }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-peer@4.1.0").unwrap(), + id: NpmPackageId::from_serialized("package-peer@4.1.0").unwrap(), copy_index: 0, dist: Default::default(), dependencies: Default::default(), @@ -1259,28 +1314,28 @@ mod test { packages, vec![ NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-a@1.0.0").unwrap(), + id: NpmPackageId::from_serialized("package-a@1.0.0").unwrap(), copy_index: 0, dependencies: HashMap::from([ ( "package-b".to_string(), - NpmPackageId::deserialize_name("package-b@2.0.0").unwrap(), + NpmPackageId::from_serialized("package-b@2.0.0").unwrap(), ), ( "package-c".to_string(), - NpmPackageId::deserialize_name("package-c@3.0.0").unwrap(), + NpmPackageId::from_serialized("package-c@3.0.0").unwrap(), ), ]), dist: Default::default(), }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-b@2.0.0").unwrap(), + id: NpmPackageId::from_serialized("package-b@2.0.0").unwrap(), copy_index: 0, dist: Default::default(), dependencies: HashMap::new(), }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-c@3.0.0").unwrap(), + id: NpmPackageId::from_serialized("package-c@3.0.0").unwrap(), copy_index: 0, dist: Default::default(), dependencies: HashMap::new(), @@ -1321,7 +1376,7 @@ mod test { packages, vec![ NpmResolutionPackage { - id: NpmPackageId::deserialize_name( + id: NpmPackageId::from_serialized( "package-a@1.0.0_package-peer@4.0.0" ) .unwrap(), @@ -1329,14 +1384,14 @@ mod test { dependencies: HashMap::from([ ( "package-b".to_string(), - NpmPackageId::deserialize_name( + NpmPackageId::from_serialized( "package-b@2.0.0_package-peer@4.0.0" ) .unwrap(), ), ( "package-c".to_string(), - NpmPackageId::deserialize_name( + NpmPackageId::from_serialized( "package-c@3.0.0_package-peer@4.0.0" ) .unwrap(), @@ -1345,7 +1400,7 @@ mod test { dist: Default::default(), }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name( + id: NpmPackageId::from_serialized( "package-b@2.0.0_package-peer@4.0.0" ) .unwrap(), @@ -1353,11 +1408,11 @@ mod test { dist: Default::default(), dependencies: HashMap::from([( "package-peer".to_string(), - NpmPackageId::deserialize_name("package-peer@4.0.0").unwrap(), + NpmPackageId::from_serialized("package-peer@4.0.0").unwrap(), )]) }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name( + id: NpmPackageId::from_serialized( "package-c@3.0.0_package-peer@4.0.0" ) .unwrap(), @@ -1365,11 +1420,11 @@ mod test { dist: Default::default(), dependencies: HashMap::from([( "package-peer".to_string(), - NpmPackageId::deserialize_name("package-peer@4.0.0").unwrap(), + NpmPackageId::from_serialized("package-peer@4.0.0").unwrap(), )]) }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-peer@4.0.0").unwrap(), + id: NpmPackageId::from_serialized("package-peer@4.0.0").unwrap(), copy_index: 0, dist: Default::default(), dependencies: Default::default(), @@ -1409,25 +1464,25 @@ mod test { packages, vec![ NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-0@1.0.0").unwrap(), + id: NpmPackageId::from_serialized("package-0@1.0.0").unwrap(), copy_index: 0, dependencies: HashMap::from([( "package-peer-a".to_string(), - NpmPackageId::deserialize_name("package-peer-a@2.0.0").unwrap(), + NpmPackageId::from_serialized("package-peer-a@2.0.0").unwrap(), )]), dist: Default::default(), }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-peer-a@2.0.0").unwrap(), + id: NpmPackageId::from_serialized("package-peer-a@2.0.0").unwrap(), copy_index: 0, dependencies: HashMap::from([( "package-peer-b".to_string(), - NpmPackageId::deserialize_name("package-peer-b@3.0.0").unwrap(), + NpmPackageId::from_serialized("package-peer-b@3.0.0").unwrap(), )]), dist: Default::default(), }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-peer-b@3.0.0").unwrap(), + id: NpmPackageId::from_serialized("package-peer-b@3.0.0").unwrap(), copy_index: 0, dependencies: HashMap::new(), dist: Default::default(), @@ -1466,7 +1521,7 @@ mod test { packages, vec![ NpmResolutionPackage { - id: NpmPackageId::deserialize_name( + id: NpmPackageId::from_serialized( "package-0@1.0.0_package-peer-a@2.0.0_package-peer-b@3.0.0" ) .unwrap(), @@ -1474,32 +1529,32 @@ mod test { dependencies: HashMap::from([ ( "package-peer-a".to_string(), - NpmPackageId::deserialize_name( + NpmPackageId::from_serialized( "package-peer-a@2.0.0_package-peer-b@3.0.0" ) .unwrap(), ), ( "package-peer-b".to_string(), - NpmPackageId::deserialize_name("package-peer-b@3.0.0").unwrap(), + NpmPackageId::from_serialized("package-peer-b@3.0.0").unwrap(), ) ]), dist: Default::default(), }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name( + id: NpmPackageId::from_serialized( "package-peer-a@2.0.0_package-peer-b@3.0.0" ) .unwrap(), copy_index: 0, dependencies: HashMap::from([( "package-peer-b".to_string(), - NpmPackageId::deserialize_name("package-peer-b@3.0.0").unwrap(), + NpmPackageId::from_serialized("package-peer-b@3.0.0").unwrap(), )]), dist: Default::default(), }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-peer-b@3.0.0").unwrap(), + id: NpmPackageId::from_serialized("package-peer-b@3.0.0").unwrap(), copy_index: 0, dependencies: HashMap::new(), dist: Default::default(), @@ -1565,11 +1620,11 @@ mod test { packages, vec![ NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-0@1.1.1").unwrap(), + id: NpmPackageId::from_serialized("package-0@1.1.1").unwrap(), copy_index: 0, dependencies: HashMap::from([( "package-a".to_string(), - NpmPackageId::deserialize_name( + NpmPackageId::from_serialized( "package-a@1.0.0_package-peer-a@4.0.0" ) .unwrap(), @@ -1577,7 +1632,7 @@ mod test { dist: Default::default(), }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name( + id: NpmPackageId::from_serialized( "package-a@1.0.0_package-peer-a@4.0.0" ) .unwrap(), @@ -1585,31 +1640,31 @@ mod test { dependencies: HashMap::from([ ( "package-b".to_string(), - NpmPackageId::deserialize_name( + NpmPackageId::from_serialized( "package-b@2.0.0_package-peer-a@4.0.0" ) .unwrap(), ), ( "package-c".to_string(), - NpmPackageId::deserialize_name( + NpmPackageId::from_serialized( "package-c@3.0.0_package-peer-a@4.0.0" ) .unwrap(), ), ( "package-d".to_string(), - NpmPackageId::deserialize_name("package-d@3.5.0").unwrap(), + NpmPackageId::from_serialized("package-d@3.5.0").unwrap(), ), ( "package-peer-a".to_string(), - NpmPackageId::deserialize_name("package-peer-a@4.0.0").unwrap(), + NpmPackageId::from_serialized("package-peer-a@4.0.0").unwrap(), ), ]), dist: Default::default(), }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name( + id: NpmPackageId::from_serialized( "package-b@2.0.0_package-peer-a@4.0.0" ) .unwrap(), @@ -1618,16 +1673,16 @@ mod test { dependencies: HashMap::from([ ( "package-peer-a".to_string(), - NpmPackageId::deserialize_name("package-peer-a@4.0.0").unwrap(), + NpmPackageId::from_serialized("package-peer-a@4.0.0").unwrap(), ), ( "package-peer-c".to_string(), - NpmPackageId::deserialize_name("package-peer-c@6.2.0").unwrap(), + NpmPackageId::from_serialized("package-peer-c@6.2.0").unwrap(), ) ]) }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name( + id: NpmPackageId::from_serialized( "package-c@3.0.0_package-peer-a@4.0.0" ) .unwrap(), @@ -1635,38 +1690,38 @@ mod test { dist: Default::default(), dependencies: HashMap::from([( "package-peer-a".to_string(), - NpmPackageId::deserialize_name("package-peer-a@4.0.0").unwrap(), + NpmPackageId::from_serialized("package-peer-a@4.0.0").unwrap(), )]) }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-d@3.5.0").unwrap(), + id: NpmPackageId::from_serialized("package-d@3.5.0").unwrap(), copy_index: 0, dependencies: HashMap::from([]), dist: Default::default(), }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-e@3.6.0").unwrap(), + id: NpmPackageId::from_serialized("package-e@3.6.0").unwrap(), copy_index: 0, dependencies: HashMap::from([]), dist: Default::default(), }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-peer-a@4.0.0").unwrap(), + id: NpmPackageId::from_serialized("package-peer-a@4.0.0").unwrap(), copy_index: 0, dist: Default::default(), dependencies: HashMap::from([( "package-peer-b".to_string(), - NpmPackageId::deserialize_name("package-peer-b@5.4.1").unwrap(), + NpmPackageId::from_serialized("package-peer-b@5.4.1").unwrap(), )]) }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-peer-b@5.4.1").unwrap(), + id: NpmPackageId::from_serialized("package-peer-b@5.4.1").unwrap(), copy_index: 0, dist: Default::default(), dependencies: Default::default(), }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-peer-c@6.2.0").unwrap(), + id: NpmPackageId::from_serialized("package-peer-c@6.2.0").unwrap(), copy_index: 0, dist: Default::default(), dependencies: Default::default(), @@ -1696,12 +1751,12 @@ mod test { packages, vec![ NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-a@1.0.0_package-a@1.0.0") + id: NpmPackageId::from_serialized("package-a@1.0.0_package-a@1.0.0") .unwrap(), copy_index: 0, dependencies: HashMap::from([( "package-b".to_string(), - NpmPackageId::deserialize_name( + NpmPackageId::from_serialized( "package-b@2.0.0_package-a@1.0.0__package-a@1.0.0" ) .unwrap(), @@ -1709,14 +1764,14 @@ mod test { dist: Default::default(), }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name( + id: NpmPackageId::from_serialized( "package-b@2.0.0_package-a@1.0.0__package-a@1.0.0" ) .unwrap(), copy_index: 0, dependencies: HashMap::from([( "package-a".to_string(), - NpmPackageId::deserialize_name("package-a@1.0.0_package-a@1.0.0") + NpmPackageId::from_serialized("package-a@1.0.0_package-a@1.0.0") .unwrap(), )]), dist: Default::default(), @@ -1733,41 +1788,119 @@ mod test { } #[tokio::test] - async fn resolve_deps_circular() { + async fn resolve_peer_deps_multiple_copies() { let api = TestNpmRegistryApi::default(); api.ensure_package_version("package-a", "1.0.0"); api.ensure_package_version("package-b", "2.0.0"); - api.add_dependency(("package-a", "1.0.0"), ("package-b", "*")); - api.add_dependency(("package-b", "2.0.0"), ("package-a", "1")); + api.ensure_package_version("package-dep", "3.0.0"); + api.ensure_package_version("package-peer", "4.0.0"); + api.ensure_package_version("package-peer", "5.0.0"); + api.add_dependency(("package-a", "1.0.0"), ("package-dep", "*")); + api.add_dependency(("package-a", "1.0.0"), ("package-peer", "4")); + api.add_dependency(("package-b", "2.0.0"), ("package-dep", "*")); + api.add_dependency(("package-b", "2.0.0"), ("package-peer", "5")); + api.add_peer_dependency(("package-dep", "3.0.0"), ("package-peer", "*")); - let (packages, package_reqs) = - run_resolver_and_get_output(api, vec!["npm:package-a@1.0"]).await; + let (packages, package_reqs) = run_resolver_and_get_output( + api, + vec!["npm:package-a@1", "npm:package-b@2"], + ) + .await; assert_eq!( packages, vec![ NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-a@1.0.0").unwrap(), + id: NpmPackageId::from_serialized( + "package-a@1.0.0_package-peer@4.0.0" + ) + .unwrap(), + copy_index: 0, + dependencies: HashMap::from([ + ( + "package-dep".to_string(), + NpmPackageId::from_serialized( + "package-dep@3.0.0_package-peer@4.0.0" + ) + .unwrap(), + ), + ( + "package-peer".to_string(), + NpmPackageId::from_serialized("package-peer@4.0.0").unwrap(), + ), + ]), + dist: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::from_serialized( + "package-b@2.0.0_package-peer@5.0.0" + ) + .unwrap(), + copy_index: 0, + dependencies: HashMap::from([ + ( + "package-dep".to_string(), + NpmPackageId::from_serialized( + "package-dep@3.0.0_package-peer@5.0.0" + ) + .unwrap(), + ), + ( + "package-peer".to_string(), + NpmPackageId::from_serialized("package-peer@5.0.0").unwrap(), + ), + ]), + dist: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::from_serialized( + "package-dep@3.0.0_package-peer@4.0.0" + ) + .unwrap(), copy_index: 0, dependencies: HashMap::from([( - "package-b".to_string(), - NpmPackageId::deserialize_name("package-b@2.0.0").unwrap(), + "package-peer".to_string(), + NpmPackageId::from_serialized("package-peer@4.0.0").unwrap(), )]), dist: Default::default(), }, NpmResolutionPackage { - id: NpmPackageId::deserialize_name("package-b@2.0.0").unwrap(), - copy_index: 0, + id: NpmPackageId::from_serialized( + "package-dep@3.0.0_package-peer@5.0.0" + ) + .unwrap(), + copy_index: 1, dependencies: HashMap::from([( - "package-a".to_string(), - NpmPackageId::deserialize_name("package-a@1.0.0").unwrap(), + "package-peer".to_string(), + NpmPackageId::from_serialized("package-peer@5.0.0").unwrap(), )]), dist: Default::default(), }, + NpmResolutionPackage { + id: NpmPackageId::from_serialized("package-peer@4.0.0").unwrap(), + copy_index: 0, + dependencies: HashMap::new(), + dist: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::from_serialized("package-peer@5.0.0").unwrap(), + copy_index: 0, + dependencies: HashMap::new(), + dist: Default::default(), + }, ] ); assert_eq!( package_reqs, - vec![("package-a@1.0".to_string(), "package-a@1.0.0".to_string())] + vec![ + ( + "package-a@1".to_string(), + "package-a@1.0.0_package-peer@4.0.0".to_string() + ), + ( + "package-b@2".to_string(), + "package-b@2.0.0_package-peer@5.0.0".to_string() + ) + ] ); } @@ -1792,7 +1925,7 @@ mod test { let mut package_reqs = snapshot .package_reqs .into_iter() - .map(|(a, b)| (a.to_string(), b.as_serializable_name())) + .map(|(a, b)| (a.to_string(), b.as_serialized())) .collect::>(); package_reqs.sort_by(|a, b| a.0.to_string().cmp(&b.0.to_string())); (packages, package_reqs) diff --git a/cli/npm/resolution/mod.rs b/cli/npm/resolution/mod.rs index cb17a3c9a933f6..1b211aae64c518 100644 --- a/cli/npm/resolution/mod.rs +++ b/cli/npm/resolution/mod.rs @@ -182,11 +182,11 @@ impl NpmPackageId { } } - pub fn as_serializable_name(&self) -> String { - self.as_serialize_name_with_level(0) + pub fn as_serialized(&self) -> String { + self.as_serialized_with_level(0) } - fn as_serialize_name_with_level(&self, level: usize) -> String { + fn as_serialized_with_level(&self, level: usize) -> String { // WARNING: This should not change because it's used in the lockfile let mut result = format!( "{}@{}", @@ -202,12 +202,12 @@ impl NpmPackageId { // this gets deep because npm package names can start // with a number result.push_str(&"_".repeat(level + 1)); - result.push_str(&peer.as_serialize_name_with_level(level + 1)); + result.push_str(&peer.as_serialized_with_level(level + 1)); } result } - pub fn deserialize_name(id: &str) -> Result { + pub fn from_serialized(id: &str) -> Result { use monch::*; fn parse_name(input: &str) -> ParseResult<&str> { @@ -662,8 +662,8 @@ mod tests { }, ], }; - let serialized = id.as_serializable_name(); + let serialized = id.as_serialized(); assert_eq!(serialized, "pkg-a@1.2.3_pkg-b@3.2.1__pkg-c@1.3.2__pkg-d@2.3.4_pkg-e@2.3.1__pkg-f@2.3.1"); - assert_eq!(NpmPackageId::deserialize_name(&serialized).unwrap(), id); + assert_eq!(NpmPackageId::from_serialized(&serialized).unwrap(), id); } } diff --git a/cli/npm/resolution/snapshot.rs b/cli/npm/resolution/snapshot.rs index 7e29f9efb17069..32b42b72b9b01c 100644 --- a/cli/npm/resolution/snapshot.rs +++ b/cli/npm/resolution/snapshot.rs @@ -224,7 +224,7 @@ impl NpmResolutionSnapshot { let mut package_reqs: HashMap; let mut packages_by_name: HashMap>; let mut packages: HashMap; - let mut copy_indexes_builder: SnapshotPackageCopyIndexesBuilder; + let mut copy_index_resolver: SnapshotPackageCopyIndexResolver; { let lockfile = lockfile.lock(); @@ -235,22 +235,22 @@ impl NpmResolutionSnapshot { let packages_len = lockfile.content.npm.packages.len(); packages = HashMap::with_capacity(packages_len); packages_by_name = HashMap::with_capacity(packages_len); // close enough - copy_indexes_builder = - SnapshotPackageCopyIndexesBuilder::with_capacity(packages_len); + copy_index_resolver = + SnapshotPackageCopyIndexResolver::with_capacity(packages_len); let mut verify_ids = HashSet::with_capacity(packages_len); // collect the specifiers to version mappings for (key, value) in &lockfile.content.npm.specifiers { let reference = NpmPackageReference::from_str(&format!("npm:{}", key)) .with_context(|| format!("Unable to parse npm specifier: {}", key))?; - let package_id = NpmPackageId::deserialize_name(value)?; + let package_id = NpmPackageId::from_serialized(value)?; package_reqs.insert(reference.req, package_id.clone()); verify_ids.insert(package_id.clone()); } // then the packages for (key, value) in &lockfile.content.npm.packages { - let package_id = NpmPackageId::deserialize_name(key)?; + let package_id = NpmPackageId::from_serialized(key)?; // collect the dependencies let mut dependencies = HashMap::default(); @@ -261,14 +261,14 @@ impl NpmResolutionSnapshot { .push(package_id.clone()); for (name, specifier) in &value.dependencies { - let dep_id = NpmPackageId::deserialize_name(specifier)?; + let dep_id = NpmPackageId::from_serialized(specifier)?; dependencies.insert(name.to_string(), dep_id.clone()); verify_ids.insert(dep_id); } let package = NpmResolutionPackage { id: package_id.clone(), - copy_index: copy_indexes_builder.add_package(&package_id), + copy_index: copy_index_resolver.resolve(&package_id), // temporary dummy value dist: NpmPackageVersionDistInfo { tarball: "foobar".to_string(), @@ -330,12 +330,12 @@ impl NpmResolutionSnapshot { } } -pub struct SnapshotPackageCopyIndexesBuilder { +pub struct SnapshotPackageCopyIndexResolver { packages_to_copy_index: HashMap, package_name_version_to_copy_count: HashMap<(String, String), usize>, } -impl SnapshotPackageCopyIndexesBuilder { +impl SnapshotPackageCopyIndexResolver { pub fn with_capacity(capacity: usize) -> Self { Self { packages_to_copy_index: HashMap::with_capacity(capacity), @@ -367,7 +367,7 @@ impl SnapshotPackageCopyIndexesBuilder { } } - pub fn add_package(&mut self, id: &NpmPackageId) -> usize { + pub fn resolve(&mut self, id: &NpmPackageId) -> usize { if let Some(index) = self.packages_to_copy_index.get(id) { *index } else { @@ -410,4 +410,48 @@ mod tests { assert_eq!(name_without_path("@foo/bar/baz"), "@foo/bar"); assert_eq!(name_without_path("@hello"), "@hello"); } + + #[test] + fn test_copy_index_resolver() { + let mut copy_index_resolver = + SnapshotPackageCopyIndexResolver::with_capacity(10); + assert_eq!( + copy_index_resolver + .resolve(&NpmPackageId::from_serialized("package@1.0.0").unwrap()), + 0 + ); + assert_eq!( + copy_index_resolver + .resolve(&NpmPackageId::from_serialized("package@1.0.0").unwrap()), + 0 + ); + assert_eq!( + copy_index_resolver.resolve( + &NpmPackageId::from_serialized("package@1.0.0_package-b@1.0.0") + .unwrap() + ), + 1 + ); + assert_eq!( + copy_index_resolver.resolve( + &NpmPackageId::from_serialized( + "package@1.0.0_package-b@1.0.0__package-c@2.0.0" + ) + .unwrap() + ), + 2 + ); + assert_eq!( + copy_index_resolver.resolve( + &NpmPackageId::from_serialized("package@1.0.0_package-b@1.0.0") + .unwrap() + ), + 1 + ); + assert_eq!( + copy_index_resolver + .resolve(&NpmPackageId::from_serialized("package-b@1.0.0").unwrap()), + 0 + ); + } } From b4979c5eda435633ff88deb56bba8d353466518c Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 7 Nov 2022 11:51:24 -0500 Subject: [PATCH 28/43] Update monch --- Cargo.lock | 8 +++---- cli/Cargo.toml | 4 ++-- cli/npm/cache.rs | 4 ++-- cli/npm/resolution/mod.rs | 3 +-- cli/npm/resolution/snapshot.rs | 3 ++- cli/npm/semver/errors.rs | 40 ---------------------------------- cli/npm/semver/mod.rs | 2 -- cli/npm/semver/specifier.rs | 1 - 8 files changed, 11 insertions(+), 54 deletions(-) delete mode 100644 cli/npm/semver/errors.rs diff --git a/Cargo.lock b/Cargo.lock index 4392d4a7e39af3..605e41db5458a8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1227,9 +1227,9 @@ dependencies = [ [[package]] name = "deno_task_shell" -version = "0.7.0" +version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a275d3f78e828b4adddf20a472d9ac1927ac311aac48dca869bb8653d5a4a0b9" +checksum = "e8ad1e1002ecf8bafcb9b968bf19856ba4fe0e6c0c73b3404565bb29b15aae2c" dependencies = [ "anyhow", "futures", @@ -2803,9 +2803,9 @@ dependencies = [ [[package]] name = "monch" -version = "0.2.1" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c5e2e282addadb529bb31700f7d184797382fa2eb18384986aad78d117eaf0c4" +checksum = "f13de1c3edc9a5b9dc3a1029f56e9ab3eba34640010aff4fc01044c42ef67afa" [[package]] name = "naga" diff --git a/cli/Cargo.toml b/cli/Cargo.toml index 9920e46ce7ed98..e2ec7cef0cb4c4 100644 --- a/cli/Cargo.toml +++ b/cli/Cargo.toml @@ -57,7 +57,7 @@ deno_emit = "0.10.0" deno_graph = "0.37.1" deno_lint = { version = "0.34.0", features = ["docs"] } deno_runtime = { version = "0.82.0", path = "../runtime" } -deno_task_shell = "0.7.0" +deno_task_shell = "0.7.2" napi_sym = { path = "./napi_sym", version = "0.4.0" } atty = "=0.2.14" @@ -86,7 +86,7 @@ libc = "=0.2.126" log = { version = "=0.4.17", features = ["serde"] } lsp-types = "=0.93.2" # used by tower-lsp and "proposed" feature is unstable in patch releases mitata = "=0.0.7" -monch = "=0.2.1" +monch = "=0.4.0" notify = "=5.0.0" once_cell = "=1.14.0" os_pipe = "=1.0.1" diff --git a/cli/npm/cache.rs b/cli/npm/cache.rs index 2f2a1fe0864a78..9b27eb025873c9 100644 --- a/cli/npm/cache.rs +++ b/cli/npm/cache.rs @@ -217,7 +217,8 @@ impl ReadonlyNpmCache { // packages. // 2. We can figure out the package id from the path. This is used // in resolve_package_id_from_specifier - // Maybe consider only supporting this if people use --node-modules-dir + // Probably use a hash of the package name at `npm/-/` then create + // a mapping for these package names. todo!("deno currently doesn't support npm package names that are not all lowercase"); } // ensure backslashes are used on windows @@ -345,7 +346,6 @@ impl NpmCache { dist: &NpmPackageVersionDistInfo, registry_url: &Url, ) -> Result<(), AnyError> { - // todo(THIS PR): callers should cache the 0-indexed version first let package_folder = self.readonly.package_folder_for_name_and_version( package.0, package.1, diff --git a/cli/npm/resolution/mod.rs b/cli/npm/resolution/mod.rs index 1b211aae64c518..0eee00fc9943f9 100644 --- a/cli/npm/resolution/mod.rs +++ b/cli/npm/resolution/mod.rs @@ -295,8 +295,7 @@ impl NpmPackageId { } } - // todo(THIS PR): move this someone re-usable - crate::npm::semver::errors::with_failure_handling(parse_id_at_level(0))(id) + with_failure_handling(parse_id_at_level(0))(id) .with_context(|| format!("Invalid npm package id '{}'.", id)) } } diff --git a/cli/npm/resolution/snapshot.rs b/cli/npm/resolution/snapshot.rs index 32b42b72b9b01c..4749fb5eef36a5 100644 --- a/cli/npm/resolution/snapshot.rs +++ b/cli/npm/resolution/snapshot.rs @@ -199,7 +199,8 @@ impl NpmResolutionSnapshot { name: &str, version_matcher: &impl NpmVersionMatcher, ) -> Option { - // todo(THIS PR): this is not correct because some ids will be better than others + // todo(dsherret): this is not exactly correct because some ids + // will be better than others due to peer dependencies let mut maybe_best_id: Option<&NpmPackageId> = None; if let Some(ids) = self.packages_by_name.get(name) { for id in ids { diff --git a/cli/npm/semver/errors.rs b/cli/npm/semver/errors.rs deleted file mode 100644 index 0f2ed1bf0ed058..00000000000000 --- a/cli/npm/semver/errors.rs +++ /dev/null @@ -1,40 +0,0 @@ -// Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. - -use deno_core::anyhow::bail; -use deno_core::error::AnyError; -use monch::ParseError; -use monch::ParseErrorFailure; -use monch::ParseResult; - -// todo(THIS PR): open an issue in monch about these - -pub fn with_failure_handling<'a, T>( - combinator: impl Fn(&'a str) -> ParseResult, -) -> impl Fn(&'a str) -> Result { - move |input| match combinator(input) { - Ok((input, result)) => { - if !input.is_empty() { - error_for_failure(fail_for_trailing_input(input)) - } else { - Ok(result) - } - } - Err(ParseError::Backtrace) => { - error_for_failure(fail_for_trailing_input(input)) - } - Err(ParseError::Failure(e)) => error_for_failure(e), - } -} - -fn error_for_failure(e: ParseErrorFailure) -> Result { - bail!( - "{}\n {}\n ~", - e.message, - // truncate the output to prevent wrapping in the console - e.input.chars().take(60).collect::() - ) -} - -fn fail_for_trailing_input(input: &str) -> ParseErrorFailure { - ParseErrorFailure::new(input, "Unexpected character.") -} diff --git a/cli/npm/semver/mod.rs b/cli/npm/semver/mod.rs index 6e1985adc12075..cd63b2a2993a4b 100644 --- a/cli/npm/semver/mod.rs +++ b/cli/npm/semver/mod.rs @@ -11,7 +11,6 @@ use serde::Serialize; use crate::npm::resolution::NpmVersionMatcher; -use self::errors::with_failure_handling; use self::range::Partial; use self::range::VersionBoundKind; use self::range::VersionRange; @@ -20,7 +19,6 @@ use self::range::VersionRangeSet; use self::range::XRange; pub use self::specifier::SpecifierVersionReq; -pub mod errors; mod range; mod specifier; diff --git a/cli/npm/semver/specifier.rs b/cli/npm/semver/specifier.rs index c3e7f716b00534..dc4fe101052721 100644 --- a/cli/npm/semver/specifier.rs +++ b/cli/npm/semver/specifier.rs @@ -6,7 +6,6 @@ use monch::*; use serde::Deserialize; use serde::Serialize; -use super::errors::with_failure_handling; use super::range::Partial; use super::range::VersionRange; use super::range::XRange; From 846a83e74fec668485c12d4b999f4c31bc4864d5 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 7 Nov 2022 12:40:30 -0500 Subject: [PATCH 29/43] Fix lack of determinism going up the tree --- cli/npm/resolution/graph.rs | 69 ++++++++++++++++++++-------------- cli/npm/resolution/mod.rs | 8 ++++ cli/npm/resolution/snapshot.rs | 5 +-- 3 files changed, 50 insertions(+), 32 deletions(-) diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index 19671623f345b2..d2797b6e5d93eb 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -1,6 +1,8 @@ // Copyright 2018-2022 the Deno authors. All rights reserved. MIT license. use std::borrow::Cow; +use std::collections::BTreeMap; +use std::collections::BTreeSet; use std::collections::HashMap; use std::collections::HashSet; use std::collections::VecDeque; @@ -77,11 +79,11 @@ impl GraphPath { } } -#[derive(Debug, Clone, PartialEq, Eq, Hash)] +#[derive(Debug, Clone, PartialEq, Eq, Hash, Ord, PartialOrd)] enum NodeParent { /// These are top of the graph npm package requirements /// as specified in Deno code. - Req(NpmPackageReq), + Req, /// A reference to another node, which is a resolved package. Node(NpmPackageId), } @@ -90,8 +92,10 @@ enum NodeParent { #[derive(Debug)] struct Node { pub id: NpmPackageId, - pub parents: HashMap>, - pub children: HashMap, + // Use BTreeMap and BTreeSet in order to create determinism + // when going up and down the tree + pub parents: BTreeMap>, + pub children: BTreeMap, pub deps: Arc>, } @@ -112,7 +116,7 @@ impl Node { #[derive(Debug, Default)] pub struct Graph { - package_reqs: HashMap, + package_reqs: HashMap, packages_by_name: HashMap>, // Ideally this value would be Rc>, but we need to use a Mutex // because the lsp requires Send and this code is executed in the lsp. @@ -151,17 +155,17 @@ impl Graph { }; for (package_req, id) in &snapshot.package_reqs { let node = fill_for_id(&mut graph, id, &snapshot.packages); - (*node).lock().add_parent( - package_req.to_string(), - NodeParent::Req(package_req.clone()), - ); - graph.package_reqs.insert(package_req.clone(), id.clone()); + let package_req_text = package_req.to_string(); + (*node) + .lock() + .add_parent(package_req_text.clone(), NodeParent::Req); + graph.package_reqs.insert(package_req_text, id.clone()); } graph } pub fn has_package_req(&self, req: &NpmPackageReq) -> bool { - self.package_reqs.contains_key(req) + self.package_reqs.contains_key(&req.to_string()) } fn get_or_create_for_id( @@ -230,12 +234,12 @@ impl Graph { NodeParent::Node(parent_id) => { self.set_child_parent_node(specifier, child, parent_id); } - NodeParent::Req(package_req) => { + NodeParent::Req => { let mut node = (*child).lock(); node.add_parent(specifier.to_string(), parent.clone()); self .package_reqs - .insert(package_req.clone(), node.id.clone()); + .insert(specifier.to_string(), node.id.clone()); } } } @@ -269,9 +273,8 @@ impl Graph { assert_eq!(removed_child_id, *child_id); } } - NodeParent::Req(req) => { - assert_eq!(req.to_string(), specifier); - if let Some(removed_child_id) = self.package_reqs.remove(req) { + NodeParent::Req => { + if let Some(removed_child_id) = self.package_reqs.remove(specifier) { assert_eq!(removed_child_id, *child_id); } } @@ -314,13 +317,23 @@ impl Graph { copy_index: copy_index_resolver.resolve(&id), id, dist, - dependencies: node.children.clone(), + dependencies: node + .children + .iter() + .map(|(key, value)| (key.clone(), value.clone())) + .collect(), }, ); } Ok(NpmResolutionSnapshot { - package_reqs: self.package_reqs, + package_reqs: self + .package_reqs + .into_iter() + .map(|(specifier, id)| { + (NpmPackageReq::from_str(&specifier).unwrap(), id) + }) + .collect(), packages_by_name: self.packages_by_name, packages, }) @@ -408,7 +421,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> self.graph.set_child_parent( &package_req.to_string(), &node, - &NodeParent::Req(package_req.clone()), + &NodeParent::Req, ); self .pending_unresolved_nodes @@ -630,19 +643,17 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> ))); } } - NodeParent::Req(req) => { + NodeParent::Req => { // in this case, the parent is the root so the children are all the package requirements if let Some(child_id) = find_matching_child(peer_dep, self.graph.package_reqs.values()) { - let old_id = self.graph.package_reqs.get(req).unwrap().clone(); let mut path = path.specifiers; - path.pop(); // go back down one level + let specifier = path.pop().unwrap(); // go back down one level + let old_id = + self.graph.package_reqs.get(&specifier).unwrap().clone(); return Ok(Some(self.set_new_peer_dep( - HashMap::from([( - req.to_string(), - HashSet::from([NodeParent::Req(req.clone())]), - )]), + BTreeMap::from([(specifier, BTreeSet::from([NodeParent::Req]))]), &old_id, &child_id, path, @@ -670,7 +681,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> fn set_new_peer_dep( &mut self, - previous_parents: HashMap>, + previous_parents: BTreeMap>, node_id: &NpmPackageId, peer_dep_id: &NpmPackageId, mut path: Vec, @@ -750,9 +761,9 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> } else { let next_node_id = old_node_children.get(&next_specifier).unwrap(); bottom_parent_id = self.set_new_peer_dep( - HashMap::from([( + BTreeMap::from([( next_specifier.to_string(), - HashSet::from([NodeParent::Node(new_id.clone())]), + BTreeSet::from([NodeParent::Node(new_id.clone())]), )]), next_node_id, &peer_dep_id, diff --git a/cli/npm/resolution/mod.rs b/cli/npm/resolution/mod.rs index 0eee00fc9943f9..5dfa02196762a2 100644 --- a/cli/npm/resolution/mod.rs +++ b/cli/npm/resolution/mod.rs @@ -133,6 +133,14 @@ impl std::fmt::Display for NpmPackageReq { } } +impl NpmPackageReq { + pub fn from_str(text: &str) -> Result { + // probably should do something more targetted in the future + let reference = NpmPackageReference::from_str(&format!("npm:{}", text))?; + Ok(reference.req) + } +} + impl NpmVersionMatcher for NpmPackageReq { fn tag(&self) -> Option<&str> { match &self.version_req { diff --git a/cli/npm/resolution/snapshot.rs b/cli/npm/resolution/snapshot.rs index 4749fb5eef36a5..6f57cd9b8122b2 100644 --- a/cli/npm/resolution/snapshot.rs +++ b/cli/npm/resolution/snapshot.rs @@ -20,7 +20,6 @@ use crate::npm::registry::NpmRegistryApi; use crate::npm::registry::RealNpmRegistryApi; use super::NpmPackageId; -use super::NpmPackageReference; use super::NpmPackageReq; use super::NpmResolutionPackage; use super::NpmVersionMatcher; @@ -242,10 +241,10 @@ impl NpmResolutionSnapshot { // collect the specifiers to version mappings for (key, value) in &lockfile.content.npm.specifiers { - let reference = NpmPackageReference::from_str(&format!("npm:{}", key)) + let package_req = NpmPackageReq::from_str(key) .with_context(|| format!("Unable to parse npm specifier: {}", key))?; let package_id = NpmPackageId::from_serialized(value)?; - package_reqs.insert(reference.req, package_id.clone()); + package_reqs.insert(package_req, package_id.clone()); verify_ids.insert(package_id.clone()); } From 422b512897129da29208eecb656969eac2341070 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 7 Nov 2022 12:57:59 -0500 Subject: [PATCH 30/43] Use hard links. --- cli/fs_util.rs | 33 ++++++ cli/npm/cache.rs | 8 +- cli/npm/resolution/graph.rs | 219 ++++++++++++++++++------------------ cli/npm/resolvers/local.rs | 3 +- 4 files changed, 151 insertions(+), 112 deletions(-) diff --git a/cli/fs_util.rs b/cli/fs_util.rs index fa1535469d94e9..cafe29af8fda3a 100644 --- a/cli/fs_util.rs +++ b/cli/fs_util.rs @@ -357,6 +357,39 @@ pub fn copy_dir_recursive(from: &Path, to: &Path) -> Result<(), AnyError> { Ok(()) } +/// Hardlinks the files in one directory to another directory. +/// +/// Note: Does not handle symlinks. +pub fn hard_link_dir_recursive(from: &Path, to: &Path) -> Result<(), AnyError> { + std::fs::create_dir_all(&to) + .with_context(|| format!("Creating {}", to.display()))?; + let read_dir = std::fs::read_dir(&from) + .with_context(|| format!("Reading {}", from.display()))?; + + for entry in read_dir { + let entry = entry?; + let file_type = entry.file_type()?; + let new_from = from.join(entry.file_name()); + let new_to = to.join(entry.file_name()); + + if file_type.is_dir() { + hard_link_dir_recursive(&new_from, &new_to).with_context(|| { + format!("Dir {} to {}", new_from.display(), new_to.display()) + })?; + } else if file_type.is_file() { + std::fs::hard_link(&new_from, &new_to).with_context(|| { + format!( + "Hard linking {} to {}", + new_from.display(), + new_to.display() + ) + })?; + } + } + + Ok(()) +} + pub fn symlink_dir(oldpath: &Path, newpath: &Path) -> Result<(), AnyError> { let err_mapper = |err: Error| { Error::new( diff --git a/cli/npm/cache.rs b/cli/npm/cache.rs index 9b27eb025873c9..e44618c8e97c5e 100644 --- a/cli/npm/cache.rs +++ b/cli/npm/cache.rs @@ -406,11 +406,15 @@ impl NpmCache { let original_package_folder = self .readonly .package_folder_for_name_and_version(&id.name, &id.version, registry_url); - // todo(THIS PR): use hard linking instead of copying with_folder_sync_lock( (id.name.as_str(), &id.version), &package_folder, - || fs_util::copy_dir_recursive(&original_package_folder, &package_folder), + || { + fs_util::hard_link_dir_recursive( + &original_package_folder, + &package_folder, + ) + }, )?; Ok(()) } diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index d2797b6e5d93eb..2ed998d4cdf73e 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -1800,119 +1800,122 @@ mod test { #[tokio::test] async fn resolve_peer_deps_multiple_copies() { - let api = TestNpmRegistryApi::default(); - api.ensure_package_version("package-a", "1.0.0"); - api.ensure_package_version("package-b", "2.0.0"); - api.ensure_package_version("package-dep", "3.0.0"); - api.ensure_package_version("package-peer", "4.0.0"); - api.ensure_package_version("package-peer", "5.0.0"); - api.add_dependency(("package-a", "1.0.0"), ("package-dep", "*")); - api.add_dependency(("package-a", "1.0.0"), ("package-peer", "4")); - api.add_dependency(("package-b", "2.0.0"), ("package-dep", "*")); - api.add_dependency(("package-b", "2.0.0"), ("package-peer", "5")); - api.add_peer_dependency(("package-dep", "3.0.0"), ("package-peer", "*")); - - let (packages, package_reqs) = run_resolver_and_get_output( - api, - vec!["npm:package-a@1", "npm:package-b@2"], - ) - .await; - assert_eq!( - packages, - vec![ - NpmResolutionPackage { - id: NpmPackageId::from_serialized( - "package-a@1.0.0_package-peer@4.0.0" - ) - .unwrap(), - copy_index: 0, - dependencies: HashMap::from([ - ( - "package-dep".to_string(), - NpmPackageId::from_serialized( - "package-dep@3.0.0_package-peer@4.0.0" - ) - .unwrap(), - ), - ( + // repeat this a few times to have a higher probability of surfacing indeterminism + for _ in 0..3 { + let api = TestNpmRegistryApi::default(); + api.ensure_package_version("package-a", "1.0.0"); + api.ensure_package_version("package-b", "2.0.0"); + api.ensure_package_version("package-dep", "3.0.0"); + api.ensure_package_version("package-peer", "4.0.0"); + api.ensure_package_version("package-peer", "5.0.0"); + api.add_dependency(("package-a", "1.0.0"), ("package-dep", "*")); + api.add_dependency(("package-a", "1.0.0"), ("package-peer", "4")); + api.add_dependency(("package-b", "2.0.0"), ("package-dep", "*")); + api.add_dependency(("package-b", "2.0.0"), ("package-peer", "5")); + api.add_peer_dependency(("package-dep", "3.0.0"), ("package-peer", "*")); + + let (packages, package_reqs) = run_resolver_and_get_output( + api, + vec!["npm:package-a@1", "npm:package-b@2"], + ) + .await; + assert_eq!( + packages, + vec![ + NpmResolutionPackage { + id: NpmPackageId::from_serialized( + "package-a@1.0.0_package-peer@4.0.0" + ) + .unwrap(), + copy_index: 0, + dependencies: HashMap::from([ + ( + "package-dep".to_string(), + NpmPackageId::from_serialized( + "package-dep@3.0.0_package-peer@4.0.0" + ) + .unwrap(), + ), + ( + "package-peer".to_string(), + NpmPackageId::from_serialized("package-peer@4.0.0").unwrap(), + ), + ]), + dist: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::from_serialized( + "package-b@2.0.0_package-peer@5.0.0" + ) + .unwrap(), + copy_index: 0, + dependencies: HashMap::from([ + ( + "package-dep".to_string(), + NpmPackageId::from_serialized( + "package-dep@3.0.0_package-peer@5.0.0" + ) + .unwrap(), + ), + ( + "package-peer".to_string(), + NpmPackageId::from_serialized("package-peer@5.0.0").unwrap(), + ), + ]), + dist: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::from_serialized( + "package-dep@3.0.0_package-peer@4.0.0" + ) + .unwrap(), + copy_index: 0, + dependencies: HashMap::from([( "package-peer".to_string(), NpmPackageId::from_serialized("package-peer@4.0.0").unwrap(), - ), - ]), - dist: Default::default(), - }, - NpmResolutionPackage { - id: NpmPackageId::from_serialized( - "package-b@2.0.0_package-peer@5.0.0" - ) - .unwrap(), - copy_index: 0, - dependencies: HashMap::from([ - ( - "package-dep".to_string(), - NpmPackageId::from_serialized( - "package-dep@3.0.0_package-peer@5.0.0" - ) - .unwrap(), - ), - ( + )]), + dist: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::from_serialized( + "package-dep@3.0.0_package-peer@5.0.0" + ) + .unwrap(), + copy_index: 1, + dependencies: HashMap::from([( "package-peer".to_string(), NpmPackageId::from_serialized("package-peer@5.0.0").unwrap(), - ), - ]), - dist: Default::default(), - }, - NpmResolutionPackage { - id: NpmPackageId::from_serialized( - "package-dep@3.0.0_package-peer@4.0.0" - ) - .unwrap(), - copy_index: 0, - dependencies: HashMap::from([( - "package-peer".to_string(), - NpmPackageId::from_serialized("package-peer@4.0.0").unwrap(), - )]), - dist: Default::default(), - }, - NpmResolutionPackage { - id: NpmPackageId::from_serialized( - "package-dep@3.0.0_package-peer@5.0.0" + )]), + dist: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::from_serialized("package-peer@4.0.0").unwrap(), + copy_index: 0, + dependencies: HashMap::new(), + dist: Default::default(), + }, + NpmResolutionPackage { + id: NpmPackageId::from_serialized("package-peer@5.0.0").unwrap(), + copy_index: 0, + dependencies: HashMap::new(), + dist: Default::default(), + }, + ] + ); + assert_eq!( + package_reqs, + vec![ + ( + "package-a@1".to_string(), + "package-a@1.0.0_package-peer@4.0.0".to_string() + ), + ( + "package-b@2".to_string(), + "package-b@2.0.0_package-peer@5.0.0".to_string() ) - .unwrap(), - copy_index: 1, - dependencies: HashMap::from([( - "package-peer".to_string(), - NpmPackageId::from_serialized("package-peer@5.0.0").unwrap(), - )]), - dist: Default::default(), - }, - NpmResolutionPackage { - id: NpmPackageId::from_serialized("package-peer@4.0.0").unwrap(), - copy_index: 0, - dependencies: HashMap::new(), - dist: Default::default(), - }, - NpmResolutionPackage { - id: NpmPackageId::from_serialized("package-peer@5.0.0").unwrap(), - copy_index: 0, - dependencies: HashMap::new(), - dist: Default::default(), - }, - ] - ); - assert_eq!( - package_reqs, - vec![ - ( - "package-a@1".to_string(), - "package-a@1.0.0_package-peer@4.0.0".to_string() - ), - ( - "package-b@2".to_string(), - "package-b@2.0.0_package-peer@5.0.0".to_string() - ) - ] - ); + ] + ); + } } async fn run_resolver_and_get_output( diff --git a/cli/npm/resolvers/local.rs b/cli/npm/resolvers/local.rs index c9c9d7e4a417a4..66b65226d2483c 100644 --- a/cli/npm/resolvers/local.rs +++ b/cli/npm/resolvers/local.rs @@ -349,8 +349,7 @@ async fn sync_resolution_with_fs( .join("node_modules"), &package.id.name, ); - // todo(THIS PR): hard link instead - fs_util::copy_dir_recursive(&source_path, &package_path)?; + fs_util::hard_link_dir_recursive(&source_path, &package_path)?; // write out a file that indicates this folder has been initialized fs::write(initialized_file, "")?; } From d37eab6e5bb659cbe2e96fddb81f20a38d80e595 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 7 Nov 2022 14:48:45 -0500 Subject: [PATCH 31/43] Failing refactor. --- cli/npm/resolution/graph.rs | 203 +++++++++++++++++++++++------------- 1 file changed, 131 insertions(+), 72 deletions(-) diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index 2ed998d4cdf73e..71131071b8c2f3 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -4,7 +4,6 @@ use std::borrow::Cow; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::collections::HashMap; -use std::collections::HashSet; use std::collections::VecDeque; use std::sync::Arc; @@ -33,49 +32,77 @@ use super::NpmResolutionPackage; use super::NpmVersionMatcher; #[derive(Default, Clone)] -struct VisitedVersions(HashSet); +struct VisitedVersionsPath { + previous_node: Option>, + visited_version: (String, NpmVersion), +} -impl VisitedVersions { - pub fn add(&mut self, id: &NpmPackageId) -> bool { - self.0.insert(Self::id_as_key(id)) +impl VisitedVersionsPath { + pub fn new(id: &NpmPackageId) -> Arc { + Arc::new(Self { + previous_node: None, + visited_version: Self::id_as_name_and_version(id), + }) } - pub fn has_visited(&self, id: &NpmPackageId) -> bool { - self.0.contains(&Self::id_as_key(id)) + pub fn with_id( + self: &Arc, + id: &NpmPackageId, + ) -> Option> { + if self.has_visited(id) { + None + } else { + Some(Arc::new(Self { + previous_node: Some(self.clone()), + visited_version: Self::id_as_name_and_version(id), + })) + } + } + + pub fn has_visited(self: &Arc, id: &NpmPackageId) -> bool { + let mut maybe_next_node = Some(self); + let name_and_version = Self::id_as_name_and_version(id); + while let Some(next_node) = maybe_next_node { + if next_node.visited_version == name_and_version { + return true; + } + maybe_next_node = next_node.previous_node.as_ref(); + } + false } - fn id_as_key(id: &NpmPackageId) -> String { - // we only key on name and version in the id and not peer dependencies - // because the peer dependencies could change above and below us, - // but the names and versions won't - format!("{}@{}", id.name, id.version) + fn id_as_name_and_version(id: &NpmPackageId) -> (String, NpmVersion) { + (id.name.clone(), id.version.clone()) } } #[derive(Default, Clone)] struct GraphPath { - // todo(THIS PR): investigate if this should use a singly linked list too - visited_versions: VisitedVersions, - // todo(THIS PR): switch to a singly linked list here - specifiers: Vec, + previous_node: Option>, + specifier: String, } impl GraphPath { - pub fn with_step(&self, specifier: &str, id: &NpmPackageId) -> GraphPath { - let mut copy = self.clone(); - assert!(copy.visited_versions.add(id)); - copy.specifiers.push(specifier.to_string()); - copy + pub fn new(specifier: String) -> Arc { + Arc::new(Self { + previous_node: None, + specifier, + }) + } + + pub fn with_specifier(self: &Arc, specifier: String) -> Arc { + Arc::new(Self { + previous_node: Some(self.clone()), + specifier, + }) } - pub fn with_specifier(&self, specifier: String) -> GraphPath { - let mut copy = self.clone(); - copy.specifiers.push(specifier); - copy + pub fn pop(&self) -> Option<&Arc> { + self.previous_node.as_ref() } - pub fn has_visited_version(&self, id: &NpmPackageId) -> bool { - self.visited_versions.has_visited(id) + pub fn is_last(&self) -> bool { + self.previous_node.is_none() } } @@ -343,7 +370,8 @@ impl Graph { pub struct GraphDependencyResolver<'a, TNpmRegistryApi: NpmRegistryApi> { graph: &'a mut Graph, api: &'a TNpmRegistryApi, - pending_unresolved_nodes: VecDeque<(VisitedVersions, Arc>)>, + pending_unresolved_nodes: + VecDeque<(Arc, Arc>)>, } impl<'a, TNpmRegistryApi: NpmRegistryApi> @@ -423,9 +451,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> &node, &NodeParent::Req, ); - self - .pending_unresolved_nodes - .push_back((VisitedVersions::default(), node)); + self.add_pending_unresolved_node(None, &node); Ok(()) } @@ -434,7 +460,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> entry: &NpmDependencyEntry, package_info: NpmPackageInfo, parent_id: &NpmPackageId, - visited_versions: &VisitedVersions, + visited_versions: &Arc, ) -> Result<(), AnyError> { let node = self.resolve_node_from_info( &entry.name, @@ -456,10 +482,28 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> &node, &NodeParent::Node(parent_id.clone()), ); + self.add_pending_unresolved_node(Some(visited_versions), &node); + Ok(()) + } + + fn add_pending_unresolved_node( + &mut self, + maybe_previous_visited_versions: Option<&Arc>, + node: &Arc>, + ) { + let node_id = node.lock().id.clone(); + let visited_versions = match maybe_previous_visited_versions { + Some(previous_visited_versions) => { + match previous_visited_versions.with_id(&node_id) { + Some(visited_versions) => visited_versions, + None => return, // circular, don't visit this node + } + } + None => VisitedVersionsPath::new(&node_id), + }; self .pending_unresolved_nodes - .push_back((visited_versions.clone(), node)); - Ok(()) + .push_back((visited_versions, node.clone())); } fn resolve_node_from_info( @@ -503,7 +547,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> pub async fn resolve_pending(&mut self) -> Result<(), AnyError> { while !self.pending_unresolved_nodes.is_empty() { // now go down through the dependencies by tree depth - while let Some((mut visited_versions, parent_node)) = + while let Some((visited_versions, parent_node)) = self.pending_unresolved_nodes.pop_front() { let (mut parent_id, deps) = { @@ -511,10 +555,6 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> (parent_node.id.clone(), parent_node.deps.clone()) }; - if !visited_versions.add(&parent_id) { - continue; // circular - } - // cache all the dependencies' registry infos in parallel if should if !should_sync_download() { let handles = deps @@ -578,7 +618,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> parent_id: &NpmPackageId, peer_dep: &NpmDependencyEntry, peer_package_info: NpmPackageInfo, - visited_ancestor_versions: &VisitedVersions, + visited_ancestor_versions: &Arc, ) -> Result, AnyError> { fn find_matching_child<'a>( peer_dep: &NpmDependencyEntry, @@ -597,7 +637,8 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> // Peer dependencies are resolved based on its ancestors' siblings. // If not found, then it resolves based on the version requirement if non-optional. let mut pending_ancestors = VecDeque::new(); // go up the tree by depth - let path = GraphPath::default().with_step(specifier, parent_id); + let path = GraphPath::new(specifier.to_string()); + let visited_versions = VisitedVersionsPath::new(parent_id); // skip over the current node for (specifier, grand_parents) in @@ -605,28 +646,50 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> { let path = path.with_specifier(specifier); for grand_parent in grand_parents { - pending_ancestors.push_back((grand_parent, path.clone())); + let visited_versions = match &grand_parent { + NodeParent::Node(id) => visited_versions.with_id(id), + NodeParent::Req => Some(visited_versions.clone()), + }; + if let Some(visited_versions) = visited_versions { + pending_ancestors.push_back(( + grand_parent, + path.clone(), + visited_versions, + )); + } } } - while let Some((ancestor, path)) = pending_ancestors.pop_front() { + while let Some((ancestor, path, visited_versions)) = + pending_ancestors.pop_front() + { match &ancestor { NodeParent::Node(ancestor_node_id) => { - // we've gone in a full circle, so don't keep looking - if path.has_visited_version(ancestor_node_id) { - continue; - } - let maybe_peer_dep_id = if ancestor_node_id.name == peer_dep.name && peer_dep.version_req.satisfies(&ancestor_node_id.version) { + // found it Some(ancestor_node_id.clone()) } else { + // continue searching up let ancestor = self.graph.borrow_node(ancestor_node_id); for (specifier, parents) in &ancestor.parents { - let new_path = path.with_step(specifier, ancestor_node_id); for parent in parents { - pending_ancestors.push_back((parent.clone(), new_path.clone())); + let path = path.with_specifier(specifier.to_string()); + let visited_versions = match parent { + NodeParent::Node(id) => visited_versions.with_id(id), + NodeParent::Req => Some(visited_versions.clone()), + }; + if let Some(visited_versions) = visited_versions { + pending_ancestors.push_back(( + parent.clone(), + path, + visited_versions, + )); + } else { + // we've gone in a full circle, so don't keep looking + continue; + } } } find_matching_child(peer_dep, ancestor.children.values()) @@ -638,7 +701,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> parents, ancestor_node_id, &peer_dep_id, - path.specifiers, + &path, visited_ancestor_versions, ))); } @@ -648,8 +711,8 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> if let Some(child_id) = find_matching_child(peer_dep, self.graph.package_reqs.values()) { - let mut path = path.specifiers; - let specifier = path.pop().unwrap(); // go back down one level + let specifier = path.specifier.clone(); + let path = path.pop().unwrap(); // go back down one level from the package requirement let old_id = self.graph.package_reqs.get(&specifier).unwrap().clone(); return Ok(Some(self.set_new_peer_dep( @@ -684,8 +747,8 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> previous_parents: BTreeMap>, node_id: &NpmPackageId, peer_dep_id: &NpmPackageId, - mut path: Vec, - visited_ancestor_versions: &VisitedVersions, + path: &GraphPath, + visited_ancestor_versions: &Arc, ) -> NpmPackageId { let mut peer_dep_id = Cow::Borrowed(peer_dep_id); let old_id = node_id; @@ -708,15 +771,12 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> } // remove the previous parents from the old node - let old_node_children = { - for (specifier, parents) in &previous_parents { - for parent in parents { - self.graph.remove_child_parent(specifier, old_id, parent); - } + for (specifier, parents) in &previous_parents { + for parent in parents { + self.graph.remove_child_parent(specifier, old_id, parent); } - let old_node = self.graph.borrow_node(old_id); - old_node.children.clone() - }; + } + let old_node_children = self.graph.borrow_node(old_id).children.clone(); let (_, new_node) = self.graph.get_or_create_for_id(&new_id); @@ -743,26 +803,25 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> let mut bottom_parent_id = new_id.clone(); // continue going down the path - if let Some(next_specifier) = path.pop() { - if path.is_empty() { + if let Some(path) = path.pop() { + if path.is_last() { // this means we're at the peer dependency now debug!( "Resolved peer dependency for {} in {} to {}", - &next_specifier, &new_id, &peer_dep_id, + &path.specifier, &new_id, &peer_dep_id, ); - assert!(!old_node_children.contains_key(&next_specifier)); + assert!(!old_node_children.contains_key(&path.specifier)); let node = self.graph.get_or_create_for_id(&peer_dep_id).1; self - .pending_unresolved_nodes - .push_back((visited_ancestor_versions.clone(), node.clone())); + .add_pending_unresolved_node(Some(visited_ancestor_versions), &node); self .graph - .set_child_parent_node(&next_specifier, &node, &new_id); + .set_child_parent_node(&path.specifier, &node, &new_id); } else { - let next_node_id = old_node_children.get(&next_specifier).unwrap(); + let next_node_id = old_node_children.get(&path.specifier).unwrap(); bottom_parent_id = self.set_new_peer_dep( BTreeMap::from([( - next_specifier.to_string(), + path.specifier.to_string(), BTreeSet::from([NodeParent::Node(new_id.clone())]), )]), next_node_id, From 9a40e57dc06b4aae224153ab5361d52f6e54dccd Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 7 Nov 2022 14:49:27 -0500 Subject: [PATCH 32/43] Revert "Failing refactor." This reverts commit d37eab6e5bb659cbe2e96fddb81f20a38d80e595. --- cli/npm/resolution/graph.rs | 203 +++++++++++++----------------------- 1 file changed, 72 insertions(+), 131 deletions(-) diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index 71131071b8c2f3..2ed998d4cdf73e 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -4,6 +4,7 @@ use std::borrow::Cow; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::collections::HashMap; +use std::collections::HashSet; use std::collections::VecDeque; use std::sync::Arc; @@ -32,77 +33,49 @@ use super::NpmResolutionPackage; use super::NpmVersionMatcher; #[derive(Default, Clone)] -struct VisitedVersionsPath { - previous_node: Option>, - visited_version: (String, NpmVersion), -} +struct VisitedVersions(HashSet); -impl VisitedVersionsPath { - pub fn new(id: &NpmPackageId) -> Arc { - Arc::new(Self { - previous_node: None, - visited_version: Self::id_as_name_and_version(id), - }) +impl VisitedVersions { + pub fn add(&mut self, id: &NpmPackageId) -> bool { + self.0.insert(Self::id_as_key(id)) } - pub fn with_id( - self: &Arc, - id: &NpmPackageId, - ) -> Option> { - if self.has_visited(id) { - None - } else { - Some(Arc::new(Self { - previous_node: Some(self.clone()), - visited_version: Self::id_as_name_and_version(id), - })) - } - } - - pub fn has_visited(self: &Arc, id: &NpmPackageId) -> bool { - let mut maybe_next_node = Some(self); - let name_and_version = Self::id_as_name_and_version(id); - while let Some(next_node) = maybe_next_node { - if next_node.visited_version == name_and_version { - return true; - } - maybe_next_node = next_node.previous_node.as_ref(); - } - false + pub fn has_visited(&self, id: &NpmPackageId) -> bool { + self.0.contains(&Self::id_as_key(id)) } - fn id_as_name_and_version(id: &NpmPackageId) -> (String, NpmVersion) { - (id.name.clone(), id.version.clone()) + fn id_as_key(id: &NpmPackageId) -> String { + // we only key on name and version in the id and not peer dependencies + // because the peer dependencies could change above and below us, + // but the names and versions won't + format!("{}@{}", id.name, id.version) } } #[derive(Default, Clone)] struct GraphPath { - previous_node: Option>, - specifier: String, + // todo(THIS PR): investigate if this should use a singly linked list too + visited_versions: VisitedVersions, + // todo(THIS PR): switch to a singly linked list here + specifiers: Vec, } impl GraphPath { - pub fn new(specifier: String) -> Arc { - Arc::new(Self { - previous_node: None, - specifier, - }) - } - - pub fn with_specifier(self: &Arc, specifier: String) -> Arc { - Arc::new(Self { - previous_node: Some(self.clone()), - specifier, - }) + pub fn with_step(&self, specifier: &str, id: &NpmPackageId) -> GraphPath { + let mut copy = self.clone(); + assert!(copy.visited_versions.add(id)); + copy.specifiers.push(specifier.to_string()); + copy } - pub fn pop(&self) -> Option<&Arc> { - self.previous_node.as_ref() + pub fn with_specifier(&self, specifier: String) -> GraphPath { + let mut copy = self.clone(); + copy.specifiers.push(specifier); + copy } - pub fn is_last(&self) -> bool { - self.previous_node.is_none() + pub fn has_visited_version(&self, id: &NpmPackageId) -> bool { + self.visited_versions.has_visited(id) } } @@ -370,8 +343,7 @@ impl Graph { pub struct GraphDependencyResolver<'a, TNpmRegistryApi: NpmRegistryApi> { graph: &'a mut Graph, api: &'a TNpmRegistryApi, - pending_unresolved_nodes: - VecDeque<(Arc, Arc>)>, + pending_unresolved_nodes: VecDeque<(VisitedVersions, Arc>)>, } impl<'a, TNpmRegistryApi: NpmRegistryApi> @@ -451,7 +423,9 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> &node, &NodeParent::Req, ); - self.add_pending_unresolved_node(None, &node); + self + .pending_unresolved_nodes + .push_back((VisitedVersions::default(), node)); Ok(()) } @@ -460,7 +434,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> entry: &NpmDependencyEntry, package_info: NpmPackageInfo, parent_id: &NpmPackageId, - visited_versions: &Arc, + visited_versions: &VisitedVersions, ) -> Result<(), AnyError> { let node = self.resolve_node_from_info( &entry.name, @@ -482,28 +456,10 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> &node, &NodeParent::Node(parent_id.clone()), ); - self.add_pending_unresolved_node(Some(visited_versions), &node); - Ok(()) - } - - fn add_pending_unresolved_node( - &mut self, - maybe_previous_visited_versions: Option<&Arc>, - node: &Arc>, - ) { - let node_id = node.lock().id.clone(); - let visited_versions = match maybe_previous_visited_versions { - Some(previous_visited_versions) => { - match previous_visited_versions.with_id(&node_id) { - Some(visited_versions) => visited_versions, - None => return, // circular, don't visit this node - } - } - None => VisitedVersionsPath::new(&node_id), - }; self .pending_unresolved_nodes - .push_back((visited_versions, node.clone())); + .push_back((visited_versions.clone(), node)); + Ok(()) } fn resolve_node_from_info( @@ -547,7 +503,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> pub async fn resolve_pending(&mut self) -> Result<(), AnyError> { while !self.pending_unresolved_nodes.is_empty() { // now go down through the dependencies by tree depth - while let Some((visited_versions, parent_node)) = + while let Some((mut visited_versions, parent_node)) = self.pending_unresolved_nodes.pop_front() { let (mut parent_id, deps) = { @@ -555,6 +511,10 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> (parent_node.id.clone(), parent_node.deps.clone()) }; + if !visited_versions.add(&parent_id) { + continue; // circular + } + // cache all the dependencies' registry infos in parallel if should if !should_sync_download() { let handles = deps @@ -618,7 +578,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> parent_id: &NpmPackageId, peer_dep: &NpmDependencyEntry, peer_package_info: NpmPackageInfo, - visited_ancestor_versions: &Arc, + visited_ancestor_versions: &VisitedVersions, ) -> Result, AnyError> { fn find_matching_child<'a>( peer_dep: &NpmDependencyEntry, @@ -637,8 +597,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> // Peer dependencies are resolved based on its ancestors' siblings. // If not found, then it resolves based on the version requirement if non-optional. let mut pending_ancestors = VecDeque::new(); // go up the tree by depth - let path = GraphPath::new(specifier.to_string()); - let visited_versions = VisitedVersionsPath::new(parent_id); + let path = GraphPath::default().with_step(specifier, parent_id); // skip over the current node for (specifier, grand_parents) in @@ -646,50 +605,28 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> { let path = path.with_specifier(specifier); for grand_parent in grand_parents { - let visited_versions = match &grand_parent { - NodeParent::Node(id) => visited_versions.with_id(id), - NodeParent::Req => Some(visited_versions.clone()), - }; - if let Some(visited_versions) = visited_versions { - pending_ancestors.push_back(( - grand_parent, - path.clone(), - visited_versions, - )); - } + pending_ancestors.push_back((grand_parent, path.clone())); } } - while let Some((ancestor, path, visited_versions)) = - pending_ancestors.pop_front() - { + while let Some((ancestor, path)) = pending_ancestors.pop_front() { match &ancestor { NodeParent::Node(ancestor_node_id) => { + // we've gone in a full circle, so don't keep looking + if path.has_visited_version(ancestor_node_id) { + continue; + } + let maybe_peer_dep_id = if ancestor_node_id.name == peer_dep.name && peer_dep.version_req.satisfies(&ancestor_node_id.version) { - // found it Some(ancestor_node_id.clone()) } else { - // continue searching up let ancestor = self.graph.borrow_node(ancestor_node_id); for (specifier, parents) in &ancestor.parents { + let new_path = path.with_step(specifier, ancestor_node_id); for parent in parents { - let path = path.with_specifier(specifier.to_string()); - let visited_versions = match parent { - NodeParent::Node(id) => visited_versions.with_id(id), - NodeParent::Req => Some(visited_versions.clone()), - }; - if let Some(visited_versions) = visited_versions { - pending_ancestors.push_back(( - parent.clone(), - path, - visited_versions, - )); - } else { - // we've gone in a full circle, so don't keep looking - continue; - } + pending_ancestors.push_back((parent.clone(), new_path.clone())); } } find_matching_child(peer_dep, ancestor.children.values()) @@ -701,7 +638,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> parents, ancestor_node_id, &peer_dep_id, - &path, + path.specifiers, visited_ancestor_versions, ))); } @@ -711,8 +648,8 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> if let Some(child_id) = find_matching_child(peer_dep, self.graph.package_reqs.values()) { - let specifier = path.specifier.clone(); - let path = path.pop().unwrap(); // go back down one level from the package requirement + let mut path = path.specifiers; + let specifier = path.pop().unwrap(); // go back down one level let old_id = self.graph.package_reqs.get(&specifier).unwrap().clone(); return Ok(Some(self.set_new_peer_dep( @@ -747,8 +684,8 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> previous_parents: BTreeMap>, node_id: &NpmPackageId, peer_dep_id: &NpmPackageId, - path: &GraphPath, - visited_ancestor_versions: &Arc, + mut path: Vec, + visited_ancestor_versions: &VisitedVersions, ) -> NpmPackageId { let mut peer_dep_id = Cow::Borrowed(peer_dep_id); let old_id = node_id; @@ -771,12 +708,15 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> } // remove the previous parents from the old node - for (specifier, parents) in &previous_parents { - for parent in parents { - self.graph.remove_child_parent(specifier, old_id, parent); + let old_node_children = { + for (specifier, parents) in &previous_parents { + for parent in parents { + self.graph.remove_child_parent(specifier, old_id, parent); + } } - } - let old_node_children = self.graph.borrow_node(old_id).children.clone(); + let old_node = self.graph.borrow_node(old_id); + old_node.children.clone() + }; let (_, new_node) = self.graph.get_or_create_for_id(&new_id); @@ -803,25 +743,26 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> let mut bottom_parent_id = new_id.clone(); // continue going down the path - if let Some(path) = path.pop() { - if path.is_last() { + if let Some(next_specifier) = path.pop() { + if path.is_empty() { // this means we're at the peer dependency now debug!( "Resolved peer dependency for {} in {} to {}", - &path.specifier, &new_id, &peer_dep_id, + &next_specifier, &new_id, &peer_dep_id, ); - assert!(!old_node_children.contains_key(&path.specifier)); + assert!(!old_node_children.contains_key(&next_specifier)); let node = self.graph.get_or_create_for_id(&peer_dep_id).1; self - .add_pending_unresolved_node(Some(visited_ancestor_versions), &node); + .pending_unresolved_nodes + .push_back((visited_ancestor_versions.clone(), node.clone())); self .graph - .set_child_parent_node(&path.specifier, &node, &new_id); + .set_child_parent_node(&next_specifier, &node, &new_id); } else { - let next_node_id = old_node_children.get(&path.specifier).unwrap(); + let next_node_id = old_node_children.get(&next_specifier).unwrap(); bottom_parent_id = self.set_new_peer_dep( BTreeMap::from([( - path.specifier.to_string(), + next_specifier.to_string(), BTreeSet::from([NodeParent::Node(new_id.clone())]), )]), next_node_id, From 15a06af526e7edd14c39c8135f256e3d0e241880 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 7 Nov 2022 15:06:59 -0500 Subject: [PATCH 33/43] Not sure if faster... maybe slower? --- cli/npm/resolution/graph.rs | 153 ++++++++++++++++++++++++------------ 1 file changed, 101 insertions(+), 52 deletions(-) diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index 2ed998d4cdf73e..02ee2cbd690024 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -4,7 +4,6 @@ use std::borrow::Cow; use std::collections::BTreeMap; use std::collections::BTreeSet; use std::collections::HashMap; -use std::collections::HashSet; use std::collections::VecDeque; use std::sync::Arc; @@ -33,50 +32,72 @@ use super::NpmResolutionPackage; use super::NpmVersionMatcher; #[derive(Default, Clone)] -struct VisitedVersions(HashSet); +struct VisitedVersionsPath { + previous_node: Option>, + visited_version: (String, NpmVersion), +} -impl VisitedVersions { - pub fn add(&mut self, id: &NpmPackageId) -> bool { - self.0.insert(Self::id_as_key(id)) +impl VisitedVersionsPath { + pub fn new(id: &NpmPackageId) -> Arc { + Arc::new(Self { + previous_node: None, + visited_version: Self::id_as_name_and_version(id), + }) } - pub fn has_visited(&self, id: &NpmPackageId) -> bool { - self.0.contains(&Self::id_as_key(id)) + pub fn with_parent( + self: &Arc, + parent: &NodeParent, + ) -> Option> { + match parent { + NodeParent::Node(id) => self.with_id(id), + NodeParent::Req => Some(self.clone()), + } } - fn id_as_key(id: &NpmPackageId) -> String { - // we only key on name and version in the id and not peer dependencies - // because the peer dependencies could change above and below us, - // but the names and versions won't - format!("{}@{}", id.name, id.version) + pub fn with_id( + self: &Arc, + id: &NpmPackageId, + ) -> Option> { + if self.has_visited(id) { + None + } else { + Some(Arc::new(Self { + previous_node: Some(self.clone()), + visited_version: Self::id_as_name_and_version(id), + })) + } + } + + pub fn has_visited(self: &Arc, id: &NpmPackageId) -> bool { + let mut maybe_next_node = Some(self); + let name_and_version = Self::id_as_name_and_version(id); + while let Some(next_node) = maybe_next_node { + if next_node.visited_version == name_and_version { + return true; + } + maybe_next_node = next_node.previous_node.as_ref(); + } + false + } + + fn id_as_name_and_version(id: &NpmPackageId) -> (String, NpmVersion) { + (id.name.clone(), id.version.clone()) } } #[derive(Default, Clone)] struct GraphPath { - // todo(THIS PR): investigate if this should use a singly linked list too - visited_versions: VisitedVersions, // todo(THIS PR): switch to a singly linked list here specifiers: Vec, } impl GraphPath { - pub fn with_step(&self, specifier: &str, id: &NpmPackageId) -> GraphPath { - let mut copy = self.clone(); - assert!(copy.visited_versions.add(id)); - copy.specifiers.push(specifier.to_string()); - copy - } - pub fn with_specifier(&self, specifier: String) -> GraphPath { let mut copy = self.clone(); copy.specifiers.push(specifier); copy } - - pub fn has_visited_version(&self, id: &NpmPackageId) -> bool { - self.visited_versions.has_visited(id) - } } #[derive(Debug, Clone, PartialEq, Eq, Hash, Ord, PartialOrd)] @@ -343,7 +364,8 @@ impl Graph { pub struct GraphDependencyResolver<'a, TNpmRegistryApi: NpmRegistryApi> { graph: &'a mut Graph, api: &'a TNpmRegistryApi, - pending_unresolved_nodes: VecDeque<(VisitedVersions, Arc>)>, + pending_unresolved_nodes: + VecDeque<(Arc, Arc>)>, } impl<'a, TNpmRegistryApi: NpmRegistryApi> @@ -423,9 +445,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> &node, &NodeParent::Req, ); - self - .pending_unresolved_nodes - .push_back((VisitedVersions::default(), node)); + self.try_add_pending_unresolved_node(None, &node); Ok(()) } @@ -434,7 +454,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> entry: &NpmDependencyEntry, package_info: NpmPackageInfo, parent_id: &NpmPackageId, - visited_versions: &VisitedVersions, + visited_versions: &Arc, ) -> Result<(), AnyError> { let node = self.resolve_node_from_info( &entry.name, @@ -456,10 +476,28 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> &node, &NodeParent::Node(parent_id.clone()), ); + self.try_add_pending_unresolved_node(Some(&visited_versions), &node); + Ok(()) + } + + fn try_add_pending_unresolved_node( + &mut self, + maybe_previous_visited_versions: Option<&Arc>, + node: &Arc>, + ) { + let node_id = node.lock().id.clone(); + let visited_versions = match maybe_previous_visited_versions { + Some(previous_visited_versions) => { + match previous_visited_versions.with_id(&node_id) { + Some(visited_versions) => visited_versions, + None => return, // circular, don't visit this node + } + } + None => VisitedVersionsPath::new(&node_id), + }; self .pending_unresolved_nodes - .push_back((visited_versions.clone(), node)); - Ok(()) + .push_back((visited_versions, node.clone())); } fn resolve_node_from_info( @@ -503,7 +541,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> pub async fn resolve_pending(&mut self) -> Result<(), AnyError> { while !self.pending_unresolved_nodes.is_empty() { // now go down through the dependencies by tree depth - while let Some((mut visited_versions, parent_node)) = + while let Some((visited_versions, parent_node)) = self.pending_unresolved_nodes.pop_front() { let (mut parent_id, deps) = { @@ -511,10 +549,6 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> (parent_node.id.clone(), parent_node.deps.clone()) }; - if !visited_versions.add(&parent_id) { - continue; // circular - } - // cache all the dependencies' registry infos in parallel if should if !should_sync_download() { let handles = deps @@ -578,7 +612,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> parent_id: &NpmPackageId, peer_dep: &NpmDependencyEntry, peer_package_info: NpmPackageInfo, - visited_ancestor_versions: &VisitedVersions, + visited_ancestor_versions: &Arc, ) -> Result, AnyError> { fn find_matching_child<'a>( peer_dep: &NpmDependencyEntry, @@ -597,7 +631,8 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> // Peer dependencies are resolved based on its ancestors' siblings. // If not found, then it resolves based on the version requirement if non-optional. let mut pending_ancestors = VecDeque::new(); // go up the tree by depth - let path = GraphPath::default().with_step(specifier, parent_id); + let path = GraphPath::default().with_specifier(specifier.to_string()); + let visited_versions = VisitedVersionsPath::new(parent_id); // skip over the current node for (specifier, grand_parents) in @@ -605,18 +640,23 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> { let path = path.with_specifier(specifier); for grand_parent in grand_parents { - pending_ancestors.push_back((grand_parent, path.clone())); + if let Some(visited_versions) = + visited_versions.with_parent(&grand_parent) + { + pending_ancestors.push_back(( + grand_parent, + path.clone(), + visited_versions, + )); + } } } - while let Some((ancestor, path)) = pending_ancestors.pop_front() { + while let Some((ancestor, path, visited_versions)) = + pending_ancestors.pop_front() + { match &ancestor { NodeParent::Node(ancestor_node_id) => { - // we've gone in a full circle, so don't keep looking - if path.has_visited_version(ancestor_node_id) { - continue; - } - let maybe_peer_dep_id = if ancestor_node_id.name == peer_dep.name && peer_dep.version_req.satisfies(&ancestor_node_id.version) { @@ -624,9 +664,17 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> } else { let ancestor = self.graph.borrow_node(ancestor_node_id); for (specifier, parents) in &ancestor.parents { - let new_path = path.with_step(specifier, ancestor_node_id); + let new_path = path.with_specifier(specifier.clone()); for parent in parents { - pending_ancestors.push_back((parent.clone(), new_path.clone())); + if let Some(visited_versions) = + visited_versions.with_parent(parent) + { + pending_ancestors.push_back(( + parent.clone(), + new_path.clone(), + visited_versions, + )); + } } } find_matching_child(peer_dep, ancestor.children.values()) @@ -685,7 +733,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> node_id: &NpmPackageId, peer_dep_id: &NpmPackageId, mut path: Vec, - visited_ancestor_versions: &VisitedVersions, + visited_ancestor_versions: &Arc, ) -> NpmPackageId { let mut peer_dep_id = Cow::Borrowed(peer_dep_id); let old_id = node_id; @@ -752,9 +800,10 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> ); assert!(!old_node_children.contains_key(&next_specifier)); let node = self.graph.get_or_create_for_id(&peer_dep_id).1; - self - .pending_unresolved_nodes - .push_back((visited_ancestor_versions.clone(), node.clone())); + self.try_add_pending_unresolved_node( + Some(&visited_ancestor_versions), + &node, + ); self .graph .set_child_parent_node(&next_specifier, &node, &new_id); From 0e929fefec775d8682b46a34d6a13641af9b0744 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 7 Nov 2022 15:22:28 -0500 Subject: [PATCH 34/43] Add `GraphSpecifierPath` --- cli/npm/resolution/graph.rs | 118 +++++++++++++++++++++--------------- 1 file changed, 68 insertions(+), 50 deletions(-) diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index 02ee2cbd690024..cfb7d11a8841a9 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -31,17 +31,23 @@ use super::NpmPackageReq; use super::NpmResolutionPackage; use super::NpmVersionMatcher; +/// A memory efficient path of visited name and versions in the graph +/// which is used to detect cycles. +/// +/// Note(dsherret): although this is definitely more memory efficient +/// than a HashSet, I haven't done any tests about whether this is +/// faster in practice. #[derive(Default, Clone)] struct VisitedVersionsPath { previous_node: Option>, - visited_version: (String, NpmVersion), + visited_version_key: String, } impl VisitedVersionsPath { pub fn new(id: &NpmPackageId) -> Arc { Arc::new(Self { previous_node: None, - visited_version: Self::id_as_name_and_version(id), + visited_version_key: Self::id_to_key(id), }) } @@ -64,16 +70,16 @@ impl VisitedVersionsPath { } else { Some(Arc::new(Self { previous_node: Some(self.clone()), - visited_version: Self::id_as_name_and_version(id), + visited_version_key: Self::id_to_key(id), })) } } pub fn has_visited(self: &Arc, id: &NpmPackageId) -> bool { let mut maybe_next_node = Some(self); - let name_and_version = Self::id_as_name_and_version(id); + let key = Self::id_to_key(id); while let Some(next_node) = maybe_next_node { - if next_node.visited_version == name_and_version { + if next_node.visited_version_key == key { return true; } maybe_next_node = next_node.previous_node.as_ref(); @@ -81,22 +87,35 @@ impl VisitedVersionsPath { false } - fn id_as_name_and_version(id: &NpmPackageId) -> (String, NpmVersion) { - (id.name.clone(), id.version.clone()) + fn id_to_key(id: &NpmPackageId) -> String { + format!("{}@{}", id.name, id.version) } } +/// A memory efficient path of the visited specifiers in the tree. #[derive(Default, Clone)] -struct GraphPath { - // todo(THIS PR): switch to a singly linked list here - specifiers: Vec, +struct GraphSpecifierPath { + previous_node: Option>, + specifier: String, } -impl GraphPath { - pub fn with_specifier(&self, specifier: String) -> GraphPath { - let mut copy = self.clone(); - copy.specifiers.push(specifier); - copy +impl GraphSpecifierPath { + pub fn new(specifier: String) -> Arc { + Arc::new(Self { + previous_node: None, + specifier, + }) + } + + pub fn with_specifier(self: &Arc, specifier: String) -> Arc { + Arc::new(Self { + previous_node: Some(self.clone()), + specifier, + }) + } + + pub fn pop(&self) -> Option<&Arc> { + self.previous_node.as_ref() } } @@ -631,7 +650,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> // Peer dependencies are resolved based on its ancestors' siblings. // If not found, then it resolves based on the version requirement if non-optional. let mut pending_ancestors = VecDeque::new(); // go up the tree by depth - let path = GraphPath::default().with_specifier(specifier.to_string()); + let path = GraphSpecifierPath::new(specifier.to_string()); let visited_versions = VisitedVersionsPath::new(parent_id); // skip over the current node @@ -686,7 +705,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> parents, ancestor_node_id, &peer_dep_id, - path.specifiers, + &path, visited_ancestor_versions, ))); } @@ -696,15 +715,15 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> if let Some(child_id) = find_matching_child(peer_dep, self.graph.package_reqs.values()) { - let mut path = path.specifiers; - let specifier = path.pop().unwrap(); // go back down one level + let specifier = path.specifier.to_string(); + let path = path.pop().unwrap(); // go back down one level from the package requirement let old_id = self.graph.package_reqs.get(&specifier).unwrap().clone(); return Ok(Some(self.set_new_peer_dep( BTreeMap::from([(specifier, BTreeSet::from([NodeParent::Req]))]), &old_id, &child_id, - path, + &path, visited_ancestor_versions, ))); } @@ -732,7 +751,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> previous_parents: BTreeMap>, node_id: &NpmPackageId, peer_dep_id: &NpmPackageId, - mut path: Vec, + path: &Arc, visited_ancestor_versions: &Arc, ) -> NpmPackageId { let mut peer_dep_id = Cow::Borrowed(peer_dep_id); @@ -791,35 +810,34 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> let mut bottom_parent_id = new_id.clone(); // continue going down the path - if let Some(next_specifier) = path.pop() { - if path.is_empty() { - // this means we're at the peer dependency now - debug!( - "Resolved peer dependency for {} in {} to {}", - &next_specifier, &new_id, &peer_dep_id, - ); - assert!(!old_node_children.contains_key(&next_specifier)); - let node = self.graph.get_or_create_for_id(&peer_dep_id).1; - self.try_add_pending_unresolved_node( - Some(&visited_ancestor_versions), - &node, - ); - self - .graph - .set_child_parent_node(&next_specifier, &node, &new_id); - } else { - let next_node_id = old_node_children.get(&next_specifier).unwrap(); - bottom_parent_id = self.set_new_peer_dep( - BTreeMap::from([( - next_specifier.to_string(), - BTreeSet::from([NodeParent::Node(new_id.clone())]), - )]), - next_node_id, - &peer_dep_id, - path, - visited_ancestor_versions, - ); - } + let next_specifier = &path.specifier; + if let Some(path) = path.pop() { + let next_node_id = old_node_children.get(next_specifier).unwrap(); + bottom_parent_id = self.set_new_peer_dep( + BTreeMap::from([( + next_specifier.to_string(), + BTreeSet::from([NodeParent::Node(new_id.clone())]), + )]), + next_node_id, + &peer_dep_id, + path, + visited_ancestor_versions, + ); + } else { + // this means we're at the peer dependency now + debug!( + "Resolved peer dependency for {} in {} to {}", + next_specifier, &new_id, &peer_dep_id, + ); + assert!(!old_node_children.contains_key(next_specifier)); + let node = self.graph.get_or_create_for_id(&peer_dep_id).1; + self.try_add_pending_unresolved_node( + Some(&visited_ancestor_versions), + &node, + ); + self + .graph + .set_child_parent_node(next_specifier, &node, &new_id); } // forget the old node at this point if it has no parents From d719c5cec8c331d3a13013e02e41f3e06371f105 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 7 Nov 2022 18:37:44 -0500 Subject: [PATCH 35/43] Adding integration test and fixing bugs. --- cli/fs_util.rs | 60 ++++++- cli/lockfile.rs | 14 +- cli/npm/cache.rs | 10 ++ cli/npm/resolution/graph.rs | 2 +- cli/npm/resolution/mod.rs | 4 +- cli/npm/resolution/snapshot.rs | 27 ++-- cli/npm/resolvers/mod.rs | 10 +- cli/tests/integration/npm_tests.rs | 152 +++++++++++++++++- .../peer_deps_with_copied_folders/main.out | 10 ++ .../npm/peer_deps_with_copied_folders/main.ts | 5 + .../main_reload.out | 10 ++ .../peer-dep-test-child/1.0.0/index.js | 1 + .../peer-dep-test-child/1.0.0/package.json | 8 + .../peer-dep-test-child/2.0.0/index.js | 1 + .../peer-dep-test-child/2.0.0/package.json | 8 + .../1.0.0/dist/index.js | 1 + .../peer-dep-test-grandchild/1.0.0/index.js | 1 + .../1.0.0/package.json | 7 + .../peer-dep-test-peer/1.0.0/index.js | 1 + .../peer-dep-test-peer/1.0.0/package.json | 4 + .../peer-dep-test-peer/2.0.0/index.js | 1 + .../peer-dep-test-peer/2.0.0/package.json | 4 + 22 files changed, 314 insertions(+), 27 deletions(-) create mode 100644 cli/tests/testdata/npm/peer_deps_with_copied_folders/main.out create mode 100644 cli/tests/testdata/npm/peer_deps_with_copied_folders/main.ts create mode 100644 cli/tests/testdata/npm/peer_deps_with_copied_folders/main_reload.out create mode 100644 cli/tests/testdata/npm/registry/@denotest/peer-dep-test-child/1.0.0/index.js create mode 100644 cli/tests/testdata/npm/registry/@denotest/peer-dep-test-child/1.0.0/package.json create mode 100644 cli/tests/testdata/npm/registry/@denotest/peer-dep-test-child/2.0.0/index.js create mode 100644 cli/tests/testdata/npm/registry/@denotest/peer-dep-test-child/2.0.0/package.json create mode 100644 cli/tests/testdata/npm/registry/@denotest/peer-dep-test-grandchild/1.0.0/dist/index.js create mode 100644 cli/tests/testdata/npm/registry/@denotest/peer-dep-test-grandchild/1.0.0/index.js create mode 100644 cli/tests/testdata/npm/registry/@denotest/peer-dep-test-grandchild/1.0.0/package.json create mode 100644 cli/tests/testdata/npm/registry/@denotest/peer-dep-test-peer/1.0.0/index.js create mode 100644 cli/tests/testdata/npm/registry/@denotest/peer-dep-test-peer/1.0.0/package.json create mode 100644 cli/tests/testdata/npm/registry/@denotest/peer-dep-test-peer/2.0.0/index.js create mode 100644 cli/tests/testdata/npm/registry/@denotest/peer-dep-test-peer/2.0.0/package.json diff --git a/cli/fs_util.rs b/cli/fs_util.rs index cafe29af8fda3a..843f5e0cfe099d 100644 --- a/cli/fs_util.rs +++ b/cli/fs_util.rs @@ -15,6 +15,7 @@ use std::io::ErrorKind; use std::io::Write; use std::path::Path; use std::path::PathBuf; +use std::time::Duration; use walkdir::WalkDir; pub fn atomic_write_file>( @@ -377,13 +378,58 @@ pub fn hard_link_dir_recursive(from: &Path, to: &Path) -> Result<(), AnyError> { format!("Dir {} to {}", new_from.display(), new_to.display()) })?; } else if file_type.is_file() { - std::fs::hard_link(&new_from, &new_to).with_context(|| { - format!( - "Hard linking {} to {}", - new_from.display(), - new_to.display() - ) - })?; + // note: chance for race conditions here between attempting to create, + // then removing, then attempting to create. There doesn't seem to be + // a way to hard link with overwriting in Rust, but maybe there is some + // way with platform specific code. The workaround here is to handle + // scenarios where something else might create or remove files. + if let Err(err) = std::fs::hard_link(&new_from, &new_to) { + if err.kind() == ErrorKind::AlreadyExists { + if let Err(err) = std::fs::remove_file(&new_to) { + if err.kind() == ErrorKind::NotFound { + // Assume another process/thread created this hard link to the file we are wanting + // to remove then sleep a little bit to let the other process/thread move ahead + // faster to reduce contention. + std::thread::sleep(Duration::from_millis(10)); + } else { + return Err(err).with_context(|| { + format!( + "Removing file to hard link {} to {}", + new_from.display(), + new_to.display() + ) + }); + } + } + + // Always attempt to recreate the hardlink. In contention scenarios, the other process + // might have been killed or exited after removing the file, but before creating the hardlink + if let Err(err) = std::fs::hard_link(&new_from, &new_to) { + // Assume another process/thread created this hard link to the file we are wanting + // to now create then sleep a little bit to let the other process/thread move ahead + // faster to reduce contention. + if err.kind() == ErrorKind::AlreadyExists { + std::thread::sleep(Duration::from_millis(10)); + } else { + return Err(err).with_context(|| { + format!( + "Hard linking {} to {}", + new_from.display(), + new_to.display() + ) + }); + } + } + } else { + return Err(err).with_context(|| { + format!( + "Hard linking {} to {}", + new_from.display(), + new_to.display() + ) + }); + } + } } } diff --git a/cli/lockfile.rs b/cli/lockfile.rs index 8ee20b6fc314c9..246c9745077c08 100644 --- a/cli/lockfile.rs +++ b/cli/lockfile.rs @@ -16,6 +16,7 @@ use std::rc::Rc; use std::sync::Arc; use crate::args::ConfigFile; +use crate::npm::NpmPackageId; use crate::npm::NpmPackageReq; use crate::npm::NpmResolutionPackage; use crate::tools::fmt::format_json; @@ -40,7 +41,7 @@ pub struct NpmPackageInfo { #[derive(Clone, Debug, Default, Serialize, Deserialize)] pub struct NpmContent { - /// Mapping between requests for npm packages and resolved specifiers, eg. + /// Mapping between requests for npm packages and resolved packages, eg. /// { /// "chalk": "chalk@5.0.0" /// "react@17": "react@17.0.1" @@ -319,12 +320,13 @@ Use \"--lock-write\" flag to regenerate the lockfile at \"{}\".", pub fn insert_npm_specifier( &mut self, package_req: &NpmPackageReq, - version: String, + package_id: &NpmPackageId, ) { - self.content.npm.specifiers.insert( - package_req.to_string(), - format!("{}@{}", package_req.name, version), - ); + self + .content + .npm + .specifiers + .insert(package_req.to_string(), package_id.as_serialized()); self.has_content_changed = true; } } diff --git a/cli/npm/cache.rs b/cli/npm/cache.rs index e44618c8e97c5e..28fbf68b2da7bf 100644 --- a/cli/npm/cache.rs +++ b/cli/npm/cache.rs @@ -403,6 +403,16 @@ impl NpmCache { ) -> Result<(), AnyError> { assert_ne!(id.copy_index, 0); let package_folder = self.readonly.package_folder_for_id(id, registry_url); + + if package_folder.exists() + // if this file exists, then the package didn't successfully extract + // the first time, or another process is currently extracting the zip file + && !package_folder.join(NPM_PACKAGE_SYNC_LOCK_FILENAME).exists() + && self.cache_setting.should_use_for_npm_package(&id.name) + { + return Ok(()); + } + let original_package_folder = self .readonly .package_folder_for_name_and_version(&id.name, &id.version, registry_url); diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index cfb7d11a8841a9..bafa073b514d8e 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -34,7 +34,7 @@ use super::NpmVersionMatcher; /// A memory efficient path of visited name and versions in the graph /// which is used to detect cycles. /// -/// Note(dsherret): although this is definitely more memory efficient +/// note(dsherret): although this is definitely more memory efficient /// than a HashSet, I haven't done any tests about whether this is /// faster in practice. #[derive(Default, Clone)] diff --git a/cli/npm/resolution/mod.rs b/cli/npm/resolution/mod.rs index 5dfa02196762a2..cee2bd9bf5a9fa 100644 --- a/cli/npm/resolution/mod.rs +++ b/cli/npm/resolution/mod.rs @@ -523,8 +523,8 @@ impl NpmResolution { lockfile: &mut Lockfile, snapshot: &NpmResolutionSnapshot, ) -> Result<(), AnyError> { - for (package_req, version) in snapshot.package_reqs.iter() { - lockfile.insert_npm_specifier(package_req, version.to_string()); + for (package_req, package_id) in snapshot.package_reqs.iter() { + lockfile.insert_npm_specifier(package_req, package_id); } for package in self.all_packages() { lockfile.check_or_insert_npm_package(&package)?; diff --git a/cli/npm/resolution/snapshot.rs b/cli/npm/resolution/snapshot.rs index 6f57cd9b8122b2..1d428f4c2674e9 100644 --- a/cli/npm/resolution/snapshot.rs +++ b/cli/npm/resolution/snapshot.rs @@ -14,6 +14,7 @@ use serde::Deserialize; use serde::Serialize; use crate::lockfile::Lockfile; +use crate::npm::cache::should_sync_download; use crate::npm::cache::NpmPackageCacheFolderId; use crate::npm::registry::NpmPackageVersionDistInfo; use crate::npm::registry::NpmRegistryApi; @@ -285,8 +286,7 @@ impl NpmResolutionSnapshot { for id in &verify_ids { if !packages.contains_key(id) { bail!( - "the lockfile ({}) is corrupt. You can recreate it with --lock-write", - lockfile.filename.display(), + "the lockfile is corrupt. You can recreate it with --lock-write" ); } } @@ -295,13 +295,22 @@ impl NpmResolutionSnapshot { let mut unresolved_tasks = Vec::with_capacity(packages_by_name.len()); // cache the package names in parallel in the registry api - for package_name in packages_by_name.keys() { - let package_name = package_name.clone(); - let api = api.clone(); - unresolved_tasks.push(tokio::task::spawn(async move { - api.package_info(&package_name).await?; - Result::<_, AnyError>::Ok(()) - })); + // unless synchronous download should occur + if should_sync_download() { + let mut package_names = packages_by_name.keys().collect::>(); + package_names.sort(); + for package_name in package_names { + api.package_info(package_name).await?; + } + } else { + for package_name in packages_by_name.keys() { + let package_name = package_name.clone(); + let api = api.clone(); + unresolved_tasks.push(tokio::task::spawn(async move { + api.package_info(&package_name).await?; + Result::<_, AnyError>::Ok(()) + })); + } } for result in futures::future::join_all(unresolved_tasks).await { result??; diff --git a/cli/npm/resolvers/mod.rs b/cli/npm/resolvers/mod.rs index a0b500823537f1..6cd40594bb9d1a 100644 --- a/cli/npm/resolvers/mod.rs +++ b/cli/npm/resolvers/mod.rs @@ -6,6 +6,7 @@ mod local; use deno_ast::ModuleSpecifier; use deno_core::anyhow::bail; +use deno_core::anyhow::Context; use deno_core::error::custom_error; use deno_core::error::AnyError; use deno_core::parking_lot::Mutex; @@ -112,7 +113,14 @@ impl NpmPackageResolver { lockfile: Arc>, ) -> Result<(), AnyError> { let snapshot = - NpmResolutionSnapshot::from_lockfile(lockfile.clone(), &self.api).await?; + NpmResolutionSnapshot::from_lockfile(lockfile.clone(), &self.api) + .await + .with_context(|| { + format!( + "failed reading lockfile '{}'", + lockfile.lock().filename.display() + ) + })?; self.maybe_lockfile = Some(lockfile); if let Some(node_modules_folder) = &self.local_node_modules_path { self.inner = Arc::new(LocalNpmPackageResolver::new( diff --git a/cli/tests/integration/npm_tests.rs b/cli/tests/integration/npm_tests.rs index 72af72a76c3df6..54ca0213256416 100644 --- a/cli/tests/integration/npm_tests.rs +++ b/cli/tests/integration/npm_tests.rs @@ -994,7 +994,7 @@ fn lock_file_missing_top_level_package() { let stderr = String::from_utf8(output.stderr).unwrap(); assert_eq!( stderr, - "error: the lockfile (deno.lock) is corrupt. You can recreate it with --lock-write\n" + "error: failed reading lockfile 'deno.lock'\n\nCaused by:\n the lockfile is corrupt. You can recreate it with --lock-write\n" ); } @@ -1046,6 +1046,156 @@ fn auto_discover_lock_file() { )); } +#[test] +fn peer_deps_with_copied_folders_and_lockfile() { + let _server = http_server(); + + let deno_dir = util::new_deno_dir(); + let temp_dir = util::TempDir::new(); + + // write empty config file + temp_dir.write("deno.json", "{}"); + let test_folder_path = test_util::testdata_path() + .join("npm") + .join("peer_deps_with_copied_folders"); + let main_contents = + std::fs::read_to_string(test_folder_path.join("main.ts")).unwrap(); + temp_dir.write("./main.ts", main_contents); + + let deno = util::deno_cmd_with_deno_dir(&deno_dir) + .current_dir(temp_dir.path()) + .arg("run") + .arg("--unstable") + .arg("-A") + .arg("main.ts") + .envs(env_vars()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let output = deno.wait_with_output().unwrap(); + assert!(output.status.success()); + + let expected_output = + std::fs::read_to_string(test_folder_path.join("main.out")).unwrap(); + + assert_eq!(String::from_utf8(output.stderr).unwrap(), expected_output); + + assert!(temp_dir.path().join("deno.lock").exists()); + let grandchild_path = deno_dir + .path() + .join("npm") + .join("localhost_4545") + .join("npm") + .join("registry") + .join("@denotest") + .join("peer-dep-test-grandchild"); + assert!(grandchild_path.join("1.0.0").exists()); + assert!(grandchild_path.join("1.0.0_1").exists()); // copy folder, which is hardlinked + + // run again + let deno = util::deno_cmd_with_deno_dir(&deno_dir) + .current_dir(temp_dir.path()) + .arg("run") + .arg("--unstable") + .arg("-A") + .arg("main.ts") + .envs(env_vars()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let output = deno.wait_with_output().unwrap(); + assert_eq!(String::from_utf8(output.stderr).unwrap(), "1\n2\n"); + assert!(output.status.success()); + + // Ensure it works with reloading. This output will be slightly different + // because resolution has already occurred due to the lockfile. + let expected_reload_output = + std::fs::read_to_string(test_folder_path.join("main_reload.out")).unwrap(); + let deno = util::deno_cmd_with_deno_dir(&deno_dir) + .current_dir(temp_dir.path()) + .arg("run") + .arg("--unstable") + .arg("--reload") + .arg("-A") + .arg("main.ts") + .envs(env_vars()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let output = deno.wait_with_output().unwrap(); + assert_eq!( + String::from_utf8(output.stderr).unwrap(), + expected_reload_output + ); + assert!(output.status.success()); + + // now run with local node modules + let deno = util::deno_cmd_with_deno_dir(&deno_dir) + .current_dir(temp_dir.path()) + .arg("run") + .arg("--unstable") + .arg("--node-modules-dir") + .arg("-A") + .arg("main.ts") + .envs(env_vars()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let output = deno.wait_with_output().unwrap(); + assert!(output.status.success()); + assert_eq!(String::from_utf8(output.stderr).unwrap(), "1\n2\n"); + + let deno_folder = temp_dir.path().join("node_modules").join(".deno"); + assert!(deno_folder + .join("@denotest+peer-dep-test-grandchild@1.0.0") + .exists()); + assert!(deno_folder + .join("@denotest+peer-dep-test-grandchild@1.0.0_1") + .exists()); // copy folder + + // now again run with local node modules + let deno = util::deno_cmd_with_deno_dir(&deno_dir) + .current_dir(temp_dir.path()) + .arg("run") + .arg("--unstable") + .arg("--node-modules-dir") + .arg("-A") + .arg("main.ts") + .envs(env_vars()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let output = deno.wait_with_output().unwrap(); + assert!(output.status.success()); + assert_eq!(String::from_utf8(output.stderr).unwrap(), "1\n2\n"); + + // now ensure it works with reloading + let deno = util::deno_cmd_with_deno_dir(&deno_dir) + .current_dir(temp_dir.path()) + .arg("run") + .arg("--unstable") + .arg("--node-modules-dir") + .arg("--reload") + .arg("-A") + .arg("main.ts") + .envs(env_vars()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let output = deno.wait_with_output().unwrap(); + assert!(output.status.success()); + assert_eq!( + String::from_utf8(output.stderr).unwrap(), + expected_reload_output + ); +} + fn env_vars_no_sync_download() -> Vec<(String, String)> { vec![ ("DENO_NODE_COMPAT_URL".to_string(), util::std_file_url()), diff --git a/cli/tests/testdata/npm/peer_deps_with_copied_folders/main.out b/cli/tests/testdata/npm/peer_deps_with_copied_folders/main.out new file mode 100644 index 00000000000000..ce0dc68968e44a --- /dev/null +++ b/cli/tests/testdata/npm/peer_deps_with_copied_folders/main.out @@ -0,0 +1,10 @@ +Download http://localhost:4545/npm/registry/@denotest/peer-dep-test-child +Download http://localhost:4545/npm/registry/@denotest/peer-dep-test-grandchild +Download http://localhost:4545/npm/registry/@denotest/peer-dep-test-peer +Download http://localhost:4545/npm/registry/@denotest/peer-dep-test-child/1.0.0.tgz +Download http://localhost:4545/npm/registry/@denotest/peer-dep-test-child/2.0.0.tgz +Download http://localhost:4545/npm/registry/@denotest/peer-dep-test-grandchild/1.0.0.tgz +Download http://localhost:4545/npm/registry/@denotest/peer-dep-test-peer/1.0.0.tgz +Download http://localhost:4545/npm/registry/@denotest/peer-dep-test-peer/2.0.0.tgz +1 +2 diff --git a/cli/tests/testdata/npm/peer_deps_with_copied_folders/main.ts b/cli/tests/testdata/npm/peer_deps_with_copied_folders/main.ts new file mode 100644 index 00000000000000..a8ea8104a9daf9 --- /dev/null +++ b/cli/tests/testdata/npm/peer_deps_with_copied_folders/main.ts @@ -0,0 +1,5 @@ +import version1 from "npm:@denotest/peer-dep-test-child@1"; +import version2 from "npm:@denotest/peer-dep-test-child@2"; + +console.error(version1); +console.error(version2); diff --git a/cli/tests/testdata/npm/peer_deps_with_copied_folders/main_reload.out b/cli/tests/testdata/npm/peer_deps_with_copied_folders/main_reload.out new file mode 100644 index 00000000000000..ce0dc68968e44a --- /dev/null +++ b/cli/tests/testdata/npm/peer_deps_with_copied_folders/main_reload.out @@ -0,0 +1,10 @@ +Download http://localhost:4545/npm/registry/@denotest/peer-dep-test-child +Download http://localhost:4545/npm/registry/@denotest/peer-dep-test-grandchild +Download http://localhost:4545/npm/registry/@denotest/peer-dep-test-peer +Download http://localhost:4545/npm/registry/@denotest/peer-dep-test-child/1.0.0.tgz +Download http://localhost:4545/npm/registry/@denotest/peer-dep-test-child/2.0.0.tgz +Download http://localhost:4545/npm/registry/@denotest/peer-dep-test-grandchild/1.0.0.tgz +Download http://localhost:4545/npm/registry/@denotest/peer-dep-test-peer/1.0.0.tgz +Download http://localhost:4545/npm/registry/@denotest/peer-dep-test-peer/2.0.0.tgz +1 +2 diff --git a/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-child/1.0.0/index.js b/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-child/1.0.0/index.js new file mode 100644 index 00000000000000..636ec3c35f5848 --- /dev/null +++ b/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-child/1.0.0/index.js @@ -0,0 +1 @@ +module.exports = require("@denotest/peer-dep-test-grandchild"); diff --git a/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-child/1.0.0/package.json b/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-child/1.0.0/package.json new file mode 100644 index 00000000000000..32eb49851fa32c --- /dev/null +++ b/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-child/1.0.0/package.json @@ -0,0 +1,8 @@ +{ + "name": "@denotest/peer-dep-test-child", + "version": "1.0.0", + "dependencies": { + "@denotest/peer-dep-test-grandchild": "*", + "@denotest/peer-dep-test-peer": "^1" + } +} diff --git a/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-child/2.0.0/index.js b/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-child/2.0.0/index.js new file mode 100644 index 00000000000000..636ec3c35f5848 --- /dev/null +++ b/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-child/2.0.0/index.js @@ -0,0 +1 @@ +module.exports = require("@denotest/peer-dep-test-grandchild"); diff --git a/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-child/2.0.0/package.json b/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-child/2.0.0/package.json new file mode 100644 index 00000000000000..3c82c01f968e0e --- /dev/null +++ b/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-child/2.0.0/package.json @@ -0,0 +1,8 @@ +{ + "name": "@denotest/peer-dep-test-child", + "version": "2.0.0", + "dependencies": { + "@denotest/peer-dep-test-grandchild": "*", + "@denotest/peer-dep-test-peer": "^2" + } +} diff --git a/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-grandchild/1.0.0/dist/index.js b/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-grandchild/1.0.0/dist/index.js new file mode 100644 index 00000000000000..9a0d9730b6cd11 --- /dev/null +++ b/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-grandchild/1.0.0/dist/index.js @@ -0,0 +1 @@ +module.exports = require("@denotest/peer-dep-test-peer"); diff --git a/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-grandchild/1.0.0/index.js b/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-grandchild/1.0.0/index.js new file mode 100644 index 00000000000000..7d44863dfc69f5 --- /dev/null +++ b/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-grandchild/1.0.0/index.js @@ -0,0 +1 @@ +module.exports = require("./dist/index"); diff --git a/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-grandchild/1.0.0/package.json b/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-grandchild/1.0.0/package.json new file mode 100644 index 00000000000000..845ef414d7e115 --- /dev/null +++ b/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-grandchild/1.0.0/package.json @@ -0,0 +1,7 @@ +{ + "name": "@denotest/peer-dep-test-child-2", + "version": "1.0.0", + "peerDependencies": { + "@denotest/peer-dep-test-peer": "*" + } +} diff --git a/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-peer/1.0.0/index.js b/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-peer/1.0.0/index.js new file mode 100644 index 00000000000000..bd816eaba4ca3b --- /dev/null +++ b/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-peer/1.0.0/index.js @@ -0,0 +1 @@ +module.exports = 1; diff --git a/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-peer/1.0.0/package.json b/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-peer/1.0.0/package.json new file mode 100644 index 00000000000000..cedb3609e9ebb1 --- /dev/null +++ b/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-peer/1.0.0/package.json @@ -0,0 +1,4 @@ +{ + "name": "@denotest/peer-dep-test-peer", + "version": "1.0.0" +} diff --git a/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-peer/2.0.0/index.js b/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-peer/2.0.0/index.js new file mode 100644 index 00000000000000..4bbffde1044258 --- /dev/null +++ b/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-peer/2.0.0/index.js @@ -0,0 +1 @@ +module.exports = 2; diff --git a/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-peer/2.0.0/package.json b/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-peer/2.0.0/package.json new file mode 100644 index 00000000000000..90c24f875391f8 --- /dev/null +++ b/cli/tests/testdata/npm/registry/@denotest/peer-dep-test-peer/2.0.0/package.json @@ -0,0 +1,4 @@ +{ + "name": "@denotest/peer-dep-test-peer", + "version": "2.0.0" +} From 9f429dc8a2a1adac3c4a57e64ea628af64bb735a Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 7 Nov 2022 18:46:53 -0500 Subject: [PATCH 36/43] --reload fixes --- cli/npm/resolvers/local.rs | 14 ++++++++++++++ cli/tests/integration/npm_tests.rs | 29 +++++++++++++++++++++++++++-- 2 files changed, 41 insertions(+), 2 deletions(-) diff --git a/cli/npm/resolvers/local.rs b/cli/npm/resolvers/local.rs index 66b65226d2483c..96a7d961d1e0c9 100644 --- a/cli/npm/resolvers/local.rs +++ b/cli/npm/resolvers/local.rs @@ -32,6 +32,7 @@ use crate::npm::NpmPackageId; use crate::npm::NpmPackageReq; use crate::npm::RealNpmRegistryApi; +use super::common::cache_packages; use super::common::ensure_registry_read_permission; use super::common::InnerNpmPackageResolver; @@ -240,6 +241,7 @@ impl InnerNpmPackageResolver for LocalNpmPackageResolver { async fn sync_resolver_with_fs( resolver: &LocalNpmPackageResolver, ) -> Result<(), AnyError> { + cache_packages_to_global_cache(&resolver).await?; sync_resolution_with_fs( &resolver.resolution.snapshot(), &resolver.cache, @@ -497,3 +499,15 @@ fn get_next_node_modules_ancestor(mut path: &Path) -> &Path { } } } + +async fn cache_packages_to_global_cache( + resolver: &LocalNpmPackageResolver, +) -> Result<(), AnyError> { + let packages = resolver.resolution.all_packages_partitioned().packages; + + // only cache the non-copy packages in the global cache since + // there's no need for them when using a local node_modules + cache_packages(packages, &resolver.cache, &resolver.registry_url).await?; + + Ok(()) +} diff --git a/cli/tests/integration/npm_tests.rs b/cli/tests/integration/npm_tests.rs index 54ca0213256416..feabde36468c59 100644 --- a/cli/tests/integration/npm_tests.rs +++ b/cli/tests/integration/npm_tests.rs @@ -1076,10 +1076,13 @@ fn peer_deps_with_copied_folders_and_lockfile() { let output = deno.wait_with_output().unwrap(); assert!(output.status.success()); - let expected_output = + let expected_initial_output = std::fs::read_to_string(test_folder_path.join("main.out")).unwrap(); - assert_eq!(String::from_utf8(output.stderr).unwrap(), expected_output); + assert_eq!( + String::from_utf8(output.stderr).unwrap(), + expected_initial_output + ); assert!(temp_dir.path().join("deno.lock").exists()); let grandchild_path = deno_dir @@ -1194,6 +1197,28 @@ fn peer_deps_with_copied_folders_and_lockfile() { String::from_utf8(output.stderr).unwrap(), expected_reload_output ); + + // now ensure it works with reloading and no lockfile + let deno = util::deno_cmd_with_deno_dir(&deno_dir) + .current_dir(temp_dir.path()) + .arg("run") + .arg("--unstable") + .arg("--node-modules-dir") + .arg("--no-lock") + .arg("--reload") + .arg("-A") + .arg("main.ts") + .envs(env_vars()) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()) + .spawn() + .unwrap(); + let output = deno.wait_with_output().unwrap(); + assert_eq!( + String::from_utf8(output.stderr).unwrap(), + expected_initial_output, + ); + assert!(output.status.success()); } fn env_vars_no_sync_download() -> Vec<(String, String)> { From 500dab6815c1590c5768e5cef8b135d4ecaaf9ea Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 7 Nov 2022 19:01:20 -0500 Subject: [PATCH 37/43] Fix broken test. --- cli/npm/cache.rs | 6 +++++- cli/npm/resolvers/local.rs | 18 +++--------------- 2 files changed, 8 insertions(+), 16 deletions(-) diff --git a/cli/npm/cache.rs b/cli/npm/cache.rs index 28fbf68b2da7bf..2d983fa06e4287 100644 --- a/cli/npm/cache.rs +++ b/cli/npm/cache.rs @@ -340,6 +340,10 @@ impl NpmCache { }) } + pub fn should_use_cache_for_npm_package(&self, package_name: &str) -> bool { + self.cache_setting.should_use_for_npm_package(package_name) + } + async fn ensure_package_inner( &self, package: (&str, &NpmVersion), @@ -355,7 +359,7 @@ impl NpmCache { // if this file exists, then the package didn't successfully extract // the first time, or another process is currently extracting the zip file && !package_folder.join(NPM_PACKAGE_SYNC_LOCK_FILENAME).exists() - && self.cache_setting.should_use_for_npm_package(package.0) + && self.should_use_cache_for_npm_package(package.0) { return Ok(()); } else if self.cache_setting == CacheSetting::Only { diff --git a/cli/npm/resolvers/local.rs b/cli/npm/resolvers/local.rs index 96a7d961d1e0c9..c7d8c34a3600f8 100644 --- a/cli/npm/resolvers/local.rs +++ b/cli/npm/resolvers/local.rs @@ -32,7 +32,6 @@ use crate::npm::NpmPackageId; use crate::npm::NpmPackageReq; use crate::npm::RealNpmRegistryApi; -use super::common::cache_packages; use super::common::ensure_registry_read_permission; use super::common::InnerNpmPackageResolver; @@ -241,7 +240,6 @@ impl InnerNpmPackageResolver for LocalNpmPackageResolver { async fn sync_resolver_with_fs( resolver: &LocalNpmPackageResolver, ) -> Result<(), AnyError> { - cache_packages_to_global_cache(&resolver).await?; sync_resolution_with_fs( &resolver.resolution.snapshot(), &resolver.cache, @@ -290,7 +288,9 @@ async fn sync_resolution_with_fs( get_package_folder_name(&package.get_package_cache_folder_id()); let folder_path = deno_local_registry_dir.join(&folder_name); let initialized_file = folder_path.join(".initialized"); - if !initialized_file.exists() { + if !cache.should_use_cache_for_npm_package(&package.id.name) + || !initialized_file.exists() + { let cache = cache.clone(); let registry_url = registry_url.clone(); let package = package.clone(); @@ -499,15 +499,3 @@ fn get_next_node_modules_ancestor(mut path: &Path) -> &Path { } } } - -async fn cache_packages_to_global_cache( - resolver: &LocalNpmPackageResolver, -) -> Result<(), AnyError> { - let packages = resolver.resolution.all_packages_partitioned().packages; - - // only cache the non-copy packages in the global cache since - // there's no need for them when using a local node_modules - cache_packages(packages, &resolver.cache, &resolver.registry_url).await?; - - Ok(()) -} From ebf24a9549256480be725a7b57f57bf783e9c547 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Mon, 7 Nov 2022 19:03:59 -0500 Subject: [PATCH 38/43] Clippy --- cli/npm/resolution/graph.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index bafa073b514d8e..f588ab8bebc7dc 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -495,7 +495,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> &node, &NodeParent::Node(parent_id.clone()), ); - self.try_add_pending_unresolved_node(Some(&visited_versions), &node); + self.try_add_pending_unresolved_node(Some(visited_versions), &node); Ok(()) } @@ -723,7 +723,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> BTreeMap::from([(specifier, BTreeSet::from([NodeParent::Req]))]), &old_id, &child_id, - &path, + path, visited_ancestor_versions, ))); } @@ -832,7 +832,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> assert!(!old_node_children.contains_key(next_specifier)); let node = self.graph.get_or_create_for_id(&peer_dep_id).1; self.try_add_pending_unresolved_node( - Some(&visited_ancestor_versions), + Some(visited_ancestor_versions), &node, ); self From 7cbc8498a662cb3e01c2b88185cb9df443c92f6b Mon Sep 17 00:00:00 2001 From: David Sherret Date: Tue, 8 Nov 2022 10:52:17 -0500 Subject: [PATCH 39/43] Add more information for panic. --- cli/npm/resolution/graph.rs | 9 ++++++- cli/tests/integration/npm_tests.rs | 26 ++++--------------- .../main_reload.out | 10 ------- 3 files changed, 13 insertions(+), 32 deletions(-) delete mode 100644 cli/tests/testdata/npm/peer_deps_with_copied_folders/main_reload.out diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index f588ab8bebc7dc..a15d39bb832bd1 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -291,7 +291,14 @@ impl Graph { parent_id: &NpmPackageId, ) { let mut child = (*child).lock(); - let mut parent = (**self.packages.get(parent_id).unwrap()).lock(); + let mut parent = (**self.packages.get(parent_id).unwrap_or_else(|| { + panic!( + "could not find {} in list of packages when setting child {}", + parent_id.as_serialized(), + child.id.as_serialized() + ) + })) + .lock(); assert_ne!(parent.id, child.id); parent .children diff --git a/cli/tests/integration/npm_tests.rs b/cli/tests/integration/npm_tests.rs index 0c5523082e6151..cbc2dcbeab0744 100644 --- a/cli/tests/integration/npm_tests.rs +++ b/cli/tests/integration/npm_tests.rs @@ -1084,13 +1084,10 @@ fn peer_deps_with_copied_folders_and_lockfile() { let output = deno.wait_with_output().unwrap(); assert!(output.status.success()); - let expected_initial_output = + let expected_output = std::fs::read_to_string(test_folder_path.join("main.out")).unwrap(); - assert_eq!( - String::from_utf8(output.stderr).unwrap(), - expected_initial_output - ); + assert_eq!(String::from_utf8(output.stderr).unwrap(), expected_output); assert!(temp_dir.path().join("deno.lock").exists()); let grandchild_path = deno_dir @@ -1120,10 +1117,6 @@ fn peer_deps_with_copied_folders_and_lockfile() { assert_eq!(String::from_utf8(output.stderr).unwrap(), "1\n2\n"); assert!(output.status.success()); - // Ensure it works with reloading. This output will be slightly different - // because resolution has already occurred due to the lockfile. - let expected_reload_output = - std::fs::read_to_string(test_folder_path.join("main_reload.out")).unwrap(); let deno = util::deno_cmd_with_deno_dir(&deno_dir) .current_dir(temp_dir.path()) .arg("run") @@ -1137,10 +1130,7 @@ fn peer_deps_with_copied_folders_and_lockfile() { .spawn() .unwrap(); let output = deno.wait_with_output().unwrap(); - assert_eq!( - String::from_utf8(output.stderr).unwrap(), - expected_reload_output - ); + assert_eq!(String::from_utf8(output.stderr).unwrap(), expected_output); assert!(output.status.success()); // now run with local node modules @@ -1201,10 +1191,7 @@ fn peer_deps_with_copied_folders_and_lockfile() { .unwrap(); let output = deno.wait_with_output().unwrap(); assert!(output.status.success()); - assert_eq!( - String::from_utf8(output.stderr).unwrap(), - expected_reload_output - ); + assert_eq!(String::from_utf8(output.stderr).unwrap(), expected_output); // now ensure it works with reloading and no lockfile let deno = util::deno_cmd_with_deno_dir(&deno_dir) @@ -1222,10 +1209,7 @@ fn peer_deps_with_copied_folders_and_lockfile() { .spawn() .unwrap(); let output = deno.wait_with_output().unwrap(); - assert_eq!( - String::from_utf8(output.stderr).unwrap(), - expected_initial_output, - ); + assert_eq!(String::from_utf8(output.stderr).unwrap(), expected_output,); assert!(output.status.success()); } diff --git a/cli/tests/testdata/npm/peer_deps_with_copied_folders/main_reload.out b/cli/tests/testdata/npm/peer_deps_with_copied_folders/main_reload.out deleted file mode 100644 index ce0dc68968e44a..00000000000000 --- a/cli/tests/testdata/npm/peer_deps_with_copied_folders/main_reload.out +++ /dev/null @@ -1,10 +0,0 @@ -Download http://localhost:4545/npm/registry/@denotest/peer-dep-test-child -Download http://localhost:4545/npm/registry/@denotest/peer-dep-test-grandchild -Download http://localhost:4545/npm/registry/@denotest/peer-dep-test-peer -Download http://localhost:4545/npm/registry/@denotest/peer-dep-test-child/1.0.0.tgz -Download http://localhost:4545/npm/registry/@denotest/peer-dep-test-child/2.0.0.tgz -Download http://localhost:4545/npm/registry/@denotest/peer-dep-test-grandchild/1.0.0.tgz -Download http://localhost:4545/npm/registry/@denotest/peer-dep-test-peer/1.0.0.tgz -Download http://localhost:4545/npm/registry/@denotest/peer-dep-test-peer/2.0.0.tgz -1 -2 From 999fb28903b4cc0f047c0058f23a08faa946258a Mon Sep 17 00:00:00 2001 From: David Sherret Date: Tue, 8 Nov 2022 11:51:35 -0500 Subject: [PATCH 40/43] Fix infinite loop --- cli/npm/resolution/graph.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index a15d39bb832bd1..d7130d9c072675 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -175,11 +175,13 @@ impl Graph { packages: &HashMap, ) -> Arc> { let resolution = packages.get(id).unwrap(); - let node = graph.get_or_create_for_id(id).1; + let (created, node) = graph.get_or_create_for_id(id); + if created { for (name, child_id) in &resolution.dependencies { let child_node = fill_for_id(graph, child_id, packages); graph.set_child_parent_node(name, &child_node, id); } + } node } From e2202cde88e3d83b566e04b032c1d3fcb974dd03 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Tue, 8 Nov 2022 12:48:18 -0500 Subject: [PATCH 41/43] Add deno info support. --- cli/lockfile.rs | 2 +- cli/npm/resolution/graph.rs | 28 ++++-- cli/npm/resolution/mod.rs | 8 +- cli/npm/resolution/snapshot.rs | 10 +- cli/npm/resolvers/local.rs | 11 ++- cli/tests/integration/npm_tests.rs | 17 ++++ .../main_info.out | 14 +++ .../main_info_json.out | 95 +++++++++++++++++++ cli/tools/info.rs | 22 +++-- 9 files changed, 178 insertions(+), 29 deletions(-) create mode 100644 cli/tests/testdata/npm/peer_deps_with_copied_folders/main_info.out create mode 100644 cli/tests/testdata/npm/peer_deps_with_copied_folders/main_info_json.out diff --git a/cli/lockfile.rs b/cli/lockfile.rs index 527cc2e535f52c..8df53d07fa87d0 100644 --- a/cli/lockfile.rs +++ b/cli/lockfile.rs @@ -290,7 +290,7 @@ This could be caused by: * the source itself may be corrupt Use \"--lock-write\" flag to regenerate the lockfile at \"{}\".", - package.id, self.filename.display() + package.id.display(), self.filename.display() ))); } } else { diff --git a/cli/npm/resolution/graph.rs b/cli/npm/resolution/graph.rs index d7130d9c072675..497067925b0476 100644 --- a/cli/npm/resolution/graph.rs +++ b/cli/npm/resolution/graph.rs @@ -132,6 +132,8 @@ enum NodeParent { #[derive(Debug)] struct Node { pub id: NpmPackageId, + /// If the node was forgotten due to having no parents. + pub forgotten: bool, // Use BTreeMap and BTreeSet in order to create determinism // when going up and down the tree pub parents: BTreeMap>, @@ -177,10 +179,10 @@ impl Graph { let resolution = packages.get(id).unwrap(); let (created, node) = graph.get_or_create_for_id(id); if created { - for (name, child_id) in &resolution.dependencies { - let child_node = fill_for_id(graph, child_id, packages); - graph.set_child_parent_node(name, &child_node, id); - } + for (name, child_id) in &resolution.dependencies { + let child_node = fill_for_id(graph, child_id, packages); + graph.set_child_parent_node(name, &child_node, id); + } } node } @@ -219,6 +221,7 @@ impl Graph { } else { let node = Arc::new(Mutex::new(Node { id: id.clone(), + forgotten: false, parents: Default::default(), children: Default::default(), deps: Default::default(), @@ -242,7 +245,8 @@ impl Graph { fn forget_orphan(&mut self, node_id: &NpmPackageId) { if let Some(node) = self.packages.remove(node_id) { - let node = (*node).lock(); + let mut node = (*node).lock(); + node.forgotten = true; assert_eq!(node.parents.len(), 0); // Remove the id from the list of packages by name. @@ -548,7 +552,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> "Resolved {}@{} to {}", name, version_matcher.version_text(), - id + id.as_serialized() ); let (created, node) = self.graph.get_or_create_for_id(&id); if created { @@ -556,7 +560,7 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> let mut deps = version_and_info .info .dependencies_as_entries() - .with_context(|| format!("npm package: {}", id))?; + .with_context(|| format!("npm package: {}", id.display()))?; // Ensure name alphabetical and then version descending // so these are resolved in that order deps.sort(); @@ -574,6 +578,10 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> { let (mut parent_id, deps) = { let parent_node = parent_node.lock(); + if parent_node.forgotten { + // todo(dsherret): we should try to reproduce this scenario and write a test + continue; + } (parent_node.id.clone(), parent_node.deps.clone()) }; @@ -836,7 +844,9 @@ impl<'a, TNpmRegistryApi: NpmRegistryApi> // this means we're at the peer dependency now debug!( "Resolved peer dependency for {} in {} to {}", - next_specifier, &new_id, &peer_dep_id, + next_specifier, + &new_id.as_serialized(), + &peer_dep_id.as_serialized(), ); assert!(!old_node_children.contains_key(next_specifier)); let node = self.graph.get_or_create_for_id(&peer_dep_id).1; @@ -948,7 +958,7 @@ fn get_resolved_package_version_and_info( pkg_name, version_matcher.version_text(), match parent { - Some(id) => format!(" as specified in {}", id), + Some(id) => format!(" as specified in {}", id.display()), None => String::new(), } ), diff --git a/cli/npm/resolution/mod.rs b/cli/npm/resolution/mod.rs index cee2bd9bf5a9fa..ae2553022b065e 100644 --- a/cli/npm/resolution/mod.rs +++ b/cli/npm/resolution/mod.rs @@ -306,11 +306,11 @@ impl NpmPackageId { with_failure_handling(parse_id_at_level(0))(id) .with_context(|| format!("Invalid npm package id '{}'.", id)) } -} -impl std::fmt::Display for NpmPackageId { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{}@{}", self.name, self.version) + pub fn display(&self) -> String { + // Don't implement std::fmt::Display because we don't + // want this to be used by accident in certain scenarios. + format!("{}@{}", self.name, self.version) } } diff --git a/cli/npm/resolution/snapshot.rs b/cli/npm/resolution/snapshot.rs index 1d428f4c2674e9..d76ba8b1a6582e 100644 --- a/cli/npm/resolution/snapshot.rs +++ b/cli/npm/resolution/snapshot.rs @@ -25,10 +25,14 @@ use super::NpmPackageReq; use super::NpmResolutionPackage; use super::NpmVersionMatcher; -/// Packages partitioned by if they are "copy" packages or not. Copy -/// packages are used for peer dependencies. +/// Packages partitioned by if they are "copy" packages or not. pub struct NpmPackagesPartitioned { pub packages: Vec, + /// Since peer dependency resolution occurs based on ancestors and ancestor + /// siblings, this may sometimes cause the same package (name and version) + /// to have different dependencies based on where it appears in the tree. + /// For these packages, we create a "copy package" or duplicate of the package + /// whose dependencies are that of where in the tree they've resolved to. pub copy_packages: Vec, } @@ -325,7 +329,7 @@ impl NpmResolutionSnapshot { { Some(version_info) => version_info, None => { - bail!("could not find '{}' specified in the lockfile. Maybe try again with --reload", package.id); + bail!("could not find '{}' specified in the lockfile. Maybe try again with --reload", package.id.display()); } }; package.dist = version_info.dist; diff --git a/cli/npm/resolvers/local.rs b/cli/npm/resolvers/local.rs index c7d8c34a3600f8..647ab38b3bab90 100644 --- a/cli/npm/resolvers/local.rs +++ b/cli/npm/resolvers/local.rs @@ -115,7 +115,7 @@ impl InnerNpmPackageResolver for LocalNpmPackageResolver { // it might be at the full path if there are duplicate names let fully_resolved_folder_path = join_package_name( &self.root_node_modules_path, - &resolved_package.id.to_string(), + &resolved_package.id.as_serialized(), ); Ok(if fully_resolved_folder_path.exists() { fully_resolved_folder_path @@ -186,11 +186,14 @@ impl InnerNpmPackageResolver for LocalNpmPackageResolver { &self .root_node_modules_path .join(".deno") - .join(package.id.to_string()) + .join(package.id.as_serialized()) .join("node_modules") .join(package.id.name), )?), - None => bail!("Could not find package folder for '{}'", package_id), + None => bail!( + "Could not find package folder for '{}'", + package_id.as_serialized() + ), } } @@ -404,7 +407,7 @@ async fn sync_resolution_with_fs( let root_folder_name = if found_names.insert(package_id.name.clone()) { package_id.name.clone() } else if is_top_level { - package_id.to_string() + package_id.display() } else { continue; // skip, already handled }; diff --git a/cli/tests/integration/npm_tests.rs b/cli/tests/integration/npm_tests.rs index cbc2dcbeab0744..3e31d73952ad91 100644 --- a/cli/tests/integration/npm_tests.rs +++ b/cli/tests/integration/npm_tests.rs @@ -1213,6 +1213,23 @@ fn peer_deps_with_copied_folders_and_lockfile() { assert!(output.status.success()); } +itest!(info_peer_deps { + args: "info --quiet --unstable npm/peer_deps_with_copied_folders/main.ts", + output: "npm/peer_deps_with_copied_folders/main_info.out", + exit_code: 0, + envs: env_vars(), + http_server: true, +}); + +itest!(info_peer_deps_json { + args: + "info --quiet --unstable --json npm/peer_deps_with_copied_folders/main.ts", + output: "npm/peer_deps_with_copied_folders/main_info_json.out", + exit_code: 0, + envs: env_vars(), + http_server: true, +}); + fn env_vars_no_sync_download() -> Vec<(String, String)> { vec![ ("DENO_NODE_COMPAT_URL".to_string(), util::std_file_url()), diff --git a/cli/tests/testdata/npm/peer_deps_with_copied_folders/main_info.out b/cli/tests/testdata/npm/peer_deps_with_copied_folders/main_info.out new file mode 100644 index 00000000000000..c9c4a59c142d3d --- /dev/null +++ b/cli/tests/testdata/npm/peer_deps_with_copied_folders/main_info.out @@ -0,0 +1,14 @@ +local: [WILDCARD]main.ts +type: TypeScript +dependencies: 6 unique +size: [WILDCARD] + +file:///[WILDCARD]/testdata/npm/peer_deps_with_copied_folders/main.ts (171B) +├─┬ npm:@denotest/peer-dep-test-child@1 - 1.0.0 ([WILDCARD]) +│ ├─┬ npm:@denotest/peer-dep-test-grandchild@1.0.0_@denotest+peer-dep-test-peer@1.0.0 ([WILDCARD]) +│ │ └── npm:@denotest/peer-dep-test-peer@1.0.0 ([WILDCARD]) +│ └── npm:@denotest/peer-dep-test-peer@1.0.0 ([WILDCARD]) +└─┬ npm:@denotest/peer-dep-test-child@2 - 2.0.0 ([WILDCARD]) + ├─┬ npm:@denotest/peer-dep-test-grandchild@1.0.0_@denotest+peer-dep-test-peer@2.0.0 ([WILDCARD]) + │ └── npm:@denotest/peer-dep-test-peer@2.0.0 ([WILDCARD]) + └── npm:@denotest/peer-dep-test-peer@2.0.0 ([WILDCARD]) diff --git a/cli/tests/testdata/npm/peer_deps_with_copied_folders/main_info_json.out b/cli/tests/testdata/npm/peer_deps_with_copied_folders/main_info_json.out new file mode 100644 index 00000000000000..634ec62516e4a6 --- /dev/null +++ b/cli/tests/testdata/npm/peer_deps_with_copied_folders/main_info_json.out @@ -0,0 +1,95 @@ +{ + "roots": [ + "[WILDCARD]/npm/peer_deps_with_copied_folders/main.ts" + ], + "modules": [ + { + "dependencies": [ + { + "specifier": "npm:@denotest/peer-dep-test-child@1", + "code": { + "specifier": "npm:@denotest/peer-dep-test-child@1", + "span": { + "start": { + "line": 0, + "character": 21 + }, + "end": { + "line": 0, + "character": 58 + } + } + }, + "npmPackage": "@denotest/peer-dep-test-child@1.0.0_@denotest+peer-dep-test-peer@1.0.0" + }, + { + "specifier": "npm:@denotest/peer-dep-test-child@2", + "code": { + "specifier": "npm:@denotest/peer-dep-test-child@2", + "span": { + "start": { + "line": 1, + "character": 21 + }, + "end": { + "line": 1, + "character": 58 + } + } + }, + "npmPackage": "@denotest/peer-dep-test-child@2.0.0_@denotest+peer-dep-test-peer@2.0.0" + } + ], + "kind": "esm", + "local": "[WILDCARD]main.ts", + "emit": null, + "map": null, + "size": 171, + "mediaType": "TypeScript", + "specifier": "file://[WILDCARD]/main.ts" + } + ], + "redirects": {}, + "npmPackages": { + "@denotest/peer-dep-test-child@1.0.0_@denotest+peer-dep-test-peer@1.0.0": { + "name": "@denotest/peer-dep-test-child", + "version": "1.0.0", + "dependencies": [ + "@denotest/peer-dep-test-grandchild@1.0.0_@denotest+peer-dep-test-peer@1.0.0", + "@denotest/peer-dep-test-peer@1.0.0" + ] + }, + "@denotest/peer-dep-test-child@2.0.0_@denotest+peer-dep-test-peer@2.0.0": { + "name": "@denotest/peer-dep-test-child", + "version": "2.0.0", + "dependencies": [ + "@denotest/peer-dep-test-grandchild@1.0.0_@denotest+peer-dep-test-peer@2.0.0", + "@denotest/peer-dep-test-peer@2.0.0" + ] + }, + "@denotest/peer-dep-test-grandchild@1.0.0_@denotest+peer-dep-test-peer@1.0.0": { + "name": "@denotest/peer-dep-test-grandchild", + "version": "1.0.0", + "dependencies": [ + "@denotest/peer-dep-test-peer@1.0.0" + ] + }, + "@denotest/peer-dep-test-grandchild@1.0.0_@denotest+peer-dep-test-peer@2.0.0": { + "name": "@denotest/peer-dep-test-grandchild", + "version": "1.0.0", + "dependencies": [ + "@denotest/peer-dep-test-peer@2.0.0" + ] + }, + "@denotest/peer-dep-test-peer@1.0.0": { + "name": "@denotest/peer-dep-test-peer", + "version": "1.0.0", + "dependencies": [] + }, + "@denotest/peer-dep-test-peer@2.0.0": { + "name": "@denotest/peer-dep-test-peer", + "version": "2.0.0", + "dependencies": [] + } + } +} diff --git a/cli/tools/info.rs b/cli/tools/info.rs index 12b1ae4c312965..8850c4fa2e09ce 100644 --- a/cli/tools/info.rs +++ b/cli/tools/info.rs @@ -157,7 +157,10 @@ fn add_npm_packages_to_json( }); if let Some(pkg) = maybe_package { if let Some(module) = module.as_object_mut() { - module.insert("npmPackage".to_string(), format!("{}", pkg.id).into()); + module.insert( + "npmPackage".to_string(), + format!("{}", pkg.id.as_serialized()).into(), + ); // change the "kind" to be "npm" module.insert("kind".to_string(), "npm".into()); } @@ -190,7 +193,7 @@ fn add_npm_packages_to_json( { dep.insert( "npmPackage".to_string(), - format!("{}", pkg.id).into(), + format!("{}", pkg.id.as_serialized()).into(), ); } } @@ -212,11 +215,11 @@ fn add_npm_packages_to_json( deps.sort(); let deps = deps .into_iter() - .map(|id| serde_json::Value::String(format!("{}", id))) + .map(|id| serde_json::Value::String(format!("{}", id.as_serialized()))) .collect::>(); kv.insert("dependencies".to_string(), deps.into()); - json_packages.insert(format!("{}", &pkg.id), kv.into()); + json_packages.insert(format!("{}", pkg.id.as_serialized()), kv.into()); } json.insert("npmPackages".to_string(), json_packages.into()); @@ -504,7 +507,7 @@ impl<'a> GraphDisplayContext<'a> { None => Specifier(module.specifier.clone()), }; let was_seen = !self.seen.insert(match &package_or_specifier { - Package(package) => package.id.to_string(), + Package(package) => package.id.as_serialized(), Specifier(specifier) => specifier.to_string(), }); let header_text = if was_seen { @@ -572,11 +575,14 @@ impl<'a> GraphDisplayContext<'a> { for dep_id in deps.into_iter() { let maybe_size = self.npm_info.package_sizes.get(dep_id).cloned(); let size_str = maybe_size_to_text(maybe_size); - let mut child = - TreeNode::from_text(format!("npm:{} {}", dep_id, size_str)); + let mut child = TreeNode::from_text(format!( + "npm:{} {}", + dep_id.as_serialized(), + size_str + )); if let Some(package) = self.npm_info.packages.get(dep_id) { if !package.dependencies.is_empty() { - if self.seen.contains(&package.id.to_string()) { + if self.seen.contains(&package.id.as_serialized()) { child.text = format!("{} {}", child.text, colors::gray("*")); } else { let package = package.clone(); From 0468e83a4e236bd43ecf1f2e8462cbdfb14e892d Mon Sep 17 00:00:00 2001 From: David Sherret Date: Tue, 8 Nov 2022 12:53:03 -0500 Subject: [PATCH 42/43] Clippy. --- cli/tools/info.rs | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/cli/tools/info.rs b/cli/tools/info.rs index 8850c4fa2e09ce..99541c207726af 100644 --- a/cli/tools/info.rs +++ b/cli/tools/info.rs @@ -157,10 +157,8 @@ fn add_npm_packages_to_json( }); if let Some(pkg) = maybe_package { if let Some(module) = module.as_object_mut() { - module.insert( - "npmPackage".to_string(), - format!("{}", pkg.id.as_serialized()).into(), - ); + module + .insert("npmPackage".to_string(), pkg.id.as_serialized().into()); // change the "kind" to be "npm" module.insert("kind".to_string(), "npm".into()); } @@ -193,7 +191,7 @@ fn add_npm_packages_to_json( { dep.insert( "npmPackage".to_string(), - format!("{}", pkg.id.as_serialized()).into(), + pkg.id.as_serialized().into(), ); } } @@ -215,11 +213,11 @@ fn add_npm_packages_to_json( deps.sort(); let deps = deps .into_iter() - .map(|id| serde_json::Value::String(format!("{}", id.as_serialized()))) + .map(|id| serde_json::Value::String(id.as_serialized())) .collect::>(); kv.insert("dependencies".to_string(), deps.into()); - json_packages.insert(format!("{}", pkg.id.as_serialized()), kv.into()); + json_packages.insert(pkg.id.as_serialized(), kv.into()); } json.insert("npmPackages".to_string(), json_packages.into()); From c48394404ee8804ea02c362f482c6da00dc91ba8 Mon Sep 17 00:00:00 2001 From: David Sherret Date: Tue, 8 Nov 2022 13:34:58 -0500 Subject: [PATCH 43/43] Fix. --- cli/npm/resolution/mod.rs | 4 +- cli/npm/resolvers/local.rs | 97 ++++++++++++++++-------------- cli/tests/integration/npm_tests.rs | 2 +- 3 files changed, 55 insertions(+), 48 deletions(-) diff --git a/cli/npm/resolution/mod.rs b/cli/npm/resolution/mod.rs index ae2553022b065e..934cfb59b82e1f 100644 --- a/cli/npm/resolution/mod.rs +++ b/cli/npm/resolution/mod.rs @@ -109,9 +109,9 @@ impl NpmPackageReference { impl std::fmt::Display for NpmPackageReference { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { if let Some(sub_path) = &self.sub_path { - write!(f, "{}/{}", self.req, sub_path) + write!(f, "npm:{}/{}", self.req, sub_path) } else { - write!(f, "{}", self.req) + write!(f, "npm:{}", self.req) } } } diff --git a/cli/npm/resolvers/local.rs b/cli/npm/resolvers/local.rs index 647ab38b3bab90..678f776f3494bd 100644 --- a/cli/npm/resolvers/local.rs +++ b/cli/npm/resolvers/local.rs @@ -30,6 +30,7 @@ use crate::npm::resolution::NpmResolutionSnapshot; use crate::npm::NpmCache; use crate::npm::NpmPackageId; use crate::npm::NpmPackageReq; +use crate::npm::NpmResolutionPackage; use crate::npm::RealNpmRegistryApi; use super::common::ensure_registry_read_permission; @@ -102,6 +103,35 @@ impl LocalNpmPackageResolver { // it's within the directory, so use it specifier.to_file_path().ok() } + + fn get_package_id_folder( + &self, + package_id: &NpmPackageId, + ) -> Result { + match self.resolution.resolve_package_from_id(package_id) { + Some(package) => Ok(self.get_package_id_folder_from_package(&package)), + None => bail!( + "Could not find package information for '{}'", + package_id.as_serialized() + ), + } + } + + fn get_package_id_folder_from_package( + &self, + package: &NpmResolutionPackage, + ) -> PathBuf { + // package is stored at: + // node_modules/.deno//node_modules/ + self + .root_node_modules_path + .join(".deno") + .join(get_package_folder_id_folder_name( + &package.get_package_cache_folder_id(), + )) + .join("node_modules") + .join(&package.id.name) + } } impl InnerNpmPackageResolver for LocalNpmPackageResolver { @@ -109,19 +139,8 @@ impl InnerNpmPackageResolver for LocalNpmPackageResolver { &self, pkg_req: &NpmPackageReq, ) -> Result { - let resolved_package = - self.resolution.resolve_package_from_deno_module(pkg_req)?; - - // it might be at the full path if there are duplicate names - let fully_resolved_folder_path = join_package_name( - &self.root_node_modules_path, - &resolved_package.id.as_serialized(), - ); - Ok(if fully_resolved_folder_path.exists() { - fully_resolved_folder_path - } else { - join_package_name(&self.root_node_modules_path, &resolved_package.id.name) - }) + let package = self.resolution.resolve_package_from_deno_module(pkg_req)?; + Ok(self.get_package_id_folder_from_package(&package)) } fn resolve_package_folder_from_package( @@ -179,22 +198,9 @@ impl InnerNpmPackageResolver for LocalNpmPackageResolver { } fn package_size(&self, package_id: &NpmPackageId) -> Result { - match self.resolution.resolve_package_from_id(package_id) { - Some(package) => Ok(fs_util::dir_size( - // package is stored at: - // node_modules/.deno//node_modules/ - &self - .root_node_modules_path - .join(".deno") - .join(package.id.as_serialized()) - .join("node_modules") - .join(package.id.name), - )?), - None => bail!( - "Could not find package folder for '{}'", - package_id.as_serialized() - ), - } + let package_folder_path = self.get_package_id_folder(package_id)?; + + Ok(fs_util::dir_size(&package_folder_path)?) } fn has_packages(&self) -> bool { @@ -259,15 +265,6 @@ async fn sync_resolution_with_fs( registry_url: &Url, root_node_modules_dir_path: &Path, ) -> Result<(), AnyError> { - fn get_package_folder_name(id: &NpmPackageCacheFolderId) -> String { - let copy_str = if id.copy_index == 0 { - "".to_string() - } else { - format!("_{}", id.copy_index) - }; - format!("{}@{}{}", id.name, id.version, copy_str).replace('/', "+") - } - let deno_local_registry_dir = root_node_modules_dir_path.join(".deno"); fs::create_dir_all(&deno_local_registry_dir).with_context(|| { format!("Creating '{}'", deno_local_registry_dir.display()) @@ -276,7 +273,7 @@ async fn sync_resolution_with_fs( // 1. Write all the packages out the .deno directory. // // Copy (hardlink in future) // to - // node_modules/.deno//node_modules/ + // node_modules/.deno//node_modules/ let sync_download = should_sync_download(); let mut package_partitions = snapshot.all_packages_partitioned(); if sync_download { @@ -288,7 +285,7 @@ async fn sync_resolution_with_fs( Vec::with_capacity(package_partitions.packages.len()); for package in &package_partitions.packages { let folder_name = - get_package_folder_name(&package.get_package_cache_folder_id()); + get_package_folder_id_folder_name(&package.get_package_cache_folder_id()); let folder_path = deno_local_registry_dir.join(&folder_name); let initialized_file = folder_path.join(".initialized"); if !cache.should_use_cache_for_npm_package(&package.id.name) @@ -338,7 +335,7 @@ async fn sync_resolution_with_fs( for package in &package_partitions.copy_packages { let package_cache_folder_id = package.get_package_cache_folder_id(); let destination_path = deno_local_registry_dir - .join(&get_package_folder_name(&package_cache_folder_id)); + .join(&get_package_folder_id_folder_name(&package_cache_folder_id)); let initialized_file = destination_path.join(".initialized"); if !initialized_file.exists() { let sub_node_modules = destination_path.join("node_modules"); @@ -348,7 +345,7 @@ async fn sync_resolution_with_fs( })?; let source_path = join_package_name( &deno_local_registry_dir - .join(&get_package_folder_name( + .join(&get_package_folder_id_folder_name( &package_cache_folder_id.with_no_count(), )) .join("node_modules"), @@ -368,7 +365,7 @@ async fn sync_resolution_with_fs( // node_modules/.deno//node_modules/ for package in &all_packages { let sub_node_modules = deno_local_registry_dir - .join(&get_package_folder_name( + .join(&get_package_folder_id_folder_name( &package.get_package_cache_folder_id(), )) .join("node_modules"); @@ -377,7 +374,8 @@ async fn sync_resolution_with_fs( .package_from_id(dep_id) .unwrap() .get_package_cache_folder_id(); - let dep_folder_name = get_package_folder_name(&dep_cache_folder_id); + let dep_folder_name = + get_package_folder_id_folder_name(&dep_cache_folder_id); let dep_folder_path = join_package_name( &deno_local_registry_dir .join(dep_folder_name) @@ -414,7 +412,7 @@ async fn sync_resolution_with_fs( let package = snapshot.package_from_id(&package_id).unwrap(); let local_registry_package_path = join_package_name( &deno_local_registry_dir - .join(&get_package_folder_name( + .join(&get_package_folder_id_folder_name( &package.get_package_cache_folder_id(), )) .join("node_modules"), @@ -433,6 +431,15 @@ async fn sync_resolution_with_fs( Ok(()) } +fn get_package_folder_id_folder_name(id: &NpmPackageCacheFolderId) -> String { + let copy_str = if id.copy_index == 0 { + "".to_string() + } else { + format!("_{}", id.copy_index) + }; + format!("{}@{}{}", id.name, id.version, copy_str).replace('/', "+") +} + fn symlink_package_dir( old_path: &Path, new_path: &Path, diff --git a/cli/tests/integration/npm_tests.rs b/cli/tests/integration/npm_tests.rs index 3e31d73952ad91..87e85385054a1c 100644 --- a/cli/tests/integration/npm_tests.rs +++ b/cli/tests/integration/npm_tests.rs @@ -1147,8 +1147,8 @@ fn peer_deps_with_copied_folders_and_lockfile() { .spawn() .unwrap(); let output = deno.wait_with_output().unwrap(); - assert!(output.status.success()); assert_eq!(String::from_utf8(output.stderr).unwrap(), "1\n2\n"); + assert!(output.status.success()); let deno_folder = temp_dir.path().join("node_modules").join(".deno"); assert!(deno_folder