Skip to content

Commit

Permalink
Avoid panic when Git dependencies are included in fork markers (#8388)
Browse files Browse the repository at this point in the history
## Summary

Rather than relying on the distribution and package URL being the same
(which isn't true for Git dependencies), we can just use the
intersection of the markers directly.

Closes #8381.
  • Loading branch information
charliermarsh authored Oct 20, 2024
1 parent ab16bf0 commit 7e2822d
Show file tree
Hide file tree
Showing 3 changed files with 204 additions and 29 deletions.
33 changes: 29 additions & 4 deletions crates/uv-resolver/src/lock/mod.rs
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,21 @@ impl Lock {
let mut packages = BTreeMap::new();
let requires_python = graph.requires_python.clone();

// Determine the set of packages included at multiple versions.
let mut seen = FxHashSet::default();
let mut duplicates = FxHashSet::default();
for node_index in graph.petgraph.node_indices() {
let ResolutionGraphNode::Dist(dist) = &graph.petgraph[node_index] else {
continue;
};
if !dist.is_base() {
continue;
}
if !seen.insert(dist.name()) {
duplicates.insert(dist.name());
}
}

// Lock all base packages.
for node_index in graph.petgraph.node_indices() {
let ResolutionGraphNode::Dist(dist) = &graph.petgraph[node_index] else {
Expand All @@ -116,10 +131,20 @@ impl Lock {
if !dist.is_base() {
continue;
}
let fork_markers = graph
.fork_markers(dist.name(), &dist.version, dist.dist.version_or_url().url())
.cloned()
.unwrap_or_default();

// If there are multiple distributions for the same package, include the markers of all
// forks that included the current distribution.
let fork_markers = if duplicates.contains(dist.name()) {
graph
.fork_markers
.iter()
.filter(|fork_markers| !fork_markers.is_disjoint(&dist.marker))
.cloned()
.collect()
} else {
vec![]
};

let mut package = Package::from_annotated_dist(dist, fork_markers, root)?;
Self::remove_unreachable_wheels(graph, &requires_python, node_index, &mut package);

Expand Down
26 changes: 2 additions & 24 deletions crates/uv-resolver/src/resolution/graph.rs
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ use uv_distribution_types::{
use uv_git::GitResolver;
use uv_normalize::{ExtraName, GroupName, PackageName};
use uv_pep440::{Version, VersionSpecifier};
use uv_pep508::{MarkerEnvironment, MarkerTree, MarkerTreeKind, VerbatimUrl};
use uv_pep508::{MarkerEnvironment, MarkerTree, MarkerTreeKind};
use uv_pypi_types::{HashDigest, ParsedUrlError, Requirement, VerbatimParsedUrl, Yanked};

use crate::graph_ops::marker_reachability;
Expand All @@ -31,7 +31,7 @@ use crate::{
ResolverMarkers, VersionsResponse,
};

pub(crate) type MarkersForDistribution = FxHashMap<(Version, Option<VerbatimUrl>), Vec<MarkerTree>>;
pub(crate) type MarkersForDistribution = Vec<MarkerTree>;

/// A complete resolution graph in which every node represents a pinned package and every edge
/// represents a dependency between two pinned packages.
Expand All @@ -54,9 +54,6 @@ pub struct ResolutionGraph {
pub(crate) overrides: Overrides,
/// The options that were used to build the graph.
pub(crate) options: Options,
/// If there are multiple options for a package, track which fork they belong to so we
/// can write that to the lockfile and later get the correct preference per fork back.
pub(crate) package_markers: FxHashMap<PackageName, MarkersForDistribution>,
}

#[derive(Debug, Clone)]
Expand Down Expand Up @@ -131,8 +128,6 @@ impl ResolutionGraph {
package_markers
.entry(package.name.clone())
.or_default()
.entry((version.clone(), package.url.clone().map(|url| url.verbatim)))
.or_default()
.push(markers.clone());
}
}
Expand Down Expand Up @@ -239,7 +234,6 @@ impl ResolutionGraph {
let graph = Self {
petgraph,
requires_python,
package_markers,
diagnostics,
requirements: requirements.to_vec(),
constraints: constraints.clone(),
Expand Down Expand Up @@ -727,22 +721,6 @@ impl ResolutionGraph {
Ok(conjunction)
}

/// If there are multiple distributions for the same package name, return the markers of the
/// fork(s) that contained this distribution, otherwise return `None`.
pub fn fork_markers(
&self,
package_name: &PackageName,
version: &Version,
url: Option<&VerbatimUrl>,
) -> Option<&Vec<MarkerTree>> {
let package_markers = &self.package_markers.get(package_name)?;
if package_markers.len() == 1 {
None
} else {
Some(&package_markers[&(version.clone(), url.cloned())])
}
}

/// Returns a sequence of conflicting distribution errors from this
/// resolution.
///
Expand Down
174 changes: 173 additions & 1 deletion crates/uv/tests/it/lock.rs
Original file line number Diff line number Diff line change
Expand Up @@ -2689,7 +2689,6 @@ fn lock_preference() -> Result<()> {
fn lock_git_plus_prefix() -> Result<()> {
let context = TestContext::new("3.12");

// Lock against `main`.
let pyproject_toml = context.temp_dir.child("pyproject.toml");
pyproject_toml.write_str(
r#"
Expand Down Expand Up @@ -2775,6 +2774,167 @@ fn lock_git_plus_prefix() -> Result<()> {
Ok(())
}

#[test]
#[cfg(feature = "git")]
fn lock_partial_git() -> Result<()> {
let context = TestContext::new("3.12");

let pyproject_toml = context.temp_dir.child("pyproject.toml");
pyproject_toml.write_str(
r#"
[project]
name = "project"
version = "0.1.0"
requires-python = ">=3.10"
dependencies = [
"anyio @ git+https://github.com/agronholm/[email protected] ; python_version < '3.12'",
"anyio ; python_version >= '3.12'"
]

[build-system]
requires = ["setuptools>=42"]
build-backend = "setuptools.build_meta"
"#,
)?;

uv_snapshot!(context.filters(), context.lock(), @r###"
success: true
exit_code: 0
----- stdout -----

----- stderr -----
Resolved 7 packages in [TIME]
"###);

let lock = context.read("uv.lock");

insta::with_settings!({
filters => context.filters(),
}, {
assert_snapshot!(
lock, @r###"
version = 1
requires-python = ">=3.10"
resolution-markers = [
"python_full_version < '3.12'",
"python_full_version >= '3.12'",
]

[options]
exclude-newer = "2024-03-25T00:00:00Z"

[[package]]
name = "anyio"
version = "4.3.0"
source = { registry = "https://pypi.org/simple" }
resolution-markers = [
"python_full_version >= '3.12'",
]
dependencies = [
{ name = "idna", marker = "python_full_version >= '3.12'" },
{ name = "sniffio", marker = "python_full_version >= '3.12'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/db/4d/3970183622f0330d3c23d9b8a5f52e365e50381fd484d08e3285104333d3/anyio-4.3.0.tar.gz", hash = "sha256:f75253795a87df48568485fd18cdd2a3fa5c4f7c5be8e5e36637733fce06fed6", size = 159642 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/14/fd/2f20c40b45e4fb4324834aea24bd4afdf1143390242c0b33774da0e2e34f/anyio-4.3.0-py3-none-any.whl", hash = "sha256:048e05d0f6caeed70d731f3db756d35dcc1f35747c8c403364a8332c630441b8", size = 85584 },
]

[[package]]
name = "anyio"
version = "4.6.2"
source = { git = "https://github.com/agronholm/anyio?rev=4.6.2#c4844254e6db0cb804c240ba07405db73d810e0b" }
resolution-markers = [
"python_full_version < '3.12'",
]
dependencies = [
{ name = "exceptiongroup", marker = "python_full_version < '3.11'" },
{ name = "idna", marker = "python_full_version < '3.12'" },
{ name = "sniffio", marker = "python_full_version < '3.12'" },
{ name = "typing-extensions", marker = "python_full_version < '3.11'" },
]

[[package]]
name = "exceptiongroup"
version = "1.2.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/8e/1c/beef724eaf5b01bb44b6338c8c3494eff7cab376fab4904cfbbc3585dc79/exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68", size = 26264 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/b8/9a/5028fd52db10e600f1c4674441b968cf2ea4959085bfb5b99fb1250e5f68/exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14", size = 16210 },
]

[[package]]
name = "idna"
version = "3.6"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/bf/3f/ea4b9117521a1e9c50344b909be7886dd00a519552724809bb1f486986c2/idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca", size = 175426 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/c2/e7/a82b05cf63a603df6e68d59ae6a68bf5064484a0718ea5033660af4b54a9/idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f", size = 61567 },
]

[[package]]
name = "project"
version = "0.1.0"
source = { editable = "." }
dependencies = [
{ name = "anyio", version = "4.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" },
{ name = "anyio", version = "4.6.2", source = { git = "https://github.com/agronholm/anyio?rev=4.6.2#c4844254e6db0cb804c240ba07405db73d810e0b" }, marker = "python_full_version < '3.12'" },
]

[package.metadata]
requires-dist = [
{ name = "anyio", marker = "python_full_version < '3.12'", git = "https://github.com/agronholm/anyio?rev=4.6.2" },
{ name = "anyio", marker = "python_full_version >= '3.12'" },
]

[[package]]
name = "sniffio"
version = "1.3.1"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235 },
]

[[package]]
name = "typing-extensions"
version = "4.10.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/16/3a/0d26ce356c7465a19c9ea8814b960f8a36c3b0d07c323176620b7b483e44/typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb", size = 77558 }
wheels = [
{ url = "https://files.pythonhosted.org/packages/f9/de/dc04a3ea60b22624b51c703a84bbe0184abcd1d0b9bc8074b5d6b7ab90bb/typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475", size = 33926 },
]
"###
);
});

// Re-run with `--locked`.
uv_snapshot!(context.filters(), context.lock().arg("--locked"), @r###"
success: true
exit_code: 0
----- stdout -----

----- stderr -----
Resolved 7 packages in [TIME]
"###);

// Install from the lockfile, excluding development dependencies.
uv_snapshot!(context.filters(), context.sync().arg("--frozen").arg("--no-dev"), @r###"
success: true
exit_code: 0
----- stdout -----

----- stderr -----
Prepared 4 packages in [TIME]
Installed 4 packages in [TIME]
+ anyio==4.3.0
+ idna==3.6
+ project==0.1.0 (from file://[TEMP_DIR]/)
+ sniffio==1.3.1
"###);

Ok(())
}

/// Respect locked versions with `uv lock`, unless `--upgrade` is passed.
#[test]
#[cfg(feature = "git")]
Expand Down Expand Up @@ -14982,6 +15142,9 @@ fn lock_multiple_sources_index() -> Result<()> {
name = "jinja2"
version = "3.1.3"
source = { registry = "https://download.pytorch.org/whl/cu118" }
resolution-markers = [
"sys_platform == 'win32'",
]
dependencies = [
{ name = "markupsafe", marker = "sys_platform == 'win32'" },
]
Expand All @@ -14993,6 +15156,9 @@ fn lock_multiple_sources_index() -> Result<()> {
name = "jinja2"
version = "3.1.3"
source = { registry = "https://download.pytorch.org/whl/cu124" }
resolution-markers = [
"sys_platform != 'win32'",
]
dependencies = [
{ name = "markupsafe", marker = "sys_platform != 'win32'" },
]
Expand Down Expand Up @@ -15262,6 +15428,9 @@ fn lock_multiple_sources_index_explicit() -> Result<()> {
name = "jinja2"
version = "3.1.3"
source = { registry = "https://pypi.org/simple" }
resolution-markers = [
"sys_platform != 'win32'",
]
dependencies = [
{ name = "markupsafe", marker = "sys_platform != 'win32'" },
]
Expand All @@ -15274,6 +15443,9 @@ fn lock_multiple_sources_index_explicit() -> Result<()> {
name = "jinja2"
version = "3.1.3"
source = { registry = "https://test.pypi.org/simple" }
resolution-markers = [
"sys_platform == 'win32'",
]
dependencies = [
{ name = "markupsafe", marker = "sys_platform == 'win32'" },
]
Expand Down

0 comments on commit 7e2822d

Please sign in to comment.