From be26e47d2f47125668f22ab598f62155f7a056cd Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Sun, 20 Oct 2024 13:37:29 -0500 Subject: [PATCH 1/3] Log unfiltered snapshots on failure (#8349) Cherry-picked from https://github.com/astral-sh/uv/pull/8347 Seems generally really helpful to see the unfiltered snapshot when a test fails. Especially when debugging filters on Windows. --- crates/uv/tests/it/common/mod.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/crates/uv/tests/it/common/mod.rs b/crates/uv/tests/it/common/mod.rs index e5e5878cda7d..229a1e71a4fd 100644 --- a/crates/uv/tests/it/common/mod.rs +++ b/crates/uv/tests/it/common/mod.rs @@ -1112,6 +1112,7 @@ pub fn run_and_format>( /// Execute the command and format its output status, stdout and stderr into a snapshot string. /// /// This function is derived from `insta_cmd`s `spawn_with_info`. +#[allow(clippy::print_stderr)] pub fn run_and_format_with_status>( mut command: impl BorrowMut, filters: impl AsRef<[(T, T)]>, @@ -1141,13 +1142,21 @@ pub fn run_and_format_with_status>( .output() .unwrap_or_else(|err| panic!("Failed to spawn {program}: {err}")); + eprintln!("\n━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━ Unfiltered output ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━"); + eprintln!( + "----- stdout -----\n{}\n----- stderr -----\n{}", + String::from_utf8_lossy(&output.stdout), + String::from_utf8_lossy(&output.stderr), + ); + eprintln!("────────────────────────────────────────────────────────────────────────────────\n"); + let mut snapshot = apply_filters( format!( "success: {:?}\nexit_code: {}\n----- stdout -----\n{}\n----- stderr -----\n{}", output.status.success(), output.status.code().unwrap_or(!0), String::from_utf8_lossy(&output.stdout), - String::from_utf8_lossy(&output.stderr) + String::from_utf8_lossy(&output.stderr), ), filters, ); From ab16bf0a8c2362a698037d41c510d9deadbbf43e Mon Sep 17 00:00:00 2001 From: Zanie Blue Date: Sun, 20 Oct 2024 13:37:41 -0500 Subject: [PATCH 2/3] Set `UV_LINK_MODE=copy` for Windows test runs (#8350) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cherry-picked from #8347 Might fix https://github.com/astral-sh/uv/issues/6940 — I'm not seeing a failure over there after this change. I think there may be some problem with concurrent reads of junctioned files on the DevDrive? It's really hard to say. We might lose some important test coverage with this change. I'm not sure what to do about that either. --- .github/workflows/ci.yml | 4 ++++ crates/uv/tests/it/common/mod.rs | 6 ++++++ crates/uv/tests/it/edit.rs | 6 +++--- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 034688e43ba6..8313c1b599cf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -298,6 +298,10 @@ jobs: - name: "Cargo test" working-directory: ${{ env.UV_WORKSPACE }} + env: + # Avoid permission errors during concurrent tests + # See https://github.com/astral-sh/uv/issues/6940 + UV_LINK_MODE: copy run: | cargo nextest run --no-default-features --features python,pypi --workspace --status-level skip --failure-output immediate-final --no-fail-fast -j 20 --final-status-level slow diff --git a/crates/uv/tests/it/common/mod.rs b/crates/uv/tests/it/common/mod.rs index 229a1e71a4fd..b744c1a15179 100644 --- a/crates/uv/tests/it/common/mod.rs +++ b/crates/uv/tests/it/common/mod.rs @@ -299,6 +299,12 @@ impl TestContext { let mut filters = Vec::new(); + // Exclude `link-mode` on Windows since we set it in the remote test suite + if cfg!(windows) { + filters.push(("--link-mode ".to_string(), String::new())); + filters.push(((r#"link-mode = "copy"\n"#).to_string(), String::new())); + } + filters.extend( Self::path_patterns(&cache_dir) .into_iter() diff --git a/crates/uv/tests/it/edit.rs b/crates/uv/tests/it/edit.rs index 94607c90477c..0100cdd06737 100644 --- a/crates/uv/tests/it/edit.rs +++ b/crates/uv/tests/it/edit.rs @@ -3383,7 +3383,7 @@ fn add_reject_multiple_git_ref_flags() { let context = TestContext::new("3.12"); // --tag and --branch - uv_snapshot!(context + uv_snapshot!(context.filters(), context .add() .arg("foo") .arg("--tag") @@ -3404,7 +3404,7 @@ fn add_reject_multiple_git_ref_flags() { ); // --tag and --rev - uv_snapshot!(context + uv_snapshot!(context.filters(), context .add() .arg("foo") .arg("--tag") @@ -3425,7 +3425,7 @@ fn add_reject_multiple_git_ref_flags() { ); // --tag and --tag - uv_snapshot!(context + uv_snapshot!(context.filters(), context .add() .arg("foo") .arg("--tag") From 7e2822d694c6e806a9212b9391bcf41384c857d2 Mon Sep 17 00:00:00 2001 From: Charlie Marsh Date: Sun, 20 Oct 2024 14:42:21 -0400 Subject: [PATCH 3/3] Avoid panic when Git dependencies are included in fork markers (#8388) ## Summary Rather than relying on the distribution and package URL being the same (which isn't true for Git dependencies), we can just use the intersection of the markers directly. Closes https://github.com/astral-sh/uv/issues/8381. --- crates/uv-resolver/src/lock/mod.rs | 33 +++- crates/uv-resolver/src/resolution/graph.rs | 26 +-- crates/uv/tests/it/lock.rs | 174 ++++++++++++++++++++- 3 files changed, 204 insertions(+), 29 deletions(-) diff --git a/crates/uv-resolver/src/lock/mod.rs b/crates/uv-resolver/src/lock/mod.rs index 50694dd7e140..3ab114d7aa38 100644 --- a/crates/uv-resolver/src/lock/mod.rs +++ b/crates/uv-resolver/src/lock/mod.rs @@ -108,6 +108,21 @@ impl Lock { let mut packages = BTreeMap::new(); let requires_python = graph.requires_python.clone(); + // Determine the set of packages included at multiple versions. + let mut seen = FxHashSet::default(); + let mut duplicates = FxHashSet::default(); + for node_index in graph.petgraph.node_indices() { + let ResolutionGraphNode::Dist(dist) = &graph.petgraph[node_index] else { + continue; + }; + if !dist.is_base() { + continue; + } + if !seen.insert(dist.name()) { + duplicates.insert(dist.name()); + } + } + // Lock all base packages. for node_index in graph.petgraph.node_indices() { let ResolutionGraphNode::Dist(dist) = &graph.petgraph[node_index] else { @@ -116,10 +131,20 @@ impl Lock { if !dist.is_base() { continue; } - let fork_markers = graph - .fork_markers(dist.name(), &dist.version, dist.dist.version_or_url().url()) - .cloned() - .unwrap_or_default(); + + // If there are multiple distributions for the same package, include the markers of all + // forks that included the current distribution. + let fork_markers = if duplicates.contains(dist.name()) { + graph + .fork_markers + .iter() + .filter(|fork_markers| !fork_markers.is_disjoint(&dist.marker)) + .cloned() + .collect() + } else { + vec![] + }; + let mut package = Package::from_annotated_dist(dist, fork_markers, root)?; Self::remove_unreachable_wheels(graph, &requires_python, node_index, &mut package); diff --git a/crates/uv-resolver/src/resolution/graph.rs b/crates/uv-resolver/src/resolution/graph.rs index ff8a540de89a..9e379585000c 100644 --- a/crates/uv-resolver/src/resolution/graph.rs +++ b/crates/uv-resolver/src/resolution/graph.rs @@ -16,7 +16,7 @@ use uv_distribution_types::{ use uv_git::GitResolver; use uv_normalize::{ExtraName, GroupName, PackageName}; use uv_pep440::{Version, VersionSpecifier}; -use uv_pep508::{MarkerEnvironment, MarkerTree, MarkerTreeKind, VerbatimUrl}; +use uv_pep508::{MarkerEnvironment, MarkerTree, MarkerTreeKind}; use uv_pypi_types::{HashDigest, ParsedUrlError, Requirement, VerbatimParsedUrl, Yanked}; use crate::graph_ops::marker_reachability; @@ -31,7 +31,7 @@ use crate::{ ResolverMarkers, VersionsResponse, }; -pub(crate) type MarkersForDistribution = FxHashMap<(Version, Option), Vec>; +pub(crate) type MarkersForDistribution = Vec; /// A complete resolution graph in which every node represents a pinned package and every edge /// represents a dependency between two pinned packages. @@ -54,9 +54,6 @@ pub struct ResolutionGraph { pub(crate) overrides: Overrides, /// The options that were used to build the graph. pub(crate) options: Options, - /// If there are multiple options for a package, track which fork they belong to so we - /// can write that to the lockfile and later get the correct preference per fork back. - pub(crate) package_markers: FxHashMap, } #[derive(Debug, Clone)] @@ -131,8 +128,6 @@ impl ResolutionGraph { package_markers .entry(package.name.clone()) .or_default() - .entry((version.clone(), package.url.clone().map(|url| url.verbatim))) - .or_default() .push(markers.clone()); } } @@ -239,7 +234,6 @@ impl ResolutionGraph { let graph = Self { petgraph, requires_python, - package_markers, diagnostics, requirements: requirements.to_vec(), constraints: constraints.clone(), @@ -727,22 +721,6 @@ impl ResolutionGraph { Ok(conjunction) } - /// If there are multiple distributions for the same package name, return the markers of the - /// fork(s) that contained this distribution, otherwise return `None`. - pub fn fork_markers( - &self, - package_name: &PackageName, - version: &Version, - url: Option<&VerbatimUrl>, - ) -> Option<&Vec> { - let package_markers = &self.package_markers.get(package_name)?; - if package_markers.len() == 1 { - None - } else { - Some(&package_markers[&(version.clone(), url.cloned())]) - } - } - /// Returns a sequence of conflicting distribution errors from this /// resolution. /// diff --git a/crates/uv/tests/it/lock.rs b/crates/uv/tests/it/lock.rs index 1791353ac62a..3e9f41407635 100644 --- a/crates/uv/tests/it/lock.rs +++ b/crates/uv/tests/it/lock.rs @@ -2689,7 +2689,6 @@ fn lock_preference() -> Result<()> { fn lock_git_plus_prefix() -> Result<()> { let context = TestContext::new("3.12"); - // Lock against `main`. let pyproject_toml = context.temp_dir.child("pyproject.toml"); pyproject_toml.write_str( r#" @@ -2775,6 +2774,167 @@ fn lock_git_plus_prefix() -> Result<()> { Ok(()) } +#[test] +#[cfg(feature = "git")] +fn lock_partial_git() -> Result<()> { + let context = TestContext::new("3.12"); + + let pyproject_toml = context.temp_dir.child("pyproject.toml"); + pyproject_toml.write_str( + r#" + [project] + name = "project" + version = "0.1.0" + requires-python = ">=3.10" + dependencies = [ + "anyio @ git+https://github.com/agronholm/anyio@4.6.2 ; python_version < '3.12'", + "anyio ; python_version >= '3.12'" + ] + + [build-system] + requires = ["setuptools>=42"] + build-backend = "setuptools.build_meta" + "#, + )?; + + uv_snapshot!(context.filters(), context.lock(), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 7 packages in [TIME] + "###); + + let lock = context.read("uv.lock"); + + insta::with_settings!({ + filters => context.filters(), + }, { + assert_snapshot!( + lock, @r###" + version = 1 + requires-python = ">=3.10" + resolution-markers = [ + "python_full_version < '3.12'", + "python_full_version >= '3.12'", + ] + + [options] + exclude-newer = "2024-03-25T00:00:00Z" + + [[package]] + name = "anyio" + version = "4.3.0" + source = { registry = "https://pypi.org/simple" } + resolution-markers = [ + "python_full_version >= '3.12'", + ] + dependencies = [ + { name = "idna", marker = "python_full_version >= '3.12'" }, + { name = "sniffio", marker = "python_full_version >= '3.12'" }, + ] + sdist = { url = "https://files.pythonhosted.org/packages/db/4d/3970183622f0330d3c23d9b8a5f52e365e50381fd484d08e3285104333d3/anyio-4.3.0.tar.gz", hash = "sha256:f75253795a87df48568485fd18cdd2a3fa5c4f7c5be8e5e36637733fce06fed6", size = 159642 } + wheels = [ + { url = "https://files.pythonhosted.org/packages/14/fd/2f20c40b45e4fb4324834aea24bd4afdf1143390242c0b33774da0e2e34f/anyio-4.3.0-py3-none-any.whl", hash = "sha256:048e05d0f6caeed70d731f3db756d35dcc1f35747c8c403364a8332c630441b8", size = 85584 }, + ] + + [[package]] + name = "anyio" + version = "4.6.2" + source = { git = "https://github.com/agronholm/anyio?rev=4.6.2#c4844254e6db0cb804c240ba07405db73d810e0b" } + resolution-markers = [ + "python_full_version < '3.12'", + ] + dependencies = [ + { name = "exceptiongroup", marker = "python_full_version < '3.11'" }, + { name = "idna", marker = "python_full_version < '3.12'" }, + { name = "sniffio", marker = "python_full_version < '3.12'" }, + { name = "typing-extensions", marker = "python_full_version < '3.11'" }, + ] + + [[package]] + name = "exceptiongroup" + version = "1.2.0" + source = { registry = "https://pypi.org/simple" } + sdist = { url = "https://files.pythonhosted.org/packages/8e/1c/beef724eaf5b01bb44b6338c8c3494eff7cab376fab4904cfbbc3585dc79/exceptiongroup-1.2.0.tar.gz", hash = "sha256:91f5c769735f051a4290d52edd0858999b57e5876e9f85937691bd4c9fa3ed68", size = 26264 } + wheels = [ + { url = "https://files.pythonhosted.org/packages/b8/9a/5028fd52db10e600f1c4674441b968cf2ea4959085bfb5b99fb1250e5f68/exceptiongroup-1.2.0-py3-none-any.whl", hash = "sha256:4bfd3996ac73b41e9b9628b04e079f193850720ea5945fc96a08633c66912f14", size = 16210 }, + ] + + [[package]] + name = "idna" + version = "3.6" + source = { registry = "https://pypi.org/simple" } + sdist = { url = "https://files.pythonhosted.org/packages/bf/3f/ea4b9117521a1e9c50344b909be7886dd00a519552724809bb1f486986c2/idna-3.6.tar.gz", hash = "sha256:9ecdbbd083b06798ae1e86adcbfe8ab1479cf864e4ee30fe4e46a003d12491ca", size = 175426 } + wheels = [ + { url = "https://files.pythonhosted.org/packages/c2/e7/a82b05cf63a603df6e68d59ae6a68bf5064484a0718ea5033660af4b54a9/idna-3.6-py3-none-any.whl", hash = "sha256:c05567e9c24a6b9faaa835c4821bad0590fbb9d5779e7caa6e1cc4978e7eb24f", size = 61567 }, + ] + + [[package]] + name = "project" + version = "0.1.0" + source = { editable = "." } + dependencies = [ + { name = "anyio", version = "4.3.0", source = { registry = "https://pypi.org/simple" }, marker = "python_full_version >= '3.12'" }, + { name = "anyio", version = "4.6.2", source = { git = "https://github.com/agronholm/anyio?rev=4.6.2#c4844254e6db0cb804c240ba07405db73d810e0b" }, marker = "python_full_version < '3.12'" }, + ] + + [package.metadata] + requires-dist = [ + { name = "anyio", marker = "python_full_version < '3.12'", git = "https://github.com/agronholm/anyio?rev=4.6.2" }, + { name = "anyio", marker = "python_full_version >= '3.12'" }, + ] + + [[package]] + name = "sniffio" + version = "1.3.1" + source = { registry = "https://pypi.org/simple" } + sdist = { url = "https://files.pythonhosted.org/packages/a2/87/a6771e1546d97e7e041b6ae58d80074f81b7d5121207425c964ddf5cfdbd/sniffio-1.3.1.tar.gz", hash = "sha256:f4324edc670a0f49750a81b895f35c3adb843cca46f0530f79fc1babb23789dc", size = 20372 } + wheels = [ + { url = "https://files.pythonhosted.org/packages/e9/44/75a9c9421471a6c4805dbf2356f7c181a29c1879239abab1ea2cc8f38b40/sniffio-1.3.1-py3-none-any.whl", hash = "sha256:2f6da418d1f1e0fddd844478f41680e794e6051915791a034ff65e5f100525a2", size = 10235 }, + ] + + [[package]] + name = "typing-extensions" + version = "4.10.0" + source = { registry = "https://pypi.org/simple" } + sdist = { url = "https://files.pythonhosted.org/packages/16/3a/0d26ce356c7465a19c9ea8814b960f8a36c3b0d07c323176620b7b483e44/typing_extensions-4.10.0.tar.gz", hash = "sha256:b0abd7c89e8fb96f98db18d86106ff1d90ab692004eb746cf6eda2682f91b3cb", size = 77558 } + wheels = [ + { url = "https://files.pythonhosted.org/packages/f9/de/dc04a3ea60b22624b51c703a84bbe0184abcd1d0b9bc8074b5d6b7ab90bb/typing_extensions-4.10.0-py3-none-any.whl", hash = "sha256:69b1a937c3a517342112fb4c6df7e72fc39a38e7891a5730ed4985b5214b5475", size = 33926 }, + ] + "### + ); + }); + + // Re-run with `--locked`. + uv_snapshot!(context.filters(), context.lock().arg("--locked"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Resolved 7 packages in [TIME] + "###); + + // Install from the lockfile, excluding development dependencies. + uv_snapshot!(context.filters(), context.sync().arg("--frozen").arg("--no-dev"), @r###" + success: true + exit_code: 0 + ----- stdout ----- + + ----- stderr ----- + Prepared 4 packages in [TIME] + Installed 4 packages in [TIME] + + anyio==4.3.0 + + idna==3.6 + + project==0.1.0 (from file://[TEMP_DIR]/) + + sniffio==1.3.1 + "###); + + Ok(()) +} + /// Respect locked versions with `uv lock`, unless `--upgrade` is passed. #[test] #[cfg(feature = "git")] @@ -14982,6 +15142,9 @@ fn lock_multiple_sources_index() -> Result<()> { name = "jinja2" version = "3.1.3" source = { registry = "https://download.pytorch.org/whl/cu118" } + resolution-markers = [ + "sys_platform == 'win32'", + ] dependencies = [ { name = "markupsafe", marker = "sys_platform == 'win32'" }, ] @@ -14993,6 +15156,9 @@ fn lock_multiple_sources_index() -> Result<()> { name = "jinja2" version = "3.1.3" source = { registry = "https://download.pytorch.org/whl/cu124" } + resolution-markers = [ + "sys_platform != 'win32'", + ] dependencies = [ { name = "markupsafe", marker = "sys_platform != 'win32'" }, ] @@ -15262,6 +15428,9 @@ fn lock_multiple_sources_index_explicit() -> Result<()> { name = "jinja2" version = "3.1.3" source = { registry = "https://pypi.org/simple" } + resolution-markers = [ + "sys_platform != 'win32'", + ] dependencies = [ { name = "markupsafe", marker = "sys_platform != 'win32'" }, ] @@ -15274,6 +15443,9 @@ fn lock_multiple_sources_index_explicit() -> Result<()> { name = "jinja2" version = "3.1.3" source = { registry = "https://test.pypi.org/simple" } + resolution-markers = [ + "sys_platform == 'win32'", + ] dependencies = [ { name = "markupsafe", marker = "sys_platform == 'win32'" }, ]