From 08296cdf0554c9c0b42b4609221b9fe6455bdfa8 Mon Sep 17 00:00:00 2001 From: Chris Pryer Date: Tue, 6 Jun 2023 12:04:47 -0400 Subject: [PATCH 01/19] Add empty monotrail-utils crate --- Cargo.lock | 4 ++++ Cargo.toml | 3 ++- monotrail-utils/Cargo.toml | 8 ++++++++ monotrail-utils/src/lib.rs | 14 ++++++++++++++ 4 files changed, 28 insertions(+), 1 deletion(-) create mode 100644 monotrail-utils/Cargo.toml create mode 100644 monotrail-utils/src/lib.rs diff --git a/Cargo.lock b/Cargo.lock index bcff650..e991c45 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1149,6 +1149,10 @@ dependencies = [ "zstd", ] +[[package]] +name = "monotrail-utils" +version = "0.1.0" + [[package]] name = "nix" version = "0.26.2" diff --git a/Cargo.toml b/Cargo.toml index f3665d1..aad5b21 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -7,7 +7,8 @@ readme = "Readme.md" [workspace] members = [ - "install-wheel-rs" + "install-wheel-rs", + "monotrail-utils", ] [lib] diff --git a/monotrail-utils/Cargo.toml b/monotrail-utils/Cargo.toml new file mode 100644 index 0000000..e4a76e5 --- /dev/null +++ b/monotrail-utils/Cargo.toml @@ -0,0 +1,8 @@ +[package] +name = "monotrail-utils" +version = "0.1.0" +edition = "2021" + +# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html + +[dependencies] diff --git a/monotrail-utils/src/lib.rs b/monotrail-utils/src/lib.rs new file mode 100644 index 0000000..7d12d9a --- /dev/null +++ b/monotrail-utils/src/lib.rs @@ -0,0 +1,14 @@ +pub fn add(left: usize, right: usize) -> usize { + left + right +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn it_works() { + let result = add(2, 2); + assert_eq!(result, 4); + } +} From d23ff81b116dcae529deab50d50a305ecdd9da6a Mon Sep 17 00:00:00 2001 From: Chris Pryer Date: Tue, 6 Jun 2023 12:11:51 -0400 Subject: [PATCH 02/19] Add requirements_txt module to monotrail-utils Does not include poetry integration --- Cargo.lock | 13 + monotrail-utils/Cargo.toml | 15 +- monotrail-utils/src/lib.rs | 15 +- monotrail-utils/src/requirements_txt.rs | 610 ++++++++++++++++++++++++ 4 files changed, 638 insertions(+), 15 deletions(-) create mode 100644 monotrail-utils/src/requirements_txt.rs diff --git a/Cargo.lock b/Cargo.lock index e991c45..afba27f 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1152,6 +1152,19 @@ dependencies = [ [[package]] name = "monotrail-utils" version = "0.1.0" +dependencies = [ + "anyhow", + "fs-err", + "indoc 2.0.1", + "logtest", + "pep508_rs", + "serde", + "serde_json", + "tempfile", + "toml", + "tracing", + "unscanny", +] [[package]] name = "nix" diff --git a/monotrail-utils/Cargo.toml b/monotrail-utils/Cargo.toml index e4a76e5..3594c09 100644 --- a/monotrail-utils/Cargo.toml +++ b/monotrail-utils/Cargo.toml @@ -3,6 +3,17 @@ name = "monotrail-utils" version = "0.1.0" edition = "2021" -# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html - [dependencies] +anyhow = "1.0.65" +fs-err = "2.8.1" +pep508_rs = { git = "https://github.com/konstin/pep508_rs", rev = "df87d4ff0f0f3554780ab82680539cb190b0a585", features = ["serde"] } +serde = { version = "1.0.145", features = ["derive"] } +serde_json = "1.0.85" +toml = "0.7.2" +tracing = "0.1.36" +unscanny = "0.1.0" + +[dev-dependencies] +indoc = "2.0.0" +logtest = "2.0.0" +tempfile = "3.3.0" diff --git a/monotrail-utils/src/lib.rs b/monotrail-utils/src/lib.rs index 7d12d9a..11a7886 100644 --- a/monotrail-utils/src/lib.rs +++ b/monotrail-utils/src/lib.rs @@ -1,14 +1,3 @@ -pub fn add(left: usize, right: usize) -> usize { - left + right -} +pub use requirements_txt::RequirementsTxt; -#[cfg(test)] -mod tests { - use super::*; - - #[test] - fn it_works() { - let result = add(2, 2); - assert_eq!(result, 4); - } -} +mod requirements_txt; diff --git a/monotrail-utils/src/requirements_txt.rs b/monotrail-utils/src/requirements_txt.rs new file mode 100644 index 0000000..ece4097 --- /dev/null +++ b/monotrail-utils/src/requirements_txt.rs @@ -0,0 +1,610 @@ +//! Parses a subset of requirement.txt syntax +//! +//! +//! +//! Supported: +//! * [PEP 508 requirements](https://packaging.python.org/en/latest/specifications/dependency-specifiers/) +//! * `-r` +//! * `-c` +//! * `--hash` (postfix) +//! * `-e` +//! +//! Unsupported: +//! * `-e `. TBD +//! * ``. TBD +//! * ``. TBD +//! * Options without a requirement, such as `--find-links` or `--index-url` +//! +//! Grammar as implemented: +//! +//! ```text +//! file = (statement | empty ('#' any*)? '\n')* +//! empty = whitespace* +//! statement = constraint_include | requirements_include | editable_requirement | requirement +//! constraint_include = '-c' ('=' | wrappable_whitespaces) filepath +//! requirements_include = '-r' ('=' | wrappable_whitespaces) filepath +//! editable_requirement = '-e' ('=' | wrappable_whitespaces) requirement +//! # We check whether the line starts with a letter or a number, in that case we assume it's a +//! # PEP 508 requirement +//! # https://packaging.python.org/en/latest/specifications/name-normalization/#valid-non-normalized-names +//! # This does not (yet?) support plain files or urls, we use a letter or a number as first +//! # character to assume a PEP 508 requirement +//! requirement = [a-zA-Z0-9] pep508_grammar_tail wrappable_whitespaces hashes +//! hashes = ('--hash' ('=' | wrappable_whitespaces) [a-zA-Z0-9-_]+ ':' [a-zA-Z0-9-_] wrappable_whitespaces+)* +//! # This should indicate a single backslash before a newline +//! wrappable_whitespaces = whitespace ('\\\n' | whitespace)* +//! ``` + +use fs_err as fs; +use pep508_rs::{Pep508Error, Requirement}; +use serde::{Deserialize, Serialize}; +use std::fmt::{Display, Formatter}; +use std::io; +use std::path::{Path, PathBuf}; +use std::str::FromStr; +use tracing::warn; +use unscanny::{Pattern, Scanner}; + +/// We emit one of those for each requirements.txt entry +enum RequirementsTxtStatement { + /// `-r` inclusion filename + Requirements { + filename: String, + start: usize, + end: usize, + }, + /// `-c` inclusion filename + Constraint { + filename: String, + start: usize, + end: usize, + }, + /// PEP 508 requirement plus metadata + RequirementEntry(RequirementEntry), +} + +/// A [Requirement] with additional metadata from the requirements.txt, currently only hashes but in +/// the future also editable an similar information +#[derive(Debug, Deserialize, Clone, Eq, PartialEq, Serialize)] +pub struct RequirementEntry { + /// The actual PEP 508 requirement + pub requirement: Requirement, + /// Hashes of the downloadable packages + pub hashes: Vec, + /// Editable installation, see e.g. + pub editable: bool, +} + +impl Display for RequirementEntry { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + if self.editable { + write!(f, "-e ")?; + } + write!(f, "{}", self.requirement)?; + for hash in &self.hashes { + write!(f, " --hash {}", hash)? + } + + Ok(()) + } +} + +/// Parsed and flattened requirements.txt with requirements and constraints +#[derive(Debug, Deserialize, Clone, Default, Eq, PartialEq, Serialize)] +pub struct RequirementsTxt { + /// The actual requirements with the hashes + pub requirements: Vec, + /// Constraints included with `-c` + pub constraints: Vec, +} + +impl RequirementsTxt { + /// See module level documentation + pub fn parse( + requirements_txt: impl AsRef, + working_dir: impl AsRef, + ) -> Result { + let content = + fs::read_to_string(&requirements_txt).map_err(|err| RequirementsTxtFileError { + file: requirements_txt.as_ref().to_path_buf(), + error: RequirementsTxtParserError::IO(err), + })?; + let data = + Self::parse_inner(&content, working_dir).map_err(|err| RequirementsTxtFileError { + file: requirements_txt.as_ref().to_path_buf(), + error: err, + })?; + if data == Self::default() { + warn!( + "Requirements file {} does not contain any dependencies", + requirements_txt.as_ref().display() + ); + } + Ok(data) + } + + /// See module level documentation + /// + /// Note that all relative paths are dependent on the current working dir, not on the location + /// of the file + pub fn parse_inner( + content: &str, + working_dir: impl AsRef, + ) -> Result { + let mut s = Scanner::new(&content); + + let mut data = Self::default(); + while let Some(statement) = parse_entry(&mut s, &content)? { + match statement { + RequirementsTxtStatement::Requirements { + filename, + start, + end, + } => { + let sub_file = working_dir.as_ref().join(filename); + let sub_requirements = + Self::parse(&sub_file, working_dir.as_ref()).map_err(|err| { + RequirementsTxtParserError::Subfile { + source: Box::new(err), + start, + end, + } + })?; + // Add each to the correct category + data.update_from(sub_requirements); + } + RequirementsTxtStatement::Constraint { + filename, + start, + end, + } => { + let sub_file = working_dir.as_ref().join(filename); + let sub_constraints = + Self::parse(&sub_file, working_dir.as_ref()).map_err(|err| { + RequirementsTxtParserError::Subfile { + source: Box::new(err), + start, + end, + } + })?; + // Here we add both to constraints + data.constraints.extend( + sub_constraints + .requirements + .into_iter() + .map(|requirement_entry| requirement_entry.requirement), + ); + data.constraints.extend(sub_constraints.constraints); + } + RequirementsTxtStatement::RequirementEntry(requirement_entry) => { + data.requirements.push(requirement_entry); + } + } + } + Ok(data) + } + + /// Merges other into self + pub fn update_from(&mut self, other: RequirementsTxt) { + self.requirements.extend(other.requirements); + self.constraints.extend(other.constraints); + } +} + +/// Parse a single entry, that is a requirement, an inclusion or a comment line +/// +/// Consumes all preceding trivia (whitespace and comments). If it returns None, we've reached +/// the end of file +fn parse_entry( + s: &mut Scanner, + content: &str, +) -> Result, RequirementsTxtParserError> { + // Eat all preceding whitespace, this may run us to the end of file + eat_wrappable_whitespace(s); + while s.at(['\n', '\r', '#']) { + // skip comments + eat_trailing_line(s)?; + eat_wrappable_whitespace(s); + } + + let start = s.cursor(); + Ok(Some(if s.eat_if("-r") { + let requirements_file = parse_value(s, |c: char| !['\n', '\r', '#'].contains(&c))?; + let end = s.cursor(); + eat_trailing_line(s)?; + RequirementsTxtStatement::Requirements { + filename: requirements_file.to_string(), + start, + end, + } + } else if s.eat_if("-c") { + let constraints_file = parse_value(s, |c: char| !['\n', '\r', '#'].contains(&c))?; + let end = s.cursor(); + eat_trailing_line(s)?; + RequirementsTxtStatement::Constraint { + filename: constraints_file.to_string(), + start, + end, + } + } else if s.eat_if("-e") { + let (requirement, hashes) = parse_requirement_and_hashes(s, &content)?; + RequirementsTxtStatement::RequirementEntry(RequirementEntry { + requirement, + hashes, + editable: true, + }) + } else if s.at(char::is_ascii_alphanumeric) { + let (requirement, hashes) = parse_requirement_and_hashes(s, &content)?; + RequirementsTxtStatement::RequirementEntry(RequirementEntry { + requirement, + hashes, + editable: false, + }) + } else if let Some(char) = s.peek() { + return Err(RequirementsTxtParserError::Parser { + message: format!( + "Unexpected '{}', expected '-c', '-e', '-r' or the start of a requirement", + char + ), + location: s.cursor(), + }); + } else { + // EOF + return Ok(None); + })) +} + +/// Eat whitespace and ignore newlines escaped with a backslash +fn eat_wrappable_whitespace<'a>(s: &mut Scanner<'a>) -> &'a str { + let start = s.cursor(); + s.eat_while([' ', '\t']); + // Allow multiple escaped line breaks + // With the order we support `\n`, `\r`, `\r\n` without accidentally eating a `\n\r` + while s.eat_if("\\\n") || s.eat_if("\\\r\n") || s.eat_if("\\\r") { + s.eat_while([' ', '\t']); + } + s.from(start) +} + +/// Eats the end of line or a potential trailing comma +fn eat_trailing_line(s: &mut Scanner) -> Result<(), RequirementsTxtParserError> { + s.eat_while([' ', '\t']); + match s.eat() { + None | Some('\n') => {} // End of file or end of line, nothing to do + Some('\r') => { + s.eat_if('\n'); // `\r\n`, but just `\r` is also accepted + } + Some('#') => { + s.eat_until(['\r', '\n']); + if s.at('\r') { + s.eat_if('\n'); // `\r\n`, but just `\r` is also accepted + } + } + Some(other) => { + return Err(RequirementsTxtParserError::Parser { + message: format!("Expected comment or end-of-line, found '{}'", other), + location: s.cursor(), + }) + } + } + Ok(()) +} + +/// Parse a PEP 508 requirement with optional trailing hashes +fn parse_requirement_and_hashes( + s: &mut Scanner, + content: &str, +) -> Result<(Requirement, Vec), RequirementsTxtParserError> { + // PEP 508 requirement + let start = s.cursor(); + // Termination: s.eat() eventually becomes None + let (end, has_hashes) = loop { + let end = s.cursor(); + + // We look for the end of the line ... + if s.eat_if('\n') { + break (end, false); + } + if s.eat_if('\r') { + s.eat_if('\n'); // Support `\r\n` but also accept stray `\r` + break (end, false); + } + // ... or `--hash`, an escaped newline or a comment separated by whitespace ... + if !eat_wrappable_whitespace(s).is_empty() { + if s.after().starts_with("--") { + break (end, true); + } else if s.eat_if('#') { + s.eat_until(['\r', '\n']); + if s.at('\r') { + s.eat_if('\n'); // `\r\n`, but just `\r` is also accepted + } + break (end, false); + } else { + continue; + } + } + // ... or the end of the file, which works like the end of line + if s.eat().is_none() { + break (end, false); + } + }; + let requirement = Requirement::from_str(&content[start..end]).map_err(|err| { + RequirementsTxtParserError::Pep508 { + source: err, + start, + end, + } + })?; + let hashes = if has_hashes { + let hashes = parse_hashes(s)?; + eat_trailing_line(s)?; + hashes + } else { + Vec::new() + }; + Ok((requirement, hashes)) +} + +/// Parse `--hash=... --hash ...` after a requirement +fn parse_hashes(s: &mut Scanner) -> Result, RequirementsTxtParserError> { + let mut hashes = Vec::new(); + if s.eat_while("--hash").is_empty() { + return Err(RequirementsTxtParserError::Parser { + message: format!( + "Expected '--hash', found '{:?}'", + s.eat_while(|c: char| !c.is_whitespace()) + ), + location: s.cursor(), + }); + } + let hash = parse_value(s, |c: char| !c.is_whitespace())?; + hashes.push(hash.to_string()); + loop { + eat_wrappable_whitespace(s); + if !s.eat_if("--hash") { + break; + } + let hash = parse_value(s, |c: char| !c.is_whitespace())?; + hashes.push(hash.to_string()); + } + Ok(hashes) +} + +/// In `-=` or `- value`, this parses the part after the key +fn parse_value<'a, T>( + s: &mut Scanner<'a>, + while_pattern: impl Pattern, +) -> Result<&'a str, RequirementsTxtParserError> { + if s.eat_if('=') { + // Explicit equals sign + Ok(s.eat_while(while_pattern).trim_end()) + } else if s.eat_if(char::is_whitespace) { + // Key and value are separated by whitespace instead + s.eat_whitespace(); + Ok(s.eat_while(while_pattern).trim_end()) + } else { + Err(RequirementsTxtParserError::Parser { + message: format!("Expected '=' or whitespace, found {:?}", s.peek()), + location: s.cursor(), + }) + } +} + +/// Error parsing requirements.txt, wrapper with filename +#[derive(Debug)] +pub struct RequirementsTxtFileError { + file: PathBuf, + error: RequirementsTxtParserError, +} + +/// Error parsing requirements.txt, error disambiguation +#[derive(Debug)] +pub enum RequirementsTxtParserError { + IO(io::Error), + Parser { + message: String, + location: usize, + }, + Pep508 { + source: Pep508Error, + start: usize, + end: usize, + }, + Subfile { + source: Box, + start: usize, + end: usize, + }, +} + +impl Display for RequirementsTxtFileError { + fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { + match &self.error { + RequirementsTxtParserError::IO(err) => err.fmt(f), + RequirementsTxtParserError::Parser { message, location } => { + write!( + f, + "{} in {} position {}", + message, + self.file.display(), + location + ) + } + RequirementsTxtParserError::Pep508 { start, end, .. } => { + write!( + f, + "Couldn't parse requirement in {} position {} to {}", + self.file.display(), + start, + end, + ) + } + RequirementsTxtParserError::Subfile { start, end, .. } => { + write!( + f, + "Error parsing file included into {} at position {} to {}", + self.file.display(), + start, + end + ) + } + } + } +} + +impl std::error::Error for RequirementsTxtFileError { + fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { + match &self.error { + RequirementsTxtParserError::IO(err) => err.source(), + RequirementsTxtParserError::Pep508 { source, .. } => Some(source), + RequirementsTxtParserError::Subfile { source, .. } => Some(source.as_ref()), + _ => None, + } + } +} + +#[cfg(test)] +mod test { + use crate::requirements_txt::RequirementsTxt; + use fs_err as fs; + use indoc::indoc; + use logtest::Logger; + use std::path::Path; + use tempfile::tempdir; + use tracing::log::Level; + + #[test] + fn test_requirements_txt_parsing() { + let working_dir = Path::new("test-data").join("requirements-txt"); + for dir_entry in fs::read_dir(&working_dir).unwrap() { + let dir_entry = dir_entry.unwrap().path(); + if dir_entry.extension().unwrap_or_default().to_str().unwrap() != "txt" { + continue; + } + let actual = RequirementsTxt::parse(&dir_entry, &working_dir).unwrap(); + let fixture = dir_entry.with_extension("json"); + // Update the json fixtures + // fs::write(&fixture, &serde_json::to_string_pretty(&actual).unwrap()).unwrap(); + let snapshot = serde_json::from_str(&fs::read_to_string(fixture).unwrap()).unwrap(); + assert_eq!(actual, snapshot); + } + } + + /// Test with flipped line endings + #[test] + fn test_other_line_endings() { + let temp_dir = tempdir().unwrap(); + let mut files = Vec::new(); + let working_dir = Path::new("test-data").join("requirements-txt"); + for dir_entry in fs::read_dir(&working_dir).unwrap() { + let dir_entry = dir_entry.unwrap(); + if dir_entry + .path() + .extension() + .unwrap_or_default() + .to_str() + .unwrap() + != "txt" + { + continue; + } + let copied = temp_dir.path().join(dir_entry.file_name()); + let original = fs::read_to_string(dir_entry.path()).unwrap(); + // Replace line endings with the other choice. This works even if you use git with LF + // only on windows. + let changed = if original.contains("\r\n") { + original.replace("\r\n", "\n") + } else { + original.replace('\n', "\r\n") + }; + fs::write(&copied, &changed).unwrap(); + files.push((copied, dir_entry.path().with_extension("json"))); + } + for (file, fixture) in files { + let actual = RequirementsTxt::parse(&file, &working_dir).unwrap(); + let snapshot = serde_json::from_str(&fs::read_to_string(fixture).unwrap()).unwrap(); + assert_eq!(actual, snapshot); + } + } + + /// Pass test only - currently fails due to `-e ./` in pyproject.toml-constrained.in + #[test] + #[ignore] + fn test_pydantic() { + let working_dir = Path::new("test-data").join("requirements-pydantic"); + for basic in fs::read_dir(&working_dir).unwrap() { + let basic = basic.unwrap().path(); + if !["txt", "in"].contains(&basic.extension().unwrap_or_default().to_str().unwrap()) { + continue; + } + RequirementsTxt::parse(&basic, &working_dir).unwrap(); + } + } + + #[test] + fn test_invalid_include_missing_file() { + let working_dir = Path::new("test-data").join("requirements-txt"); + let basic = working_dir.join("invalid-include"); + let missing = working_dir.join("missing.txt"); + let err = RequirementsTxt::parse(&basic, &working_dir).unwrap_err(); + let errors = anyhow::Error::new(err) + .chain() + .map(ToString::to_string) + .collect::>(); + assert_eq!(errors.len(), 3); + assert_eq!( + errors[0], + format!( + "Error parsing file included into {} at position 0 to 14", + basic.display() + ) + ); + assert_eq!( + errors[1], + format!("failed to open file `{}`", missing.display()), + ); + // The last error message is os specific + } + + #[test] + fn test_invalid_requirement() { + let working_dir = Path::new("test-data").join("requirements-txt"); + let basic = working_dir.join("invalid-requirement"); + let err = RequirementsTxt::parse(&basic, &working_dir).unwrap_err(); + let errors = anyhow::Error::new(err) + .chain() + .map(ToString::to_string) + .collect::>(); + let expected = &[ + format!( + "Couldn't parse requirement in {} position 0 to 15", + basic.display() + ), + indoc! {" + Expected an alphanumeric character starting the extra name, found 'ö' + numpy[ö]==1.29 + ^" + } + .to_string(), + ]; + assert_eq!(errors, expected) + } + + #[test] + fn test_empty_file() { + let working_dir = Path::new("test-data").join("requirements-txt"); + let path = working_dir.join("empty.txt"); + + // TODO(konstin) I think that logger isn't thread safe + let logger = Logger::start(); + RequirementsTxt::parse(&path, &working_dir).unwrap(); + let warnings: Vec<_> = logger + .into_iter() + .filter(|message| message.level() >= Level::Warn) + .collect(); + assert_eq!(warnings.len(), 1, "{:?}", warnings); + assert!(warnings[0] + .args() + .ends_with("does not contain any dependencies")); + } +} From d64b333518630d0f7e17e6e64dd9bc5e75b171d9 Mon Sep 17 00:00:00 2001 From: Chris Pryer Date: Wed, 7 Jun 2023 13:18:03 -0400 Subject: [PATCH 03/19] Use pep508_rs v0.2.1 --- Cargo.lock | 20 ++++++++++++++++++-- monotrail-utils/Cargo.toml | 2 +- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index afba27f..83ec291 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1127,7 +1127,7 @@ dependencies = [ "mockito", "nix", "pep440_rs", - "pep508_rs", + "pep508_rs 0.2.0", "pyo3", "rayon", "regex", @@ -1157,7 +1157,7 @@ dependencies = [ "fs-err", "indoc 2.0.1", "logtest", - "pep508_rs", + "pep508_rs 0.2.1", "serde", "serde_json", "tempfile", @@ -1303,6 +1303,22 @@ dependencies = [ "url", ] +[[package]] +name = "pep508_rs" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c0713d7bb861ca2b7d4c50a38e1f31a4b63a2e2df35ef1e5855cc29e108453e2" +dependencies = [ + "once_cell", + "pep440_rs", + "regex", + "serde", + "thiserror", + "tracing", + "unicode-width", + "url", +] + [[package]] name = "percent-encoding" version = "2.2.0" diff --git a/monotrail-utils/Cargo.toml b/monotrail-utils/Cargo.toml index 3594c09..575ea8a 100644 --- a/monotrail-utils/Cargo.toml +++ b/monotrail-utils/Cargo.toml @@ -6,7 +6,7 @@ edition = "2021" [dependencies] anyhow = "1.0.65" fs-err = "2.8.1" -pep508_rs = { git = "https://github.com/konstin/pep508_rs", rev = "df87d4ff0f0f3554780ab82680539cb190b0a585", features = ["serde"] } +pep508_rs = { version = "0.2.1", features = ["serde"] } serde = { version = "1.0.145", features = ["derive"] } serde_json = "1.0.85" toml = "0.7.2" From 97987ad5ef5fe8c5b816d2b581895d96c3cc9112 Mon Sep 17 00:00:00 2001 From: Chris Pryer Date: Wed, 7 Jun 2023 13:22:40 -0400 Subject: [PATCH 04/19] Add crate doc comment to monotrail-utils --- monotrail-utils/src/lib.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/monotrail-utils/src/lib.rs b/monotrail-utils/src/lib.rs index 11a7886..1dab22e 100644 --- a/monotrail-utils/src/lib.rs +++ b/monotrail-utils/src/lib.rs @@ -1,3 +1,5 @@ +//! Implements stand-alone utilities used by `monotrail` + pub use requirements_txt::RequirementsTxt; mod requirements_txt; From 74853ff40d3d1da0b2efe225bcf2979ac3c5a06a Mon Sep 17 00:00:00 2001 From: Chris Pryer Date: Wed, 7 Jun 2023 13:48:18 -0400 Subject: [PATCH 05/19] Start with monotrail-utils v0.0.1 --- Cargo.lock | 2 +- monotrail-utils/Cargo.toml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 83ec291..146b0d2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1151,7 +1151,7 @@ dependencies = [ [[package]] name = "monotrail-utils" -version = "0.1.0" +version = "0.0.1" dependencies = [ "anyhow", "fs-err", diff --git a/monotrail-utils/Cargo.toml b/monotrail-utils/Cargo.toml index 575ea8a..e4f1688 100644 --- a/monotrail-utils/Cargo.toml +++ b/monotrail-utils/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "monotrail-utils" -version = "0.1.0" +version = "0.0.1" edition = "2021" [dependencies] From cee49b5be06385d7ce3b1c9098e6ef81b03a489a Mon Sep 17 00:00:00 2001 From: Chris Pryer Date: Wed, 7 Jun 2023 16:53:36 -0400 Subject: [PATCH 06/19] Use pep508_rs v0.2.1 in monotrail --- Cargo.lock | 20 +++----------------- Cargo.toml | 8 ++------ 2 files changed, 5 insertions(+), 23 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 146b0d2..fe3799a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1125,9 +1125,10 @@ dependencies = [ "libz-sys", "logtest", "mockito", + "monotrail-utils", "nix", "pep440_rs", - "pep508_rs 0.2.0", + "pep508_rs", "pyo3", "rayon", "regex", @@ -1157,7 +1158,7 @@ dependencies = [ "fs-err", "indoc 2.0.1", "logtest", - "pep508_rs 0.2.1", + "pep508_rs", "serde", "serde_json", "tempfile", @@ -1288,21 +1289,6 @@ dependencies = [ "unicode-width", ] -[[package]] -name = "pep508_rs" -version = "0.2.0" -source = "git+https://github.com/konstin/pep508_rs?rev=df87d4ff0f0f3554780ab82680539cb190b0a585#df87d4ff0f0f3554780ab82680539cb190b0a585" -dependencies = [ - "once_cell", - "pep440_rs", - "regex", - "serde", - "thiserror", - "tracing", - "unicode-width", - "url", -] - [[package]] name = "pep508_rs" version = "0.2.1" diff --git a/Cargo.toml b/Cargo.toml index aad5b21..7340b76 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -30,10 +30,10 @@ libc = "0.2.133" libloading = "0.8.0" # For the zig build libz-sys = { version = "1.1.8", features = ["static"] } +monotrail-utils = { version = "0.0.1", path = "monotrail-utils" } nix = { version = "0.26.1", features = ["process"] } pep440_rs = "0.3.2" -#pep508_rs = { version = "0.2.0", features = ["serde"] } -pep508_rs = { git = "https://github.com/konstin/pep508_rs", rev = "df87d4ff0f0f3554780ab82680539cb190b0a585", features = ["serde"] } +pep508_rs = { version = "0.2.1", features = ["serde"] } pyo3 = { version = "0.18.1", features = ["extension-module", "abi3-py37"], optional = true } rayon = "1.5.3" regex = "1.6.0" @@ -58,10 +58,6 @@ default = ["vendored"] python_bindings = ["pyo3", "install-wheel-rs/python_bindings"] vendored = ["git2/vendored-openssl", "git2/vendored-libgit2"] -# zip implementation -[profile.dev.package.adler] -opt-level = 3 - # https://doc.rust-lang.org/cargo/reference/profiles.html#release [profile.perf] inherits = "release" From 5cb1e076364fc88f43ff1ebf40bb4cc989deeefd Mon Sep 17 00:00:00 2001 From: Chris Pryer Date: Wed, 7 Jun 2023 16:54:14 -0400 Subject: [PATCH 07/19] Use RequirementsTxt in poetry_integration --- src/cli.rs | 2 +- src/lib.rs | 2 - src/monotrail.rs | 8 +- src/poetry_integration/read_dependencies.rs | 72 ++- src/requirements_txt.rs | 684 -------------------- 5 files changed, 76 insertions(+), 692 deletions(-) delete mode 100644 src/requirements_txt.rs diff --git a/src/cli.rs b/src/cli.rs index c92b756..fd01e86 100644 --- a/src/cli.rs +++ b/src/cli.rs @@ -6,7 +6,6 @@ use crate::package_index::download_distribution; use crate::poetry_integration::read_dependencies::{read_poetry_specs, read_toml_files}; use crate::poetry_integration::run::poetry_run; use crate::ppipx; -use crate::requirements_txt::RequirementsTxt; use crate::spec::RequestedSpec; use crate::utils::cache_dir; use crate::venv_parser::get_venv_python_version; @@ -14,6 +13,7 @@ use crate::verify_installation::verify_installation; use anyhow::{bail, Context}; use clap::Parser; use install_wheel_rs::{compatible_tags, Arch, InstallLocation, Os, WheelInstallerError}; +use monotrail_utils::RequirementsTxt; use pep440_rs::Operator; use pep508_rs::VersionOrUrl; use std::env; diff --git a/src/lib.rs b/src/lib.rs index e53385c..05b7573 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -18,7 +18,6 @@ pub use cli::{run_cli, Cli}; pub use inject_and_run::{parse_major_minor, run_python_args}; use poetry_integration::read_dependencies::read_poetry_specs; -pub use requirements_txt::RequirementsTxt; #[doc(hidden)] pub use utils::assert_cli_error; @@ -32,7 +31,6 @@ mod poetry_integration; mod ppipx; #[cfg(feature = "python_bindings")] mod python_bindings; -mod requirements_txt; mod source_distribution; mod spec; mod standalone_python; diff --git a/src/monotrail.rs b/src/monotrail.rs index 6f4e935..e3befe4 100644 --- a/src/monotrail.rs +++ b/src/monotrail.rs @@ -4,9 +4,10 @@ use crate::inject_and_run::{ }; use crate::install::{install_all, InstalledPackage}; use crate::poetry_integration::lock::poetry_resolve; -use crate::poetry_integration::read_dependencies::{poetry_spec_from_dir, specs_from_git}; +use crate::poetry_integration::read_dependencies::{ + poetry_spec_from_dir, read_requirements_for_poetry, specs_from_git, +}; use crate::read_poetry_specs; -use crate::requirements_txt::RequirementsTxt; use crate::spec::RequestedSpec; use crate::standalone_python::provision_python; use crate::utils::{cache_dir, get_dir_content}; @@ -536,8 +537,7 @@ pub fn specs_from_requirements_txt_resolved( lockfile: Option<&str>, python_context: &PythonContext, ) -> anyhow::Result<(Vec, String)> { - let requirements = RequirementsTxt::parse(&requirements_txt, ¤t_dir()?)? - .into_poetry(&requirements_txt)?; + let requirements = read_requirements_for_poetry(&requirements_txt, ¤t_dir()?)?; // We don't know whether the requirements.txt is from `pip freeze` or just a list of // version, so we let it go through poetry resolve either way. For a frozen file // there will just be no change diff --git a/src/poetry_integration/read_dependencies.rs b/src/poetry_integration/read_dependencies.rs index c74062f..c017140 100644 --- a/src/poetry_integration/read_dependencies.rs +++ b/src/poetry_integration/read_dependencies.rs @@ -11,7 +11,8 @@ use crate::utils::cache_dir; use anyhow::{bail, Context}; use fs_err as fs; use install_wheel_rs::{normalize_name, Script, WheelFilename, WheelInstallerError}; -use pep508_rs::MarkerEnvironment; +use monotrail_utils::RequirementsTxt; +use pep508_rs::{MarkerEnvironment, VersionOrUrl}; use regex::Regex; use sha2::{Digest, Sha256}; use std::collections::{BTreeMap, HashMap, HashSet, VecDeque}; @@ -427,10 +428,51 @@ pub fn poetry_spec_from_dir( Ok((specs, scripts, lockfile)) } +/// Reads and parses requirements into poetry dependencies from a requirements file. +pub fn read_requirements_for_poetry( + requirements_txt: impl AsRef, + working_dir: impl AsRef, +) -> anyhow::Result> { + let requirements_txt = requirements_txt.as_ref(); + let requirements = RequirementsTxt::parse(requirements_txt, working_dir)?; + if !requirements.constraints.is_empty() { + bail!( + "Constraints (`-c`) from {} are not supported yet", + requirements_txt.display() + ); + } + let mut poetry_requirements: BTreeMap = BTreeMap::new(); + for requirement_entry in requirements.requirements { + let version = match requirement_entry.requirement.version_or_url { + None => "*".to_string(), + Some(VersionOrUrl::Url(_)) => { + bail!( + "Unsupported url requirement in {}: '{}'", + requirements_txt.display(), + requirement_entry.requirement, + ) + } + Some(VersionOrUrl::VersionSpecifier(specifiers)) => specifiers.to_string(), + }; + + let dep = poetry_toml::Dependency::Expanded { + version: Some(version), + optional: Some(false), + extras: requirement_entry.requirement.extras.clone(), + git: None, + branch: None, + }; + poetry_requirements.insert(requirement_entry.requirement.name, dep); + } + Ok(poetry_requirements) +} + #[cfg(test)] mod test { use super::{parse_dep_extra, poetry_spec_from_dir, read_toml_files}; + use crate::poetry_integration::read_dependencies::read_requirements_for_poetry; use crate::read_poetry_specs; + use indoc::indoc; use pep508_rs::{MarkerEnvironment, StringVersion}; use std::collections::HashSet; use std::path::Path; @@ -509,6 +551,34 @@ mod test { } } + #[test] + fn test_requirements_txt_poetry() { + let expected = indoc! {r#" + [inflection] + version = "==0.5.1" + optional = false + + [numpy] + version = "*" + optional = false + + [pandas] + version = ">=1, <2" + optional = false + extras = ["tabulate"] + + [upsidedown] + version = "==0.4" + optional = false + "#}; + + let working_dir = Path::new("test-data").join("requirements-txt"); + let path = working_dir.join("for-poetry.txt"); + let reqs = read_requirements_for_poetry(&path, working_dir).unwrap(); + let poetry_toml = toml::to_string(&reqs).unwrap(); + assert_eq!(poetry_toml, expected); + } + #[test] fn test_outdated_lockfile() { let err = poetry_spec_from_dir( diff --git a/src/requirements_txt.rs b/src/requirements_txt.rs deleted file mode 100644 index 08c67ff..0000000 --- a/src/requirements_txt.rs +++ /dev/null @@ -1,684 +0,0 @@ -//! Parses a subset of requirement.txt syntax -//! -//! -//! -//! Supported: -//! * [PEP 508 requirements](https://packaging.python.org/en/latest/specifications/dependency-specifiers/) -//! * `-r` -//! * `-c` -//! * `--hash` (postfix) -//! * `-e` -//! -//! Unsupported: -//! * `-e `. TBD -//! * ``. TBD -//! * ``. TBD -//! * Options without a requirement, such as `--find-links` or `--index-url` -//! -//! Grammar as implemented: -//! -//! ```text -//! file = (statement | empty ('#' any*)? '\n')* -//! empty = whitespace* -//! statement = constraint_include | requirements_include | editable_requirement | requirement -//! constraint_include = '-c' ('=' | wrappable_whitespaces) filepath -//! requirements_include = '-r' ('=' | wrappable_whitespaces) filepath -//! editable_requirement = '-e' ('=' | wrappable_whitespaces) requirement -//! # We check whether the line starts with a letter or a number, in that case we assume it's a -//! # PEP 508 requirement -//! # https://packaging.python.org/en/latest/specifications/name-normalization/#valid-non-normalized-names -//! # This does not (yet?) support plain files or urls, we use a letter or a number as first -//! # character to assume a PEP 508 requirement -//! requirement = [a-zA-Z0-9] pep508_grammar_tail wrappable_whitespaces hashes -//! hashes = ('--hash' ('=' | wrappable_whitespaces) [a-zA-Z0-9-_]+ ':' [a-zA-Z0-9-_] wrappable_whitespaces+)* -//! # This should indicate a single backslash before a newline -//! wrappable_whitespaces = whitespace ('\\\n' | whitespace)* -//! ``` - -use crate::poetry_integration::poetry_toml; -use anyhow::bail; -use fs_err as fs; -use pep508_rs::{Pep508Error, Requirement, VersionOrUrl}; -use serde::{Deserialize, Serialize}; -use std::collections::BTreeMap; -use std::fmt::{Display, Formatter}; -use std::io; -use std::path::{Path, PathBuf}; -use std::str::FromStr; -use tracing::warn; -use unscanny::{Pattern, Scanner}; - -/// We emit one of those for each requirements.txt entry -enum RequirementsTxtStatement { - /// `-r` inclusion filename - Requirements { - filename: String, - start: usize, - end: usize, - }, - /// `-c` inclusion filename - Constraint { - filename: String, - start: usize, - end: usize, - }, - /// PEP 508 requirement plus metadata - RequirementEntry(RequirementEntry), -} - -/// A [Requirement] with additional metadata from the requirements.txt, currently only hashes but in -/// the future also editable an similar information -#[derive(Debug, Deserialize, Clone, Eq, PartialEq, Serialize)] -pub struct RequirementEntry { - /// The actual PEP 508 requirement - pub requirement: Requirement, - /// Hashes of the downloadable packages - pub hashes: Vec, - /// Editable installation, see e.g. - pub editable: bool, -} - -impl Display for RequirementEntry { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - if self.editable { - write!(f, "-e ")?; - } - write!(f, "{}", self.requirement)?; - for hash in &self.hashes { - write!(f, " --hash {}", hash)? - } - - Ok(()) - } -} - -/// Parsed and flattened requirements.txt with requirements and constraints -#[derive(Debug, Deserialize, Clone, Default, Eq, PartialEq, Serialize)] -pub struct RequirementsTxt { - /// The actual requirements with the hashes - pub requirements: Vec, - /// Constraints included with `-c` - pub constraints: Vec, -} - -impl RequirementsTxt { - /// See module level documentation - pub fn parse( - requirements_txt: impl AsRef, - working_dir: impl AsRef, - ) -> Result { - let content = - fs::read_to_string(&requirements_txt).map_err(|err| RequirementsTxtFileError { - file: requirements_txt.as_ref().to_path_buf(), - error: RequirementsTxtParserError::IO(err), - })?; - let data = - Self::parse_inner(&content, working_dir).map_err(|err| RequirementsTxtFileError { - file: requirements_txt.as_ref().to_path_buf(), - error: err, - })?; - if data == Self::default() { - warn!( - "Requirements file {} does not contain any dependencies", - requirements_txt.as_ref().display() - ); - } - Ok(data) - } - - /// See module level documentation - /// - /// Note that all relative paths are dependent on the current working dir, not on the location - /// of the file - pub fn parse_inner( - content: &str, - working_dir: impl AsRef, - ) -> Result { - let mut s = Scanner::new(&content); - - let mut data = Self::default(); - while let Some(statement) = parse_entry(&mut s, &content)? { - match statement { - RequirementsTxtStatement::Requirements { - filename, - start, - end, - } => { - let sub_file = working_dir.as_ref().join(filename); - let sub_requirements = - Self::parse(&sub_file, working_dir.as_ref()).map_err(|err| { - RequirementsTxtParserError::Subfile { - source: Box::new(err), - start, - end, - } - })?; - // Add each to the correct category - data.update_from(sub_requirements); - } - RequirementsTxtStatement::Constraint { - filename, - start, - end, - } => { - let sub_file = working_dir.as_ref().join(filename); - let sub_constraints = - Self::parse(&sub_file, working_dir.as_ref()).map_err(|err| { - RequirementsTxtParserError::Subfile { - source: Box::new(err), - start, - end, - } - })?; - // Here we add both to constraints - data.constraints.extend( - sub_constraints - .requirements - .into_iter() - .map(|requirement_entry| requirement_entry.requirement), - ); - data.constraints.extend(sub_constraints.constraints); - } - RequirementsTxtStatement::RequirementEntry(requirement_entry) => { - data.requirements.push(requirement_entry); - } - } - } - Ok(data) - } - - /// Merges other into self - pub fn update_from(&mut self, other: RequirementsTxt) { - self.requirements.extend(other.requirements); - self.constraints.extend(other.constraints); - } - - /// Method to bridge between the new parser and the poetry assumptions of the existing code - pub fn into_poetry( - self, - requirements_txt: &Path, - ) -> anyhow::Result> { - if !self.constraints.is_empty() { - bail!( - "Constraints (`-c`) from {} are not supported yet", - requirements_txt.display() - ); - } - let mut poetry_requirements: BTreeMap = BTreeMap::new(); - for requirement_entry in self.requirements { - let version = match requirement_entry.requirement.version_or_url { - None => "*".to_string(), - Some(VersionOrUrl::Url(_)) => { - bail!( - "Unsupported url requirement in {}: '{}'", - requirements_txt.display(), - requirement_entry.requirement, - ) - } - Some(VersionOrUrl::VersionSpecifier(specifiers)) => specifiers.to_string(), - }; - - let dep = poetry_toml::Dependency::Expanded { - version: Some(version), - optional: Some(false), - extras: requirement_entry.requirement.extras.clone(), - git: None, - branch: None, - }; - poetry_requirements.insert(requirement_entry.requirement.name, dep); - } - Ok(poetry_requirements) - } -} - -/// Parse a single entry, that is a requirement, an inclusion or a comment line -/// -/// Consumes all preceding trivia (whitespace and comments). If it returns None, we've reached -/// the end of file -fn parse_entry( - s: &mut Scanner, - content: &str, -) -> Result, RequirementsTxtParserError> { - // Eat all preceding whitespace, this may run us to the end of file - eat_wrappable_whitespace(s); - while s.at(['\n', '\r', '#']) { - // skip comments - eat_trailing_line(s)?; - eat_wrappable_whitespace(s); - } - - let start = s.cursor(); - Ok(Some(if s.eat_if("-r") { - let requirements_file = parse_value(s, |c: char| !['\n', '\r', '#'].contains(&c))?; - let end = s.cursor(); - eat_trailing_line(s)?; - RequirementsTxtStatement::Requirements { - filename: requirements_file.to_string(), - start, - end, - } - } else if s.eat_if("-c") { - let constraints_file = parse_value(s, |c: char| !['\n', '\r', '#'].contains(&c))?; - let end = s.cursor(); - eat_trailing_line(s)?; - RequirementsTxtStatement::Constraint { - filename: constraints_file.to_string(), - start, - end, - } - } else if s.eat_if("-e") { - let (requirement, hashes) = parse_requirement_and_hashes(s, &content)?; - RequirementsTxtStatement::RequirementEntry(RequirementEntry { - requirement, - hashes, - editable: true, - }) - } else if s.at(char::is_ascii_alphanumeric) { - let (requirement, hashes) = parse_requirement_and_hashes(s, &content)?; - RequirementsTxtStatement::RequirementEntry(RequirementEntry { - requirement, - hashes, - editable: false, - }) - } else if let Some(char) = s.peek() { - return Err(RequirementsTxtParserError::Parser { - message: format!( - "Unexpected '{}', expected '-c', '-e', '-r' or the start of a requirement", - char - ), - location: s.cursor(), - }); - } else { - // EOF - return Ok(None); - })) -} - -/// Eat whitespace and ignore newlines escaped with a backslash -fn eat_wrappable_whitespace<'a>(s: &mut Scanner<'a>) -> &'a str { - let start = s.cursor(); - s.eat_while([' ', '\t']); - // Allow multiple escaped line breaks - // With the order we support `\n`, `\r`, `\r\n` without accidentally eating a `\n\r` - while s.eat_if("\\\n") || s.eat_if("\\\r\n") || s.eat_if("\\\r") { - s.eat_while([' ', '\t']); - } - s.from(start) -} - -/// Eats the end of line or a potential trailing comma -fn eat_trailing_line(s: &mut Scanner) -> Result<(), RequirementsTxtParserError> { - s.eat_while([' ', '\t']); - match s.eat() { - None | Some('\n') => {} // End of file or end of line, nothing to do - Some('\r') => { - s.eat_if('\n'); // `\r\n`, but just `\r` is also accepted - } - Some('#') => { - s.eat_until(['\r', '\n']); - if s.at('\r') { - s.eat_if('\n'); // `\r\n`, but just `\r` is also accepted - } - } - Some(other) => { - return Err(RequirementsTxtParserError::Parser { - message: format!("Expected comment or end-of-line, found '{}'", other), - location: s.cursor(), - }) - } - } - Ok(()) -} - -/// Parse a PEP 508 requirement with optional trailing hashes -fn parse_requirement_and_hashes( - s: &mut Scanner, - content: &str, -) -> Result<(Requirement, Vec), RequirementsTxtParserError> { - // PEP 508 requirement - let start = s.cursor(); - // Termination: s.eat() eventually becomes None - let (end, has_hashes) = loop { - let end = s.cursor(); - - // We look for the end of the line ... - if s.eat_if('\n') { - break (end, false); - } - if s.eat_if('\r') { - s.eat_if('\n'); // Support `\r\n` but also accept stray `\r` - break (end, false); - } - // ... or `--hash`, an escaped newline or a comment separated by whitespace ... - if !eat_wrappable_whitespace(s).is_empty() { - if s.after().starts_with("--") { - break (end, true); - } else if s.eat_if('#') { - s.eat_until(['\r', '\n']); - if s.at('\r') { - s.eat_if('\n'); // `\r\n`, but just `\r` is also accepted - } - break (end, false); - } else { - continue; - } - } - // ... or the end of the file, which works like the end of line - if s.eat().is_none() { - break (end, false); - } - }; - let requirement = Requirement::from_str(&content[start..end]).map_err(|err| { - RequirementsTxtParserError::Pep508 { - source: err, - start, - end, - } - })?; - let hashes = if has_hashes { - let hashes = parse_hashes(s)?; - eat_trailing_line(s)?; - hashes - } else { - Vec::new() - }; - Ok((requirement, hashes)) -} - -/// Parse `--hash=... --hash ...` after a requirement -fn parse_hashes(s: &mut Scanner) -> Result, RequirementsTxtParserError> { - let mut hashes = Vec::new(); - if s.eat_while("--hash").is_empty() { - return Err(RequirementsTxtParserError::Parser { - message: format!( - "Expected '--hash', found '{:?}'", - s.eat_while(|c: char| !c.is_whitespace()) - ), - location: s.cursor(), - }); - } - let hash = parse_value(s, |c: char| !c.is_whitespace())?; - hashes.push(hash.to_string()); - loop { - eat_wrappable_whitespace(s); - if !s.eat_if("--hash") { - break; - } - let hash = parse_value(s, |c: char| !c.is_whitespace())?; - hashes.push(hash.to_string()); - } - Ok(hashes) -} - -/// In `-=` or `- value`, this parses the part after the key -fn parse_value<'a, T>( - s: &mut Scanner<'a>, - while_pattern: impl Pattern, -) -> Result<&'a str, RequirementsTxtParserError> { - if s.eat_if('=') { - // Explicit equals sign - Ok(s.eat_while(while_pattern).trim_end()) - } else if s.eat_if(char::is_whitespace) { - // Key and value are separated by whitespace instead - s.eat_whitespace(); - Ok(s.eat_while(while_pattern).trim_end()) - } else { - Err(RequirementsTxtParserError::Parser { - message: format!("Expected '=' or whitespace, found {:?}", s.peek()), - location: s.cursor(), - }) - } -} - -/// Error parsing requirements.txt, wrapper with filename -#[derive(Debug)] -pub struct RequirementsTxtFileError { - file: PathBuf, - error: RequirementsTxtParserError, -} - -/// Error parsing requirements.txt, error disambiguation -#[derive(Debug)] -pub enum RequirementsTxtParserError { - IO(io::Error), - Parser { - message: String, - location: usize, - }, - Pep508 { - source: Pep508Error, - start: usize, - end: usize, - }, - Subfile { - source: Box, - start: usize, - end: usize, - }, -} - -impl Display for RequirementsTxtFileError { - fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result { - match &self.error { - RequirementsTxtParserError::IO(err) => err.fmt(f), - RequirementsTxtParserError::Parser { message, location } => { - write!( - f, - "{} in {} position {}", - message, - self.file.display(), - location - ) - } - RequirementsTxtParserError::Pep508 { start, end, .. } => { - write!( - f, - "Couldn't parse requirement in {} position {} to {}", - self.file.display(), - start, - end, - ) - } - RequirementsTxtParserError::Subfile { start, end, .. } => { - write!( - f, - "Error parsing file included into {} at position {} to {}", - self.file.display(), - start, - end - ) - } - } - } -} - -impl std::error::Error for RequirementsTxtFileError { - fn source(&self) -> Option<&(dyn std::error::Error + 'static)> { - match &self.error { - RequirementsTxtParserError::IO(err) => err.source(), - RequirementsTxtParserError::Pep508 { source, .. } => Some(source), - RequirementsTxtParserError::Subfile { source, .. } => Some(source.as_ref()), - _ => None, - } - } -} - -#[cfg(test)] -mod test { - use crate::requirements_txt::RequirementsTxt; - use fs_err as fs; - use indoc::indoc; - use logtest::Logger; - use std::collections::BTreeMap; - use std::path::Path; - use tempfile::tempdir; - use tracing::log::Level; - - #[test] - fn test_requirements_txt_parsing() { - let working_dir = Path::new("test-data").join("requirements-txt"); - for dir_entry in fs::read_dir(&working_dir).unwrap() { - let dir_entry = dir_entry.unwrap().path(); - if dir_entry.extension().unwrap_or_default().to_str().unwrap() != "txt" { - continue; - } - let actual = RequirementsTxt::parse(&dir_entry, &working_dir).unwrap(); - let fixture = dir_entry.with_extension("json"); - // Update the json fixtures - // fs::write(&fixture, &serde_json::to_string_pretty(&actual).unwrap()).unwrap(); - let snapshot = serde_json::from_str(&fs::read_to_string(fixture).unwrap()).unwrap(); - assert_eq!(actual, snapshot); - } - } - - /// Test with flipped line endings - #[test] - fn test_other_line_endings() { - let temp_dir = tempdir().unwrap(); - let mut files = Vec::new(); - let working_dir = Path::new("test-data").join("requirements-txt"); - for dir_entry in fs::read_dir(&working_dir).unwrap() { - let dir_entry = dir_entry.unwrap(); - if dir_entry - .path() - .extension() - .unwrap_or_default() - .to_str() - .unwrap() - != "txt" - { - continue; - } - let copied = temp_dir.path().join(dir_entry.file_name()); - let original = fs::read_to_string(dir_entry.path()).unwrap(); - // Replace line endings with the other choice. This works even if you use git with LF - // only on windows. - let changed = if original.contains("\r\n") { - original.replace("\r\n", "\n") - } else { - original.replace('\n', "\r\n") - }; - fs::write(&copied, &changed).unwrap(); - files.push((copied, dir_entry.path().with_extension("json"))); - } - for (file, fixture) in files { - let actual = RequirementsTxt::parse(&file, &working_dir).unwrap(); - let snapshot = serde_json::from_str(&fs::read_to_string(fixture).unwrap()).unwrap(); - assert_eq!(actual, snapshot); - } - } - - /// Pass test only - currently fails due to `-e ./` in pyproject.toml-constrained.in - #[test] - #[ignore] - fn test_pydantic() { - let working_dir = Path::new("test-data").join("requirements-pydantic"); - for basic in fs::read_dir(&working_dir).unwrap() { - let basic = basic.unwrap().path(); - if !["txt", "in"].contains(&basic.extension().unwrap_or_default().to_str().unwrap()) { - continue; - } - RequirementsTxt::parse(&basic, &working_dir).unwrap(); - } - } - - #[test] - fn test_invalid_include_missing_file() { - let working_dir = Path::new("test-data").join("requirements-txt"); - let basic = working_dir.join("invalid-include"); - let missing = working_dir.join("missing.txt"); - let err = RequirementsTxt::parse(&basic, &working_dir).unwrap_err(); - let errors = anyhow::Error::new(err) - .chain() - .map(ToString::to_string) - .collect::>(); - assert_eq!(errors.len(), 3); - assert_eq!( - errors[0], - format!( - "Error parsing file included into {} at position 0 to 14", - basic.display() - ) - ); - assert_eq!( - errors[1], - format!("failed to open file `{}`", missing.display()), - ); - // The last error message is os specific - } - - #[test] - fn test_invalid_requirement() { - let working_dir = Path::new("test-data").join("requirements-txt"); - let basic = working_dir.join("invalid-requirement"); - let err = RequirementsTxt::parse(&basic, &working_dir).unwrap_err(); - let errors = anyhow::Error::new(err) - .chain() - .map(ToString::to_string) - .collect::>(); - let expected = &[ - format!( - "Couldn't parse requirement in {} position 0 to 15", - basic.display() - ), - indoc! {" - Expected an alphanumeric character starting the extra name, found 'ö' - numpy[ö]==1.29 - ^" - } - .to_string(), - ]; - assert_eq!(errors, expected) - } - - #[test] - fn test_requirements_txt_poetry() { - let expected = indoc! {r#" - [inflection] - version = "==0.5.1" - optional = false - - [numpy] - version = "*" - optional = false - - [pandas] - version = ">=1, <2" - optional = false - extras = ["tabulate"] - - [upsidedown] - version = "==0.4" - optional = false - "#}; - - let working_dir = Path::new("test-data").join("requirements-txt"); - let path = working_dir.join("for-poetry.txt"); - let reqs = RequirementsTxt::parse(&path, &working_dir) - .unwrap() - .into_poetry(&path) - .unwrap(); - // sort lines - let reqs = BTreeMap::from_iter(&reqs); - let poetry_toml = toml::to_string(&reqs).unwrap(); - assert_eq!(poetry_toml, expected); - } - - #[test] - fn test_empty_file() { - let working_dir = Path::new("test-data").join("requirements-txt"); - let path = working_dir.join("empty.txt"); - - // TODO(konstin) I think that logger isn't thread safe - let logger = Logger::start(); - RequirementsTxt::parse(&path, &working_dir).unwrap(); - let warnings: Vec<_> = logger - .into_iter() - .filter(|message| message.level() >= Level::Warn) - .collect(); - assert_eq!(warnings.len(), 1, "{:?}", warnings); - assert!(warnings[0] - .args() - .ends_with("does not contain any dependencies")); - } -} From 3ee53b067c096d164bc11002627983abc4be1b5c Mon Sep 17 00:00:00 2001 From: Chris Pryer Date: Wed, 7 Jun 2023 17:01:12 -0400 Subject: [PATCH 08/19] Add Cargo.toml profile config ('zip implementation') back This was removed accidentally in #53 --- Cargo.toml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Cargo.toml b/Cargo.toml index 7340b76..f86672f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -58,6 +58,10 @@ default = ["vendored"] python_bindings = ["pyo3", "install-wheel-rs/python_bindings"] vendored = ["git2/vendored-openssl", "git2/vendored-libgit2"] +# zip implementation +[profile.dev.package.adler] +opt-level = 3 + # https://doc.rust-lang.org/cargo/reference/profiles.html#release [profile.perf] inherits = "release" From 06aaba0e856785585a8a596d250528ff150577a6 Mon Sep 17 00:00:00 2001 From: Chris Pryer Date: Wed, 7 Jun 2023 17:04:00 -0400 Subject: [PATCH 09/19] Collapse import statements --- src/poetry_integration/read_dependencies.rs | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/poetry_integration/read_dependencies.rs b/src/poetry_integration/read_dependencies.rs index c017140..121277d 100644 --- a/src/poetry_integration/read_dependencies.rs +++ b/src/poetry_integration/read_dependencies.rs @@ -469,8 +469,9 @@ pub fn read_requirements_for_poetry( #[cfg(test)] mod test { - use super::{parse_dep_extra, poetry_spec_from_dir, read_toml_files}; - use crate::poetry_integration::read_dependencies::read_requirements_for_poetry; + use super::{ + parse_dep_extra, poetry_spec_from_dir, read_requirements_for_poetry, read_toml_files, + }; use crate::read_poetry_specs; use indoc::indoc; use pep508_rs::{MarkerEnvironment, StringVersion}; From 9c804ba1e041f0905bad9bd5da8df322f193f281 Mon Sep 17 00:00:00 2001 From: Chris Pryer Date: Wed, 7 Jun 2023 17:17:21 -0400 Subject: [PATCH 10/19] Small chore --- src/poetry_integration/read_dependencies.rs | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/poetry_integration/read_dependencies.rs b/src/poetry_integration/read_dependencies.rs index 121277d..184d7ec 100644 --- a/src/poetry_integration/read_dependencies.rs +++ b/src/poetry_integration/read_dependencies.rs @@ -430,19 +430,18 @@ pub fn poetry_spec_from_dir( /// Reads and parses requirements into poetry dependencies from a requirements file. pub fn read_requirements_for_poetry( - requirements_txt: impl AsRef, - working_dir: impl AsRef, + requirements_txt: &Path, + working_dir: &Path, ) -> anyhow::Result> { - let requirements_txt = requirements_txt.as_ref(); - let requirements = RequirementsTxt::parse(requirements_txt, working_dir)?; - if !requirements.constraints.is_empty() { + let data = RequirementsTxt::parse(requirements_txt, working_dir)?; + if !data.constraints.is_empty() { bail!( "Constraints (`-c`) from {} are not supported yet", requirements_txt.display() ); } let mut poetry_requirements: BTreeMap = BTreeMap::new(); - for requirement_entry in requirements.requirements { + for requirement_entry in data.requirements { let version = match requirement_entry.requirement.version_or_url { None => "*".to_string(), Some(VersionOrUrl::Url(_)) => { @@ -575,7 +574,7 @@ mod test { let working_dir = Path::new("test-data").join("requirements-txt"); let path = working_dir.join("for-poetry.txt"); - let reqs = read_requirements_for_poetry(&path, working_dir).unwrap(); + let reqs = read_requirements_for_poetry(&path, &working_dir).unwrap(); let poetry_toml = toml::to_string(&reqs).unwrap(); assert_eq!(poetry_toml, expected); } From a71d8d969d91a44ccc87b5d8d842a64d295d656c Mon Sep 17 00:00:00 2001 From: Chris Pryer Date: Sat, 10 Jun 2023 12:16:06 -0400 Subject: [PATCH 11/19] Cargo clippy --fix --- monotrail-utils/src/requirements_txt.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/monotrail-utils/src/requirements_txt.rs b/monotrail-utils/src/requirements_txt.rs index ece4097..cedbef0 100644 --- a/monotrail-utils/src/requirements_txt.rs +++ b/monotrail-utils/src/requirements_txt.rs @@ -131,10 +131,10 @@ impl RequirementsTxt { content: &str, working_dir: impl AsRef, ) -> Result { - let mut s = Scanner::new(&content); + let mut s = Scanner::new(content); let mut data = Self::default(); - while let Some(statement) = parse_entry(&mut s, &content)? { + while let Some(statement) = parse_entry(&mut s, content)? { match statement { RequirementsTxtStatement::Requirements { filename, @@ -227,14 +227,14 @@ fn parse_entry( end, } } else if s.eat_if("-e") { - let (requirement, hashes) = parse_requirement_and_hashes(s, &content)?; + let (requirement, hashes) = parse_requirement_and_hashes(s, content)?; RequirementsTxtStatement::RequirementEntry(RequirementEntry { requirement, hashes, editable: true, }) } else if s.at(char::is_ascii_alphanumeric) { - let (requirement, hashes) = parse_requirement_and_hashes(s, &content)?; + let (requirement, hashes) = parse_requirement_and_hashes(s, content)?; RequirementsTxtStatement::RequirementEntry(RequirementEntry { requirement, hashes, @@ -597,7 +597,7 @@ mod test { // TODO(konstin) I think that logger isn't thread safe let logger = Logger::start(); - RequirementsTxt::parse(&path, &working_dir).unwrap(); + RequirementsTxt::parse(path, &working_dir).unwrap(); let warnings: Vec<_> = logger .into_iter() .filter(|message| message.level() >= Level::Warn) From 175f7b321d895632d0f5c35e8a2954a217de0b0f Mon Sep 17 00:00:00 2001 From: Chris Pryer Date: Sat, 10 Jun 2023 12:34:54 -0400 Subject: [PATCH 12/19] Use shared workspace dependencies Does not include dev deps --- Cargo.toml | 26 ++++++++++++++++++-------- monotrail-utils/Cargo.toml | 16 ++++++++-------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index f86672f..7f437b9 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -16,12 +16,12 @@ crate-type = ["rlib", "cdylib"] name = "monotrail" [dependencies] -anyhow = "1.0.65" +anyhow = { workspace = true } base64 = "0.21.0" clap = { version = "4.0.2", features = ["derive"] } cpufeatures = "0.2.5" dirs = "5.0.0" -fs-err = "2.8.1" +fs-err = { workspace = true } fs2 = "0.4.3" git2 = "0.17.1" indicatif = "0.17.1" @@ -33,26 +33,36 @@ libz-sys = { version = "1.1.8", features = ["static"] } monotrail-utils = { version = "0.0.1", path = "monotrail-utils" } nix = { version = "0.26.1", features = ["process"] } pep440_rs = "0.3.2" -pep508_rs = { version = "0.2.1", features = ["serde"] } +pep508_rs = { workspace = true, features = ["serde"] } pyo3 = { version = "0.18.1", features = ["extension-module", "abi3-py37"], optional = true } rayon = "1.5.3" regex = "1.6.0" -serde = { version = "1.0.145", features = ["derive"] } -serde_json = "1.0.85" +serde = { workspace = true } +serde_json = { workspace = true } sha2 = "0.10.6" tar = "0.4.38" target-lexicon = "0.12.4" tempfile = "3.3.0" thiserror = "1.0.37" -toml = "0.7.2" -tracing = "0.1.36" +toml = { workspace = true } +tracing = { workspace = true } tracing-subscriber = "0.3.15" -unscanny = "0.1.0" +unscanny = { workspace = true } ureq = { version = "2.5.0", features = ["json"] } walkdir = "2.3.2" widestring = "1.0.2" zstd = "0.12.1" +[workspace.dependencies] +anyhow = "1.0.65" +fs-err = "2.8.1" +pep508_rs = { version = "0.2.1", features = ["serde"] } +serde = { version = "1.0.145", features = ["derive"] } +serde_json = "1.0.85" +toml = "0.7.2" +tracing = "0.1.36" +unscanny = "0.1.0" + [features] default = ["vendored"] python_bindings = ["pyo3", "install-wheel-rs/python_bindings"] diff --git a/monotrail-utils/Cargo.toml b/monotrail-utils/Cargo.toml index e4f1688..1f7316d 100644 --- a/monotrail-utils/Cargo.toml +++ b/monotrail-utils/Cargo.toml @@ -4,14 +4,14 @@ version = "0.0.1" edition = "2021" [dependencies] -anyhow = "1.0.65" -fs-err = "2.8.1" -pep508_rs = { version = "0.2.1", features = ["serde"] } -serde = { version = "1.0.145", features = ["derive"] } -serde_json = "1.0.85" -toml = "0.7.2" -tracing = "0.1.36" -unscanny = "0.1.0" +anyhow = { workspace = true } +fs-err = { workspace = true } +pep508_rs = { workspace = true } +serde = { workspace = true } +serde_json = { workspace = true } +toml = { workspace = true } +tracing = { workspace = true } +unscanny = { workspace = true } [dev-dependencies] indoc = "2.0.0" From 9880144d81def4526d41274b3801cd4d53f1b592 Mon Sep 17 00:00:00 2001 From: Chris Pryer Date: Sat, 10 Jun 2023 12:51:00 -0400 Subject: [PATCH 13/19] Use shared dependencies from install-wheel-rs --- Cargo.toml | 24 ++++++++++++++++-------- install-wheel-rs/Cargo.toml | 20 ++++++++++---------- 2 files changed, 26 insertions(+), 18 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 7f437b9..6cddd55 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,12 +17,12 @@ name = "monotrail" [dependencies] anyhow = { workspace = true } -base64 = "0.21.0" +base64 = { workspace = true } clap = { version = "4.0.2", features = ["derive"] } cpufeatures = "0.2.5" dirs = "5.0.0" fs-err = { workspace = true } -fs2 = "0.4.3" +fs2 = { workspace = true } git2 = "0.17.1" indicatif = "0.17.1" install-wheel-rs = { version = "0.0.3", path = "install-wheel-rs" } @@ -34,34 +34,42 @@ monotrail-utils = { version = "0.0.1", path = "monotrail-utils" } nix = { version = "0.26.1", features = ["process"] } pep440_rs = "0.3.2" pep508_rs = { workspace = true, features = ["serde"] } -pyo3 = { version = "0.18.1", features = ["extension-module", "abi3-py37"], optional = true } +pyo3 = { workspace = true, features = ["extension-module", "abi3-py37"], optional = true } rayon = "1.5.3" -regex = "1.6.0" +regex = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } sha2 = "0.10.6" tar = "0.4.38" -target-lexicon = "0.12.4" -tempfile = "3.3.0" +target-lexicon = { workspace = true } +tempfile = { workspace = true } thiserror = "1.0.37" toml = { workspace = true } tracing = { workspace = true } -tracing-subscriber = "0.3.15" +tracing-subscriber = { workspace = true } unscanny = { workspace = true } ureq = { version = "2.5.0", features = ["json"] } -walkdir = "2.3.2" +walkdir = { workspace = true } widestring = "1.0.2" zstd = "0.12.1" [workspace.dependencies] anyhow = "1.0.65" +base64 = "0.21.0" fs-err = "2.8.1" +fs2 = "0.4.3" pep508_rs = { version = "0.2.1", features = ["serde"] } +pyo3 = { version = "0.18.1", features = ["extension-module", "abi3-py37"] } +regex = "1.6.0" serde = { version = "1.0.145", features = ["derive"] } serde_json = "1.0.85" +target-lexicon = "0.12.4" +tempfile = "3.3.0" toml = "0.7.2" tracing = "0.1.36" +tracing-subscriber = "0.3.15" unscanny = "0.1.0" +walkdir = "2.3.2" [features] default = ["vendored"] diff --git a/install-wheel-rs/Cargo.toml b/install-wheel-rs/Cargo.toml index d1e2ad3..8eefc18 100644 --- a/install-wheel-rs/Cargo.toml +++ b/install-wheel-rs/Cargo.toml @@ -10,27 +10,27 @@ name = "install_wheel_rs" #crate-type = ["cdylib", "rlib"] [dependencies] -base64 = "0.21.0" +base64 = { workspace = true } configparser = "3.0.1" csv = "1.1.6" -fs-err = "2.8.1" -fs2 = "0.4.3" +fs-err = { workspace = true } +fs2 = { workspace = true } glibc_version = "0.1.2" goblin = "0.6.0" platform-info = "1.0.0" plist = "1.3.1" -pyo3 = { version = "0.18.1", features = ["extension-module", "abi3-py37"], optional = true } +pyo3 = { workspace = true, features = ["extension-module", "abi3-py37"], optional = true } python-pkginfo = "0.5.4" -regex = "1.6.0" +regex = { workspace = true } serde = { version = "1.0.144", features = ["derive"] } -serde_json = "1.0.85" +serde_json = { workspace = true } sha2 = "0.10.5" target-lexicon = "0.12.4" -tempfile = "3.3.0" +tempfile = { workspace = true } thiserror = "1.0.34" -tracing = "0.1.36" -tracing-subscriber = { version = "0.3.15", optional = true } -walkdir = "2.3.2" +tracing = { workspace = true } +tracing-subscriber = { workspace = true, optional = true } +walkdir = { workspace = true } zip = { version = "0.6.2", default-features = false, features = ["deflate"] } # no default features for zstd [features] From 67bcc2313e69322ec05b451738e6ed622c077fdb Mon Sep 17 00:00:00 2001 From: Chris Pryer Date: Sat, 10 Jun 2023 12:54:31 -0400 Subject: [PATCH 14/19] Bump install-wheel-rs thiserror to v1.0.37 --- Cargo.toml | 3 ++- install-wheel-rs/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index 6cddd55..e2a60a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -43,7 +43,7 @@ sha2 = "0.10.6" tar = "0.4.38" target-lexicon = { workspace = true } tempfile = { workspace = true } -thiserror = "1.0.37" +thiserror = { workspace = true } toml = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true } @@ -65,6 +65,7 @@ serde = { version = "1.0.145", features = ["derive"] } serde_json = "1.0.85" target-lexicon = "0.12.4" tempfile = "3.3.0" +thiserror = "1.0.37" toml = "0.7.2" tracing = "0.1.36" tracing-subscriber = "0.3.15" diff --git a/install-wheel-rs/Cargo.toml b/install-wheel-rs/Cargo.toml index 8eefc18..f63cca1 100644 --- a/install-wheel-rs/Cargo.toml +++ b/install-wheel-rs/Cargo.toml @@ -27,7 +27,7 @@ serde_json = { workspace = true } sha2 = "0.10.5" target-lexicon = "0.12.4" tempfile = { workspace = true } -thiserror = "1.0.34" +thiserror = { workspace = true } tracing = { workspace = true } tracing-subscriber = { workspace = true, optional = true } walkdir = { workspace = true } From b9d93ea716ba78019d87b10c5a6b87d9e8479803 Mon Sep 17 00:00:00 2001 From: Chris Pryer Date: Sat, 10 Jun 2023 12:55:59 -0400 Subject: [PATCH 15/19] Bump install-wheel-rs sha2 to v0.10.6 --- Cargo.toml | 3 ++- install-wheel-rs/Cargo.toml | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Cargo.toml b/Cargo.toml index e2a60a1..dd2e2a1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -39,7 +39,7 @@ rayon = "1.5.3" regex = { workspace = true } serde = { workspace = true } serde_json = { workspace = true } -sha2 = "0.10.6" +sha2 = { workspace = true } tar = "0.4.38" target-lexicon = { workspace = true } tempfile = { workspace = true } @@ -63,6 +63,7 @@ pyo3 = { version = "0.18.1", features = ["extension-module", "abi3-py37"] } regex = "1.6.0" serde = { version = "1.0.145", features = ["derive"] } serde_json = "1.0.85" +sha2 = "0.10.6" target-lexicon = "0.12.4" tempfile = "3.3.0" thiserror = "1.0.37" diff --git a/install-wheel-rs/Cargo.toml b/install-wheel-rs/Cargo.toml index f63cca1..e8ad263 100644 --- a/install-wheel-rs/Cargo.toml +++ b/install-wheel-rs/Cargo.toml @@ -24,7 +24,7 @@ python-pkginfo = "0.5.4" regex = { workspace = true } serde = { version = "1.0.144", features = ["derive"] } serde_json = { workspace = true } -sha2 = "0.10.5" +sha2 = { workspace = true } target-lexicon = "0.12.4" tempfile = { workspace = true } thiserror = { workspace = true } From a0cad0176bd98c99f3f6e1bf707a959a781c98e6 Mon Sep 17 00:00:00 2001 From: Chris Pryer Date: Sat, 10 Jun 2023 12:57:09 -0400 Subject: [PATCH 16/19] Bump install-wheel-rs serde to v1.0.145 --- install-wheel-rs/Cargo.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install-wheel-rs/Cargo.toml b/install-wheel-rs/Cargo.toml index e8ad263..4d8c900 100644 --- a/install-wheel-rs/Cargo.toml +++ b/install-wheel-rs/Cargo.toml @@ -22,7 +22,7 @@ plist = "1.3.1" pyo3 = { workspace = true, features = ["extension-module", "abi3-py37"], optional = true } python-pkginfo = "0.5.4" regex = { workspace = true } -serde = { version = "1.0.144", features = ["derive"] } +serde = { workspace = true, features = ["derive"] } serde_json = { workspace = true } sha2 = { workspace = true } target-lexicon = "0.12.4" From ee7a182d4bfa31123778662658c8df2e037eb472 Mon Sep 17 00:00:00 2001 From: Chris Pryer Date: Sat, 10 Jun 2023 17:10:20 -0400 Subject: [PATCH 17/19] Use shared dev dependencies --- Cargo.lock | 15 +++++++-------- Cargo.toml | 14 +++++++++----- install-wheel-rs/Cargo.toml | 2 +- monotrail-utils/Cargo.toml | 6 +++--- 4 files changed, 20 insertions(+), 17 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index fe3799a..709de95 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -679,9 +679,9 @@ dependencies = [ [[package]] name = "h2" -version = "0.3.18" +version = "0.3.19" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "17f8a914c2987b688368b5138aa05321db91f4090cf26118185672ad588bce21" +checksum = "d357c7ae988e7d2182f7d7871d0b963962420b0678b0997ce7de72001aeab782" dependencies = [ "bytes", "fnv", @@ -1075,14 +1075,13 @@ dependencies = [ [[package]] name = "mio" -version = "0.8.6" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b9d9a46eff5b4ff64b45a9e316a6d1e0bc719ef429cbec4dc630684212bfdf9" +checksum = "927a765cd3fc26206e66b296465fa9d3e5ab003e651c1b3c060e7956d96b19d2" dependencies = [ "libc", - "log", "wasi", - "windows-sys 0.45.0", + "windows-sys 0.48.0", ] [[package]] @@ -1967,9 +1966,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.28.0" +version = "1.28.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c3c786bf8134e5a3a166db9b29ab8f48134739014a3eca7bc6bfa95d673b136f" +checksum = "94d7b1cfd2aa4011f2de74c2c4c63665e27a71006b0a192dcd2710272e73dfa2" dependencies = [ "autocfg", "bytes", diff --git a/Cargo.toml b/Cargo.toml index dd2e2a1..f3e0f65 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -58,6 +58,9 @@ anyhow = "1.0.65" base64 = "0.21.0" fs-err = "2.8.1" fs2 = "0.4.3" +indoc = "2.0.0" +logtest = "2.0.0" +mockito = "1.0.0" pep508_rs = { version = "0.2.1", features = ["serde"] } pyo3 = { version = "0.18.1", features = ["extension-module", "abi3-py37"] } regex = "1.6.0" @@ -72,6 +75,7 @@ tracing = "0.1.36" tracing-subscriber = "0.3.15" unscanny = "0.1.0" walkdir = "2.3.2" +which = "4.3.0" [features] default = ["vendored"] @@ -88,8 +92,8 @@ inherits = "release" debug = 1 [dev-dependencies] -indoc = "2.0.0" -logtest = "2.0.0" -mockito = "1.0.0" -tempfile = "3.3.0" -which = "4.3.0" +indoc = { workspace = true } +logtest = { workspace = true } +mockito = { workspace = true } +tempfile = { workspace = true } +which = { workspace = true } diff --git a/install-wheel-rs/Cargo.toml b/install-wheel-rs/Cargo.toml index 4d8c900..51f96b1 100644 --- a/install-wheel-rs/Cargo.toml +++ b/install-wheel-rs/Cargo.toml @@ -37,4 +37,4 @@ zip = { version = "0.6.2", default-features = false, features = ["deflate"] } # python_bindings = ["pyo3", "tracing-subscriber"] [dev-dependencies] -indoc = "2.0.0" +indoc = { workspace = true } diff --git a/monotrail-utils/Cargo.toml b/monotrail-utils/Cargo.toml index 1f7316d..5121d12 100644 --- a/monotrail-utils/Cargo.toml +++ b/monotrail-utils/Cargo.toml @@ -14,6 +14,6 @@ tracing = { workspace = true } unscanny = { workspace = true } [dev-dependencies] -indoc = "2.0.0" -logtest = "2.0.0" -tempfile = "3.3.0" +indoc = { workspace = true } +logtest = { workspace = true } +tempfile = { workspace = true } From 938ce41f7b2de5c752da048f121910a48ef2d2b8 Mon Sep 17 00:00:00 2001 From: Chris Pryer Date: Sat, 10 Jun 2023 17:39:42 -0400 Subject: [PATCH 18/19] Fix requirements_txt tests --- monotrail-utils/src/requirements_txt.rs | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/monotrail-utils/src/requirements_txt.rs b/monotrail-utils/src/requirements_txt.rs index cedbef0..3056f79 100644 --- a/monotrail-utils/src/requirements_txt.rs +++ b/monotrail-utils/src/requirements_txt.rs @@ -469,13 +469,13 @@ mod test { use fs_err as fs; use indoc::indoc; use logtest::Logger; - use std::path::Path; + use std::path::{Path, PathBuf}; use tempfile::tempdir; use tracing::log::Level; #[test] fn test_requirements_txt_parsing() { - let working_dir = Path::new("test-data").join("requirements-txt"); + let working_dir = workspace_test_data_dir().join("requirements-txt"); for dir_entry in fs::read_dir(&working_dir).unwrap() { let dir_entry = dir_entry.unwrap().path(); if dir_entry.extension().unwrap_or_default().to_str().unwrap() != "txt" { @@ -495,7 +495,7 @@ mod test { fn test_other_line_endings() { let temp_dir = tempdir().unwrap(); let mut files = Vec::new(); - let working_dir = Path::new("test-data").join("requirements-txt"); + let working_dir = workspace_test_data_dir().join("requirements-txt"); for dir_entry in fs::read_dir(&working_dir).unwrap() { let dir_entry = dir_entry.unwrap(); if dir_entry @@ -531,7 +531,7 @@ mod test { #[test] #[ignore] fn test_pydantic() { - let working_dir = Path::new("test-data").join("requirements-pydantic"); + let working_dir = workspace_test_data_dir().join("requirments-pydantic"); for basic in fs::read_dir(&working_dir).unwrap() { let basic = basic.unwrap().path(); if !["txt", "in"].contains(&basic.extension().unwrap_or_default().to_str().unwrap()) { @@ -543,7 +543,7 @@ mod test { #[test] fn test_invalid_include_missing_file() { - let working_dir = Path::new("test-data").join("requirements-txt"); + let working_dir = workspace_test_data_dir().join("requirements-txt"); let basic = working_dir.join("invalid-include"); let missing = working_dir.join("missing.txt"); let err = RequirementsTxt::parse(&basic, &working_dir).unwrap_err(); @@ -568,7 +568,7 @@ mod test { #[test] fn test_invalid_requirement() { - let working_dir = Path::new("test-data").join("requirements-txt"); + let working_dir = workspace_test_data_dir().join("requirements-txt"); let basic = working_dir.join("invalid-requirement"); let err = RequirementsTxt::parse(&basic, &working_dir).unwrap_err(); let errors = anyhow::Error::new(err) @@ -592,7 +592,7 @@ mod test { #[test] fn test_empty_file() { - let working_dir = Path::new("test-data").join("requirements-txt"); + let working_dir = workspace_test_data_dir().join("requirements-txt"); let path = working_dir.join("empty.txt"); // TODO(konstin) I think that logger isn't thread safe @@ -607,4 +607,11 @@ mod test { .args() .ends_with("does not contain any dependencies")); } + + fn workspace_test_data_dir() -> PathBuf { + Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .join("test-data") + } } From e6526ad3dd9700a05b5a6b3708ee12a8c54fe017 Mon Sep 17 00:00:00 2001 From: Chris Pryer Date: Sun, 11 Jun 2023 04:11:16 -0400 Subject: [PATCH 19/19] Fix logtest issue on macos Closes #56 --- monotrail-utils/src/requirements_txt.rs | 20 -------------- .../tests/test_requirements_txt_empty_file.rs | 27 +++++++++++++++++++ 2 files changed, 27 insertions(+), 20 deletions(-) create mode 100644 monotrail-utils/tests/test_requirements_txt_empty_file.rs diff --git a/monotrail-utils/src/requirements_txt.rs b/monotrail-utils/src/requirements_txt.rs index 3056f79..0e4fc22 100644 --- a/monotrail-utils/src/requirements_txt.rs +++ b/monotrail-utils/src/requirements_txt.rs @@ -468,10 +468,8 @@ mod test { use crate::requirements_txt::RequirementsTxt; use fs_err as fs; use indoc::indoc; - use logtest::Logger; use std::path::{Path, PathBuf}; use tempfile::tempdir; - use tracing::log::Level; #[test] fn test_requirements_txt_parsing() { @@ -590,24 +588,6 @@ mod test { assert_eq!(errors, expected) } - #[test] - fn test_empty_file() { - let working_dir = workspace_test_data_dir().join("requirements-txt"); - let path = working_dir.join("empty.txt"); - - // TODO(konstin) I think that logger isn't thread safe - let logger = Logger::start(); - RequirementsTxt::parse(path, &working_dir).unwrap(); - let warnings: Vec<_> = logger - .into_iter() - .filter(|message| message.level() >= Level::Warn) - .collect(); - assert_eq!(warnings.len(), 1, "{:?}", warnings); - assert!(warnings[0] - .args() - .ends_with("does not contain any dependencies")); - } - fn workspace_test_data_dir() -> PathBuf { Path::new(env!("CARGO_MANIFEST_DIR")) .parent() diff --git a/monotrail-utils/tests/test_requirements_txt_empty_file.rs b/monotrail-utils/tests/test_requirements_txt_empty_file.rs new file mode 100644 index 0000000..95f2944 --- /dev/null +++ b/monotrail-utils/tests/test_requirements_txt_empty_file.rs @@ -0,0 +1,27 @@ +use logtest::Logger; +use monotrail_utils::RequirementsTxt; +use std::path::Path; +use tracing::log::Level; + +// NOTE: Prevent race conditions by running isolated from other tests +// See https://github.com/yoshuawuyts/logtest/blob/a6da0057fb52ec702e89eadf4689e3a56a97099b/src/lib.rs#L12-L16 +#[test] +fn test_empty_requirements_file() { + let working_dir = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .join("test-data") + .join("requirements-txt"); + let path = working_dir.join("empty.txt"); + + let logger = Logger::start(); + RequirementsTxt::parse(path, &working_dir).unwrap(); + let warnings: Vec<_> = logger + .into_iter() + .filter(|message| message.level() >= Level::Warn) + .collect(); + assert_eq!(warnings.len(), 1, "{:?}", warnings); + assert!(warnings[0] + .args() + .ends_with("does not contain any dependencies")); +}