From 0f8a634942319597ed456ae12c2a3331fcd28e01 Mon Sep 17 00:00:00 2001 From: zongz Date: Fri, 10 Nov 2023 15:09:53 +0800 Subject: [PATCH 1/9] feat: add file pattern as the compile entry Signed-off-by: zongz --- kclvm/Cargo.lock | 1 + kclvm/driver/Cargo.toml | 1 + kclvm/driver/src/lib.rs | 42 +++++- .../src/test_data/expand_file_pattern/kcl.mod | 5 + .../expand_file_pattern/kcl1/kcl.mod | 5 + .../expand_file_pattern/kcl1/kcl2/kcl.mod | 5 + .../expand_file_pattern/kcl1/kcl2/main.k | 1 + .../expand_file_pattern/kcl1/kcl4/kcl.mod | 5 + .../expand_file_pattern/kcl1/kcl4/main.k | 1 + .../test_data/expand_file_pattern/kcl1/main.k | 1 + .../expand_file_pattern/kcl3/kcl.mod | 5 + .../test_data/expand_file_pattern/kcl3/main.k | 1 + .../src/test_data/expand_file_pattern/main.k | 1 + kclvm/driver/src/tests.rs | 126 ++++++++++++++++++ .../runner/src/test_file_pattern/kcl1/kcl.mod | 5 + .../src/test_file_pattern/kcl1/kcl3/kcl.mod | 5 + .../src/test_file_pattern/kcl1/kcl3/main.k | 1 + .../runner/src/test_file_pattern/kcl1/main.k | 1 + .../runner/src/test_file_pattern/kcl2/kcl.mod | 5 + .../runner/src/test_file_pattern/kcl2/main.k | 1 + kclvm/runner/src/tests.rs | 19 +++ 21 files changed, 231 insertions(+), 6 deletions(-) create mode 100644 kclvm/driver/src/test_data/expand_file_pattern/kcl.mod create mode 100644 kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl.mod create mode 100644 kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl2/kcl.mod create mode 100644 kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl2/main.k create mode 100644 kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl4/kcl.mod create mode 100644 kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl4/main.k create mode 100644 kclvm/driver/src/test_data/expand_file_pattern/kcl1/main.k create mode 100644 kclvm/driver/src/test_data/expand_file_pattern/kcl3/kcl.mod create mode 100644 kclvm/driver/src/test_data/expand_file_pattern/kcl3/main.k create mode 100644 kclvm/driver/src/test_data/expand_file_pattern/main.k create mode 100644 kclvm/runner/src/test_file_pattern/kcl1/kcl.mod create mode 100644 kclvm/runner/src/test_file_pattern/kcl1/kcl3/kcl.mod create mode 100644 kclvm/runner/src/test_file_pattern/kcl1/kcl3/main.k create mode 100644 kclvm/runner/src/test_file_pattern/kcl1/main.k create mode 100644 kclvm/runner/src/test_file_pattern/kcl2/kcl.mod create mode 100644 kclvm/runner/src/test_file_pattern/kcl2/main.k diff --git a/kclvm/Cargo.lock b/kclvm/Cargo.lock index 041f66817..9578e9ded 100644 --- a/kclvm/Cargo.lock +++ b/kclvm/Cargo.lock @@ -1594,6 +1594,7 @@ name = "kclvm-driver" version = "0.7.0-alpha.2" dependencies = [ "anyhow", + "glob", "kclvm-ast", "kclvm-config", "kclvm-parser", diff --git a/kclvm/driver/Cargo.toml b/kclvm/driver/Cargo.toml index 96c8be259..fe867169b 100644 --- a/kclvm/driver/Cargo.toml +++ b/kclvm/driver/Cargo.toml @@ -17,3 +17,4 @@ walkdir = "2" serde = { version = "1.0", features = ["derive"] } anyhow = { version = "1.0.70", features = ["backtrace"] } +glob = "0.3.1" diff --git a/kclvm/driver/src/lib.rs b/kclvm/driver/src/lib.rs index 6acab4029..f7ed55ef5 100644 --- a/kclvm/driver/src/lib.rs +++ b/kclvm/driver/src/lib.rs @@ -6,6 +6,7 @@ pub const DEFAULT_PROJECT_FILE: &str = "project.yaml"; #[cfg(test)] mod tests; +use glob::glob; use kclvm_ast::ast; use kclvm_config::{ modfile::{KCL_FILE_EXTENSION, KCL_FILE_SUFFIX, KCL_MOD_PATH_ENV}, @@ -16,12 +17,28 @@ use kclvm_parser::LoadProgramOptions; use kclvm_utils::path::PathPrefix; use kpm_metadata::fill_pkg_maps_for_k_file; use std::{ - fs::read_dir, + collections::VecDeque, + fs::{self, read_dir}, io::{self, ErrorKind}, path::{Path, PathBuf}, }; use walkdir::WalkDir; +/// Expand the file pattern to a list of files and canonicalize the files. +pub fn expand_if_file_pattern(file_pattern: String) -> Result, String> { + let paths = glob(&file_pattern).map_err(|_| "Invalid file pattern")?; + let mut matched_files = vec![]; + + for path in paths { + if let Ok(path) = path { + let abs_path = fs::canonicalize(path).map_err(|err| err.to_string())?; + matched_files.push(abs_path.to_string_lossy().to_string()); + } + } + + Ok(matched_files) +} + /// Normalize input files with the working directory and replace ${KCL_MOD} with the module root path. pub fn canonicalize_input_files( k_files: &[String], @@ -29,17 +46,30 @@ pub fn canonicalize_input_files( check_exist: bool, ) -> Result, String> { let mut kcl_paths = Vec::::new(); + let mut k_files_queue: VecDeque = k_files.iter().map(|f| f.to_string()).collect(); + + // The first traversal expands the file pattern and converts the relative path to an absolute path. + while !k_files_queue.is_empty() { + let file = k_files_queue.pop_front().unwrap(); + let path = Path::new(&file); + if !path.exists() { + if let Ok(res) = expand_if_file_pattern(file.to_string()) { + res.iter().for_each(|p| { + if !kcl_paths.contains(p) { + k_files_queue.push_back(p.to_string()); + } + }); + continue; + } + } - // The first traversal changes the relative path to an absolute path - for (_, file) in k_files.iter().enumerate() { - let path = Path::new(file); let is_absolute = path.is_absolute(); let is_exist_maybe_symlink = path.exists(); // If the input file or path is a relative path and it is not a absolute path in the KCL module VFS, // join with the work directory path and convert it to a absolute path. let path = ModRelativePath::from(file.to_string()); let abs_path = if !is_absolute && !path.is_relative_path().map_err(|err| err.to_string())? { - let filepath = Path::new(&work_dir).join(file); + let filepath = Path::new(&work_dir).join(file.to_string()); match filepath.canonicalize() { Ok(path) => Some(path.adjust_canonicalization()), Err(_) => { @@ -57,7 +87,7 @@ pub fn canonicalize_input_files( }; // If the input file or path is a symlink, convert it to a real path. let real_path = if is_exist_maybe_symlink { - match PathBuf::from(file).canonicalize() { + match PathBuf::from(file.to_string()).canonicalize() { Ok(real_path) => Some(String::from(real_path.to_str().unwrap())), Err(_) => { if check_exist { diff --git a/kclvm/driver/src/test_data/expand_file_pattern/kcl.mod b/kclvm/driver/src/test_data/expand_file_pattern/kcl.mod new file mode 100644 index 000000000..f647ff5c5 --- /dev/null +++ b/kclvm/driver/src/test_data/expand_file_pattern/kcl.mod @@ -0,0 +1,5 @@ +[package] +name = "expand_file_pattern" +edition = "0.0.1" +version = "0.0.1" + diff --git a/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl.mod b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl.mod new file mode 100644 index 000000000..ab572545d --- /dev/null +++ b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl.mod @@ -0,0 +1,5 @@ +[package] +name = "kcl1" +edition = "0.0.1" +version = "0.0.1" + diff --git a/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl2/kcl.mod b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl2/kcl.mod new file mode 100644 index 000000000..5fb058acd --- /dev/null +++ b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl2/kcl.mod @@ -0,0 +1,5 @@ +[package] +name = "kcl2" +edition = "0.0.1" +version = "0.0.1" + diff --git a/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl2/main.k b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl2/main.k new file mode 100644 index 000000000..fa7048e63 --- /dev/null +++ b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl2/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl4/kcl.mod b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl4/kcl.mod new file mode 100644 index 000000000..98f429002 --- /dev/null +++ b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl4/kcl.mod @@ -0,0 +1,5 @@ +[package] +name = "kcl4" +edition = "0.0.1" +version = "0.0.1" + diff --git a/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl4/main.k b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl4/main.k new file mode 100644 index 000000000..fa7048e63 --- /dev/null +++ b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl4/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/kclvm/driver/src/test_data/expand_file_pattern/kcl1/main.k b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/main.k new file mode 100644 index 000000000..fa7048e63 --- /dev/null +++ b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/kclvm/driver/src/test_data/expand_file_pattern/kcl3/kcl.mod b/kclvm/driver/src/test_data/expand_file_pattern/kcl3/kcl.mod new file mode 100644 index 000000000..b99124970 --- /dev/null +++ b/kclvm/driver/src/test_data/expand_file_pattern/kcl3/kcl.mod @@ -0,0 +1,5 @@ +[package] +name = "kcl3" +edition = "0.0.1" +version = "0.0.1" + diff --git a/kclvm/driver/src/test_data/expand_file_pattern/kcl3/main.k b/kclvm/driver/src/test_data/expand_file_pattern/kcl3/main.k new file mode 100644 index 000000000..fa7048e63 --- /dev/null +++ b/kclvm/driver/src/test_data/expand_file_pattern/kcl3/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/kclvm/driver/src/test_data/expand_file_pattern/main.k b/kclvm/driver/src/test_data/expand_file_pattern/main.k new file mode 100644 index 000000000..fa7048e63 --- /dev/null +++ b/kclvm/driver/src/test_data/expand_file_pattern/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/kclvm/driver/src/tests.rs b/kclvm/driver/src/tests.rs index 3fcd7be44..b10377e37 100644 --- a/kclvm/driver/src/tests.rs +++ b/kclvm/driver/src/tests.rs @@ -25,6 +25,132 @@ fn test_canonicalize_input_files() { assert!(canonicalize_input_files(&input_files, work_dir, true).is_err()); } +#[test] +fn test_expand_if_file_pattern() { + let input_files = vec!["./src/test_data/expand_file_pattern/**/main.k".to_string()]; + let work_dir = ".".to_string(); + let expected_files = vec![ + Path::new("./src/test_data/expand_file_pattern/kcl1/kcl2/main.k") + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string(), + Path::new("./src/test_data/expand_file_pattern/kcl1/kcl4/main.k") + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string(), + Path::new("./src/test_data/expand_file_pattern/kcl1/main.k") + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string(), + Path::new("./src/test_data/expand_file_pattern/kcl3/main.k") + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string(), + Path::new("./src/test_data/expand_file_pattern/main.k") + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string(), + ]; + assert_eq!( + canonicalize_input_files(&input_files, work_dir.clone(), false).unwrap(), + expected_files + ); + + let input_files = vec![ + "./src/test_data/expand_file_pattern/**/main.k".to_string(), + "${KCL_MOD/aaa".to_string(), + ]; + let work_dir = ".".to_string(); + let expected_files = vec![ + Path::new("./src/test_data/expand_file_pattern/kcl1/kcl2/main.k") + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string(), + Path::new("./src/test_data/expand_file_pattern/kcl1/kcl4/main.k") + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string(), + Path::new("./src/test_data/expand_file_pattern/kcl1/main.k") + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string(), + Path::new("./src/test_data/expand_file_pattern/kcl3/main.k") + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string(), + Path::new("./src/test_data/expand_file_pattern/main.k") + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string(), + ]; + assert_eq!( + canonicalize_input_files(&input_files, work_dir.clone(), false).unwrap(), + expected_files + ); + + let input_files = vec![ + "./src/test_data/expand_file_pattern/kcl1/main.k".to_string(), + "${KCL_MOD/aaa".to_string(), + ]; + let work_dir = ".".to_string(); + let expected_files = vec![Path::new("./src/test_data/expand_file_pattern/kcl1/main.k") + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string()]; + assert_eq!( + canonicalize_input_files(&input_files, work_dir.clone(), false).unwrap(), + expected_files + ); + + let input_files = vec![ + "./src/test_data/expand_file_pattern/kcl1/main.k".to_string(), + "./src/test_data/expand_file_pattern/**/main.k".to_string(), + ]; + let work_dir = ".".to_string(); + let expected_files = vec![ + Path::new("./src/test_data/expand_file_pattern/kcl1/main.k") + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string(), + Path::new("./src/test_data/expand_file_pattern/kcl1/kcl2/main.k") + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string(), + Path::new("./src/test_data/expand_file_pattern/kcl1/kcl4/main.k") + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string(), + Path::new("./src/test_data/expand_file_pattern/kcl3/main.k") + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string(), + Path::new("./src/test_data/expand_file_pattern/main.k") + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string(), + ]; + assert_eq!( + canonicalize_input_files(&input_files, work_dir.clone(), false).unwrap(), + expected_files + ); +} + #[test] fn test_parse_key_value_pair() { let cases = [ diff --git a/kclvm/runner/src/test_file_pattern/kcl1/kcl.mod b/kclvm/runner/src/test_file_pattern/kcl1/kcl.mod new file mode 100644 index 000000000..ab572545d --- /dev/null +++ b/kclvm/runner/src/test_file_pattern/kcl1/kcl.mod @@ -0,0 +1,5 @@ +[package] +name = "kcl1" +edition = "0.0.1" +version = "0.0.1" + diff --git a/kclvm/runner/src/test_file_pattern/kcl1/kcl3/kcl.mod b/kclvm/runner/src/test_file_pattern/kcl1/kcl3/kcl.mod new file mode 100644 index 000000000..b99124970 --- /dev/null +++ b/kclvm/runner/src/test_file_pattern/kcl1/kcl3/kcl.mod @@ -0,0 +1,5 @@ +[package] +name = "kcl3" +edition = "0.0.1" +version = "0.0.1" + diff --git a/kclvm/runner/src/test_file_pattern/kcl1/kcl3/main.k b/kclvm/runner/src/test_file_pattern/kcl1/kcl3/main.k new file mode 100644 index 000000000..0acce2f52 --- /dev/null +++ b/kclvm/runner/src/test_file_pattern/kcl1/kcl3/main.k @@ -0,0 +1 @@ +k3 = 'Hello World!' \ No newline at end of file diff --git a/kclvm/runner/src/test_file_pattern/kcl1/main.k b/kclvm/runner/src/test_file_pattern/kcl1/main.k new file mode 100644 index 000000000..ac6689514 --- /dev/null +++ b/kclvm/runner/src/test_file_pattern/kcl1/main.k @@ -0,0 +1 @@ +k1 = 'Hello World!' \ No newline at end of file diff --git a/kclvm/runner/src/test_file_pattern/kcl2/kcl.mod b/kclvm/runner/src/test_file_pattern/kcl2/kcl.mod new file mode 100644 index 000000000..5fb058acd --- /dev/null +++ b/kclvm/runner/src/test_file_pattern/kcl2/kcl.mod @@ -0,0 +1,5 @@ +[package] +name = "kcl2" +edition = "0.0.1" +version = "0.0.1" + diff --git a/kclvm/runner/src/test_file_pattern/kcl2/main.k b/kclvm/runner/src/test_file_pattern/kcl2/main.k new file mode 100644 index 000000000..fe6900bb4 --- /dev/null +++ b/kclvm/runner/src/test_file_pattern/kcl2/main.k @@ -0,0 +1 @@ +k2 = 'Hello World!' \ No newline at end of file diff --git a/kclvm/runner/src/tests.rs b/kclvm/runner/src/tests.rs index 58659e2fe..98512a657 100644 --- a/kclvm/runner/src/tests.rs +++ b/kclvm/runner/src/tests.rs @@ -564,6 +564,9 @@ fn test_exec() { test_indent_error(); println!("test_indent_error - PASS"); + + test_compile_with_file_pattern(); + println!("test_compile_with_file_pattern - PASS"); } fn test_indent_error() { @@ -662,3 +665,19 @@ fn get_files>( } files } + +fn test_compile_with_file_pattern() { + let test_path = PathBuf::from("./src/test_file_pattern/**/main.k"); + let mut args = ExecProgramArgs::default(); + args.k_filename_list.push(test_path.display().to_string()); + let res = exec_program(Arc::new(ParseSession::default()), &args); + assert!(res.is_ok()); + assert_eq!( + res.clone().unwrap().yaml_result, + "k3: Hello World!\nk1: Hello World!\nk2: Hello World!" + ); + assert_eq!( + res.unwrap().json_result, + "[{\"k3\": \"Hello World!\", \"k1\": \"Hello World!\", \"k2\": \"Hello World!\"}]" + ); +} From 86e9c206b67507a647d720e0c5fa0e6095b74c75 Mon Sep 17 00:00:00 2001 From: zongz Date: Fri, 10 Nov 2023 15:52:57 +0800 Subject: [PATCH 2/9] fix: fix grammar test Signed-off-by: zongz --- kclvm/driver/src/lib.rs | 14 ++++--- .../src/test_data/expand_file_pattern/KCL_MOD | 0 kclvm/driver/src/tests.rs | 41 +++++++------------ 3 files changed, 23 insertions(+), 32 deletions(-) create mode 100644 kclvm/driver/src/test_data/expand_file_pattern/KCL_MOD diff --git a/kclvm/driver/src/lib.rs b/kclvm/driver/src/lib.rs index f7ed55ef5..86014fd22 100644 --- a/kclvm/driver/src/lib.rs +++ b/kclvm/driver/src/lib.rs @@ -54,12 +54,14 @@ pub fn canonicalize_input_files( let path = Path::new(&file); if !path.exists() { if let Ok(res) = expand_if_file_pattern(file.to_string()) { - res.iter().for_each(|p| { - if !kcl_paths.contains(p) { - k_files_queue.push_back(p.to_string()); - } - }); - continue; + if !res.is_empty() { + res.iter().for_each(|p| { + if !kcl_paths.contains(p) { + k_files_queue.push_back(p.to_string()); + } + }); + continue; + } } } diff --git a/kclvm/driver/src/test_data/expand_file_pattern/KCL_MOD b/kclvm/driver/src/test_data/expand_file_pattern/KCL_MOD new file mode 100644 index 000000000..e69de29bb diff --git a/kclvm/driver/src/tests.rs b/kclvm/driver/src/tests.rs index b10377e37..3314634a6 100644 --- a/kclvm/driver/src/tests.rs +++ b/kclvm/driver/src/tests.rs @@ -26,31 +26,35 @@ fn test_canonicalize_input_files() { } #[test] -fn test_expand_if_file_pattern() { - let input_files = vec!["./src/test_data/expand_file_pattern/**/main.k".to_string()]; +fn test_expand_if_file_pattern_with_kcl_mod() { + let path = PathBuf::from("./src/test_data/expand_file_pattern"); + let input_files = vec![ + path.join("**").join("main.k").to_string_lossy().to_string(), + "${KCL_MOD}/src/test_data/expand_file_pattern/KCL_MOD".to_string(), + ]; let work_dir = ".".to_string(); let expected_files = vec![ - Path::new("./src/test_data/expand_file_pattern/kcl1/kcl2/main.k") + path.join("kcl1/kcl2/main.k") .canonicalize() .unwrap() .to_string_lossy() .to_string(), - Path::new("./src/test_data/expand_file_pattern/kcl1/kcl4/main.k") + path.join("kcl1/kcl4/main.k") .canonicalize() .unwrap() .to_string_lossy() .to_string(), - Path::new("./src/test_data/expand_file_pattern/kcl1/main.k") + path.join("kcl1/main.k") .canonicalize() .unwrap() .to_string_lossy() .to_string(), - Path::new("./src/test_data/expand_file_pattern/kcl3/main.k") + path.join("kcl3/main.k") .canonicalize() .unwrap() .to_string_lossy() .to_string(), - Path::new("./src/test_data/expand_file_pattern/main.k") + path.join("main.k") .canonicalize() .unwrap() .to_string_lossy() @@ -60,11 +64,11 @@ fn test_expand_if_file_pattern() { canonicalize_input_files(&input_files, work_dir.clone(), false).unwrap(), expected_files ); +} - let input_files = vec![ - "./src/test_data/expand_file_pattern/**/main.k".to_string(), - "${KCL_MOD/aaa".to_string(), - ]; +#[test] +fn test_expand_if_file_pattern() { + let input_files = vec!["./src/test_data/expand_file_pattern/**/main.k".to_string()]; let work_dir = ".".to_string(); let expected_files = vec![ Path::new("./src/test_data/expand_file_pattern/kcl1/kcl2/main.k") @@ -98,21 +102,6 @@ fn test_expand_if_file_pattern() { expected_files ); - let input_files = vec![ - "./src/test_data/expand_file_pattern/kcl1/main.k".to_string(), - "${KCL_MOD/aaa".to_string(), - ]; - let work_dir = ".".to_string(); - let expected_files = vec![Path::new("./src/test_data/expand_file_pattern/kcl1/main.k") - .canonicalize() - .unwrap() - .to_string_lossy() - .to_string()]; - assert_eq!( - canonicalize_input_files(&input_files, work_dir.clone(), false).unwrap(), - expected_files - ); - let input_files = vec![ "./src/test_data/expand_file_pattern/kcl1/main.k".to_string(), "./src/test_data/expand_file_pattern/**/main.k".to_string(), From b3989095bbdb7e50c880249ca070474253c5412f Mon Sep 17 00:00:00 2001 From: zongz Date: Fri, 10 Nov 2023 17:45:24 +0800 Subject: [PATCH 3/9] fix: fix failed test case Signed-off-by: zongz --- kclvm/driver/src/lib.rs | 33 +++++++++++++++------------------ kclvm/runner/src/lib.rs | 5 +++-- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/kclvm/driver/src/lib.rs b/kclvm/driver/src/lib.rs index 86014fd22..a3faf9a75 100644 --- a/kclvm/driver/src/lib.rs +++ b/kclvm/driver/src/lib.rs @@ -39,6 +39,18 @@ pub fn expand_if_file_pattern(file_pattern: String) -> Result, Strin Ok(matched_files) } +pub fn expand_input_files(k_files: &[String]) -> Vec { + let mut res = vec![]; + for file in k_files { + if let Ok(files) = expand_if_file_pattern(file.to_string()) { + res.extend(files); + } else { + res.push(file.to_string()); + } + } + res +} + /// Normalize input files with the working directory and replace ${KCL_MOD} with the module root path. pub fn canonicalize_input_files( k_files: &[String], @@ -46,24 +58,9 @@ pub fn canonicalize_input_files( check_exist: bool, ) -> Result, String> { let mut kcl_paths = Vec::::new(); - let mut k_files_queue: VecDeque = k_files.iter().map(|f| f.to_string()).collect(); - - // The first traversal expands the file pattern and converts the relative path to an absolute path. - while !k_files_queue.is_empty() { - let file = k_files_queue.pop_front().unwrap(); - let path = Path::new(&file); - if !path.exists() { - if let Ok(res) = expand_if_file_pattern(file.to_string()) { - if !res.is_empty() { - res.iter().for_each(|p| { - if !kcl_paths.contains(p) { - k_files_queue.push_back(p.to_string()); - } - }); - continue; - } - } - } + // The first traversal changes the relative path to an absolute path + for (_, file) in k_files.iter().enumerate() { + let path = Path::new(file); let is_absolute = path.is_absolute(); let is_exist_maybe_symlink = path.exists(); diff --git a/kclvm/runner/src/lib.rs b/kclvm/runner/src/lib.rs index 8142e5d26..7c86b07a5 100644 --- a/kclvm/runner/src/lib.rs +++ b/kclvm/runner/src/lib.rs @@ -6,7 +6,7 @@ use kclvm_ast::{ ast::{Module, Program}, MAIN_PKG, }; -use kclvm_driver::canonicalize_input_files; +use kclvm_driver::{canonicalize_input_files, expand_input_files}; use kclvm_error::{Diagnostic, Handler}; use kclvm_parser::{load_program, ParseSession}; use kclvm_query::apply_overrides; @@ -79,7 +79,8 @@ pub fn exec_program( let opts = args.get_load_program_options(); let k_files = &args.k_filename_list; let work_dir = args.work_dir.clone().unwrap_or_default(); - let kcl_paths = canonicalize_input_files(k_files, work_dir, false)?; + let k_files = expand_input_files(k_files); + let kcl_paths = canonicalize_input_files(&k_files, work_dir, false)?; let kcl_paths_str = kcl_paths.iter().map(|s| s.as_str()).collect::>(); From 385cb2225537984c52cd634a1c22333a279ae60b Mon Sep 17 00:00:00 2001 From: zongz Date: Fri, 10 Nov 2023 19:05:16 +0800 Subject: [PATCH 4/9] fix: fix failed test case Signed-off-by: zongz --- kclvm/driver/src/lib.rs | 12 ++++++++---- kclvm/driver/src/main.k | 10 ++++++++++ kclvm/driver/src/tests.rs | 38 ++++++++++++++------------------------ 3 files changed, 32 insertions(+), 28 deletions(-) create mode 100644 kclvm/driver/src/main.k diff --git a/kclvm/driver/src/lib.rs b/kclvm/driver/src/lib.rs index a3faf9a75..da2d5d49d 100644 --- a/kclvm/driver/src/lib.rs +++ b/kclvm/driver/src/lib.rs @@ -17,7 +17,7 @@ use kclvm_parser::LoadProgramOptions; use kclvm_utils::path::PathPrefix; use kpm_metadata::fill_pkg_maps_for_k_file; use std::{ - collections::VecDeque, + collections::{HashSet, VecDeque}, fs::{self, read_dir}, io::{self, ErrorKind}, path::{Path, PathBuf}, @@ -31,7 +31,7 @@ pub fn expand_if_file_pattern(file_pattern: String) -> Result, Strin for path in paths { if let Ok(path) = path { - let abs_path = fs::canonicalize(path).map_err(|err| err.to_string())?; + let abs_path: PathBuf = fs::canonicalize(path).map_err(|err| err.to_string())?; matched_files.push(abs_path.to_string_lossy().to_string()); } } @@ -43,9 +43,13 @@ pub fn expand_input_files(k_files: &[String]) -> Vec { let mut res = vec![]; for file in k_files { if let Ok(files) = expand_if_file_pattern(file.to_string()) { - res.extend(files); + if !files.is_empty() { + res.extend(files); + } else { + res.push(file.to_string()) + } } else { - res.push(file.to_string()); + res.push(file.to_string()) } } res diff --git a/kclvm/driver/src/main.k b/kclvm/driver/src/main.k new file mode 100644 index 000000000..1960d15f4 --- /dev/null +++ b/kclvm/driver/src/main.k @@ -0,0 +1,10 @@ +[item | { + if item.kind == "Job": + spec.template.metadata.labels = { + meta.k8s.alipay.com/zone = tydd + sigma.ali/app-name = sigmaetcd + sigma.ali/deploy-unit = sigmaetcd-prod + sigma.ali/instance-group = sigma3-alipay-master + sigma.ali/site = tydd + } + } for item in option("items")] \ No newline at end of file diff --git a/kclvm/driver/src/tests.rs b/kclvm/driver/src/tests.rs index 3314634a6..151735e5d 100644 --- a/kclvm/driver/src/tests.rs +++ b/kclvm/driver/src/tests.rs @@ -7,8 +7,8 @@ use kclvm_parser::LoadProgramOptions; use walkdir::WalkDir; use crate::arguments::parse_key_value_pair; -use crate::canonicalize_input_files; use crate::kpm_metadata::{fetch_metadata, fill_pkg_maps_for_k_file, lookup_the_nearest_file_dir}; +use crate::{canonicalize_input_files, expand_input_files}; #[test] fn test_canonicalize_input_files() { @@ -26,13 +26,12 @@ fn test_canonicalize_input_files() { } #[test] -fn test_expand_if_file_pattern_with_kcl_mod() { +fn test_expand_input_files_with_kcl_mod() { let path = PathBuf::from("./src/test_data/expand_file_pattern"); let input_files = vec![ path.join("**").join("main.k").to_string_lossy().to_string(), "${KCL_MOD}/src/test_data/expand_file_pattern/KCL_MOD".to_string(), ]; - let work_dir = ".".to_string(); let expected_files = vec![ path.join("kcl1/kcl2/main.k") .canonicalize() @@ -59,55 +58,46 @@ fn test_expand_if_file_pattern_with_kcl_mod() { .unwrap() .to_string_lossy() .to_string(), + "${KCL_MOD}/src/test_data/expand_file_pattern/KCL_MOD".to_string(), ]; - assert_eq!( - canonicalize_input_files(&input_files, work_dir.clone(), false).unwrap(), - expected_files - ); + assert_eq!(expand_input_files(&input_files), expected_files); } #[test] -fn test_expand_if_file_pattern() { +fn test_expand_input_files() { let input_files = vec!["./src/test_data/expand_file_pattern/**/main.k".to_string()]; - let work_dir = ".".to_string(); - let expected_files = vec![ + let mut expected_files = vec![ Path::new("./src/test_data/expand_file_pattern/kcl1/kcl2/main.k") .canonicalize() .unwrap() .to_string_lossy() .to_string(), - Path::new("./src/test_data/expand_file_pattern/kcl1/kcl4/main.k") - .canonicalize() - .unwrap() - .to_string_lossy() - .to_string(), - Path::new("./src/test_data/expand_file_pattern/kcl1/main.k") + Path::new("./src/test_data/expand_file_pattern/kcl3/main.k") .canonicalize() .unwrap() .to_string_lossy() .to_string(), - Path::new("./src/test_data/expand_file_pattern/kcl3/main.k") + Path::new("./src/test_data/expand_file_pattern/main.k") .canonicalize() .unwrap() .to_string_lossy() .to_string(), - Path::new("./src/test_data/expand_file_pattern/main.k") + Path::new("./src/test_data/expand_file_pattern/kcl1/kcl4/main.k") .canonicalize() .unwrap() .to_string_lossy() .to_string(), ]; assert_eq!( - canonicalize_input_files(&input_files, work_dir.clone(), false).unwrap(), - expected_files + expand_input_files(&input_files).sort(), + expected_files.sort() ); let input_files = vec![ "./src/test_data/expand_file_pattern/kcl1/main.k".to_string(), "./src/test_data/expand_file_pattern/**/main.k".to_string(), ]; - let work_dir = ".".to_string(); - let expected_files = vec![ + let mut expected_files = vec![ Path::new("./src/test_data/expand_file_pattern/kcl1/main.k") .canonicalize() .unwrap() @@ -135,8 +125,8 @@ fn test_expand_if_file_pattern() { .to_string(), ]; assert_eq!( - canonicalize_input_files(&input_files, work_dir.clone(), false).unwrap(), - expected_files + expand_input_files(&input_files).sort(), + expected_files.sort() ); } From 1bd74f0e2009f005e8fc153f1e5250e2712d92b5 Mon Sep 17 00:00:00 2001 From: zongz Date: Sat, 11 Nov 2023 13:51:53 +0800 Subject: [PATCH 5/9] fix: rm useless files Signed-off-by: zongz --- kclvm/driver/src/main.k | 10 ---------- 1 file changed, 10 deletions(-) delete mode 100644 kclvm/driver/src/main.k diff --git a/kclvm/driver/src/main.k b/kclvm/driver/src/main.k deleted file mode 100644 index 1960d15f4..000000000 --- a/kclvm/driver/src/main.k +++ /dev/null @@ -1,10 +0,0 @@ -[item | { - if item.kind == "Job": - spec.template.metadata.labels = { - meta.k8s.alipay.com/zone = tydd - sigma.ali/app-name = sigmaetcd - sigma.ali/deploy-unit = sigmaetcd-prod - sigma.ali/instance-group = sigma3-alipay-master - sigma.ali/site = tydd - } - } for item in option("items")] \ No newline at end of file From fcb29797388fea9631d8649978eef0d2139c54cf Mon Sep 17 00:00:00 2001 From: zongz Date: Mon, 13 Nov 2023 12:56:49 +0800 Subject: [PATCH 6/9] fix: fix windows test cases Signed-off-by: zongz --- kclvm/driver/src/lib.rs | 8 +++----- kclvm/driver/src/tests.rs | 32 ++++++-------------------------- 2 files changed, 9 insertions(+), 31 deletions(-) diff --git a/kclvm/driver/src/lib.rs b/kclvm/driver/src/lib.rs index da2d5d49d..4ead1d703 100644 --- a/kclvm/driver/src/lib.rs +++ b/kclvm/driver/src/lib.rs @@ -17,22 +17,20 @@ use kclvm_parser::LoadProgramOptions; use kclvm_utils::path::PathPrefix; use kpm_metadata::fill_pkg_maps_for_k_file; use std::{ - collections::{HashSet, VecDeque}, - fs::{self, read_dir}, + fs::read_dir, io::{self, ErrorKind}, path::{Path, PathBuf}, }; use walkdir::WalkDir; -/// Expand the file pattern to a list of files and canonicalize the files. +/// Expand the file pattern to a list of files. pub fn expand_if_file_pattern(file_pattern: String) -> Result, String> { let paths = glob(&file_pattern).map_err(|_| "Invalid file pattern")?; let mut matched_files = vec![]; for path in paths { if let Ok(path) = path { - let abs_path: PathBuf = fs::canonicalize(path).map_err(|err| err.to_string())?; - matched_files.push(abs_path.to_string_lossy().to_string()); + matched_files.push(path.to_string_lossy().to_string()); } } diff --git a/kclvm/driver/src/tests.rs b/kclvm/driver/src/tests.rs index 151735e5d..fddec3088 100644 --- a/kclvm/driver/src/tests.rs +++ b/kclvm/driver/src/tests.rs @@ -27,37 +27,17 @@ fn test_canonicalize_input_files() { #[test] fn test_expand_input_files_with_kcl_mod() { - let path = PathBuf::from("./src/test_data/expand_file_pattern"); + let path = PathBuf::from("src/test_data/expand_file_pattern"); let input_files = vec![ path.join("**").join("main.k").to_string_lossy().to_string(), "${KCL_MOD}/src/test_data/expand_file_pattern/KCL_MOD".to_string(), ]; let expected_files = vec![ - path.join("kcl1/kcl2/main.k") - .canonicalize() - .unwrap() - .to_string_lossy() - .to_string(), - path.join("kcl1/kcl4/main.k") - .canonicalize() - .unwrap() - .to_string_lossy() - .to_string(), - path.join("kcl1/main.k") - .canonicalize() - .unwrap() - .to_string_lossy() - .to_string(), - path.join("kcl3/main.k") - .canonicalize() - .unwrap() - .to_string_lossy() - .to_string(), - path.join("main.k") - .canonicalize() - .unwrap() - .to_string_lossy() - .to_string(), + path.join("kcl1/kcl2/main.k").to_string_lossy().to_string(), + path.join("kcl1/kcl4/main.k").to_string_lossy().to_string(), + path.join("kcl1/main.k").to_string_lossy().to_string(), + path.join("kcl3/main.k").to_string_lossy().to_string(), + path.join("main.k").to_string_lossy().to_string(), "${KCL_MOD}/src/test_data/expand_file_pattern/KCL_MOD".to_string(), ]; assert_eq!(expand_input_files(&input_files), expected_files); From 1fd17f3d02216b879d874afa2fbb5ff7b549b2ce Mon Sep 17 00:00:00 2001 From: zongz Date: Mon, 13 Nov 2023 13:00:17 +0800 Subject: [PATCH 7/9] fix: fix CR comments Signed-off-by: zongz --- kclvm/driver/src/lib.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kclvm/driver/src/lib.rs b/kclvm/driver/src/lib.rs index 4ead1d703..d9595b6ae 100644 --- a/kclvm/driver/src/lib.rs +++ b/kclvm/driver/src/lib.rs @@ -25,7 +25,7 @@ use walkdir::WalkDir; /// Expand the file pattern to a list of files. pub fn expand_if_file_pattern(file_pattern: String) -> Result, String> { - let paths = glob(&file_pattern).map_err(|_| "Invalid file pattern")?; + let paths = glob(&file_pattern).map_err(|_| format!("invalid file pattern {file_pattern}"))?; let mut matched_files = vec![]; for path in paths { From 0191adedaefa66ff7f24fcd0d16806ee8844f999 Mon Sep 17 00:00:00 2001 From: zongz Date: Mon, 13 Nov 2023 14:04:16 +0800 Subject: [PATCH 8/9] fix: fix windows test case Signed-off-by: zongz --- kclvm/driver/src/tests.rs | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/kclvm/driver/src/tests.rs b/kclvm/driver/src/tests.rs index fddec3088..753272649 100644 --- a/kclvm/driver/src/tests.rs +++ b/kclvm/driver/src/tests.rs @@ -1,6 +1,7 @@ use std::path::{Path, PathBuf}; use std::{env, fs, panic}; +use kclvm_ast::expr_as; use kclvm_config::modfile::get_vendor_home; use kclvm_config::settings::KeyValuePair; use kclvm_parser::LoadProgramOptions; @@ -40,7 +41,15 @@ fn test_expand_input_files_with_kcl_mod() { path.join("main.k").to_string_lossy().to_string(), "${KCL_MOD}/src/test_data/expand_file_pattern/KCL_MOD".to_string(), ]; - assert_eq!(expand_input_files(&input_files), expected_files); + let got_paths: Vec = expand_input_files(&input_files) + .iter() + .map(|s| s.replace("/", "").replace("\\", "")) + .collect(); + let expect_paths: Vec = expected_files + .iter() + .map(|s| s.replace("/", "").replace("\\", "")) + .collect(); + assert_eq!(got_paths, expect_paths); } #[test] From 4485fdd354f4de06420e49f70774017dfd08e02e Mon Sep 17 00:00:00 2001 From: zongz Date: Mon, 13 Nov 2023 15:05:34 +0800 Subject: [PATCH 9/9] fix: rm useless import Signed-off-by: zongz --- kclvm/driver/src/tests.rs | 1 - 1 file changed, 1 deletion(-) diff --git a/kclvm/driver/src/tests.rs b/kclvm/driver/src/tests.rs index 753272649..3849582ae 100644 --- a/kclvm/driver/src/tests.rs +++ b/kclvm/driver/src/tests.rs @@ -1,7 +1,6 @@ use std::path::{Path, PathBuf}; use std::{env, fs, panic}; -use kclvm_ast::expr_as; use kclvm_config::modfile::get_vendor_home; use kclvm_config::settings::KeyValuePair; use kclvm_parser::LoadProgramOptions;