diff --git a/kclvm/Cargo.lock b/kclvm/Cargo.lock index cd207f202..5804760e2 100644 --- a/kclvm/Cargo.lock +++ b/kclvm/Cargo.lock @@ -1594,6 +1594,7 @@ name = "kclvm-driver" version = "0.7.0-alpha.2" dependencies = [ "anyhow", + "glob", "kclvm-ast", "kclvm-config", "kclvm-parser", diff --git a/kclvm/driver/Cargo.toml b/kclvm/driver/Cargo.toml index 96c8be259..fe867169b 100644 --- a/kclvm/driver/Cargo.toml +++ b/kclvm/driver/Cargo.toml @@ -17,3 +17,4 @@ walkdir = "2" serde = { version = "1.0", features = ["derive"] } anyhow = { version = "1.0.70", features = ["backtrace"] } +glob = "0.3.1" diff --git a/kclvm/driver/src/lib.rs b/kclvm/driver/src/lib.rs index 6acab4029..d9595b6ae 100644 --- a/kclvm/driver/src/lib.rs +++ b/kclvm/driver/src/lib.rs @@ -6,6 +6,7 @@ pub const DEFAULT_PROJECT_FILE: &str = "project.yaml"; #[cfg(test)] mod tests; +use glob::glob; use kclvm_ast::ast; use kclvm_config::{ modfile::{KCL_FILE_EXTENSION, KCL_FILE_SUFFIX, KCL_MOD_PATH_ENV}, @@ -22,6 +23,36 @@ use std::{ }; use walkdir::WalkDir; +/// Expand the file pattern to a list of files. +pub fn expand_if_file_pattern(file_pattern: String) -> Result, String> { + let paths = glob(&file_pattern).map_err(|_| format!("invalid file pattern {file_pattern}"))?; + let mut matched_files = vec![]; + + for path in paths { + if let Ok(path) = path { + matched_files.push(path.to_string_lossy().to_string()); + } + } + + Ok(matched_files) +} + +pub fn expand_input_files(k_files: &[String]) -> Vec { + let mut res = vec![]; + for file in k_files { + if let Ok(files) = expand_if_file_pattern(file.to_string()) { + if !files.is_empty() { + res.extend(files); + } else { + res.push(file.to_string()) + } + } else { + res.push(file.to_string()) + } + } + res +} + /// Normalize input files with the working directory and replace ${KCL_MOD} with the module root path. pub fn canonicalize_input_files( k_files: &[String], @@ -29,17 +60,17 @@ pub fn canonicalize_input_files( check_exist: bool, ) -> Result, String> { let mut kcl_paths = Vec::::new(); - // The first traversal changes the relative path to an absolute path for (_, file) in k_files.iter().enumerate() { let path = Path::new(file); + let is_absolute = path.is_absolute(); let is_exist_maybe_symlink = path.exists(); // If the input file or path is a relative path and it is not a absolute path in the KCL module VFS, // join with the work directory path and convert it to a absolute path. let path = ModRelativePath::from(file.to_string()); let abs_path = if !is_absolute && !path.is_relative_path().map_err(|err| err.to_string())? { - let filepath = Path::new(&work_dir).join(file); + let filepath = Path::new(&work_dir).join(file.to_string()); match filepath.canonicalize() { Ok(path) => Some(path.adjust_canonicalization()), Err(_) => { @@ -57,7 +88,7 @@ pub fn canonicalize_input_files( }; // If the input file or path is a symlink, convert it to a real path. let real_path = if is_exist_maybe_symlink { - match PathBuf::from(file).canonicalize() { + match PathBuf::from(file.to_string()).canonicalize() { Ok(real_path) => Some(String::from(real_path.to_str().unwrap())), Err(_) => { if check_exist { diff --git a/kclvm/driver/src/test_data/expand_file_pattern/KCL_MOD b/kclvm/driver/src/test_data/expand_file_pattern/KCL_MOD new file mode 100644 index 000000000..e69de29bb diff --git a/kclvm/driver/src/test_data/expand_file_pattern/kcl.mod b/kclvm/driver/src/test_data/expand_file_pattern/kcl.mod new file mode 100644 index 000000000..f647ff5c5 --- /dev/null +++ b/kclvm/driver/src/test_data/expand_file_pattern/kcl.mod @@ -0,0 +1,5 @@ +[package] +name = "expand_file_pattern" +edition = "0.0.1" +version = "0.0.1" + diff --git a/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl.mod b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl.mod new file mode 100644 index 000000000..ab572545d --- /dev/null +++ b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl.mod @@ -0,0 +1,5 @@ +[package] +name = "kcl1" +edition = "0.0.1" +version = "0.0.1" + diff --git a/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl2/kcl.mod b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl2/kcl.mod new file mode 100644 index 000000000..5fb058acd --- /dev/null +++ b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl2/kcl.mod @@ -0,0 +1,5 @@ +[package] +name = "kcl2" +edition = "0.0.1" +version = "0.0.1" + diff --git a/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl2/main.k b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl2/main.k new file mode 100644 index 000000000..fa7048e63 --- /dev/null +++ b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl2/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl4/kcl.mod b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl4/kcl.mod new file mode 100644 index 000000000..98f429002 --- /dev/null +++ b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl4/kcl.mod @@ -0,0 +1,5 @@ +[package] +name = "kcl4" +edition = "0.0.1" +version = "0.0.1" + diff --git a/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl4/main.k b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl4/main.k new file mode 100644 index 000000000..fa7048e63 --- /dev/null +++ b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl4/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/kclvm/driver/src/test_data/expand_file_pattern/kcl1/main.k b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/main.k new file mode 100644 index 000000000..fa7048e63 --- /dev/null +++ b/kclvm/driver/src/test_data/expand_file_pattern/kcl1/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/kclvm/driver/src/test_data/expand_file_pattern/kcl3/kcl.mod b/kclvm/driver/src/test_data/expand_file_pattern/kcl3/kcl.mod new file mode 100644 index 000000000..b99124970 --- /dev/null +++ b/kclvm/driver/src/test_data/expand_file_pattern/kcl3/kcl.mod @@ -0,0 +1,5 @@ +[package] +name = "kcl3" +edition = "0.0.1" +version = "0.0.1" + diff --git a/kclvm/driver/src/test_data/expand_file_pattern/kcl3/main.k b/kclvm/driver/src/test_data/expand_file_pattern/kcl3/main.k new file mode 100644 index 000000000..fa7048e63 --- /dev/null +++ b/kclvm/driver/src/test_data/expand_file_pattern/kcl3/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/kclvm/driver/src/test_data/expand_file_pattern/main.k b/kclvm/driver/src/test_data/expand_file_pattern/main.k new file mode 100644 index 000000000..fa7048e63 --- /dev/null +++ b/kclvm/driver/src/test_data/expand_file_pattern/main.k @@ -0,0 +1 @@ +The_first_kcl_program = 'Hello World!' \ No newline at end of file diff --git a/kclvm/driver/src/tests.rs b/kclvm/driver/src/tests.rs index 3fcd7be44..3849582ae 100644 --- a/kclvm/driver/src/tests.rs +++ b/kclvm/driver/src/tests.rs @@ -7,8 +7,8 @@ use kclvm_parser::LoadProgramOptions; use walkdir::WalkDir; use crate::arguments::parse_key_value_pair; -use crate::canonicalize_input_files; use crate::kpm_metadata::{fetch_metadata, fill_pkg_maps_for_k_file, lookup_the_nearest_file_dir}; +use crate::{canonicalize_input_files, expand_input_files}; #[test] fn test_canonicalize_input_files() { @@ -25,6 +25,99 @@ fn test_canonicalize_input_files() { assert!(canonicalize_input_files(&input_files, work_dir, true).is_err()); } +#[test] +fn test_expand_input_files_with_kcl_mod() { + let path = PathBuf::from("src/test_data/expand_file_pattern"); + let input_files = vec![ + path.join("**").join("main.k").to_string_lossy().to_string(), + "${KCL_MOD}/src/test_data/expand_file_pattern/KCL_MOD".to_string(), + ]; + let expected_files = vec![ + path.join("kcl1/kcl2/main.k").to_string_lossy().to_string(), + path.join("kcl1/kcl4/main.k").to_string_lossy().to_string(), + path.join("kcl1/main.k").to_string_lossy().to_string(), + path.join("kcl3/main.k").to_string_lossy().to_string(), + path.join("main.k").to_string_lossy().to_string(), + "${KCL_MOD}/src/test_data/expand_file_pattern/KCL_MOD".to_string(), + ]; + let got_paths: Vec = expand_input_files(&input_files) + .iter() + .map(|s| s.replace("/", "").replace("\\", "")) + .collect(); + let expect_paths: Vec = expected_files + .iter() + .map(|s| s.replace("/", "").replace("\\", "")) + .collect(); + assert_eq!(got_paths, expect_paths); +} + +#[test] +fn test_expand_input_files() { + let input_files = vec!["./src/test_data/expand_file_pattern/**/main.k".to_string()]; + let mut expected_files = vec![ + Path::new("./src/test_data/expand_file_pattern/kcl1/kcl2/main.k") + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string(), + Path::new("./src/test_data/expand_file_pattern/kcl3/main.k") + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string(), + Path::new("./src/test_data/expand_file_pattern/main.k") + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string(), + Path::new("./src/test_data/expand_file_pattern/kcl1/kcl4/main.k") + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string(), + ]; + assert_eq!( + expand_input_files(&input_files).sort(), + expected_files.sort() + ); + + let input_files = vec![ + "./src/test_data/expand_file_pattern/kcl1/main.k".to_string(), + "./src/test_data/expand_file_pattern/**/main.k".to_string(), + ]; + let mut expected_files = vec![ + Path::new("./src/test_data/expand_file_pattern/kcl1/main.k") + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string(), + Path::new("./src/test_data/expand_file_pattern/kcl1/kcl2/main.k") + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string(), + Path::new("./src/test_data/expand_file_pattern/kcl1/kcl4/main.k") + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string(), + Path::new("./src/test_data/expand_file_pattern/kcl3/main.k") + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string(), + Path::new("./src/test_data/expand_file_pattern/main.k") + .canonicalize() + .unwrap() + .to_string_lossy() + .to_string(), + ]; + assert_eq!( + expand_input_files(&input_files).sort(), + expected_files.sort() + ); +} + #[test] fn test_parse_key_value_pair() { let cases = [ diff --git a/kclvm/runner/src/lib.rs b/kclvm/runner/src/lib.rs index 8142e5d26..7c86b07a5 100644 --- a/kclvm/runner/src/lib.rs +++ b/kclvm/runner/src/lib.rs @@ -6,7 +6,7 @@ use kclvm_ast::{ ast::{Module, Program}, MAIN_PKG, }; -use kclvm_driver::canonicalize_input_files; +use kclvm_driver::{canonicalize_input_files, expand_input_files}; use kclvm_error::{Diagnostic, Handler}; use kclvm_parser::{load_program, ParseSession}; use kclvm_query::apply_overrides; @@ -79,7 +79,8 @@ pub fn exec_program( let opts = args.get_load_program_options(); let k_files = &args.k_filename_list; let work_dir = args.work_dir.clone().unwrap_or_default(); - let kcl_paths = canonicalize_input_files(k_files, work_dir, false)?; + let k_files = expand_input_files(k_files); + let kcl_paths = canonicalize_input_files(&k_files, work_dir, false)?; let kcl_paths_str = kcl_paths.iter().map(|s| s.as_str()).collect::>(); diff --git a/kclvm/runner/src/test_file_pattern/kcl1/kcl.mod b/kclvm/runner/src/test_file_pattern/kcl1/kcl.mod new file mode 100644 index 000000000..ab572545d --- /dev/null +++ b/kclvm/runner/src/test_file_pattern/kcl1/kcl.mod @@ -0,0 +1,5 @@ +[package] +name = "kcl1" +edition = "0.0.1" +version = "0.0.1" + diff --git a/kclvm/runner/src/test_file_pattern/kcl1/kcl3/kcl.mod b/kclvm/runner/src/test_file_pattern/kcl1/kcl3/kcl.mod new file mode 100644 index 000000000..b99124970 --- /dev/null +++ b/kclvm/runner/src/test_file_pattern/kcl1/kcl3/kcl.mod @@ -0,0 +1,5 @@ +[package] +name = "kcl3" +edition = "0.0.1" +version = "0.0.1" + diff --git a/kclvm/runner/src/test_file_pattern/kcl1/kcl3/main.k b/kclvm/runner/src/test_file_pattern/kcl1/kcl3/main.k new file mode 100644 index 000000000..0acce2f52 --- /dev/null +++ b/kclvm/runner/src/test_file_pattern/kcl1/kcl3/main.k @@ -0,0 +1 @@ +k3 = 'Hello World!' \ No newline at end of file diff --git a/kclvm/runner/src/test_file_pattern/kcl1/main.k b/kclvm/runner/src/test_file_pattern/kcl1/main.k new file mode 100644 index 000000000..ac6689514 --- /dev/null +++ b/kclvm/runner/src/test_file_pattern/kcl1/main.k @@ -0,0 +1 @@ +k1 = 'Hello World!' \ No newline at end of file diff --git a/kclvm/runner/src/test_file_pattern/kcl2/kcl.mod b/kclvm/runner/src/test_file_pattern/kcl2/kcl.mod new file mode 100644 index 000000000..5fb058acd --- /dev/null +++ b/kclvm/runner/src/test_file_pattern/kcl2/kcl.mod @@ -0,0 +1,5 @@ +[package] +name = "kcl2" +edition = "0.0.1" +version = "0.0.1" + diff --git a/kclvm/runner/src/test_file_pattern/kcl2/main.k b/kclvm/runner/src/test_file_pattern/kcl2/main.k new file mode 100644 index 000000000..fe6900bb4 --- /dev/null +++ b/kclvm/runner/src/test_file_pattern/kcl2/main.k @@ -0,0 +1 @@ +k2 = 'Hello World!' \ No newline at end of file diff --git a/kclvm/runner/src/tests.rs b/kclvm/runner/src/tests.rs index 58659e2fe..98512a657 100644 --- a/kclvm/runner/src/tests.rs +++ b/kclvm/runner/src/tests.rs @@ -564,6 +564,9 @@ fn test_exec() { test_indent_error(); println!("test_indent_error - PASS"); + + test_compile_with_file_pattern(); + println!("test_compile_with_file_pattern - PASS"); } fn test_indent_error() { @@ -662,3 +665,19 @@ fn get_files>( } files } + +fn test_compile_with_file_pattern() { + let test_path = PathBuf::from("./src/test_file_pattern/**/main.k"); + let mut args = ExecProgramArgs::default(); + args.k_filename_list.push(test_path.display().to_string()); + let res = exec_program(Arc::new(ParseSession::default()), &args); + assert!(res.is_ok()); + assert_eq!( + res.clone().unwrap().yaml_result, + "k3: Hello World!\nk1: Hello World!\nk2: Hello World!" + ); + assert_eq!( + res.unwrap().json_result, + "[{\"k3\": \"Hello World!\", \"k1\": \"Hello World!\", \"k2\": \"Hello World!\"}]" + ); +}