feat: add file pattern as the compile entry (#861)

* feat: add file pattern as the compile entry

Signed-off-by: zongz <[email protected]>

* fix: fix grammar test

Signed-off-by: zongz <[email protected]>

* fix: fix failed test case

Signed-off-by: zongz <[email protected]>

* fix: fix failed test case

Signed-off-by: zongz <[email protected]>

* fix: rm useless files

Signed-off-by: zongz <[email protected]>

* fix: fix windows test cases

Signed-off-by: zongz <[email protected]>

* fix: fix CR comments

Signed-off-by: zongz <[email protected]>

* fix: fix windows test case

Signed-off-by: zongz <[email protected]>

* fix: rm useless import

Signed-off-by: zongz <[email protected]>

---------

Signed-off-by: zongz <[email protected]>

kclvm/driver/Cargo.toml

@@ -17,3 +17,4 @@ walkdir = "2"
 
 serde = { version = "1.0", features = ["derive"] }
 anyhow = { version = "1.0.70", features = ["backtrace"] }
+glob = "0.3.1"

kclvm/driver/src/lib.rs

@@ -6,6 +6,7 @@ pub const DEFAULT_PROJECT_FILE: &str = "project.yaml";
 #[cfg(test)]
 mod tests;
 
+use glob::glob;
 use kclvm_ast::ast;
 use kclvm_config::{
     modfile::{KCL_FILE_EXTENSION, KCL_FILE_SUFFIX, KCL_MOD_PATH_ENV},
@@ -22,24 +23,54 @@ use std::{
 };
 use walkdir::WalkDir;
 
+/// Expand the file pattern to a list of files.
+pub fn expand_if_file_pattern(file_pattern: String) -> Result<Vec<String>, String> {
+    let paths = glob(&file_pattern).map_err(|_| format!("invalid file pattern {file_pattern}"))?;
+    let mut matched_files = vec![];
+
+    for path in paths {
+        if let Ok(path) = path {
+            matched_files.push(path.to_string_lossy().to_string());
+        }
+    }
+
+    Ok(matched_files)
+}
+
+pub fn expand_input_files(k_files: &[String]) -> Vec<String> {
+    let mut res = vec![];
+    for file in k_files {
+        if let Ok(files) = expand_if_file_pattern(file.to_string()) {
+            if !files.is_empty() {
+                res.extend(files);
+            } else {
+                res.push(file.to_string())
+            }
+        } else {
+            res.push(file.to_string())
+        }
+    }
+    res
+}
+
 /// Normalize input files with the working directory and replace ${KCL_MOD} with the module root path.
 pub fn canonicalize_input_files(
     k_files: &[String],
     work_dir: String,
     check_exist: bool,
 ) -> Result<Vec<String>, String> {
     let mut kcl_paths = Vec::<String>::new();
-
     // The first traversal changes the relative path to an absolute path
     for (_, file) in k_files.iter().enumerate() {
         let path = Path::new(file);
+
         let is_absolute = path.is_absolute();
         let is_exist_maybe_symlink = path.exists();
         // If the input file or path is a relative path and it is not a absolute path in the KCL module VFS,
         // join with the work directory path and convert it to a absolute path.
         let path = ModRelativePath::from(file.to_string());
         let abs_path = if !is_absolute && !path.is_relative_path().map_err(|err| err.to_string())? {
-            let filepath = Path::new(&work_dir).join(file);
+            let filepath = Path::new(&work_dir).join(file.to_string());
             match filepath.canonicalize() {
                 Ok(path) => Some(path.adjust_canonicalization()),
                 Err(_) => {
@@ -57,7 +88,7 @@ pub fn canonicalize_input_files(
         };
         // If the input file or path is a symlink, convert it to a real path.
         let real_path = if is_exist_maybe_symlink {
-            match PathBuf::from(file).canonicalize() {
+            match PathBuf::from(file.to_string()).canonicalize() {
                 Ok(real_path) => Some(String::from(real_path.to_str().unwrap())),
                 Err(_) => {
                     if check_exist {

kclvm/driver/src/test_data/expand_file_pattern/kcl.mod

@@ -0,0 +1,5 @@
+[package]
+name = "expand_file_pattern"
+edition = "0.0.1"
+version = "0.0.1"
+

kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl.mod

@@ -0,0 +1,5 @@
+[package]
+name = "kcl1"
+edition = "0.0.1"
+version = "0.0.1"
+

kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl2/kcl.mod

@@ -0,0 +1,5 @@
+[package]
+name = "kcl2"
+edition = "0.0.1"
+version = "0.0.1"
+

kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl2/main.k

@@ -0,0 +1 @@
+The_first_kcl_program = 'Hello World!'

kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl4/kcl.mod

@@ -0,0 +1,5 @@
+[package]
+name = "kcl4"
+edition = "0.0.1"
+version = "0.0.1"
+

kclvm/driver/src/test_data/expand_file_pattern/kcl1/kcl4/main.k

@@ -0,0 +1 @@
+The_first_kcl_program = 'Hello World!'

kclvm/driver/src/test_data/expand_file_pattern/kcl1/main.k

@@ -0,0 +1 @@
+The_first_kcl_program = 'Hello World!'

kclvm/driver/src/test_data/expand_file_pattern/kcl3/kcl.mod

@@ -0,0 +1,5 @@
+[package]
+name = "kcl3"
+edition = "0.0.1"
+version = "0.0.1"
+

kclvm/driver/src/test_data/expand_file_pattern/kcl3/main.k

@@ -0,0 +1 @@
+The_first_kcl_program = 'Hello World!'

kclvm/driver/src/test_data/expand_file_pattern/main.k

@@ -0,0 +1 @@
+The_first_kcl_program = 'Hello World!'

kclvm/driver/src/tests.rs

@@ -7,8 +7,8 @@ use kclvm_parser::LoadProgramOptions;
 use walkdir::WalkDir;
 
 use crate::arguments::parse_key_value_pair;
-use crate::canonicalize_input_files;
 use crate::kpm_metadata::{fetch_metadata, fill_pkg_maps_for_k_file, lookup_the_nearest_file_dir};
+use crate::{canonicalize_input_files, expand_input_files};
 
 #[test]
 fn test_canonicalize_input_files() {
@@ -25,6 +25,99 @@ fn test_canonicalize_input_files() {
     assert!(canonicalize_input_files(&input_files, work_dir, true).is_err());
 }
 
+#[test]
+fn test_expand_input_files_with_kcl_mod() {
+    let path = PathBuf::from("src/test_data/expand_file_pattern");
+    let input_files = vec![
+        path.join("**").join("main.k").to_string_lossy().to_string(),
+        "${KCL_MOD}/src/test_data/expand_file_pattern/KCL_MOD".to_string(),
+    ];
+    let expected_files = vec![
+        path.join("kcl1/kcl2/main.k").to_string_lossy().to_string(),
+        path.join("kcl1/kcl4/main.k").to_string_lossy().to_string(),
+        path.join("kcl1/main.k").to_string_lossy().to_string(),
+        path.join("kcl3/main.k").to_string_lossy().to_string(),
+        path.join("main.k").to_string_lossy().to_string(),
+        "${KCL_MOD}/src/test_data/expand_file_pattern/KCL_MOD".to_string(),
+    ];
+    let got_paths: Vec<String> = expand_input_files(&input_files)
+        .iter()
+        .map(|s| s.replace("/", "").replace("\\", ""))
+        .collect();
+    let expect_paths: Vec<String> = expected_files
+        .iter()
+        .map(|s| s.replace("/", "").replace("\\", ""))
+        .collect();
+    assert_eq!(got_paths, expect_paths);
+}
+
+#[test]
+fn test_expand_input_files() {
+    let input_files = vec!["./src/test_data/expand_file_pattern/**/main.k".to_string()];
+    let mut expected_files = vec![
+        Path::new("./src/test_data/expand_file_pattern/kcl1/kcl2/main.k")
+            .canonicalize()
+            .unwrap()
+            .to_string_lossy()
+            .to_string(),
+        Path::new("./src/test_data/expand_file_pattern/kcl3/main.k")
+            .canonicalize()
+            .unwrap()
+            .to_string_lossy()
+            .to_string(),
+        Path::new("./src/test_data/expand_file_pattern/main.k")
+            .canonicalize()
+            .unwrap()
+            .to_string_lossy()
+            .to_string(),
+        Path::new("./src/test_data/expand_file_pattern/kcl1/kcl4/main.k")
+            .canonicalize()
+            .unwrap()
+            .to_string_lossy()
+            .to_string(),
+    ];
+    assert_eq!(
+        expand_input_files(&input_files).sort(),
+        expected_files.sort()
+    );
+
+    let input_files = vec![
+        "./src/test_data/expand_file_pattern/kcl1/main.k".to_string(),
+        "./src/test_data/expand_file_pattern/**/main.k".to_string(),
+    ];
+    let mut expected_files = vec![
+        Path::new("./src/test_data/expand_file_pattern/kcl1/main.k")
+            .canonicalize()
+            .unwrap()
+            .to_string_lossy()
+            .to_string(),
+        Path::new("./src/test_data/expand_file_pattern/kcl1/kcl2/main.k")
+            .canonicalize()
+            .unwrap()
+            .to_string_lossy()
+            .to_string(),
+        Path::new("./src/test_data/expand_file_pattern/kcl1/kcl4/main.k")
+            .canonicalize()
+            .unwrap()
+            .to_string_lossy()
+            .to_string(),
+        Path::new("./src/test_data/expand_file_pattern/kcl3/main.k")
+            .canonicalize()
+            .unwrap()
+            .to_string_lossy()
+            .to_string(),
+        Path::new("./src/test_data/expand_file_pattern/main.k")
+            .canonicalize()
+            .unwrap()
+            .to_string_lossy()
+            .to_string(),
+    ];
+    assert_eq!(
+        expand_input_files(&input_files).sort(),
+        expected_files.sort()
+    );
+}
+
 #[test]
 fn test_parse_key_value_pair() {
     let cases = [

kclvm/runner/src/lib.rs

@@ -6,7 +6,7 @@ use kclvm_ast::{
     ast::{Module, Program},
     MAIN_PKG,
 };
-use kclvm_driver::canonicalize_input_files;
+use kclvm_driver::{canonicalize_input_files, expand_input_files};
 use kclvm_error::{Diagnostic, Handler};
 use kclvm_parser::{load_program, ParseSession};
 use kclvm_query::apply_overrides;
@@ -79,7 +79,8 @@ pub fn exec_program(
     let opts = args.get_load_program_options();
     let k_files = &args.k_filename_list;
     let work_dir = args.work_dir.clone().unwrap_or_default();
-    let kcl_paths = canonicalize_input_files(k_files, work_dir, false)?;
+    let k_files = expand_input_files(k_files);
+    let kcl_paths = canonicalize_input_files(&k_files, work_dir, false)?;
 
     let kcl_paths_str = kcl_paths.iter().map(|s| s.as_str()).collect::<Vec<&str>>();
 

kclvm/runner/src/test_file_pattern/kcl1/kcl.mod

@@ -0,0 +1,5 @@
+[package]
+name = "kcl1"
+edition = "0.0.1"
+version = "0.0.1"
+

kclvm/runner/src/test_file_pattern/kcl1/kcl3/kcl.mod

@@ -0,0 +1,5 @@
+[package]
+name = "kcl3"
+edition = "0.0.1"
+version = "0.0.1"
+

kclvm/runner/src/test_file_pattern/kcl1/kcl3/main.k

@@ -0,0 +1 @@
+k3 = 'Hello World!'

kclvm/runner/src/test_file_pattern/kcl1/main.k

@@ -0,0 +1 @@
+k1 = 'Hello World!'

kclvm/runner/src/test_file_pattern/kcl2/kcl.mod

@@ -0,0 +1,5 @@
+[package]
+name = "kcl2"
+edition = "0.0.1"
+version = "0.0.1"
+

kclvm/runner/src/test_file_pattern/kcl2/main.k

@@ -0,0 +1 @@
+k2 = 'Hello World!'

kclvm/runner/src/tests.rs

@@ -564,6 +564,9 @@ fn test_exec() {
 
     test_indent_error();
     println!("test_indent_error - PASS");
+
+    test_compile_with_file_pattern();
+    println!("test_compile_with_file_pattern - PASS");
 }
 
 fn test_indent_error() {
@@ -662,3 +665,19 @@ fn get_files<P: AsRef<Path>>(
     }
     files
 }
+
+fn test_compile_with_file_pattern() {
+    let test_path = PathBuf::from("./src/test_file_pattern/**/main.k");
+    let mut args = ExecProgramArgs::default();
+    args.k_filename_list.push(test_path.display().to_string());
+    let res = exec_program(Arc::new(ParseSession::default()), &args);
+    assert!(res.is_ok());
+    assert_eq!(
+        res.clone().unwrap().yaml_result,
+        "k3: Hello World!\nk1: Hello World!\nk2: Hello World!"
+    );
+    assert_eq!(
+        res.unwrap().json_result,
+        "[{\"k3\": \"Hello World!\", \"k1\": \"Hello World!\", \"k2\": \"Hello World!\"}]"
+    );
+}