refactor/traverse: introduce unit tests for module traversal

Author: drahnr

demo/src/main.rs

@@ -1,3 +1,4 @@
+//! Just a lil somethin somethin
 mod lib;
 
 pub mod nested;

src/literalset.rs

@@ -256,7 +256,7 @@ fn find_coverage<'a>(
 /// A set of consecutive literals.
 ///
 /// Provides means to render them as a code block
-#[derive(Clone, Default, Debug, PartialEq, Eq)]
+#[derive(Clone, Default, Debug, Hash, PartialEq, Eq)]
 pub struct LiteralSet {
     /// consecutive set of literals mapped by line number
     literals: Vec<TrimmedLiteral>,

src/traverse/iter.rs

@@ -21,7 +21,7 @@ pub struct TraverseModulesIter {
 impl Default for TraverseModulesIter {
     fn default() -> Self {
         Self {
-            mex_depth: usize::MAX,
+            max_depth: usize::MAX,
             queue: VecDeque::with_capacity(128),
         }
     }
@@ -36,7 +36,7 @@ impl TraverseModulesIter {
         let path = path.canonicalize().unwrap();
         let meta = path.metadata().unwrap();
         if meta.is_file() {
-            self.queue.push((path, level));
+            self.queue.push_back((path, level));
         } else if meta.is_dir() {
             walkdir::WalkDir::new(path)
                 .max_depth(1)
@@ -55,7 +55,7 @@ impl TraverseModulesIter {
                         .is_some()
                 })
                 .try_for_each::<_, Result<()>>(|path| {
-                    self.queue.push((path, level));
+                    self.queue.push_back((path, level));
                     Ok(())
                 })?;
         }

src/traverse/mod.rs

@@ -267,22 +267,21 @@ pub(crate) fn extract(
         Markdown(PathBuf),
     }
 
-    // convert all `Cargo.toml` manifest files to their respective product files
-    // so after this conversion all of them are considered
+    // stage 1 - obtain canonical paths
     let mut flow = VecDeque::<PathBuf>::with_capacity(32);
     flow.extend(paths.into_iter().filter_map(|path_in| {
-        if path_in.is_absolute() {
+        let path = if path_in.is_absolute() {
             path_in.to_owned()
         } else {
             cwd.join(&path_in)
-        }
-        .canonicalize()
-        .ok()
+        };
+        info!("Processing {} -> {}", path_in.display(), path.display());
+        path.canonicalize().ok()
     }));
 
+    // stage 2 - check for manifest, .rs , .md files and directories
     let mut files_to_check = Vec::with_capacity(64);
-    for path in flow {
-        info!("Processing {} -> {}", path_in.display(), path.display());
+    while let Some(path) = flow.pop_front() {
         files_to_check.push(if let Ok(meta) = path.metadata() {
             if meta.is_file() {
                 match path.file_name().map(|x| x.to_str()).flatten() {
@@ -295,16 +294,16 @@ pub(crate) fn extract(
                     }
                 }
             } else if meta.is_dir() {
-                let cargo_toml = to_manifest_dir(path).unwrap().join("Cargo.toml");
+                let cargo_toml = to_manifest_dir(&path).unwrap().join("Cargo.toml");
                 if cargo_toml.is_file() {
-
                     Extraction::Manifest(cargo_toml)
                 } else {
                     // @todo should we just collect all .rs files here instead?
 
                     // we know it's a directory, and we limit the entries to 0 levels,
-                    // will cause to yield all .rs files in that dir
-                    flow.extend(traverse_with_depth_limit(path, 0)?.into_iter());
+                    // will cause to yield all "^.*\.rs$" files in that dir
+                    // which is what we want in this case
+                    flow.extend(TraverseModulesIter::with_depth_limit(&path, 0)?);
                     continue;
                 }
             } else {
@@ -315,26 +314,30 @@ pub(crate) fn extract(
         })
     }
 
-    let files_to_check = files_to_check.into_iter().try_fold::<Vec<_>, _, Result<_>>(Vec::with_capacity(64), |mut acc, tagged_path| {
-        match tagged_path {
-            Extraction::Manifest(ref cargo_toml_path) => {
-                let manifest_list = handle_manifest(cargo_toml_path)?;
-                acc.extend(manifest_list);
+    // stage 3 - resolve the manifest products and workspaces, warn about missing
+    let files_to_check = files_to_check
+        .into_iter()
+        .try_fold::<Vec<_>, _, Result<_>>(Vec::with_capacity(64), |mut acc, tagged_path| {
+            match tagged_path {
+                Extraction::Manifest(ref cargo_toml_path) => {
+                    let manifest_list = handle_manifest(cargo_toml_path)?;
+                    acc.extend(manifest_list);
+                }
+                Extraction::Missing(ref missing_path) => warn!(
+                    "File passed as argument or listed in Cargo.toml manifest does not exist: {}",
+                    missing_path.display()
+                ),
+                Extraction::Source(path) => acc.push(CheckEntity::Source(path)),
+                Extraction::Markdown(path) => acc.push(CheckEntity::Markdown(path)),
             }
-            Extraction::Missing(ref missing_path) => warn!(
-                "File passed as argument or listed in Cargo.toml manifest does not exist: {}",
-                missing_path.display()
-            ),
-            Extraction::Source(path) => acc.push(CheckEntity::Source(path)),
-            Extraction::Markdown(path) => acc.push(CheckEntity::Markdown(path)),
-        }
-        Ok(acc)
-    })?;
+            Ok(acc)
+        })?;
 
+    // stage 4 - expand from the passed source files, if recursive, recurse down the module train
     let docs: Vec<Documentation> = files_to_check
         .iter()
         .try_fold::<Vec<Documentation>, _, Result<Vec<Documentation>>>(
-            Vec::with_capacity(items.len()),
+            Vec::with_capacity(files_to_check.len()),
             |mut acc, item| {
                 match item {
                     CheckEntity::Source(path) => {