diff --git a/README.md b/README.md index 8784c48..c8eb4e7 100644 --- a/README.md +++ b/README.md @@ -2,7 +2,7 @@ [![Actions Status](https://github.com/bcomnes/async-folder-walker/workflows/tests/badge.svg)](https://github.com/bcomnes/async-folder-walker/actions) [![Exports](https://img.shields.io/badge/exports-esm-blue)](https://github.com/standard-things/esm) -WIP - nothing to see here +A recursive async iterator of the files and directories in a given folder. Can take multiple folders, limit walk depth and filter based on path names and stat results. ``` npm install async-folder-walker @@ -11,9 +11,95 @@ npm install async-folder-walker ## Usage ``` js -import { async-folder-walker } from 'async-folder-walker' +import { asyncFolderWalker, allFiles } from 'async-folder-walker'; + +async function iterateFiles () { + const walker = asyncFolderWalker(['.git', 'node_modules']); + for await (const file of walker) { + console.log(file); // logs the file path! + } +} + +async function getAllFiles () { + const allFilepaths = await allFiles(['.git', 'node_modules']); + console.log(allFilepaths); +} + +iterateFiles().then(() => getAllFiles()); +``` + +## API + +### `import { asyncFolderWalker } from 'async-folder-walker'` + +Import `asyncFolderWalker`. + +### `async-gen = asyncFolderWalker(paths, [opts])` + +Return an async generator that will iterate over all of files inside of a directory. `paths` can be a string path or an Array of string paths. + +You can iterate over each file and directory individually using a `for-await...of` loop. Note, you must be inside an [async function statement](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/async_function). + +```js +import { asyncFolderWalker } from 'async-folder-walker'; +async function iterateFiles () { + const walker = asyncFolderWalker(['.git', 'node_modules']); + for await (const file of walker) { + console.log(file); // logs the file path! + } +} + +iterateFiles(); +``` + +Opts include: + +```js +{ + fs: require('fs'), + pathFilter: filepath => true, + statFilter st => true, + maxDepth: Infinity, + shaper: ({ root, filepath, stat, relname, basename }) => filepath +} ``` +The `pathFilter` function allows you to filter files from additional async stat operations. Return false to filter the file. + +```js +{ // exclude node_modules + pathFilter: filepath => !filepath.includes(node_modules) +} +``` + +The `statFilter` function allows you to filter files based on the internal stat operation. Return false to filter the file. + +```js +{ // exclude all directories: + statFilter: st => !st.isDirectory() +} +``` + +The `shaper` function lets you change the shape of the returned value based on data accumulaed during the iteration. To return the same shape as [okdistribute/folder-walker](https://github.com/okdistribute/folder-walker) use the following function: + +```js +{ // Return the same shape as folder-walker + shaper: fwData => fwData +} +```` + +### `files = await allFiles(paths, [opts])` + +Get an Array of all files inside of a directory. `paths` can be a single string path or an array of string paths. + +`opts` Is the same as `asyncFolderWalker`. + +## See also + +This module is effectivly a rewrite of [okdistribute/folder-walker](https://github.com/okdistribute/folder-walker) using async generators instead of Node streams, and a few tweaks to the underlying options to make the results a bit more flexible. + +- [for-await...of](https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Statements/for-await...of) + ## License MIT diff --git a/main.js b/main.js index 79091c4..9019102 100644 --- a/main.js +++ b/main.js @@ -3,13 +3,73 @@ import path from 'path'; const { readdir, lstat } = fs.promises; +/** + * pathFilter lets you filter files based on a resolved `filepath`. + * @callback pathFilter + * @param {String} filepath - The resolved `filepath` of the file to test for filtering. + * + * @return {Boolean} Return false to filter the given `filepath` and true to include it. + */ +const pathFilter = filepath => true; + +/** + * statFilter lets you filter files based on a lstat object. + * @callback statFilter + * @param {Object} st - A fs.Stats instance. + * + * @return {Boolean} Return false to filter the given `filepath` and true to include it. + */ +const statFilter = filepath => true; + +/** + * FWStats is the object that the okdistribute/folder-walker module returns by default. + * + * @typedef FWStats + * @property {String} root - The filepath of the directory where the walk started. + * @property {String} filepath - The resolved filepath. + * @property {Object} stat - A fs.Stats instance. + * @property {String} relname - The relative path to `root`. + * @property {String} basename - The resolved filepath of the files containing directory. + */ + +/** + * shaper lets you change the shape of the returned file data from walk-time stats. + * @callback shaper + * @param {FWStats} fwStats - The same status object returned from folder-walker. + * + * @return {*} - Whatever you want returned from the directory walk. + */ +const shaper = ({ root, filepath, stat, relname, basename }) => filepath; + +/** + * Options object + * + * @typedef Opts + * @property {pathFilter} [pathFilter] - A pathFilter cb. + * @property {statFilter} [statFilter] - A statFilter cb. + * @property {Number} [maxDepth=Infinity] - The maximum number of folders to walk down into. + * @property {shaper} [shaper] - A shaper cb. + */ + +/** + * Create an async generator that iterates over all folders and directories inside of `dirs`. + * + * @async + * @generator + * @function + * @public + * @param {String|String[]} dirs - The path of the directory to walk, or an array of directory paths. + * @param {?(Opts)} opts - Options used for the directory walk. + * + * @yields {Promise} - An async iterator that returns anything. + */ export async function * asyncFolderWalker (dirs, opts) { opts = Object.assign({ fs, - pathFilter: filename => true, - statFilter: st => true, + pathFilter, + statFilter, maxDepth: Infinity, - shapeFn: ({ root, filepath, stat, relname, basename }) => filepath + shaper }, opts); const roots = [dirs].flat().filter(opts.pathFilter); @@ -24,7 +84,7 @@ export async function * asyncFolderWalker (dirs, opts) { if (typeof current === 'undefined') continue; const st = await lstat(current); if ((!st.isDirectory() || depthLimiter(current, root, opts.maxDepth)) && opts.statFilter(st)) { - yield opts.shapeFn(fwShape(root, current, st)); + yield opts.shaper(fwShape(root, current, st)); continue; } @@ -36,11 +96,22 @@ export async function * asyncFolderWalker (dirs, opts) { if (opts.pathFilter(next)) pending.unshift(next); } if (current === root || !opts.statFilter(st)) continue; - else yield opts.shapeFn(fwShape(root, current, st)); + else yield opts.shaper(fwShape(root, current, st)); } } } +/** + * Generates the same shape as the folder-walker module. + * + * @function + * @private + * @param {String} root - Root filepath. + * @param {String} name - Target filepath. + * @param {Object} st - fs.Stat object. + * + * @return {FWStats} - Folder walker object. + */ function fwShape (root, name, st) { return { root: root, @@ -51,6 +122,17 @@ function fwShape (root, name, st) { }; } +/** + * Test if we are at maximum directory depth. + * + * @private + * @function + * @param {String} filePath - The resolved path of the target fille. + * @param {String} relativeTo - The root directory of the current walk. + * @param {Number} maxDepth - The maximum number of folders to descend into. + * + * @returns {Boolean} - Return true to signal stop descending. + */ function depthLimiter (filePath, relativeTo, maxDepth) { if (maxDepth === Infinity) return false; const rootDepth = relativeTo.split(path.sep).length; @@ -58,6 +140,13 @@ function depthLimiter (filePath, relativeTo, maxDepth) { return fileDepth - rootDepth > maxDepth; } +/** + * Async iterable collector + * + * @async + * @function + * @private + */ export async function all (iterator) { const collect = []; @@ -68,6 +157,17 @@ export async function all (iterator) { return collect; } +/** + * allFiles gives you all files from the directory walk as an array. + * + * @async + * @function + * @public + * @param {String|String[]} dirs - The path of the directory to walk, or an array of directory paths. + * @param {?(Opts)} opts - Options used for the directory walk. + * + * @returns {Promise} - An async iterator that returns anything. + */ export async function allFiles (...args) { return all(asyncFolderWalker(...args)); } diff --git a/test.js b/test.js index a2c367f..0698973 100644 --- a/test.js +++ b/test.js @@ -56,17 +56,39 @@ test('When you just pass a file', async t => { }); test('pathFilter works', async t => { - t.fail(); -}); + const filterStrig = 'sub-folder'; + const files = await allFiles(fixtures, { + pathFilter: p => !p.includes(filterStrig) + }); -test('statFilter works', async t => { - t.fail(); + t.false(files.some(f => f.includes(filterStrig)), 'No paths include the excluded string'); }); -test('dont include root directory in response', function (t) { +test('statFilter works', async t => { + const stats = await allFiles(fixtures, { + statFilter: st => !st.isDirectory(), // Exclude files + shaper: ({ root, filepath, stat, relname, basename }) => stat // Lets get the stats instead of paths + }); + for (const st of stats) { + t.false(st.isDirectory(), 'none of the files are directories'); + } }); -test('dont walk past the maxDepth', function (t) { +test('dont include root directory in response', async (t) => { + const root = process.cwd(); + for await (const file of asyncFolderWalker(root)) { + if (file === root) t.fail('root directory should not be in results'); + } + t.pass('The root was not included in results.'); +}); +test('dont walk past the maxDepth', async t => { + const maxDepth = 3; + const walker = asyncFolderWalker(['.git', 'node_modules'], { maxDepth }); + for await (const file of walker) { + const correctLength = file.split(path.sep).length - process.cwd().split(path.sep).length <= maxDepth; + if (!correctLength) t.fail('walker walked past the depth it was supposed to'); + } + t.pass('Walker was depth limited'); });