elastic · spalger · Oct 29, 2018 · Oct 26, 2018
diff --git a/package.json b/package.json
@@ -231,6 +231,7 @@
     "@types/classnames": "^2.2.3",
     "@types/d3": "^5.0.0",
     "@types/dedent": "^0.7.0",
+    "@types/del": "^3.0.1",
     "@types/elasticsearch": "^5.0.26",
     "@types/enzyme": "^3.1.12",
     "@types/eslint": "^4.16.2",

diff --git a/src/dev/build/lib/fs.js b/src/dev/build/lib/fs.js
@@ -39,7 +39,7 @@ const readFileAsync = promisify(fs.readFile);
 const readdirAsync = promisify(fs.readdir);
 const utimesAsync = promisify(fs.utimes);
 
-function assertAbsolute(path) {
+export function assertAbsolute(path) {
   if (!isAbsolute(path)) {
     throw new TypeError(
       'Please use absolute paths to keep things explicit. You probably want to use `build.resolvePath()` or `config.resolveFromRepo()`.'

diff --git a/src/dev/build/lib/index.js b/src/dev/build/lib/index.js
@@ -31,3 +31,4 @@ export {
   untar,
   deleteAll,
 } from './fs';
+export { scanDelete } from './scan_delete';
diff --git a/src/dev/build/lib/scan_delete.test.ts b/src/dev/build/lib/scan_delete.test.ts
@@ -0,0 +1,64 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { readdirSync } from 'fs';
+import { relative, resolve } from 'path';
+
+import del from 'del';
+
+// @ts-ignore
+import { mkdirp, write } from './fs';
+import { scanDelete } from './scan_delete';
+
+const TMP = resolve(__dirname, '__tests__/__tmp__');
+
+// clean and recreate TMP directory
+beforeEach(async () => {
+  await del(TMP);
+  await mkdirp(resolve(TMP, 'foo/bar/baz'));
+  await mkdirp(resolve(TMP, 'foo/bar/box'));
+  await mkdirp(resolve(TMP, 'a/b/c/d/e'));
+  await write(resolve(TMP, 'a/bar'), 'foo');
+});
+
+// cleanup TMP directory
+afterAll(async () => {
+  await del(TMP);
+});
+
+it('requires an absolute directory', async () => {
+  await expect(
+    scanDelete({
+      directory: relative(process.cwd(), TMP),
+      regularExpressions: [],
+    })
+  ).rejects.toMatchInlineSnapshot(
+    `[TypeError: Please use absolute paths to keep things explicit. You probably want to use \`build.resolvePath()\` or \`config.resolveFromRepo()\`.]`
+  );
+});
+
+it('deletes files/folders matching regular expression', async () => {
+  await scanDelete({
+    directory: TMP,
+    regularExpressions: [/^.*[\/\\](bar|c)([\/\\]|$)/],
+  });
+  expect(readdirSync(resolve(TMP, 'foo'))).toEqual([]);
+  expect(readdirSync(resolve(TMP, 'a'))).toEqual(['b']);
+  expect(readdirSync(resolve(TMP, 'a/b'))).toEqual([]);
+});
diff --git a/src/dev/build/lib/scan_delete.ts b/src/dev/build/lib/scan_delete.ts
@@ -0,0 +1,80 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import Fs from 'fs';
+
+import del from 'del';
+import { join } from 'path';
+import * as Rx from 'rxjs';
+import { count, map, mergeAll, mergeMap } from 'rxjs/operators';
+
+// @ts-ignore
+import { assertAbsolute } from './fs';
+
+const getStat$ = Rx.bindNodeCallback(Fs.stat);
+const getReadDir$ = Rx.bindNodeCallback(Fs.readdir);
+
+interface Options {
+  directory: string;
+  regularExpressions: RegExp[];
+  concurrency?: 20;
+}
+
+/**
+ * Scan the files in a directory and delete the directories/files that
+ * are matched by an array of regular expressions.
+ *
+ * @param options.directory the directory to scan, all files including dot files will be checked
+ * @param options.regularExpressions an array of regular expressions, if any matches the file/directory will be deleted
+ * @param options.concurrency optional concurrency to run deletes, defaults to 20
+ */
+export async function scanDelete(options: Options) {
+  const { directory, regularExpressions, concurrency = 20 } = options;
+
+  assertAbsolute(directory);
+
+  // get an observable of absolute paths within a directory
+  const getChildPath$ = (path: string) =>
+    getReadDir$(path).pipe(
+      mergeAll(),
+      map(name => join(path, name))
+    );
+
+  // get an observable of all paths to be deleted, by starting with the arg
+  // and recursively iterating through all children, unless a child matches
+  // one of the supplied regular expressions
+  const getPathsToDelete$ = (path: string): Rx.Observable<string> => {
+    if (regularExpressions.some(re => re.test(path))) {
+      return Rx.of(path);
+    }
+
+    return getStat$(path).pipe(
+      mergeMap(stat => (stat.isDirectory() ? getChildPath$(path) : Rx.EMPTY)),
+      mergeMap(getPathsToDelete$)
+    );
+  };
+
+  return await Rx.of(directory)
+    .pipe(
+      mergeMap(getPathsToDelete$),
+      mergeMap(async path => await del(path), concurrency),
+      count()
+    )
+    .toPromise();
+}
diff --git a/src/dev/build/tasks/clean_tasks.js b/src/dev/build/tasks/clean_tasks.js
@@ -17,7 +17,9 @@
  * under the License.
  */
 
-import { deleteAll } from '../lib';
+import minimatch from 'minimatch';
+
+import { deleteAll, scanDelete } from '../lib';
 
 export const CleanTask = {
   global: true,
@@ -49,105 +51,122 @@ export const CleanTypescriptTask = {
     'Cleaning typescript source files that have been transpiled to JS',
 
   async run(config, log, build) {
-    await deleteAll(log, [
-      build.resolvePath('**/*.{ts,tsx,d.ts}'),
-      build.resolvePath('**/tsconfig*.json'),
-    ]);
+    log.info('Deleted %d files', await scanDelete({
+      directory: build.resolvePath(),
+      regularExpressions: [
+        /\.(ts|tsx|d\.ts)$/,
+        /tsconfig.*\.json$/
+      ]
+    }));
   },
 };
 
 export const CleanExtraFilesFromModulesTask = {
   description: 'Cleaning tests, examples, docs, etc. from node_modules',
 
   async run(config, log, build) {
-    const deleteFromNodeModules = globs => {
-      return deleteAll(
-        log,
-        globs.map(p => build.resolvePath(`node_modules/**/${p}`))
+    const makeRegexps = patterns =>
+      patterns.map(pattern =>
+        minimatch.makeRe(pattern, { nocase: true })
       );
-    };
 
-    const tests = [
-      'test',
-      'tests',
-      '__tests__',
-      'mocha.opts',
-      '*.test.js',
-      '*.snap',
-      'coverage',
-    ];
-    const docs = [
-      'doc',
-      'docs',
-      'CONTRIBUTING.md',
-      'Contributing.md',
-      'contributing.md',
-      'History.md',
-      'HISTORY.md',
-      'history.md',
-      'CHANGELOG.md',
-      'Changelog.md',
-      'changelog.md',
-    ];
-    const examples = ['example', 'examples', 'demo', 'samples'];
-    const bins = ['.bin'];
-    const linters = [
-      '.eslintrc',
-      '.eslintrc.js',
-      '.eslintrc.yml',
-      '.prettierrc',
-      '.jshintrc',
-      '.babelrc',
-      '.jscs.json',
-      '.lint',
-    ];
-    const hints = ['*.flow', '*.webidl', '*.map', '@types'];
-    const scripts = [
-      '*.sh',
-      '*.bat',
-      '*.exe',
-      'Gruntfile.js',
-      'gulpfile.js',
-      'Makefile',
-    ];
-    const untranspiledSources = ['*.coffee', '*.scss', '*.sass', '.ts', '.tsx'];
-    const editors = ['.editorconfig', '.vscode'];
-    const git = [
-      '.gitattributes',
-      '.gitkeep',
-      '.gitempty',
-      '.gitmodules',
-      '.keep',
-      '.empty',
-    ];
-    const ci = [
-      '.travis.yml',
-      '.coveralls.yml',
-      '.instanbul.yml',
-      'appveyor.yml',
-      '.zuul.yml',
-    ];
-    const meta = [
-      'package-lock.json',
-      'component.json',
-      'bower.json',
-      'yarn.lock',
-    ];
-    const misc = ['.*ignore', '.DS_Store', 'Dockerfile', 'docker-compose.yml'];
-
-    await deleteFromNodeModules(tests);
-    await deleteFromNodeModules(docs);
-    await deleteFromNodeModules(examples);
-    await deleteFromNodeModules(bins);
-    await deleteFromNodeModules(linters);
-    await deleteFromNodeModules(hints);
-    await deleteFromNodeModules(scripts);
-    await deleteFromNodeModules(untranspiledSources);
-    await deleteFromNodeModules(editors);
-    await deleteFromNodeModules(git);
-    await deleteFromNodeModules(ci);
-    await deleteFromNodeModules(meta);
-    await deleteFromNodeModules(misc);
+    log.info('Deleted %d files', await scanDelete({
+      directory: build.resolvePath('node_modules'),
+      regularExpressions: makeRegexps([
+        // tests
+        '**/test',
+        '**/tests',
+        '**/__tests__',
+        '**/mocha.opts',
+        '**/*.test.js',
+        '**/*.snap',
+        '**/coverage',
+
+        // docs
+        '**/doc',
+        '**/docs',
+        '**/CONTRIBUTING.md',
+        '**/Contributing.md',
+        '**/contributing.md',
+        '**/History.md',
+        '**/HISTORY.md',
+        '**/history.md',
+        '**/CHANGELOG.md',
+        '**/Changelog.md',
+        '**/changelog.md',
+
+        // examples
+        '**/example',
+        '**/examples',
+        '**/demo',
+        '**/samples',
+
+        // bins
+        '**/.bin',
+
+        // linters
+        '**/.eslintrc',
+        '**/.eslintrc.js',
+        '**/.eslintrc.yml',
+        '**/.prettierrc',
+        '**/.jshintrc',
+        '**/.babelrc',
+        '**/.jscs.json',
+        '**/.lint',
+
+        // hints
+        '**/*.flow',
+        '**/*.webidl',
+        '**/*.map',
+        '**/@types',
+
+        // scripts
+        '**/*.sh',
+        '**/*.bat',
+        '**/*.exe',
+        '**/Gruntfile.js',
+        '**/gulpfile.js',
+        '**/Makefile',
+
+        // untranspiled sources
+        '**/*.coffee',
+        '**/*.scss',
+        '**/*.sass',
+        '**/.ts',
+        '**/.tsx',
+
+        // editors
+        '**/.editorconfig',
+        '**/.vscode',
+
+        // git
+        '**/.gitattributes',
+        '**/.gitkeep',
+        '**/.gitempty',
+        '**/.gitmodules',
+        '**/.keep',
+        '**/.empty',
+
+        // ci
+        '**/.travis.yml',
+        '**/.coveralls.yml',
+        '**/.instanbul.yml',
+        '**/appveyor.yml',
+        '**/.zuul.yml',
+
+        // metadata
+        '**/package-lock.json',
+        '**/component.json',
+        '**/bower.json',
+        '**/yarn.lock',
+
+        // misc
+        '**/.*ignore',
+        '**/.DS_Store',
+        '**/Dockerfile',
+        '**/docker-compose.yml'
+      ])
+    }));
   },
 };