[dev/build] use more performant copy implementation (elastic#26109)

* [dev/build] use more performant copy implementation

* [dev/build] cleanup coments, install task

src/dev/build/lib/index.js

@@ -32,3 +32,4 @@ export {
   deleteAll,
 } from './fs';
 export { scanDelete } from './scan_delete';
+export { scanCopy } from './scan_copy';

src/dev/build/lib/scan_copy.test.ts

@@ -0,0 +1,133 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { chmodSync, statSync } from 'fs';
+import { resolve } from 'path';
+
+import del from 'del';
+
+// @ts-ignore
+import { getChildPaths, mkdirp, write } from './fs';
+import { scanCopy } from './scan_copy';
+
+const IS_WINDOWS = process.platform === 'win32';
+const FIXTURES = resolve(__dirname, '__tests__/fixtures');
+const WORLD_EXECUTABLE = resolve(FIXTURES, 'bin/world_executable');
+const TMP = resolve(__dirname, '__tests__/__tmp__');
+
+const getCommonMode = (path: string) =>
+  statSync(path)
+    .mode.toString(8)
+    .slice(-3);
+
+// ensure WORLD_EXECUTABLE is actually executable by all
+beforeAll(async () => {
+  chmodSync(WORLD_EXECUTABLE, 0o777);
+});
+
+// cleanup TMP directory
+afterEach(async () => {
+  await del(TMP);
+});
+
+it('rejects if source path is not absolute', async () => {
+  await expect(
+    scanCopy({
+      source: 'foo/bar',
+      destination: __dirname,
+    })
+  ).rejects.toThrowErrorMatchingInlineSnapshot(
+    `"Please use absolute paths to keep things explicit. You probably want to use \`build.resolvePath()\` or \`config.resolveFromRepo()\`."`
+  );
+});
+
+it('rejects if destination path is not absolute', async () => {
+  await expect(
+    scanCopy({
+      source: __dirname,
+      destination: 'foo/bar',
+    })
+  ).rejects.toThrowErrorMatchingInlineSnapshot(
+    `"Please use absolute paths to keep things explicit. You probably want to use \`build.resolvePath()\` or \`config.resolveFromRepo()\`."`
+  );
+});
+
+it('rejects if neither path is absolute', async () => {
+  await expect(
+    scanCopy({
+      source: 'foo/bar',
+      destination: 'foo/bar',
+    })
+  ).rejects.toThrowErrorMatchingInlineSnapshot(
+    `"Please use absolute paths to keep things explicit. You probably want to use \`build.resolvePath()\` or \`config.resolveFromRepo()\`."`
+  );
+});
+
+it('copies files and directories from source to dest, including dot files, creating dest if necessary, respecting mode', async () => {
+  const destination = resolve(TMP, 'a/b/c');
+  await scanCopy({
+    source: FIXTURES,
+    destination,
+  });
+
+  expect((await getChildPaths(resolve(destination, 'foo_dir'))).sort()).toEqual([
+    resolve(destination, 'foo_dir/.bar'),
+    resolve(destination, 'foo_dir/bar.txt'),
+    resolve(destination, 'foo_dir/foo'),
+  ]);
+
+  expect(getCommonMode(resolve(destination, 'bin/world_executable'))).toBe(
+    IS_WINDOWS ? '666' : '777'
+  );
+
+  expect(getCommonMode(resolve(destination, 'foo_dir/bar.txt'))).toBe(IS_WINDOWS ? '666' : '644');
+});
+
+it('applies filter function specified', async () => {
+  const destination = resolve(TMP, 'a/b/c/d');
+  await scanCopy({
+    source: FIXTURES,
+    destination,
+    filter: record => !record.name.includes('bar'),
+  });
+
+  expect((await getChildPaths(resolve(destination, 'foo_dir'))).sort()).toEqual([
+    resolve(destination, 'foo_dir/foo'),
+  ]);
+});
+
+it('supports atime and mtime', async () => {
+  const destination = resolve(TMP, 'a/b/c/d/e');
+  const time = new Date(1425298511000);
+
+  await scanCopy({
+    source: FIXTURES,
+    destination,
+    time,
+  });
+
+  const barTxt = statSync(resolve(destination, 'foo_dir/bar.txt'));
+  const fooDir = statSync(resolve(destination, 'foo_dir'));
+
+  // precision is platform specific
+  const oneDay = 86400000;
+  expect(Math.abs(barTxt.atimeMs - time.getTime())).toBeLessThan(oneDay);
+  expect(Math.abs(barTxt.mtimeMs - time.getTime())).toBeLessThan(oneDay);
+  expect(Math.abs(fooDir.atimeMs - time.getTime())).toBeLessThan(oneDay);
+});

src/dev/build/lib/scan_copy.ts

@@ -0,0 +1,110 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import Fs from 'fs';
+import { basename, join } from 'path';
+import { promisify } from 'util';
+
+// @ts-ignore
+import { assertAbsolute, mkdirp } from './fs';
+
+const statAsync = promisify(Fs.stat);
+const mkdirAsync = promisify(Fs.mkdir);
+const utimesAsync = promisify(Fs.utimes);
+const copyFileAsync = promisify(Fs.copyFile);
+const readdirAsync = promisify(Fs.readdir);
+
+interface Options {
+  /**
+   * directory to copy from
+   */
+  source: string;
+  /**
+   * path to copy to
+   */
+  destination: string;
+  /**
+   * function that is called with each Record
+   */
+  filter?: (record: Record) => boolean;
+  /**
+   * Date to use for atime/mtime
+   */
+  time?: Date;
+}
+
+class Record {
+  constructor(
+    public isDirectory: boolean,
+    public name: string,
+    public absolute: string,
+    public absoluteDest: string
+  ) {}
+}
+
+/**
+ * Copy all of the files from one directory to another, optionally filtered with a
+ * function or modifying mtime/atime for each file.
+ */
+export async function scanCopy(options: Options) {
+  const { source, destination, filter, time } = options;
+
+  assertAbsolute(source);
+  assertAbsolute(destination);
+
+  // get filtered Records for files/directories within a directory
+  const getChildRecords = async (parent: Record) => {
+    const names = await readdirAsync(parent.absolute);
+    const records = await Promise.all(
+      names.map(async name => {
+        const absolute = join(parent.absolute, name);
+        const stat = await statAsync(absolute);
+        return new Record(stat.isDirectory(), name, absolute, join(parent.absoluteDest, name));
+      })
+    );
+
+    return records.filter(record => (filter ? filter(record) : true));
+  };
+
+  // create or copy each child of a directory
+  const copyChildren = async (record: Record) => {
+    const children = await getChildRecords(record);
+    await Promise.all(children.map(async child => await copy(child)));
+  };
+
+  // create or copy a record and recurse into directories
+  const copy = async (record: Record) => {
+    if (record.isDirectory) {
+      await mkdirAsync(record.absoluteDest);
+    } else {
+      await copyFileAsync(record.absolute, record.absoluteDest, Fs.constants.COPYFILE_EXCL);
+    }
+
+    if (time) {
+      await utimesAsync(record.absoluteDest, time, time);
+    }
+
+    if (record.isDirectory) {
+      await copyChildren(record);
+    }
+  };
+
+  await mkdirp(destination);
+  await copyChildren(new Record(true, basename(source), source, destination));
+}

src/dev/build/tasks/create_archives_sources_task.js

@@ -17,42 +17,34 @@
  * under the License.
  */
 
-import { copyAll } from '../lib';
+import { scanCopy } from '../lib';
 import { getNodeDownloadInfo } from './nodejs';
 
 export const CreateArchivesSourcesTask = {
   description: 'Creating platform-specific archive source directories',
   async run(config, log, build) {
     await Promise.all(config.getTargetPlatforms().map(async platform => {
       // copy all files from generic build source directory into platform-specific build directory
-      await copyAll(
-        build.resolvePath('.'),
-        build.resolvePathForPlatform(platform, '.'),
-        {
-          select: [
-            '**/*',
-            '!node_modules/**',
-          ],
-          dot: true,
-        },
-      );
+      await scanCopy({
+        source: build.resolvePath(),
+        destination: build.resolvePathForPlatform(platform),
+        filter: record => !(record.isDirectory && record.name === 'node_modules')
+      });
+
+      await scanCopy({
+        source: build.resolvePath('node_modules'),
+        destination: build.resolvePathForPlatform(platform, 'node_modules'),
+        time: new Date()
+      });
 
-      const currentTime = new Date();
-      await copyAll(
-        build.resolvePath('node_modules'),
-        build.resolvePathForPlatform(platform, 'node_modules'),
-        {
-          dot: true,
-          time: currentTime
-        },
-      );
       log.debug('Generic build source copied into', platform.getName(), 'specific build directory');
 
       // copy node.js install
-      await copyAll(
-        getNodeDownloadInfo(config, platform).extractDir,
-        build.resolvePathForPlatform(platform, 'node')
-      );
+      await scanCopy({
+        source: getNodeDownloadInfo(config, platform).extractDir,
+        destination: build.resolvePathForPlatform(platform, 'node'),
+      });
+
       log.debug('Node.js copied into', platform.getName(), 'specific build directory');
     }));
   }