From 35dc4695e759b361642754cb2bd6c4adeabc4674 Mon Sep 17 00:00:00 2001 From: Spencer Date: Fri, 23 Nov 2018 09:50:39 -0700 Subject: [PATCH] [dev/build] use more performant copy implementation (#26109) * [dev/build] use more performant copy implementation * [dev/build] cleanup coments, install task --- src/dev/build/lib/index.js | 1 + src/dev/build/lib/scan_copy.test.ts | 133 ++++++++++++++++++ src/dev/build/lib/scan_copy.ts | 110 +++++++++++++++ .../tasks/create_archives_sources_task.js | 42 +++--- 4 files changed, 261 insertions(+), 25 deletions(-) create mode 100644 src/dev/build/lib/scan_copy.test.ts create mode 100644 src/dev/build/lib/scan_copy.ts diff --git a/src/dev/build/lib/index.js b/src/dev/build/lib/index.js index 952519773d6ba..7e318567316a5 100644 --- a/src/dev/build/lib/index.js +++ b/src/dev/build/lib/index.js @@ -32,3 +32,4 @@ export { deleteAll, } from './fs'; export { scanDelete } from './scan_delete'; +export { scanCopy } from './scan_copy'; diff --git a/src/dev/build/lib/scan_copy.test.ts b/src/dev/build/lib/scan_copy.test.ts new file mode 100644 index 0000000000000..cda009f9137f8 --- /dev/null +++ b/src/dev/build/lib/scan_copy.test.ts @@ -0,0 +1,133 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import { chmodSync, statSync } from 'fs'; +import { resolve } from 'path'; + +import del from 'del'; + +// @ts-ignore +import { getChildPaths, mkdirp, write } from './fs'; +import { scanCopy } from './scan_copy'; + +const IS_WINDOWS = process.platform === 'win32'; +const FIXTURES = resolve(__dirname, '__tests__/fixtures'); +const WORLD_EXECUTABLE = resolve(FIXTURES, 'bin/world_executable'); +const TMP = resolve(__dirname, '__tests__/__tmp__'); + +const getCommonMode = (path: string) => + statSync(path) + .mode.toString(8) + .slice(-3); + +// ensure WORLD_EXECUTABLE is actually executable by all +beforeAll(async () => { + chmodSync(WORLD_EXECUTABLE, 0o777); +}); + +// cleanup TMP directory +afterEach(async () => { + await del(TMP); +}); + +it('rejects if source path is not absolute', async () => { + await expect( + scanCopy({ + source: 'foo/bar', + destination: __dirname, + }) + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"Please use absolute paths to keep things explicit. You probably want to use \`build.resolvePath()\` or \`config.resolveFromRepo()\`."` + ); +}); + +it('rejects if destination path is not absolute', async () => { + await expect( + scanCopy({ + source: __dirname, + destination: 'foo/bar', + }) + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"Please use absolute paths to keep things explicit. You probably want to use \`build.resolvePath()\` or \`config.resolveFromRepo()\`."` + ); +}); + +it('rejects if neither path is absolute', async () => { + await expect( + scanCopy({ + source: 'foo/bar', + destination: 'foo/bar', + }) + ).rejects.toThrowErrorMatchingInlineSnapshot( + `"Please use absolute paths to keep things explicit. You probably want to use \`build.resolvePath()\` or \`config.resolveFromRepo()\`."` + ); +}); + +it('copies files and directories from source to dest, including dot files, creating dest if necessary, respecting mode', async () => { + const destination = resolve(TMP, 'a/b/c'); + await scanCopy({ + source: FIXTURES, + destination, + }); + + expect((await getChildPaths(resolve(destination, 'foo_dir'))).sort()).toEqual([ + resolve(destination, 'foo_dir/.bar'), + resolve(destination, 'foo_dir/bar.txt'), + resolve(destination, 'foo_dir/foo'), + ]); + + expect(getCommonMode(resolve(destination, 'bin/world_executable'))).toBe( + IS_WINDOWS ? '666' : '777' + ); + + expect(getCommonMode(resolve(destination, 'foo_dir/bar.txt'))).toBe(IS_WINDOWS ? '666' : '644'); +}); + +it('applies filter function specified', async () => { + const destination = resolve(TMP, 'a/b/c/d'); + await scanCopy({ + source: FIXTURES, + destination, + filter: record => !record.name.includes('bar'), + }); + + expect((await getChildPaths(resolve(destination, 'foo_dir'))).sort()).toEqual([ + resolve(destination, 'foo_dir/foo'), + ]); +}); + +it('supports atime and mtime', async () => { + const destination = resolve(TMP, 'a/b/c/d/e'); + const time = new Date(1425298511000); + + await scanCopy({ + source: FIXTURES, + destination, + time, + }); + + const barTxt = statSync(resolve(destination, 'foo_dir/bar.txt')); + const fooDir = statSync(resolve(destination, 'foo_dir')); + + // precision is platform specific + const oneDay = 86400000; + expect(Math.abs(barTxt.atimeMs - time.getTime())).toBeLessThan(oneDay); + expect(Math.abs(barTxt.mtimeMs - time.getTime())).toBeLessThan(oneDay); + expect(Math.abs(fooDir.atimeMs - time.getTime())).toBeLessThan(oneDay); +}); diff --git a/src/dev/build/lib/scan_copy.ts b/src/dev/build/lib/scan_copy.ts new file mode 100644 index 0000000000000..0a4bfdc8d0b4f --- /dev/null +++ b/src/dev/build/lib/scan_copy.ts @@ -0,0 +1,110 @@ +/* + * Licensed to Elasticsearch B.V. under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch B.V. licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +import Fs from 'fs'; +import { basename, join } from 'path'; +import { promisify } from 'util'; + +// @ts-ignore +import { assertAbsolute, mkdirp } from './fs'; + +const statAsync = promisify(Fs.stat); +const mkdirAsync = promisify(Fs.mkdir); +const utimesAsync = promisify(Fs.utimes); +const copyFileAsync = promisify(Fs.copyFile); +const readdirAsync = promisify(Fs.readdir); + +interface Options { + /** + * directory to copy from + */ + source: string; + /** + * path to copy to + */ + destination: string; + /** + * function that is called with each Record + */ + filter?: (record: Record) => boolean; + /** + * Date to use for atime/mtime + */ + time?: Date; +} + +class Record { + constructor( + public isDirectory: boolean, + public name: string, + public absolute: string, + public absoluteDest: string + ) {} +} + +/** + * Copy all of the files from one directory to another, optionally filtered with a + * function or modifying mtime/atime for each file. + */ +export async function scanCopy(options: Options) { + const { source, destination, filter, time } = options; + + assertAbsolute(source); + assertAbsolute(destination); + + // get filtered Records for files/directories within a directory + const getChildRecords = async (parent: Record) => { + const names = await readdirAsync(parent.absolute); + const records = await Promise.all( + names.map(async name => { + const absolute = join(parent.absolute, name); + const stat = await statAsync(absolute); + return new Record(stat.isDirectory(), name, absolute, join(parent.absoluteDest, name)); + }) + ); + + return records.filter(record => (filter ? filter(record) : true)); + }; + + // create or copy each child of a directory + const copyChildren = async (record: Record) => { + const children = await getChildRecords(record); + await Promise.all(children.map(async child => await copy(child))); + }; + + // create or copy a record and recurse into directories + const copy = async (record: Record) => { + if (record.isDirectory) { + await mkdirAsync(record.absoluteDest); + } else { + await copyFileAsync(record.absolute, record.absoluteDest, Fs.constants.COPYFILE_EXCL); + } + + if (time) { + await utimesAsync(record.absoluteDest, time, time); + } + + if (record.isDirectory) { + await copyChildren(record); + } + }; + + await mkdirp(destination); + await copyChildren(new Record(true, basename(source), source, destination)); +} diff --git a/src/dev/build/tasks/create_archives_sources_task.js b/src/dev/build/tasks/create_archives_sources_task.js index d54f295ea7a2d..5697b1784e32b 100644 --- a/src/dev/build/tasks/create_archives_sources_task.js +++ b/src/dev/build/tasks/create_archives_sources_task.js @@ -17,7 +17,7 @@ * under the License. */ -import { copyAll } from '../lib'; +import { scanCopy } from '../lib'; import { getNodeDownloadInfo } from './nodejs'; export const CreateArchivesSourcesTask = { @@ -25,34 +25,26 @@ export const CreateArchivesSourcesTask = { async run(config, log, build) { await Promise.all(config.getTargetPlatforms().map(async platform => { // copy all files from generic build source directory into platform-specific build directory - await copyAll( - build.resolvePath('.'), - build.resolvePathForPlatform(platform, '.'), - { - select: [ - '**/*', - '!node_modules/**', - ], - dot: true, - }, - ); + await scanCopy({ + source: build.resolvePath(), + destination: build.resolvePathForPlatform(platform), + filter: record => !(record.isDirectory && record.name === 'node_modules') + }); + + await scanCopy({ + source: build.resolvePath('node_modules'), + destination: build.resolvePathForPlatform(platform, 'node_modules'), + time: new Date() + }); - const currentTime = new Date(); - await copyAll( - build.resolvePath('node_modules'), - build.resolvePathForPlatform(platform, 'node_modules'), - { - dot: true, - time: currentTime - }, - ); log.debug('Generic build source copied into', platform.getName(), 'specific build directory'); // copy node.js install - await copyAll( - getNodeDownloadInfo(config, platform).extractDir, - build.resolvePathForPlatform(platform, 'node') - ); + await scanCopy({ + source: getNodeDownloadInfo(config, platform).extractDir, + destination: build.resolvePathForPlatform(platform, 'node'), + }); + log.debug('Node.js copied into', platform.getName(), 'specific build directory'); })); }