diff --git a/package-lock.json b/package-lock.json index a40518a276b05..a001d3fc72414 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10502,10 +10502,13 @@ "chalk": "^2.4.2", "copy-dir": "^1.2.0", "docker-compose": "^0.22.2", + "extract-zip": "^1.6.7", "inquirer": "^7.0.4", "js-yaml": "^3.13.1", "nodegit": "^0.26.2", "ora": "^4.0.2", + "request": "^2.88.2", + "request-progress": "^3.0.0", "rimraf": "^3.0.2", "terminal-link": "^2.0.0", "yargs": "^14.0.0" @@ -10591,6 +10594,21 @@ "integrity": "sha512-zymm5+u+sCsSWyD9qNaejV3DFvhCKclKdizYaJUuHA83RLjb7nSuGnddCHGv0hk+KY7BMAlsWeK4Ueg6EV6XQg==", "dev": true }, + "mime-db": { + "version": "1.43.0", + "resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.43.0.tgz", + "integrity": "sha512-+5dsGEEovYbT8UY9yD7eE4XTc4UwJ1jBYlgaQQF38ENsKR3wj/8q8RFZrF9WIZpB2V1ArTVFUva8sAul1NzRzQ==", + "dev": true + }, + "mime-types": { + "version": "2.1.26", + "resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.26.tgz", + "integrity": "sha512-01paPWYgLrkqAyrlDorC1uDwl2p3qZT7yl806vW7DvDoxwXi46jsjFbg+WdwotBIk6/MbEhO/dh5aZ5sNj/dWQ==", + "dev": true, + "requires": { + "mime-db": "1.43.0" + } + }, "mimic-fn": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", @@ -10612,6 +10630,34 @@ "mimic-fn": "^2.1.0" } }, + "request": { + "version": "2.88.2", + "resolved": "https://registry.npmjs.org/request/-/request-2.88.2.tgz", + "integrity": "sha512-MsvtOrfG9ZcrOwAW+Qi+F6HbD0CWXEh9ou77uOb7FM2WPhwT7smM833PzanhJLsgXjN89Ir6V2PczXNnMpwKhw==", + "dev": true, + "requires": { + "aws-sign2": "~0.7.0", + "aws4": "^1.8.0", + "caseless": "~0.12.0", + "combined-stream": "~1.0.6", + "extend": "~3.0.2", + "forever-agent": "~0.6.1", + "form-data": "~2.3.2", + "har-validator": "~5.1.3", + "http-signature": "~1.2.0", + "is-typedarray": "~1.0.0", + "isstream": "~0.1.2", + "json-stringify-safe": "~5.0.1", + "mime-types": "~2.1.19", + "oauth-sign": "~0.9.0", + "performance-now": "^2.1.0", + "qs": "~6.5.2", + "safe-buffer": "^5.1.2", + "tough-cookie": "~2.5.0", + "tunnel-agent": "^0.6.0", + "uuid": "^3.3.2" + } + }, "restore-cursor": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", @@ -10679,6 +10725,16 @@ } } }, + "tough-cookie": { + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.5.0.tgz", + "integrity": "sha512-nlLsUzgm1kfLXSXfRZMc1KLAugd4hqJHDTvc2hDIwS3mZAfMEuMbc03SujMF+GEcpaX/qboeycw6iO8JwVv2+g==", + "dev": true, + "requires": { + "psl": "^1.1.28", + "punycode": "^2.1.1" + } + }, "type-fest": { "version": "0.8.1", "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.8.1.tgz", @@ -19704,7 +19760,7 @@ }, "node-pre-gyp": { "version": "0.12.0", - "resolved": "https://registry.npmjs.org/node-pre-gyp/-/node-pre-gyp-0.12.0.tgz", + "resolved": false, "integrity": "sha512-4KghwV8vH5k+g2ylT+sLTjy5wmUOb9vPhnM8NHvRf9dHmnW/CndrFXy2aRPaPST6dugXSdHXfeaHQm77PIz/1A==", "dev": true, "optional": true, @@ -19723,7 +19779,7 @@ }, "nopt": { "version": "4.0.1", - "resolved": "https://registry.npmjs.org/nopt/-/nopt-4.0.1.tgz", + "resolved": false, "integrity": "sha1-0NRoWv1UFRk8jHUFYC0NF81kR00=", "dev": true, "optional": true, @@ -35001,6 +35057,15 @@ } } }, + "request-progress": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/request-progress/-/request-progress-3.0.0.tgz", + "integrity": "sha1-TKdUCBx/7GP1BeT6qCWqBs1mnb4=", + "dev": true, + "requires": { + "throttleit": "^1.0.0" + } + }, "request-promise-core": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/request-promise-core/-/request-promise-core-1.1.1.tgz", @@ -38508,6 +38573,12 @@ "integrity": "sha512-AOvyNahXQuU7NN+VVvOOX+uW6FPaWdAOdRP5HfwYxAfCzXTFKRMoIMk+n+po318+ktcChx+F1Dd91G3YHeMKyg==", "dev": true }, + "throttleit": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/throttleit/-/throttleit-1.0.0.tgz", + "integrity": "sha1-nnhYNtr0Z0MUWlmEtiaNgoUorGw=", + "dev": true + }, "through": { "version": "2.3.8", "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", diff --git a/packages/env/CHANGELOG.md b/packages/env/CHANGELOG.md index 904815433d2d2..774afe9e90411 100644 --- a/packages/env/CHANGELOG.md +++ b/packages/env/CHANGELOG.md @@ -2,6 +2,7 @@ ### New Feature +- URLs for ZIP files are now supported as core, plugin, and theme sources. - The `.wp-env.json` coniguration file now accepts a `config` object for setting `wp-config.php` values. - A `.wp-env.override.json` configuration file can now be used to override fields from `.wp-env.json`. - You may now override the directory in which `wp-env` creates generated files with the `WP_ENV_HOME` environment variable. The default directory is `~/.wp-env/` (or `~/wp-env/` on Linux). diff --git a/packages/env/README.md b/packages/env/README.md index 1331d3019ac37..dadb8e406a20f 100644 --- a/packages/env/README.md +++ b/packages/env/README.md @@ -194,11 +194,12 @@ _Note: the port number environment variables (`WP_ENV_PORT` and `WP_ENV_TESTS_PO Several types of strings can be passed into the `core`, `plugins`, and `themes` fields: -| Type | Format | Example(s) | -| ----------------- | -------------------------- | -------------------------------------------------------- | -| Relative path | `.|~` | `"./a/directory"`, `"../a/directory"`, `"~/a/directory"` | -| Absolute path | `/|:\` | `"/a/directory"`, `"C:\\a\\directory"` | -| GitHub repository | `/[#]` | `"WordPress/WordPress"`, `"WordPress/gutenberg#master"` | +| Type | Format | Example(s) | +| ----------------- | ----------------------------- | -------------------------------------------------------- | +| Relative path | `.|~` | `"./a/directory"`, `"../a/directory"`, `"~/a/directory"` | +| Absolute path | `/|:\` | `"/a/directory"`, `"C:\\a\\directory"` | +| GitHub repository | `/[#]` | `"WordPress/WordPress"`, `"WordPress/gutenberg#master"` | +| ZIP File | `http[s]:///.zip` | `"https://wordpress.org/wordpress-5.4-beta2.zip"` | Remote sources will be downloaded into a temporary directory located in `~/.wp-env`. diff --git a/packages/env/lib/config.js b/packages/env/lib/config.js index bf7adf2f0af43..31e0d4bbb3715 100644 --- a/packages/env/lib/config.js +++ b/packages/env/lib/config.js @@ -257,6 +257,21 @@ function parseSourceString( sourceString, { workDirectoryPath } ) { }; } + const zipFields = sourceString.match( + /^https?:\/\/([^\s$.?#].[^\s]*)\.zip$/ + ); + if ( zipFields ) { + return { + type: 'zip', + url: sourceString, + path: path.resolve( + workDirectoryPath, + encodeURIComponent( zipFields[ 1 ] ) + ), + basename: encodeURIComponent( zipFields[ 1 ] ), + }; + } + const gitHubFields = sourceString.match( /^([^\/]+)\/([^#]+)(?:#(.+))?$/ ); if ( gitHubFields ) { return { diff --git a/packages/env/lib/download-source.js b/packages/env/lib/download-source.js index f3c8d5c5cbfd3..74bf5f1c4fb56 100644 --- a/packages/env/lib/download-source.js +++ b/packages/env/lib/download-source.js @@ -2,7 +2,20 @@ /** * External dependencies */ +const util = require( 'util' ); const NodeGit = require( 'nodegit' ); +const fs = require( 'fs' ); +const requestProgress = require( 'request-progress' ); +const request = require( 'request' ); +const path = require( 'path' ); + +/** + * Promisified dependencies + */ +const finished = util.promisify( require( 'stream' ).finished ); +const extractZip = util.promisify( require( 'extract-zip' ) ); +const rimraf = util.promisify( require( 'rimraf' ) ); +const copyDir = util.promisify( require( 'copy-dir' ) ); /** * @typedef {import('./config').Source} Source @@ -21,6 +34,8 @@ const NodeGit = require( 'nodegit' ); module.exports = async function downloadSource( source, options ) { if ( source.type === 'git' ) { await downloadGitSource( source, options ); + } else if ( source.type === 'zip' ) { + await downloadZipSource( source, options ); } }; @@ -36,8 +51,7 @@ module.exports = async function downloadSource( source, options ) { */ async function downloadGitSource( source, { onProgress, spinner, debug } ) { const log = debug - ? // eslint-disable-next-line no-console - ( message ) => { + ? ( message ) => { spinner.info( `NodeGit: ${ message }` ); spinner.start(); } @@ -96,3 +110,46 @@ async function downloadGitSource( source, { onProgress, spinner, debug } ) { onProgress( 1 ); } + +/** + * Downloads and extracts the zip file at `source.url` into `source.path`. + * + * @param {Source} source The source to download. + * @param {Object} options + * @param {Function} options.onProgress A function called with download progress. Will be invoked with one argument: a number that ranges from 0 to 1 which indicates current download progress for this source. + * @param {Object} options.spinner A CLI spinner which indicates progress. + * @param {boolean} options.debug True if debug mode is enabled. + */ +async function downloadZipSource( source, { onProgress, spinner, debug } ) { + const log = debug + ? ( message ) => { + spinner.info( `NodeGit: ${ message }` ); + spinner.start(); + } + : () => {}; + onProgress( 0 ); + + log( 'Downloading zip file.' ); + const zipName = `${ source.path }.zip`; + const zipFile = fs.createWriteStream( zipName ); + await finished( + requestProgress( request( source.url ) ) + .on( 'progress', ( { percent } ) => onProgress( percent ) ) + .pipe( zipFile ) + ); + + log( 'Extracting to temporary folder.' ); + const dirName = `${ source.path }.temp`; + await extractZip( zipName, { dir: dirName } ); + + log( 'Copying to mounted folder and cleaning up.' ); + await Promise.all( [ + rimraf( zipName ), + ...( await fs.promises.readdir( dirName ) ).map( ( file ) => + copyDir( path.join( dirName, file ), source.path ) + ), + ] ); + await rimraf( dirName ); + + onProgress( 1 ); +} diff --git a/packages/env/package.json b/packages/env/package.json index b6aaf13ab0266..1cdd5d80bb412 100644 --- a/packages/env/package.json +++ b/packages/env/package.json @@ -35,10 +35,13 @@ "chalk": "^2.4.2", "copy-dir": "^1.2.0", "docker-compose": "^0.22.2", + "extract-zip": "^1.6.7", "inquirer": "^7.0.4", "js-yaml": "^3.13.1", "nodegit": "^0.26.2", "ora": "^4.0.2", + "request": "^2.88.2", + "request-progress": "^3.0.0", "rimraf": "^3.0.2", "terminal-link": "^2.0.0", "yargs": "^14.0.0"