devops: remake downloading logic (#1419)

This patch:
- removes `browserType.downloadBrowserIfNeeded()` method. The method
  turned out to be ill-behaving and cannot not be used as we'd like to (see #1085)
- adds a `browserType.setExecutablePath` method to set a browser
  exectuable.

With this patch, we take the following approach towards managing browser downloads:
- `playwright-core` doesn't download any browsers. In `playwright-core`, `playwright.chromium.executablePath()` returns `null` (same for firefox and webkit).
- clients of `playwright-core` (e.g. `playwright` and others) download browsers one way or another.
They can then configure `playwright` with executable paths and re-export the `playwright` object to their clients.
- `playwright`, `playwright-firefox`, `playwright-chromium` and `playwright-webkit` download 
browsers. Once browsers are downloaded, their executable paths are saved to a `.downloaded-browsers.json` file. This file is read in `playwright/index.js` to configure browser executable paths and re-export the API.
- special case is `install-from-github.js` that also cleans up old browsers.

Author: aslushnikov

.circleci/config.yml

@@ -15,9 +15,7 @@ jobs:
       - save_cache:
           key: dependency-cache-{{ checksum "package.json" }}
           paths:
-            - ./.local-chromium
-            - ./.local-firefox
-            - ./.local-webkit
+            - ./.local-browsers
 
       - run:
           command: |

.gitignore

@@ -3,11 +3,10 @@
 /test/output-firefox
 /test/output-webkit
 /test/test-user-data-dir*
-/.local-chromium/
-/.local-firefox/
-/.local-webkit/
+/.local-browsers/
 /.dev_profile*
 .DS_Store
+.downloaded-browsers.json
 *.swp
 *.pyc
 .vscode

docs/api.md

@@ -3645,7 +3645,6 @@ const { chromium } = require('playwright');  // Or 'firefox' or 'webkit'.
 
 <!-- GEN:toc -->
 - [browserType.connect(options)](#browsertypeconnectoptions)
-- [browserType.downloadBrowserIfNeeded([progress])](#browsertypedownloadbrowserifneededprogress)
 - [browserType.executablePath()](#browsertypeexecutablepath)
 - [browserType.launch([options])](#browsertypelaunchoptions)
 - [browserType.launchPersistentContext(userDataDir, [options])](#browsertypelaunchpersistentcontextuserdatadir-options)
@@ -3661,14 +3660,8 @@ const { chromium } = require('playwright');  // Or 'firefox' or 'webkit'.
 
 This methods attaches Playwright to an existing browser instance.
 
-#### browserType.downloadBrowserIfNeeded([progress])
-- `progress` <[function]> If download is initiated, this function is called with two parameters: `downloadedBytes` and `totalBytes`.
-- returns: <[Promise]> promise that resolves when browser is successfully downloaded.
-
-Download browser binary if it is missing.
-
 #### browserType.executablePath()
-- returns: <[string]> A path where Playwright expects to find a bundled browser.
+- returns: <[string]> A path where Playwright expects to find a bundled browser executable.
 
 #### browserType.launch([options])
 - `options` <[Object]>  Set of configurable options to set on the browser. Can have the following fields:

docs/troubleshooting.md

@@ -164,7 +164,7 @@ done only once per host environment:
 
 ```bash
 # cd to the downloaded instance
-cd <project-dir-path>/node_modules/playwright/.local-chromium/linux-<revision>/chrome-linux/
+cd <project-dir-path>/node_modules/playwright/.local-browsers/chromium-<revision>/
 sudo chown root:root chrome_sandbox
 sudo chmod 4755 chrome_sandbox
 # copy sandbox executable to a shared location

download-browser.js

@@ -13,37 +13,39 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+const fs = require('fs');
+const browserFetcher = require('./lib/server/browserFetcher.js');
+const packageJSON = require('./package.json');
 
-async function downloadBrowser(browserType) {
-  const browser = browserType.name();
+async function downloadBrowserWithProgressBar(downloadPath, browser, version = '') {
   let progressBar = null;
   let lastDownloadedBytes = 0;
-  function onProgress(downloadedBytes, totalBytes) {
+  const revision = packageJSON.playwright[`${browser}_revision`];
+  function progress(downloadedBytes, totalBytes) {
     if (!progressBar) {
       const ProgressBar = require('progress');
-      progressBar = new ProgressBar(`Downloading ${browser} ${browserType._revision} - ${toMegabytes(totalBytes)} [:bar] :percent :etas `, {
+      progressBar = new ProgressBar(`Downloading ${browser} r${revision} - ${toMegabytes(totalBytes)} [:bar] :percent :etas `, {
         complete: '=',
         incomplete: ' ',
         width: 20,
         total: totalBytes,
+        host: getFromENV('PLAYWRIGHT_DOWNLOAD_HOST'),
       });
     }
     const delta = downloadedBytes - lastDownloadedBytes;
     lastDownloadedBytes = downloadedBytes;
     progressBar.tick(delta);
   }
-
-  const fetcher = browserType._createBrowserFetcher();
-  const revisionInfo = fetcher.revisionInfo();
-  // Do nothing if the revision is already downloaded.
-  if (revisionInfo.local)
-    return revisionInfo;
-  await browserType.downloadBrowserIfNeeded(onProgress);
-  logPolitely(`${browser} downloaded to ${revisionInfo.folderPath}`);
-  return revisionInfo;
+  const executablePath = await browserFetcher.downloadBrowser({
+    downloadPath,
+    browser,
+    revision,
+    progress,
+  });
+  logPolitely(`${browser} downloaded to ${downloadPath}`);
+  return executablePath;
 }
 
-
 function toMegabytes(bytes) {
   const mb = bytes / 1024 / 1024;
   return `${Math.round(mb * 10) / 10} Mb`;
@@ -57,4 +59,11 @@ function logPolitely(toBeLogged) {
     console.log(toBeLogged);
 }
 
-module.exports = {downloadBrowser};
+function getFromENV(name) {
+  let value = process.env[name];
+  value = value || process.env[`npm_config_${name.toLowerCase()}`];
+  value = value || process.env[`npm_package_config_${name.toLowerCase()}`];
+  return value;
+}
+
+module.exports = {downloadBrowserWithProgressBar};

index.js

@@ -15,9 +15,17 @@
  */
 const {Playwright} = require('./lib/server/playwright.js');
 
-module.exports = new Playwright({
-  downloadPath: __dirname,
+const playwright = new Playwright({
   browsers: ['webkit', 'chromium', 'firefox'],
-  respectEnvironmentVariables: false,
 });
 
+try {
+  const downloadedBrowsers = require('./.downloaded-browsers.json');
+  playwright.chromium._executablePath = downloadedBrowsers.crExecutablePath;
+  playwright.firefox._executablePath = downloadedBrowsers.ffExecutablePath;
+  playwright.webkit._executablePath = downloadedBrowsers.wkExecutablePath;
+} catch (e) {
+}
+
+module.exports = playwright;
+

install-from-github.js

@@ -24,41 +24,69 @@ try {
   });
 } catch (e) {
 }
-const {downloadBrowser} = require('./download-browser');
-const playwright = require('.');
+
+const path = require('path');
+const fs = require('fs');
+const util = require('util');
+const rmAsync = util.promisify(require('rimraf'));
+const existsAsync = path => fs.promises.access(path).then(() => true, e => false);
+const {downloadBrowserWithProgressBar} = require('./download-browser');
+const protocolGenerator = require('./utils/protocol-types-generator');
+const packageJSON = require('./package.json');
+
+const DOWNLOADED_BROWSERS_JSON_PATH = path.join(__dirname, '.downloaded-browsers.json');
+const DOWNLOAD_PATHS = {
+  chromium: path.join(__dirname, '.local-browsers', `chromium-${packageJSON.playwright.chromium_revision}`),
+  firefox: path.join(__dirname, '.local-browsers', `firefox-${packageJSON.playwright.firefox_revision}`),
+  webkit: path.join(__dirname, '.local-browsers', `webkit-${packageJSON.playwright.webkit_revision}`),
+};
 
 (async function() {
-  const protocolGenerator = require('./utils/protocol-types-generator');
+  const downloadedBrowsersJSON = await fs.promises.readFile(DOWNLOADED_BROWSERS_JSON_PATH, 'utf8').then(json => JSON.parse(json)).catch(() => ({}));
   try {
-    const chromeRevision = await downloadAndCleanup(playwright.chromium);
-    await protocolGenerator.generateChromiunProtocol(chromeRevision);
+    if (!(await existsAsync(DOWNLOAD_PATHS.chromium))) {
+      const crExecutablePath = await downloadBrowserWithProgressBar(DOWNLOAD_PATHS.chromium, 'chromium');
+      downloadedBrowsersJSON.crExecutablePath = crExecutablePath;
+      await protocolGenerator.generateChromiumProtocol(crExecutablePath);
+      await fs.promises.writeFile(DOWNLOADED_BROWSERS_JSON_PATH, JSON.stringify(downloadedBrowsersJSON));
+    }
   } catch (e) {
     console.warn(e.message);
   }
-
   try {
-    const firefoxRevision = await downloadAndCleanup(playwright.firefox);
-    await protocolGenerator.generateFirefoxProtocol(firefoxRevision);
+    if (!(await existsAsync(DOWNLOAD_PATHS.firefox))) {
+      const ffExecutablePath = await downloadBrowserWithProgressBar(DOWNLOAD_PATHS.firefox, 'firefox');
+      downloadedBrowsersJSON.ffExecutablePath = ffExecutablePath;
+      await protocolGenerator.generateFirefoxProtocol(ffExecutablePath);
+      await fs.promises.writeFile(DOWNLOADED_BROWSERS_JSON_PATH, JSON.stringify(downloadedBrowsersJSON));
+    }
   } catch (e) {
     console.warn(e.message);
   }
-
   try {
-    const webkitRevision = await downloadAndCleanup(playwright.webkit);
-    await protocolGenerator.generateWebKitProtocol(webkitRevision);
+    if (!(await existsAsync(DOWNLOAD_PATHS.webkit))) {
+      const wkExecutablePath = await downloadBrowserWithProgressBar(DOWNLOAD_PATHS.webkit, 'webkit');
+      downloadedBrowsersJSON.wkExecutablePath = wkExecutablePath;
+      await protocolGenerator.generateWebKitProtocol(path.dirname(wkExecutablePath));
+      await fs.promises.writeFile(DOWNLOADED_BROWSERS_JSON_PATH, JSON.stringify(downloadedBrowsersJSON));
+    }
   } catch (e) {
     console.warn(e.message);
   }
-})();
 
-async function downloadAndCleanup(browserType) {
-  const revisionInfo = await downloadBrowser(browserType);
+  // Cleanup stale revisions.
+  const directories = new Set(await readdirAsync(path.join(__dirname, '.local-browsers')));
+  directories.delete(DOWNLOAD_PATHS.chromium);
+  directories.delete(DOWNLOAD_PATHS.firefox);
+  directories.delete(DOWNLOAD_PATHS.webkit);
+  // cleanup old browser directories.
+  directories.add(path.join(__dirname, '.local-chromium'));
+  directories.add(path.join(__dirname, '.local-firefox'));
+  directories.add(path.join(__dirname, '.local-webkit'));
+  await Promise.all([...directories].map(directory => rmAsync(directory)));
 
-  // Remove previous revisions.
-  const fetcher = browserType._createBrowserFetcher();
-  const localRevisions = await fetcher.localRevisions();
-  const cleanupOldVersions = localRevisions.filter(revision => revision !== revisionInfo.revision).map(revision => fetcher.remove(revision));
-  await Promise.all([...cleanupOldVersions]);
+  async function readdirAsync(dirpath) {
+    return fs.promises.readdir(dirpath).then(dirs => dirs.map(dir => path.join(dirpath, dir)));
+  }
+})();
 
-  return revisionInfo;
-}

packages/playwright-chromium/index.js

@@ -13,12 +13,17 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
+const path = require('path');
 const {Playwright} = require('playwright-core/lib/server/playwright.js');
 
-module.exports = new Playwright({
-  downloadPath: __dirname,
+const playwright = new Playwright({
   browsers: ['chromium'],
-  respectEnvironmentVariables: true,
 });
+module.exports = playwright;
 
+try {
+  const downloadedBrowsers = require('./.downloaded-browsers.json');
+  playwright.chromium._executablePath = downloadedBrowsers.crExecutablePath;
+} catch (e) {
+  throw new Error('playwright-chromium has not downloaded Chromium.');
+}

packages/playwright-chromium/install.js

@@ -13,6 +13,10 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-const {downloadBrowser} = require('playwright-core/download-browser');
-const playwright = require('.');
-downloadBrowser(playwright.chromium);
+const path = require('path');
+const fs = require('fs');
+const {downloadBrowserWithProgressBar} = require('playwright-core/download-browser');
+(async function() {
+  const crExecutablePath = await downloadBrowserWithProgressBar(path.join(__dirname, '.local-browsers', 'chromium'), 'chromium');
+  await fs.promises.writeFile(path.join(__dirname, '.downloaded-browsers.json'), JSON.stringify({crExecutablePath}));
+})();

packages/playwright-firefox/index.js

@@ -13,12 +13,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
+const path = require('path');
 const {Playwright} = require('playwright-core/lib/server/playwright.js');
 
-module.exports = new Playwright({
-  downloadPath: __dirname,
+const playwright = new Playwright({
   browsers: ['firefox'],
-  respectEnvironmentVariables: true,
 });
+module.exports = playwright;
+
+try {
+  const downloadedBrowsers = require(path.join(__dirname, '.downloaded-browsers.json'));
+  playwright.firefox._executablePath = downloadedBrowsers.ffExecutablePath;
+} catch (e) {
+  throw new Error('playwright-firefox has not downloaded Firefox.');
+}
 

packages/playwright-firefox/install.js

@@ -13,6 +13,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-const {downloadBrowser} = require('playwright-core/download-browser');
-const playwright = require('.');
-downloadBrowser(playwright.firefox);
+const path = require('path');
+const fs = require('fs');
+const {downloadBrowserWithProgressBar} = require('playwright-core/download-browser');
+
+(async function() {
+  const ffExecutablePath = await downloadBrowserWithProgressBar(path.join(__dirname, '.local-browsers', 'firefox'), 'firefox');
+  await fs.promises.writeFile(path.join(__dirname, '.downloaded-browsers.json'), JSON.stringify({ffExecutablePath, }));
+})();

packages/playwright-webkit/index.js

@@ -13,12 +13,18 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-
+const path = require('path');
 const {Playwright} = require('playwright-core/lib/server/playwright.js');
 
-module.exports = new Playwright({
-  downloadPath: __dirname,
+const playwright = new Playwright({
   browsers: ['webkit'],
-  respectEnvironmentVariables: true,
 });
+module.exports = playwright;
+
+try {
+  const downloadedBrowsers = require(path.join(__dirname, '.downloaded-browsers.json'));
+  playwright.webkit._executablePath = downloadedBrowsers.wkExecutablePath;
+} catch (e) {
+  throw new Error('playwright-webkit has not downloaded WebKit.');
+}
 

packages/playwright-webkit/install.js

@@ -13,6 +13,11 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-const {downloadBrowser} = require('playwright-core/download-browser');
-const playwright = require('.');
-downloadBrowser(playwright.webkit);
+const path = require('path');
+const fs = require('fs');
+const {downloadBrowserWithProgressBar} = require('playwright-core/download-browser');
+
+(async function() {
+  const wkExecutablePath = await downloadBrowserWithProgressBar(path.join(__dirname, '.local-browsers', 'webkit'), 'webkit');
+  await fs.promises.writeFile(path.join(__dirname, '.downloaded-browsers.json'), JSON.stringify({wkExecutablePath, }));
+})();

packages/playwright/index.js

@@ -13,11 +13,21 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+const path = require('path');
 const {Playwright} = require('playwright-core/lib/server/playwright.js');
 
-module.exports = new Playwright({
-  downloadPath: __dirname,
+const playwright = new Playwright({
   browsers: ['webkit', 'chromium', 'firefox'],
-  respectEnvironmentVariables: true,
 });
+module.exports = playwright;
+
+try {
+  const downloadedBrowsers = require(path.join(__dirname, '.downloaded-browsers.json'));
+  playwright.chromium._executablePath = downloadedBrowsers.crExecutablePath;
+  playwright.firefox._executablePath = downloadedBrowsers.ffExecutablePath;
+  playwright.webkit._executablePath = downloadedBrowsers.wkExecutablePath;
+} catch (e) {
+  throw new Error('ERROR: Playwright did not download browsers');
+}
+
 

packages/playwright/install.js

@@ -13,10 +13,13 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
-const {downloadBrowser} = require('playwright-core/download-browser');
-const playwright = require('.');
+const path = require('path');
+const fs = require('fs');
+const {downloadBrowserWithProgressBar} = require('playwright-core/download-browser');
+
 (async function() {
-  await downloadBrowser(playwright.chromium);
-  await downloadBrowser(playwright.firefox);
-  await downloadBrowser(playwright.webkit);
+  const crExecutablePath = await downloadBrowserWithProgressBar(path.join(__dirname, '.local-browsers', 'chromium'), 'chromium');
+  const ffExecutablePath = await downloadBrowserWithProgressBar(path.join(__dirname, '.local-browsers', 'firefox'), 'firefox');
+  const wkExecutablePath = await downloadBrowserWithProgressBar(path.join(__dirname, '.local-browsers', 'webkit'), 'webkit');
+  await fs.promises.writeFile(path.join(__dirname, '.downloaded-browsers.json'), JSON.stringify({crExecutablePath, ffExecutablePath, wkExecutablePath, }));
 })();