From 32817642bbf2abf33003406fa98328a755353c20 Mon Sep 17 00:00:00 2001 From: Niek van der Maas Date: Tue, 12 Jan 2021 21:26:58 +0100 Subject: [PATCH] Support service workers/shared workers (#401) * Added test for inconsistencies between page and service workers * Wrong assert * Added CreepJS test * Fix plugin leak * Add evaluateOnNewDocument() alias * Use value instead of getter for navigator.hardwareConcurrency * Use util functions for navigator.languages * Nicer code in navigator.permissions * Disable SW test * Typo fix * Fix tests * Skip SW fixture in ava * Add workflow matrix entry for pptr 5.5.0 * Revert cat & mouse test * Add separate SW test (skipped for now) * Disable 5.5.0 again --- .../navigator.hardwareConcurrency/index.js | 20 ++-- .../evasions/navigator.languages/index.js | 7 +- .../evasions/navigator.permissions/index.js | 2 +- .../evasions/navigator.plugins/index.js | 5 +- .../package.json | 3 +- .../fixtures/dummy-with-service-worker.html | 22 ++++ .../test/fixtures/sw.js | 1 + .../test/service-worker.test.js | 112 ++++++++++++++++++ packages/puppeteer-extra-plugin/src/index.ts | 13 ++ 9 files changed, 168 insertions(+), 17 deletions(-) create mode 100644 packages/puppeteer-extra-plugin-stealth/test/fixtures/dummy-with-service-worker.html create mode 100644 packages/puppeteer-extra-plugin-stealth/test/fixtures/sw.js create mode 100644 packages/puppeteer-extra-plugin-stealth/test/service-worker.test.js diff --git a/packages/puppeteer-extra-plugin-stealth/evasions/navigator.hardwareConcurrency/index.js b/packages/puppeteer-extra-plugin-stealth/evasions/navigator.hardwareConcurrency/index.js index a88093d9..7cdfe468 100644 --- a/packages/puppeteer-extra-plugin-stealth/evasions/navigator.hardwareConcurrency/index.js +++ b/packages/puppeteer-extra-plugin-stealth/evasions/navigator.hardwareConcurrency/index.js @@ -2,8 +2,6 @@ const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin') -const withUtils = require('../_utils/withUtils') - /** * Set the hardwareConcurrency to 4 (optionally configurable with `hardwareConcurrency`) * @@ -23,15 +21,15 @@ class Plugin extends PuppeteerExtraPlugin { } async onPageCreated(page) { - await withUtils(page).evaluateOnNewDocument((utils, opts) => { - const patchNavigator = (name, value) => - utils.replaceProperty(Object.getPrototypeOf(navigator), name, { - get() { - return value - } - }) - - patchNavigator('hardwareConcurrency', opts.hardwareConcurrency || 4) + await page.evaluateOnNewDocument(opts => { + Object.defineProperty( + Object.getPrototypeOf(navigator), + 'hardwareConcurrency', + { + value: opts.hardwareConcurrency || 4, + writable: false + } + ) }, this.opts) } } diff --git a/packages/puppeteer-extra-plugin-stealth/evasions/navigator.languages/index.js b/packages/puppeteer-extra-plugin-stealth/evasions/navigator.languages/index.js index ccde566d..c0031259 100644 --- a/packages/puppeteer-extra-plugin-stealth/evasions/navigator.languages/index.js +++ b/packages/puppeteer-extra-plugin-stealth/evasions/navigator.languages/index.js @@ -1,6 +1,7 @@ 'use strict' const { PuppeteerExtraPlugin } = require('puppeteer-extra-plugin') +const withUtils = require('../_utils/withUtils') /** * Pass the Languages Test. Allows setting custom languages. @@ -17,10 +18,10 @@ class Plugin extends PuppeteerExtraPlugin { return 'stealth/evasions/navigator.languages' } + // Overwrite the `languages` property to use a custom getter. async onPageCreated(page) { - await page.evaluateOnNewDocument(opts => { - // Overwrite the `languages` property to use a custom getter. - Object.defineProperty(Object.getPrototypeOf(navigator), 'languages', { + await withUtils(page).evaluateOnNewDocument((utils, opts) => { + utils.replaceProperty(Object.getPrototypeOf(navigator), 'languages', { get: () => opts.languages || ['en-US', 'en'] }) }, this.opts) diff --git a/packages/puppeteer-extra-plugin-stealth/evasions/navigator.permissions/index.js b/packages/puppeteer-extra-plugin-stealth/evasions/navigator.permissions/index.js index 958a8fbe..aea861f3 100644 --- a/packages/puppeteer-extra-plugin-stealth/evasions/navigator.permissions/index.js +++ b/packages/puppeteer-extra-plugin-stealth/evasions/navigator.permissions/index.js @@ -35,7 +35,7 @@ class Plugin extends PuppeteerExtraPlugin { } utils.replaceWithProxy( - window.navigator.permissions.__proto__, // eslint-disable-line no-proto + Object.getPrototypeOf(navigator.permissions), 'query', handler ) diff --git a/packages/puppeteer-extra-plugin-stealth/evasions/navigator.plugins/index.js b/packages/puppeteer-extra-plugin-stealth/evasions/navigator.plugins/index.js index 06daa23a..2e8f5df8 100644 --- a/packages/puppeteer-extra-plugin-stealth/evasions/navigator.plugins/index.js +++ b/packages/puppeteer-extra-plugin-stealth/evasions/navigator.plugins/index.js @@ -59,7 +59,10 @@ class Plugin extends PuppeteerExtraPlugin { configurable: true }) Object.defineProperty(mimeTypes[type], 'enabledPlugin', { - value: new Proxy(plugins[pluginData.name], {}), // Prevent circular references + value: + type === 'application/x-pnacl' + ? mimeTypes['application/x-nacl'].enabledPlugin // these reference the same plugin, so we need to re-use the Proxy in order to avoid leaks + : new Proxy(plugins[pluginData.name], {}), // Prevent circular references writable: false, enumerable: false, // Important: `JSON.stringify(navigator.plugins)` configurable: true diff --git a/packages/puppeteer-extra-plugin-stealth/package.json b/packages/puppeteer-extra-plugin-stealth/package.json index 3a107ca3..c3aff18b 100644 --- a/packages/puppeteer-extra-plugin-stealth/package.json +++ b/packages/puppeteer-extra-plugin-stealth/package.json @@ -37,7 +37,8 @@ ], "ava": { "files": [ - "!test/util.js" + "!test/util.js", + "!test/fixtures/sw.js" ] }, "devDependencies": { diff --git a/packages/puppeteer-extra-plugin-stealth/test/fixtures/dummy-with-service-worker.html b/packages/puppeteer-extra-plugin-stealth/test/fixtures/dummy-with-service-worker.html new file mode 100644 index 00000000..ae19a2d0 --- /dev/null +++ b/packages/puppeteer-extra-plugin-stealth/test/fixtures/dummy-with-service-worker.html @@ -0,0 +1,22 @@ + + + + + title foo + + + + +

Test page with service worker

+ + diff --git a/packages/puppeteer-extra-plugin-stealth/test/fixtures/sw.js b/packages/puppeteer-extra-plugin-stealth/test/fixtures/sw.js new file mode 100644 index 00000000..df4dab9f --- /dev/null +++ b/packages/puppeteer-extra-plugin-stealth/test/fixtures/sw.js @@ -0,0 +1 @@ +// Left empty diff --git a/packages/puppeteer-extra-plugin-stealth/test/service-worker.test.js b/packages/puppeteer-extra-plugin-stealth/test/service-worker.test.js new file mode 100644 index 00000000..b27e4e6d --- /dev/null +++ b/packages/puppeteer-extra-plugin-stealth/test/service-worker.test.js @@ -0,0 +1,112 @@ +const test = require('ava') + +const { vanillaPuppeteer, addExtra } = require('./util') +const Plugin = require('..') +const http = require('http') +const fs = require('fs') +const path = require('path') + +// Create a simple HTTP server. Service Workers cannot be served from file:// URIs +const httpServer = async () => { + const server = await http + .createServer((req, res) => { + let contents, type + + if (req.url === '/sw.js') { + contents = fs.readFileSync(path.join(__dirname, './fixtures/sw.js')) + type = 'application/javascript' + } else { + contents = fs.readFileSync( + path.join(__dirname, './fixtures/dummy-with-service-worker.html') + ) + type = 'text/html' + } + + res.setHeader('Content-Type', type) + res.writeHead(200) + res.end(contents) + }) + .listen(0) // random free port + + return `http://127.0.0.1:${server.address().port}/` +} + +let browser, page, worker + +test.before(async t => { + const address = await httpServer() + console.log(`Server is running on port ${address}`) + + browser = await addExtra(vanillaPuppeteer) + .use(Plugin()) + .launch({ headless: true }) + page = await browser.newPage() + + worker = new Promise(resolve => { + browser.on('targetcreated', async target => { + if (target.type() === 'service_worker') { + resolve(target.worker()) + } + }) + }) + + await page.goto(address) + worker = await worker +}) + +test.after(async t => { + await browser.close() +}) + +test.skip('stealth: inconsistencies between page and worker', async t => { + const pageFP = await page.evaluate(detectFingerprint) + const workerFP = await worker.evaluate(detectFingerprint) + + t.deepEqual(pageFP, workerFP) +}) + +test.serial.skip('stealth: creepjs has good trust score', async t => { + page.goto('https://abrahamjuliot.github.io/creepjs/') + + const score = await ( + await ( + await page.waitForSelector('#fingerprint-data .unblurred') + ).getProperty('textContent') + ).jsonValue() + + t.true( + parseInt(score) > 80, + `The creepjs score is: ${parseInt(score)}% but it should be at least 80%` + ) +}) + +/* global OffscreenCanvas */ +function detectFingerprint() { + const results = {} + + const props = [ + 'userAgent', + 'language', + 'hardwareConcurrency', + 'deviceMemory', + 'languages', + 'platform' + ] + props.forEach(el => { + results[el] = navigator[el].toString() + }) + + const canvasOffscreenWebgl = new OffscreenCanvas(256, 256) + const contextWebgl = canvasOffscreenWebgl.getContext('webgl') + const rendererInfo = contextWebgl.getExtension('WEBGL_debug_renderer_info') + results.webglVendor = contextWebgl.getParameter( + rendererInfo.UNMASKED_VENDOR_WEBGL + ) + results.webglRenderer = contextWebgl.getParameter( + rendererInfo.UNMASKED_RENDERER_WEBGL + ) + + results.timeZone = Intl.DateTimeFormat().resolvedOptions().timeZone + + return results +} diff --git a/packages/puppeteer-extra-plugin/src/index.ts b/packages/puppeteer-extra-plugin/src/index.ts index 7b6b6157..4afe0202 100644 --- a/packages/puppeteer-extra-plugin/src/index.ts +++ b/packages/puppeteer-extra-plugin/src/index.ts @@ -372,6 +372,10 @@ export abstract class PuppeteerExtraPlugin { // noop } + async onWorkerCreated(worker: Puppeteer.Worker) { + // noop + } + /** * Called when the url of a target changes. * @@ -538,6 +542,15 @@ export abstract class PuppeteerExtraPlugin { if (this.onPageCreated) { await this.onPageCreated(page) } + } else if (target.type() === 'service_worker' || target.type() === 'shared_worker') { + const worker = (await target.worker() as any) + if (worker) { + // Fixme: find some nicer way to add the method + worker.evaluateOnNewDocument = worker.evaluate + if (this.onWorkerCreated) { + await this.onWorkerCreated(worker) + } + } } }