From b776cab0e37665b69f42e375a01e601ba186e9f6 Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Thu, 16 Jan 2020 14:50:49 -0800 Subject: [PATCH 1/3] WIP Fixing map tiles and such --- .../chromium/driver/chromium_driver.ts | 31 ++++++++++++++----- x-pack/legacy/plugins/reporting/types.d.ts | 15 +++++++++ 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts b/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts index c1777038cc7d4..305d1d462767a 100644 --- a/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts +++ b/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts @@ -15,6 +15,7 @@ import { ConditionalHeaders, ConditionalHeadersConditions, ElementPosition, + InterceptedRequest, NetworkPolicy, } from '../../../../types'; @@ -59,35 +60,49 @@ export class HeadlessChromiumDriver { }: { conditionalHeaders: ConditionalHeaders; waitForSelector: string }, logger: LevelLogger ) { - await this.page.setRequestInterception(true); logger.info(`opening url ${url}`); + // @ts-ignore + const client = this.page._client; let interceptedCount = 0; - this.page.on('request', interceptedRequest => { - const interceptedUrl = interceptedRequest.url(); + await this.page.setRequestInterception(true); + client.on('Fetch.requestPaused', (interceptedRequest: InterceptedRequest) => { + const requestId = interceptedRequest.requestId; + const interceptedUrl = interceptedRequest.request.url; const allowed = !interceptedUrl.startsWith('file://'); const isData = interceptedUrl.startsWith('data:'); // We should never ever let file protocol requests go through if (!allowed || !this.allowRequest(interceptedUrl)) { logger.error(`Got bad URL: "${interceptedUrl}", closing browser.`); - interceptedRequest.abort('blockedbyclient'); + client.send('Fetch.failRequest', { + errorReason: 'Aborted', + requestId, + }); this.page.browser().close(); throw new Error(`Received disallowed outgoing URL: "${interceptedUrl}", exiting`); } if (this._shouldUseCustomHeaders(conditionalHeaders.conditions, interceptedUrl)) { logger.debug(`Using custom headers for ${interceptedUrl}`); - interceptedRequest.continue({ - headers: { - ...interceptedRequest.headers(), + const headers = _.map( + { + ...interceptedRequest.request.headers, ...conditionalHeaders.headers, }, + (value, name) => ({ + name, + value, + }) + ); + client.send('Fetch.continueRequest', { + requestId, + headers, }); } else { const loggedUrl = isData ? this.truncateUrl(interceptedUrl) : interceptedUrl; logger.debug(`No custom headers for ${loggedUrl}`); - interceptedRequest.continue(); + client.send('Fetch.continueRequest', { requestId }); } interceptedCount = interceptedCount + (isData ? 0 : 1); }); diff --git a/x-pack/legacy/plugins/reporting/types.d.ts b/x-pack/legacy/plugins/reporting/types.d.ts index c17b969d5d7fa..5adabd785679c 100644 --- a/x-pack/legacy/plugins/reporting/types.d.ts +++ b/x-pack/legacy/plugins/reporting/types.d.ts @@ -341,3 +341,18 @@ export interface AbsoluteURLFactoryOptions { hostname: string; port: string | number; } + +export interface InterceptedRequest { + requestId: string; + request: { + url: string; + method: string; + headers: { + [key: string]: string; + }; + initialPriority: string; + referrerPolicy: string; + }; + frameId: string; + resourceType: string; +} From d088c320de9ac8d7022c16ace0084e6bd730dff5 Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Thu, 16 Jan 2020 15:12:33 -0800 Subject: [PATCH 2/3] Small comment and importing map from dolash --- .../server/browsers/chromium/driver/chromium_driver.ts | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts b/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts index 305d1d462767a..46f4867e505f2 100644 --- a/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts +++ b/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts @@ -4,7 +4,7 @@ * you may not use this file except in compliance with the Elastic License. */ -import { trunc } from 'lodash'; +import { trunc, map } from 'lodash'; import open from 'opn'; import { parse as parseUrl } from 'url'; import { Page, SerializableOrJSHandle, EvaluateFn } from 'puppeteer'; @@ -66,6 +66,9 @@ export class HeadlessChromiumDriver { let interceptedCount = 0; await this.page.setRequestInterception(true); + + // We have to reach into the Chrome Remote Protocol to apply headers as using + // puppeteer's API will render map tiles as blanks :/ client.on('Fetch.requestPaused', (interceptedRequest: InterceptedRequest) => { const requestId = interceptedRequest.requestId; const interceptedUrl = interceptedRequest.request.url; @@ -85,7 +88,7 @@ export class HeadlessChromiumDriver { if (this._shouldUseCustomHeaders(conditionalHeaders.conditions, interceptedUrl)) { logger.debug(`Using custom headers for ${interceptedUrl}`); - const headers = _.map( + const headers = map( { ...interceptedRequest.request.headers, ...conditionalHeaders.headers, From f7b5b34c7c4849a8111907d649ca7cad8f4382a5 Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Fri, 17 Jan 2020 10:09:49 -0800 Subject: [PATCH 3/3] Better destructuring and comments --- .../browsers/chromium/driver/chromium_driver.ts | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts b/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts index 46f4867e505f2..de8449ff29132 100644 --- a/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts +++ b/x-pack/legacy/plugins/reporting/server/browsers/chromium/driver/chromium_driver.ts @@ -67,11 +67,16 @@ export class HeadlessChromiumDriver { await this.page.setRequestInterception(true); - // We have to reach into the Chrome Remote Protocol to apply headers as using - // puppeteer's API will render map tiles as blanks :/ + // We have to reach into the Chrome Devtools Protocol to apply headers as using + // puppeteer's API will cause map tile requests to hang indefinitely: + // https://github.com/puppeteer/puppeteer/issues/5003 + // Docs on this client/protocol can be found here: + // https://chromedevtools.github.io/devtools-protocol/tot/Fetch client.on('Fetch.requestPaused', (interceptedRequest: InterceptedRequest) => { - const requestId = interceptedRequest.requestId; - const interceptedUrl = interceptedRequest.request.url; + const { + requestId, + request: { url: interceptedUrl }, + } = interceptedRequest; const allowed = !interceptedUrl.startsWith('file://'); const isData = interceptedUrl.startsWith('data:');