From f8280216b74b9fe8f3b71d3809a1aa52bf6670ba Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Fri, 11 Jan 2019 14:26:02 -0800 Subject: [PATCH 1/6] Record failed jobs when saving the result fails (dont retry) --- x-pack/plugins/reporting/server/lib/esqueue/worker.js | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugins/reporting/server/lib/esqueue/worker.js b/x-pack/plugins/reporting/server/lib/esqueue/worker.js index 8c6bd5125e676..c5fa020fb76c6 100644 --- a/x-pack/plugins/reporting/server/lib/esqueue/worker.js +++ b/x-pack/plugins/reporting/server/lib/esqueue/worker.js @@ -259,6 +259,7 @@ export class Worker extends events.EventEmitter { if (err.statusCode === 409) return false; this.warn(`Failure saving job output ${job._id}`, err); this.emit(constants.EVENT_WORKER_JOB_UPDATE_ERROR, this._formatErrorParams(err, job)); + return this._failJob(job, (err.message) ? err.message : false); }); }, (jobErr) => { if (!jobErr) { From 98d7f51bd6c6d67e96550017ebe67073a8b87a44 Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Tue, 15 Jan 2019 14:50:45 -0800 Subject: [PATCH 2/6] Adding a test for the catch on ES save failiure --- .../server/lib/esqueue/__tests__/worker.js | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/x-pack/plugins/reporting/server/lib/esqueue/__tests__/worker.js b/x-pack/plugins/reporting/server/lib/esqueue/__tests__/worker.js index d86125d6c4c94..5b4589938e12c 100644 --- a/x-pack/plugins/reporting/server/lib/esqueue/__tests__/worker.js +++ b/x-pack/plugins/reporting/server/lib/esqueue/__tests__/worker.js @@ -867,6 +867,32 @@ describe('Worker class', function () { return sinon.stub(workerWithFailure, '_failJob').returns(Promise.resolve()); } + describe('saving output failure', () => { + it('should mark the job as failed if saving to ES fails', async () => { + const job = { + _id: 'shouldSucced', + _source: { + timeout: 1000, + payload: 'test' + } + }; + + sinon.stub(mockQueue.client, 'update').returns(Promise.reject({ statusCode: 413 })); + + const workerFn = function (jobPayload) { + return new Promise(function (resolve) { + setTimeout(() => resolve(jobPayload), 10); + }); + }; + const worker = new Worker(mockQueue, 'test', workerFn, defaultWorkerOptions); + const failStub = getFailStub(worker); + + await worker._performJob(job); + + sinon.assert.called(failStub); + }); + }); + describe('search failure', function () { it('causes _processPendingJobs to reject the Promise', function () { sinon.stub(mockQueue.client, 'search').returns(Promise.reject(new Error('test error'))); From 8b7dfe3459acb9bfc70150a197606632dd025584 Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Tue, 22 Jan 2019 12:02:29 -0800 Subject: [PATCH 3/6] Seeing if this is causing jenkins to hang --- x-pack/plugins/reporting/server/lib/esqueue/worker.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/reporting/server/lib/esqueue/worker.js b/x-pack/plugins/reporting/server/lib/esqueue/worker.js index c5fa020fb76c6..8058da426691a 100644 --- a/x-pack/plugins/reporting/server/lib/esqueue/worker.js +++ b/x-pack/plugins/reporting/server/lib/esqueue/worker.js @@ -259,7 +259,7 @@ export class Worker extends events.EventEmitter { if (err.statusCode === 409) return false; this.warn(`Failure saving job output ${job._id}`, err); this.emit(constants.EVENT_WORKER_JOB_UPDATE_ERROR, this._formatErrorParams(err, job)); - return this._failJob(job, (err.message) ? err.message : false); + // this._failJob(job, (err.message) ? err.message : false); }); }, (jobErr) => { if (!jobErr) { From d5d5a5e24a3f52c897e08bc76888285f5a65aa1c Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Tue, 22 Jan 2019 12:49:07 -0800 Subject: [PATCH 4/6] Maybe this is hanging? --- x-pack/plugins/reporting/server/lib/esqueue/__tests__/worker.js | 2 +- x-pack/plugins/reporting/server/lib/esqueue/worker.js | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/x-pack/plugins/reporting/server/lib/esqueue/__tests__/worker.js b/x-pack/plugins/reporting/server/lib/esqueue/__tests__/worker.js index 5b4589938e12c..ac954d84018ca 100644 --- a/x-pack/plugins/reporting/server/lib/esqueue/__tests__/worker.js +++ b/x-pack/plugins/reporting/server/lib/esqueue/__tests__/worker.js @@ -867,7 +867,7 @@ describe('Worker class', function () { return sinon.stub(workerWithFailure, '_failJob').returns(Promise.resolve()); } - describe('saving output failure', () => { + describe.skip('saving output failure', () => { it('should mark the job as failed if saving to ES fails', async () => { const job = { _id: 'shouldSucced', diff --git a/x-pack/plugins/reporting/server/lib/esqueue/worker.js b/x-pack/plugins/reporting/server/lib/esqueue/worker.js index 8058da426691a..37f9ef41e83a0 100644 --- a/x-pack/plugins/reporting/server/lib/esqueue/worker.js +++ b/x-pack/plugins/reporting/server/lib/esqueue/worker.js @@ -259,7 +259,7 @@ export class Worker extends events.EventEmitter { if (err.statusCode === 409) return false; this.warn(`Failure saving job output ${job._id}`, err); this.emit(constants.EVENT_WORKER_JOB_UPDATE_ERROR, this._formatErrorParams(err, job)); - // this._failJob(job, (err.message) ? err.message : false); + this._failJob(job, (err.message) ? err.message : false); }); }, (jobErr) => { if (!jobErr) { From c49323169f5d785c43fae4eabafaa8dd98add446 Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Wed, 23 Jan 2019 09:59:08 -0800 Subject: [PATCH 5/6] Returning _failJob results --- x-pack/plugins/reporting/server/lib/esqueue/worker.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugins/reporting/server/lib/esqueue/worker.js b/x-pack/plugins/reporting/server/lib/esqueue/worker.js index 37f9ef41e83a0..c5fa020fb76c6 100644 --- a/x-pack/plugins/reporting/server/lib/esqueue/worker.js +++ b/x-pack/plugins/reporting/server/lib/esqueue/worker.js @@ -259,7 +259,7 @@ export class Worker extends events.EventEmitter { if (err.statusCode === 409) return false; this.warn(`Failure saving job output ${job._id}`, err); this.emit(constants.EVENT_WORKER_JOB_UPDATE_ERROR, this._formatErrorParams(err, job)); - this._failJob(job, (err.message) ? err.message : false); + return this._failJob(job, (err.message) ? err.message : false); }); }, (jobErr) => { if (!jobErr) { From 661de8878a4ccc1635c1bd829db7b085f5348264 Mon Sep 17 00:00:00 2001 From: Joel Griffith Date: Wed, 23 Jan 2019 11:36:13 -0800 Subject: [PATCH 6/6] Destroy the worker on end --- .../plugins/reporting/server/lib/esqueue/__tests__/worker.js | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/x-pack/plugins/reporting/server/lib/esqueue/__tests__/worker.js b/x-pack/plugins/reporting/server/lib/esqueue/__tests__/worker.js index ac954d84018ca..15fa81954d7dd 100644 --- a/x-pack/plugins/reporting/server/lib/esqueue/__tests__/worker.js +++ b/x-pack/plugins/reporting/server/lib/esqueue/__tests__/worker.js @@ -867,7 +867,7 @@ describe('Worker class', function () { return sinon.stub(workerWithFailure, '_failJob').returns(Promise.resolve()); } - describe.skip('saving output failure', () => { + describe('saving output failure', () => { it('should mark the job as failed if saving to ES fails', async () => { const job = { _id: 'shouldSucced', @@ -888,6 +888,7 @@ describe('Worker class', function () { const failStub = getFailStub(worker); await worker._performJob(job); + worker.destroy(); sinon.assert.called(failStub); });