From f15badf1dc8780b6b9226697426b568f527c8984 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Mon, 7 Oct 2024 17:58:49 +0200 Subject: [PATCH 01/12] feat: jsonata caching, better logging --- lib/modules/datasource/custom/index.spec.ts | 33 ++++++++++++++++++--- lib/modules/datasource/custom/index.ts | 29 ++++++++++++++---- lib/util/jsonata.ts | 18 +++++++++++ 3 files changed, 71 insertions(+), 9 deletions(-) create mode 100644 lib/util/jsonata.ts diff --git a/lib/modules/datasource/custom/index.spec.ts b/lib/modules/datasource/custom/index.spec.ts index 2eb0f46212fa22..bdcf284c6a3bc0 100644 --- a/lib/modules/datasource/custom/index.spec.ts +++ b/lib/modules/datasource/custom/index.spec.ts @@ -229,7 +229,7 @@ describe('modules/datasource/custom/index', () => { expect(result).toEqual(expected); }); - it('returns null if transformation using jsonata rules fail', async () => { + it('returns null if transformation compiation using jsonata fails', async () => { httpMock .scope('https://example.com') .get('/v1') @@ -248,9 +248,34 @@ describe('modules/datasource/custom/index', () => { }, }); expect(result).toBeNull(); - expect(logger.debug).toHaveBeenCalledWith( - { err: expect.any(Object), transformTemplate: '$[.name = "Alice" and' }, - 'Error while transforming response', + expect(logger.once.warn).toHaveBeenCalledWith( + { expression: expect.any(Object) }, + 'Error while compiling JSONata expression: $[.name = "Alice" and', + ); + }); + + it('returns null if jsonata expression evaluation fails', async () => { + httpMock + .scope('https://example.com') + .get('/v1') + .reply(200, '1.0.0 \n2.0.0 \n 3.0.0 ', { + 'Content-Type': 'text/plain', + }); + const result = await getPkgReleases({ + datasource: `${CustomDatasource.id}.foo`, + packageName: 'myPackage', + customDatasources: { + foo: { + defaultRegistryUrlTemplate: 'https://example.com/v1', + transformTemplates: ['$notafunction()'], + format: 'plain', + }, + }, + }); + expect(result).toBeNull(); + expect(logger.once.warn).toHaveBeenCalledWith( + { err: expect.any(Object) }, + 'Error while evaluating JSONata expression: $notafunction()', ); }); diff --git a/lib/modules/datasource/custom/index.ts b/lib/modules/datasource/custom/index.ts index 0ecdc11bc8547c..f0bc02d7be458d 100644 --- a/lib/modules/datasource/custom/index.ts +++ b/lib/modules/datasource/custom/index.ts @@ -1,6 +1,6 @@ import is from '@sindresorhus/is'; -import jsonata from 'jsonata'; import { logger } from '../../../logger'; +import * as jsonata from '../../../util/jsonata'; import { Datasource } from '../datasource'; import type { DigestConfig, GetReleasesConfig, ReleaseResult } from '../types'; import { fetchers } from './formats'; @@ -46,13 +46,32 @@ export class CustomDatasource extends Datasource { logger.trace({ data }, `Custom manager fetcher '${format}' returned data.`); for (const transformTemplate of transformTemplates) { + const expression = jsonata.getExpression(transformTemplate); + + // istanbul ignore if: JSONata doesn't seem to ever throw for this, despite the docs + if (expression instanceof Error) { + logger.once.warn( + { err: expression }, + `Error while compiling JSONata expression ${JSON.stringify(transformTemplate)}`, + ); + return null; + } + + // Check if JSONata returned a working expression + if (!expression.evaluate) { + logger.once.warn( + { expression }, + `Error while compiling JSONata expression: ${transformTemplate}`, + ); + return null; + } + try { - const expression = jsonata(transformTemplate); data = await expression.evaluate(data); } catch (err) { - logger.debug( - { err, transformTemplate }, - 'Error while transforming response', + logger.once.warn( + { err }, + `Error while evaluating JSONata expression: ${transformTemplate}`, ); return null; } diff --git a/lib/util/jsonata.ts b/lib/util/jsonata.ts new file mode 100644 index 00000000000000..b481890aefc403 --- /dev/null +++ b/lib/util/jsonata.ts @@ -0,0 +1,18 @@ +import jsonata from 'jsonata'; +import * as memCache from './cache/memory'; + +export function getExpression(input: string): jsonata.Expression | Error { + const cacheKey = `jsonata:${input}`; + const cachedExpression = memCache.get(cacheKey); + if (cachedExpression) { + return cachedExpression; + } + let result: jsonata.Expression | Error; + try { + result = jsonata(input); + } catch (err) { + result = err; + } + memCache.set(cacheKey, result); + return result; +} From 846fb2ae375caab902da9947ba5b91ced1783b39 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Mon, 7 Oct 2024 18:00:35 +0200 Subject: [PATCH 02/12] typo --- lib/modules/datasource/custom/index.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/modules/datasource/custom/index.spec.ts b/lib/modules/datasource/custom/index.spec.ts index bdcf284c6a3bc0..ebb594fe447e53 100644 --- a/lib/modules/datasource/custom/index.spec.ts +++ b/lib/modules/datasource/custom/index.spec.ts @@ -229,7 +229,7 @@ describe('modules/datasource/custom/index', () => { expect(result).toEqual(expected); }); - it('returns null if transformation compiation using jsonata fails', async () => { + it('returns null if transformation compilation using jsonata fails', async () => { httpMock .scope('https://example.com') .get('/v1') From 39067b44048b9ba90b8ac6d5a44d424eaaf03297 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Mon, 7 Oct 2024 18:08:05 +0200 Subject: [PATCH 03/12] fix/simplify ereror case --- lib/modules/datasource/custom/index.spec.ts | 2 +- lib/modules/datasource/custom/index.ts | 12 +----------- lib/util/jsonata.ts | 3 ++- 3 files changed, 4 insertions(+), 13 deletions(-) diff --git a/lib/modules/datasource/custom/index.spec.ts b/lib/modules/datasource/custom/index.spec.ts index ebb594fe447e53..9fc6d32ab244a0 100644 --- a/lib/modules/datasource/custom/index.spec.ts +++ b/lib/modules/datasource/custom/index.spec.ts @@ -249,7 +249,7 @@ describe('modules/datasource/custom/index', () => { }); expect(result).toBeNull(); expect(logger.once.warn).toHaveBeenCalledWith( - { expression: expect.any(Object) }, + { errorMessage: 'The symbol "." cannot be used as a unary operator' }, 'Error while compiling JSONata expression: $[.name = "Alice" and', ); }); diff --git a/lib/modules/datasource/custom/index.ts b/lib/modules/datasource/custom/index.ts index f0bc02d7be458d..a9a111af857d1a 100644 --- a/lib/modules/datasource/custom/index.ts +++ b/lib/modules/datasource/custom/index.ts @@ -48,19 +48,9 @@ export class CustomDatasource extends Datasource { for (const transformTemplate of transformTemplates) { const expression = jsonata.getExpression(transformTemplate); - // istanbul ignore if: JSONata doesn't seem to ever throw for this, despite the docs if (expression instanceof Error) { logger.once.warn( - { err: expression }, - `Error while compiling JSONata expression ${JSON.stringify(transformTemplate)}`, - ); - return null; - } - - // Check if JSONata returned a working expression - if (!expression.evaluate) { - logger.once.warn( - { expression }, + { errorMessage: expression.message }, `Error while compiling JSONata expression: ${transformTemplate}`, ); return null; diff --git a/lib/util/jsonata.ts b/lib/util/jsonata.ts index b481890aefc403..6867b7dc1ac964 100644 --- a/lib/util/jsonata.ts +++ b/lib/util/jsonata.ts @@ -11,7 +11,8 @@ export function getExpression(input: string): jsonata.Expression | Error { try { result = jsonata(input); } catch (err) { - result = err; + // JSONata errors aren't detected as TypeOf Error + result = new Error(err.message ?? 'Unknown JSONata error'); } memCache.set(cacheKey, result); return result; From 56e3e24b82edb43a29962ebd7be9fb2a91a146da Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Mon, 7 Oct 2024 18:18:15 +0200 Subject: [PATCH 04/12] coverage --- lib/util/jsonata.ts | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/lib/util/jsonata.ts b/lib/util/jsonata.ts index 6867b7dc1ac964..bfc2636ce0d535 100644 --- a/lib/util/jsonata.ts +++ b/lib/util/jsonata.ts @@ -3,7 +3,8 @@ import * as memCache from './cache/memory'; export function getExpression(input: string): jsonata.Expression | Error { const cacheKey = `jsonata:${input}`; - const cachedExpression = memCache.get(cacheKey); + const cachedExpression = memCache.get(cacheKey); + // istanbul ignore if: cannot test if (cachedExpression) { return cachedExpression; } @@ -12,7 +13,7 @@ export function getExpression(input: string): jsonata.Expression | Error { result = jsonata(input); } catch (err) { // JSONata errors aren't detected as TypeOf Error - result = new Error(err.message ?? 'Unknown JSONata error'); + result = new Error(err.message); } memCache.set(cacheKey, result); return result; From 271f723af72133ac2d68d75a5dc4d49777f6577a Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Mon, 7 Oct 2024 18:24:33 +0200 Subject: [PATCH 05/12] abstract evaluate --- lib/modules/datasource/custom/index.ts | 6 +++--- lib/util/jsonata.ts | 7 +++++++ 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/modules/datasource/custom/index.ts b/lib/modules/datasource/custom/index.ts index a9a111af857d1a..8614668e0e03d4 100644 --- a/lib/modules/datasource/custom/index.ts +++ b/lib/modules/datasource/custom/index.ts @@ -1,6 +1,6 @@ import is from '@sindresorhus/is'; import { logger } from '../../../logger'; -import * as jsonata from '../../../util/jsonata'; +import { evaluateExpression, getExpression } from '../../../util/jsonata'; import { Datasource } from '../datasource'; import type { DigestConfig, GetReleasesConfig, ReleaseResult } from '../types'; import { fetchers } from './formats'; @@ -46,7 +46,7 @@ export class CustomDatasource extends Datasource { logger.trace({ data }, `Custom manager fetcher '${format}' returned data.`); for (const transformTemplate of transformTemplates) { - const expression = jsonata.getExpression(transformTemplate); + const expression = getExpression(transformTemplate); if (expression instanceof Error) { logger.once.warn( @@ -57,7 +57,7 @@ export class CustomDatasource extends Datasource { } try { - data = await expression.evaluate(data); + data = await evaluateExpression(expression, data); } catch (err) { logger.once.warn( { err }, diff --git a/lib/util/jsonata.ts b/lib/util/jsonata.ts index bfc2636ce0d535..2e47a5aaaf6770 100644 --- a/lib/util/jsonata.ts +++ b/lib/util/jsonata.ts @@ -18,3 +18,10 @@ export function getExpression(input: string): jsonata.Expression | Error { memCache.set(cacheKey, result); return result; } + +export function evaluateExpression( + expression: jsonata.Expression, + data: unknown, +): Promise { + return expression.evaluate(data); +} From 08bf7ec15380fb818c1b9e204761b34236ec9ef7 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Mon, 7 Oct 2024 18:31:49 +0200 Subject: [PATCH 06/12] less abstraction --- lib/modules/datasource/custom/index.spec.ts | 2 +- lib/modules/datasource/custom/index.ts | 6 +++--- lib/util/jsonata.ts | 7 ------- 3 files changed, 4 insertions(+), 11 deletions(-) diff --git a/lib/modules/datasource/custom/index.spec.ts b/lib/modules/datasource/custom/index.spec.ts index 9fc6d32ab244a0..62574bdf774c59 100644 --- a/lib/modules/datasource/custom/index.spec.ts +++ b/lib/modules/datasource/custom/index.spec.ts @@ -250,7 +250,7 @@ describe('modules/datasource/custom/index', () => { expect(result).toBeNull(); expect(logger.once.warn).toHaveBeenCalledWith( { errorMessage: 'The symbol "." cannot be used as a unary operator' }, - 'Error while compiling JSONata expression: $[.name = "Alice" and', + 'Invalid JSONata expression: $[.name = "Alice" and', ); }); diff --git a/lib/modules/datasource/custom/index.ts b/lib/modules/datasource/custom/index.ts index 8614668e0e03d4..372d15e97ab685 100644 --- a/lib/modules/datasource/custom/index.ts +++ b/lib/modules/datasource/custom/index.ts @@ -1,6 +1,6 @@ import is from '@sindresorhus/is'; import { logger } from '../../../logger'; -import { evaluateExpression, getExpression } from '../../../util/jsonata'; +import { getExpression } from '../../../util/jsonata'; import { Datasource } from '../datasource'; import type { DigestConfig, GetReleasesConfig, ReleaseResult } from '../types'; import { fetchers } from './formats'; @@ -51,13 +51,13 @@ export class CustomDatasource extends Datasource { if (expression instanceof Error) { logger.once.warn( { errorMessage: expression.message }, - `Error while compiling JSONata expression: ${transformTemplate}`, + `Invalid JSONata expression: ${transformTemplate}`, ); return null; } try { - data = await evaluateExpression(expression, data); + data = await expression.evaluate(data); } catch (err) { logger.once.warn( { err }, diff --git a/lib/util/jsonata.ts b/lib/util/jsonata.ts index 2e47a5aaaf6770..bfc2636ce0d535 100644 --- a/lib/util/jsonata.ts +++ b/lib/util/jsonata.ts @@ -18,10 +18,3 @@ export function getExpression(input: string): jsonata.Expression | Error { memCache.set(cacheKey, result); return result; } - -export function evaluateExpression( - expression: jsonata.Expression, - data: unknown, -): Promise { - return expression.evaluate(data); -} From a38d6e4a8e2e22c2175a515619b2fa8f18963941 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Mon, 7 Oct 2024 20:54:33 +0200 Subject: [PATCH 07/12] hash input --- lib/util/jsonata.ts | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/util/jsonata.ts b/lib/util/jsonata.ts index bfc2636ce0d535..8cab49ac944337 100644 --- a/lib/util/jsonata.ts +++ b/lib/util/jsonata.ts @@ -1,8 +1,9 @@ import jsonata from 'jsonata'; +import { toSha256 } from './hash'; import * as memCache from './cache/memory'; export function getExpression(input: string): jsonata.Expression | Error { - const cacheKey = `jsonata:${input}`; + const cacheKey = `jsonata:${toSha256(input)}`; const cachedExpression = memCache.get(cacheKey); // istanbul ignore if: cannot test if (cachedExpression) { From b743319f7f936567934183b3727a93c2f74cf592 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Tue, 8 Oct 2024 09:01:30 +0200 Subject: [PATCH 08/12] fix lint --- lib/util/jsonata.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/util/jsonata.ts b/lib/util/jsonata.ts index 8cab49ac944337..2811f4e4075e5c 100644 --- a/lib/util/jsonata.ts +++ b/lib/util/jsonata.ts @@ -1,6 +1,6 @@ import jsonata from 'jsonata'; -import { toSha256 } from './hash'; import * as memCache from './cache/memory'; +import { toSha256 } from './hash'; export function getExpression(input: string): jsonata.Expression | Error { const cacheKey = `jsonata:${toSha256(input)}`; From 3b9f60ad0811bc10c01e38eb4d8fa2717bce5051 Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Tue, 8 Oct 2024 09:39:49 +0200 Subject: [PATCH 09/12] add validation --- lib/config/validation.spec.ts | 9 +++++++++ lib/config/validation.ts | 13 ++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/lib/config/validation.spec.ts b/lib/config/validation.spec.ts index 4eb07aa8cafb15..e7dc924ca5711f 100644 --- a/lib/config/validation.spec.ts +++ b/lib/config/validation.spec.ts @@ -306,10 +306,19 @@ describe('config/validation', () => { defaultRegistryUrlTemplate: [], transformTemplates: [{}], }, + bar: { + description: 'foo', + defaultRegistryUrlTemplate: 'bar', + transformTemplates: ['foo = "bar"', 'bar[0'], + }, }, } as any; const { errors } = await configValidation.validateConfig('repo', config); expect(errors).toMatchObject([ + { + message: + 'Invalid JSONata expression for customDatasources: Expected "]" before end of expression', + }, { message: 'Invalid `customDatasources.defaultRegistryUrlTemplate` configuration: is a string', diff --git a/lib/config/validation.ts b/lib/config/validation.ts index fdcf487a20a52f..b6218b84be9697 100644 --- a/lib/config/validation.ts +++ b/lib/config/validation.ts @@ -8,6 +8,7 @@ import type { } from '../modules/manager/custom/regex/types'; import type { CustomManager } from '../modules/manager/custom/types'; import type { HostRule } from '../types'; +import { getExpression } from '../util/jsonata'; import { regEx } from '../util/regex'; import { getRegexPredicate, @@ -745,7 +746,17 @@ export async function validateConfig( message: `Invalid \`${currentPath}.${subKey}\` configuration: key is not allowed`, }); } else if (subKey === 'transformTemplates') { - if (!is.array(subValue, is.string)) { + if (is.array(subValue, is.string)) { + for (const expression of subValue) { + const res = getExpression(expression); + if (res instanceof Error) { + errors.push({ + topic: 'Configuration Error', + message: `Invalid JSONata expression for ${currentPath}: ${res.message}`, + }); + } + } + } else { errors.push({ topic: 'Configuration Error', message: `Invalid \`${currentPath}.${subKey}\` configuration: is not an array of string`, From f2c892e6dcf11f4a536029ca665179af2319cf9d Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Tue, 8 Oct 2024 09:55:38 +0200 Subject: [PATCH 10/12] add tests --- lib/util/jsonata.spec.ts | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 lib/util/jsonata.spec.ts diff --git a/lib/util/jsonata.spec.ts b/lib/util/jsonata.spec.ts new file mode 100644 index 00000000000000..b559d1ddc13c35 --- /dev/null +++ b/lib/util/jsonata.spec.ts @@ -0,0 +1,13 @@ +import { getExpression } from './jsonata'; + +describe('lib/util/jsonata', () => { + describe('getExpression', () => { + it('should return an expression', () => { + expect(getExpression('foo')).not.toBeInstanceOf(Error); + }); + + it('should return an error', () => { + expect(getExpression('foo[')).toBeInstanceOf(Error); + }); + }); +} From 4004953338663892f42cd95f458e0c8682dfc5ac Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Tue, 8 Oct 2024 09:59:35 +0200 Subject: [PATCH 11/12] lint-fix --- lib/util/jsonata.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/util/jsonata.spec.ts b/lib/util/jsonata.spec.ts index b559d1ddc13c35..3e9a0675449e3d 100644 --- a/lib/util/jsonata.spec.ts +++ b/lib/util/jsonata.spec.ts @@ -10,4 +10,4 @@ describe('lib/util/jsonata', () => { expect(getExpression('foo[')).toBeInstanceOf(Error); }); }); -} +}); From 46065030be35eb6a64b81f3869d496aac56c350d Mon Sep 17 00:00:00 2001 From: Rhys Arkins Date: Tue, 8 Oct 2024 11:06:26 +0200 Subject: [PATCH 12/12] lint-fix --- lib/util/jsonata.spec.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/util/jsonata.spec.ts b/lib/util/jsonata.spec.ts index 3e9a0675449e3d..f5a6651637b4e5 100644 --- a/lib/util/jsonata.spec.ts +++ b/lib/util/jsonata.spec.ts @@ -1,6 +1,6 @@ import { getExpression } from './jsonata'; -describe('lib/util/jsonata', () => { +describe('util/jsonata', () => { describe('getExpression', () => { it('should return an expression', () => { expect(getExpression('foo')).not.toBeInstanceOf(Error);