feat(jsonata): validation, caching, better logging

lib/config/validation.spec.ts

@@ -306,10 +306,19 @@ describe('config/validation', () => {
             defaultRegistryUrlTemplate: [],
             transformTemplates: [{}],
           },
+          bar: {
+            description: 'foo',
+            defaultRegistryUrlTemplate: 'bar',
+            transformTemplates: ['foo = "bar"', 'bar[0'],
+          },
         },
       } as any;
       const { errors } = await configValidation.validateConfig('repo', config);
       expect(errors).toMatchObject([
+        {
+          message:
+            'Invalid JSONata expression for customDatasources: Expected "]" before end of expression',
+        },
         {
           message:
             'Invalid `customDatasources.defaultRegistryUrlTemplate` configuration: is a string',

lib/config/validation.ts

@@ -8,6 +8,7 @@ import type {
 } from '../modules/manager/custom/regex/types';
 import type { CustomManager } from '../modules/manager/custom/types';
 import type { HostRule } from '../types';
+import { getExpression } from '../util/jsonata';
 import { regEx } from '../util/regex';
 import {
   getRegexPredicate,
@@ -745,7 +746,17 @@ export async function validateConfig(
                       message: `Invalid \`${currentPath}.${subKey}\` configuration: key is not allowed`,
                     });
                   } else if (subKey === 'transformTemplates') {
-                    if (!is.array(subValue, is.string)) {
+                    if (is.array(subValue, is.string)) {
+                      for (const expression of subValue) {
+                        const res = getExpression(expression);
+                        if (res instanceof Error) {
+                          errors.push({
+                            topic: 'Configuration Error',
+                            message: `Invalid JSONata expression for ${currentPath}: ${res.message}`,
+                          });
+                        }
+                      }
+                    } else {
                       errors.push({
                         topic: 'Configuration Error',
                         message: `Invalid \`${currentPath}.${subKey}\` configuration: is not an array of string`,

lib/modules/datasource/custom/index.spec.ts

@@ -229,7 +229,7 @@ describe('modules/datasource/custom/index', () => {
       expect(result).toEqual(expected);
     });
 
-    it('returns null if transformation using jsonata rules fail', async () => {
+    it('returns null if transformation compilation using jsonata fails', async () => {
       httpMock
         .scope('https://example.com')
         .get('/v1')
@@ -248,9 +248,34 @@ describe('modules/datasource/custom/index', () => {
         },
       });
       expect(result).toBeNull();
-      expect(logger.debug).toHaveBeenCalledWith(
-        { err: expect.any(Object), transformTemplate: '$[.name = "Alice" and' },
-        'Error while transforming response',
+      expect(logger.once.warn).toHaveBeenCalledWith(
+        { errorMessage: 'The symbol "." cannot be used as a unary operator' },
+        'Invalid JSONata expression: $[.name = "Alice" and',
+      );
+    });
+
+    it('returns null if jsonata expression evaluation fails', async () => {
+      httpMock
+        .scope('https://example.com')
+        .get('/v1')
+        .reply(200, '1.0.0 \n2.0.0 \n 3.0.0 ', {
+          'Content-Type': 'text/plain',
+        });
+      const result = await getPkgReleases({
+        datasource: `${CustomDatasource.id}.foo`,
+        packageName: 'myPackage',
+        customDatasources: {
+          foo: {
+            defaultRegistryUrlTemplate: 'https://example.com/v1',
+            transformTemplates: ['$notafunction()'],
+            format: 'plain',
+          },
+        },
+      });
+      expect(result).toBeNull();
+      expect(logger.once.warn).toHaveBeenCalledWith(
+        { err: expect.any(Object) },
+        'Error while evaluating JSONata expression: $notafunction()',
       );
     });
 

lib/modules/datasource/custom/index.ts

@@ -1,6 +1,6 @@
 import is from '@sindresorhus/is';
-import jsonata from 'jsonata';
 import { logger } from '../../../logger';
+import { getExpression } from '../../../util/jsonata';
 import { Datasource } from '../datasource';
 import type { DigestConfig, GetReleasesConfig, ReleaseResult } from '../types';
 import { fetchers } from './formats';
@@ -46,13 +46,22 @@ export class CustomDatasource extends Datasource {
     logger.trace({ data }, `Custom manager fetcher '${format}' returned data.`);
 
     for (const transformTemplate of transformTemplates) {
+      const expression = getExpression(transformTemplate);
+
+      if (expression instanceof Error) {
+        logger.once.warn(
+          { errorMessage: expression.message },
+          `Invalid JSONata expression: ${transformTemplate}`,
+        );
+        return null;
+      }
+
       try {
-        const expression = jsonata(transformTemplate);
         data = await expression.evaluate(data);
       } catch (err) {
-        logger.debug(
-          { err, transformTemplate },
-          'Error while transforming response',
+        logger.once.warn(
+          { err },
+          `Error while evaluating JSONata expression: ${transformTemplate}`,
         );
         return null;
       }

lib/util/jsonata.spec.ts

@@ -0,0 +1,13 @@
+import { getExpression } from './jsonata';
+
+describe('util/jsonata', () => {
+  describe('getExpression', () => {
+    it('should return an expression', () => {
+      expect(getExpression('foo')).not.toBeInstanceOf(Error);
+    });
+
+    it('should return an error', () => {
+      expect(getExpression('foo[')).toBeInstanceOf(Error);
+    });
+  });
+});

lib/util/jsonata.ts

@@ -0,0 +1,21 @@
+import jsonata from 'jsonata';
+import * as memCache from './cache/memory';
+import { toSha256 } from './hash';
+
+export function getExpression(input: string): jsonata.Expression | Error {
+  const cacheKey = `jsonata:${toSha256(input)}`;
+  const cachedExpression = memCache.get<jsonata.Expression | Error>(cacheKey);
+  // istanbul ignore if: cannot test
+  if (cachedExpression) {
+    return cachedExpression;
+  }
+  let result: jsonata.Expression | Error;
+  try {
+    result = jsonata(input);
+  } catch (err) {
+    // JSONata errors aren't detected as TypeOf Error
+    result = new Error(err.message);
+  }
+  memCache.set(cacheKey, result);
+  return result;
+}