feat: make babel parsing options configurable by the user

The user can now overwrite the options provided to `@babel/parser`. The default is `source-type = null` and `parsing = null` which has the same behavior as before and attempt to guess suitable values from file url and content.

.eslintignore

@@ -5,4 +5,5 @@
 /doc
 /node_modules
 /test/pack
+/test/cases/apply-source/source.mjs
 /tmp

REFERENCE.md

@@ -314,6 +314,8 @@ The agent filter code objects (functions or objects/classes) based on a format c
         * `enabled <boolean>` Indicates whether the filtered file should be instrumented or not. *Default*: `true`.
         * `shallow <boolean>` Indicates whether the filtered file should
         * `exclude <Exclusion[]>` Additional code object filtering for the matched file.
+        * `source-type "script" | "module" | null` The `sourceType` options given to `@babel/parser`. *Default*: `null` attempt to retrieve this information from the extension of the file, else provide `"unambiguous"`.
+        * `parsing <array>` A set of plugin names to give to `@babel/parser`. Options can be given to plugins by providing a pair instead of just the name -- eg: `["recordAndtuple", {syntaxType: bar}]`. *Default*: `null` attempt to guess the plugins based on the extension and content of the file.
         * `... <Specifier>` Extends from any specifier format.
 * `exclude <Exclusion[]>` Code object filtering to apply to every file.
 * `source <boolean>` Indicates whether to include source code in the appmap file. *Default* `false`.

components/configuration-accessor/default/index.test.mjs

@@ -154,6 +154,8 @@ assertDeepEqual(
     shallow: false,
     exclude: [],
     "inline-source": null,
+    "source-type": null,
+    parsing: null,
   },
 );
 
@@ -186,6 +188,8 @@ assertDeepEqual(
       },
     ],
     "inline-source": null,
+    "source-type": null,
+    parsing: null,
   },
 );
 

components/configuration-process/node/index.test.mjs

@@ -128,6 +128,8 @@ assertThrow(
       shallow: false,
       exclude: [],
       "inline-source": null,
+      "source-type": null,
+      parsing: null,
     },
   );
   assertDeepEqual(
@@ -140,6 +142,8 @@ assertThrow(
       shallow: false,
       exclude: [],
       "inline-source": null,
+      "source-type": null,
+      parsing: null,
     },
   );
   assertDeepEqual(
@@ -152,6 +156,8 @@ assertThrow(
       shallow: false,
       exclude: [],
       "inline-source": null,
+      "source-type": null,
+      parsing: null,
     },
   );
 }

components/configuration/default/index.mjs

@@ -115,6 +115,8 @@ const normalizeDefaultPackage = (package_, _base) => {
     shallow: false,
     exclude: [],
     "inline-source": null,
+    "source-type": null,
+    parsing: null,
     ...package_,
   };
 };
@@ -188,23 +190,29 @@ const normalizePackageMatcher = (matcher, base) => {
   const {
     enabled,
     shallow,
-    "inline-source": inline,
+    "inline-source": inline_source,
     exclude,
+    "source-type": source_type,
+    parsing,
     ...rest
   } = {
     enabled: true,
     "inline-source": null,
     shallow: hasOwnProperty(matcher, "dist"),
     exclude: [],
+    "source-type": null,
+    parsing: null,
     ...matcher,
   };
   return [
     createMatcher(rest, base),
     {
       enabled,
-      "inline-source": inline,
+      "inline-source": inline_source,
       shallow,
       exclude: exclude.map(normalizeExclusion),
+      "source-type": source_type,
+      parsing,
     },
   ];
 };
@@ -535,6 +543,8 @@ export const createConfiguration = (home) => ({
     shallow: false,
     exclude: [],
     "inline-source": null,
+    "source-type": null,
+    parsing: null,
   },
   packages: [],
   exclude: [

components/configuration/default/index.test.mjs

@@ -44,11 +44,14 @@ assertDeepEqual(
 );
 
 // default-package //
+
 assertDeepEqual(extend("default-package", true, "protocol://host/base/"), {
   enabled: true,
   shallow: false,
   "inline-source": null,
   exclude: [],
+  "source-type": null,
+  parsing: null,
 });
 
 // packages //
@@ -64,6 +67,8 @@ assertDeepEqual(extend("default-package", true, "protocol://host/base/"), {
     enabled: true,
     exclude: [],
     shallow: false,
+    "source-type": null,
+    parsing: null,
   });
   assertEqual(matchUrl(matcher, "protocol://host/base/lib/foo.js"), true);
   assertEqual(matchUrl(matcher, "protocol://host/base/lib/foo.mjs"), false);

components/instrumentation/default/source.mjs

@@ -16,15 +16,17 @@ export const createSource = (
     "postmortem-function-exclusion": postmortem,
   },
 ) => {
-  const { enabled, exclude: local_criteria } = lookupUrl(
-    package_matcher_array,
-    url,
-    default_package,
-  );
+  const {
+    enabled,
+    exclude: local_criteria,
+    "source-type": source,
+    parsing: plugins,
+  } = lookupUrl(package_matcher_array, url, default_package);
   if (enabled) {
     return {
       postmortem,
       enabled: true,
+      parsing: { source, plugins },
       criteria: [...local_criteria, ...global_criteria],
       url,
       content,
@@ -36,6 +38,7 @@ export const createSource = (
     return {
       postmortem,
       enabled: false,
+      parsing: { source, plugins },
       criteria: [],
       url,
       content,
@@ -52,8 +55,8 @@ export const createSource = (
 
 export const parseSource = (source) => {
   if (source.program === null) {
-    const { url, content } = source;
-    const program = parseEstree({ url, content });
+    const { url, content, parsing } = source;
+    const program = parseEstree({ url, content }, parsing);
     source.program = program;
     return program;
   } else {

components/instrumentation/default/visit.test.mjs

@@ -18,7 +18,7 @@ const {
 
 const instrument = (options) =>
   generate(
-    visit(parseEstree(options.source), {
+    visit(parseEstree(options.source, { source: null, plugins: null }), {
       url: options.source.url,
       apply: "APPLY",
       eval: { hidden: "EVAL", aliases: ["eval"] },

components/parse/babel/index.mjs

@@ -1,6 +1,6 @@
 import BabelParser from "@babel/parser";
 import { InternalAppmapError } from "../../error/index.mjs";
-import { getLastUrlExtension } from "../../url/index.mjs";
+import { getLastUrlExtension, getUrlExtensionArray } from "../../url/index.mjs";
 import { assert, coalesce } from "../../util/index.mjs";
 import { logWarning, logError } from "../../log/index.mjs";
 
@@ -46,44 +46,88 @@ export const extractCommentLabelArray = ({ value: text }) => {
   return maybe_lines === null ? [] : maybe_lines.flatMap(extractLineLabel);
 };
 
-export const parseEstree = ({ url, content }) => {
-  const extension = getLastUrlExtension(url);
-  let source_type = "unambiguous";
-  if (extension === ".cjs" || extension === ".node") {
-    source_type = "script";
-  } else if (extension === ".mjs") {
-    source_type = "module";
+const resolveSource = (source, { url }) => {
+  if (source === null) {
+    const extension = getLastUrlExtension(url);
+    if (extension === ".cjs" || extension === ".cts" || extension === ".node") {
+      return "script";
+    } else if (extension === ".mjs" || extension === ".mts") {
+      return "module";
+    } else {
+      return "unambiguous";
+    }
+  } else {
+    return source;
   }
-  let plugins = [];
-  if (extension === ".ts" || extension === ".tsx") {
-    plugins = ["typescript"];
-  } else if (/^[ \t\n]*\/(\/[ \t]*|\*[ \t\n]*)@flow/u.test(content)) {
-    plugins = ["flow"];
+};
+
+const resolvePluginArray = (plugins, { url, content }) => {
+  if (plugins === null) {
+    const extensions = getUrlExtensionArray(url);
+    const plugins = [];
+    if (extensions.includes(".jsx")) {
+      plugins.push(["jsx", {}]);
+    }
+    if (
+      extensions.includes(".ts") ||
+      extensions.includes(".mts") ||
+      extensions.includes(".cts")
+    ) {
+      plugins.push(["typescript", {}]);
+    }
+    if (
+      extensions.includes(".tsx") ||
+      extensions.includes(".mtsx") ||
+      extensions.includes(".ctsx")
+    ) {
+      plugins.push(["jsx", {}], ["typescript", {}]);
+    }
+    if (
+      extensions.includes(".flow") ||
+      /^[ \t\n]*\/(\/[ \t]*|\*[ \t\n]*)@flow/u.test(content)
+    ) {
+      plugins.push(["flow", {}]);
+    }
+    return plugins;
+  } else {
+    return plugins;
   }
-  plugins.push(["estree", { classFeatures: true }], "jsx");
-  let result;
+};
+
+const parseSafe = ({ url, content }, options) => {
   try {
-    result = parseBabel(content, {
-      plugins,
-      sourceFilename: url,
-      sourceType: source_type,
-      errorRecovery: true,
-      attachComment: true,
-    });
+    return parseBabel(content, options);
   } catch (error) {
     logError("Unrecoverable parsing error at file %j >> %O", url, error);
+    const { sourceType: source_type } = options;
     return {
-      type: "Program",
-      body: [],
-      sourceType: "script",
-      loc: {
-        start: { line: 0, column: 0 },
-        end: { line: 0, column: 0 },
-        filename: url,
+      errors: [],
+      program: {
+        type: "Program",
+        body: [],
+        sourceType: source_type === "unambiguous" ? "script" : source_type,
+        loc: {
+          start: { line: 0, column: 0 },
+          end: { line: 0, column: 0 },
+          filename: url,
+        },
       },
     };
   }
-  const { errors, program: node } = result;
+};
+
+export const parseEstree = (file, { source, plugins }) => {
+  const { url } = file;
+  const { errors, program: node } = parseSafe(file, {
+    sourceFilename: url,
+    sourceType: resolveSource(source, file),
+    plugins: [
+      ["estree", { classFeatures: true }],
+      ...resolvePluginArray(plugins, file),
+    ],
+    errorRecovery: true,
+    attachComment: true,
+  });
   for (const error of errors) {
     logWarning("Recoverable parsing error at file %j >> %O", url, error);
   }