DXCDT-388: Enabling preservation of array replace markers in YAML (#760)

* Renaming load to loadAssetsFromLocal * Renaming load to loadAssetsFromAuth0 * integrating into export process * Adding config interpreter * Disabling keyword replacement in certain cases * Fixing directory load * More forgiving lookup logic if properties and/or addresses do not exist * Fixing directory load again * Adding case that would have previously tripped up process * Adding e2e tests for keyword preservation * Removing old tests, updating recordings, removing console. logs * Commenting-out test to fix later on * Fixing workdirectory for e2e tests * Adding eventual cases for auxillary files * Adding TODO * Adding preqrequisite checks * Fixing test * Standardizing definitions of resource identifiers * Removing incompatible id from email templates * Readding identifiers field for resource server * Fixing email templates identifier field * Reformulating arguments into object * Adding e2e case * Init * Removing console log * Adding more robust e2e test * Updating recordings * Fixing database directory handler, test * Initial work at preserving array keyword markers in YAML * Fixing unit test * Removing console log * Simplifying e2e test and re-recording * Working for string replace markers now too * Continuing * Continuing * Making test slightly more clear * Removing extraneous changes * Being flexible for both single and double quotes YAML array replacement * Renaming function, adding comment --------- Co-authored-by: Will Vedder <[email protected]>
auth0 · Mar 3, 2023 · 025193b · 025193b
1 parent bdc06aa
commit 025193b
Show file tree

Hide file tree

Showing 9 changed files with 225 additions and 97 deletions.
diff --git a/src/context/yaml/handlers/actions.ts b/src/context/yaml/handlers/actions.ts
@@ -86,7 +86,8 @@ async function dump(context: YAMLContext): Promise<ParsedActions> {
       runtime: action.runtime,
       dependencies: action.dependencies || [],
       status: action.status,
-      secrets: mapSecrets(action.secrets || []),
+      secrets:
+        typeof action.secrets === 'string' ? action.secrets : mapSecrets(action.secrets || []), //Enables keyword preservation to operate on action secrets
       supported_triggers: action.supported_triggers,
     })),
   };

diff --git a/src/context/yaml/index.ts b/src/context/yaml/index.ts
@@ -1,7 +1,12 @@
 import fs from 'fs-extra';
 import yaml from 'js-yaml';
 import path from 'path';
-import { loadFileAndReplaceKeywords, keywordReplace, Auth0 } from '../../tools';
+import {
+  loadFileAndReplaceKeywords,
+  keywordReplace,
+  wrapArrayReplaceMarkersInQuotes,
+  Auth0,
+} from '../../tools';
 
 import log from '../../logger';
 import { isFile, toConfigFn, stripIdentifiers, formatResults, recordsSorter } from '../../utils';
@@ -71,7 +76,7 @@ export default class YAMLContext {
           this.assets,
           yaml.load(
             opts.disableKeywordReplacement
-              ? fs.readFileSync(fPath, 'utf8')
+              ? wrapArrayReplaceMarkersInQuotes(fs.readFileSync(fPath, 'utf8'), this.mappings)
               : keywordReplace(fs.readFileSync(fPath, 'utf8'), this.mappings)
           ) || {}
         );
@@ -104,7 +109,7 @@ export default class YAMLContext {
 
     // Run initial schema check to ensure valid YAML
     const auth0 = new Auth0(this.mgmtClient, this.assets, toConfigFn(this.config));
-    await auth0.validate();
+    if (!opts.disableKeywordReplacement) await auth0.validate(); //The schema validation needs to be disabled during keyword-preserved export because a field may be enforced as an array but will be expressed with an array replace marker (string).
 
     // Allow handlers to process the assets such as loading files etc
     await Promise.all(

diff --git a/src/tools/index.ts b/src/tools/index.ts
@@ -1,14 +1,26 @@
 import constants from './constants';
 import deploy from './deploy';
 import Auth0 from './auth0';
-import { keywordReplace, loadFileAndReplaceKeywords } from './utils';
+import {
+  keywordReplace,
+  loadFileAndReplaceKeywords,
+  wrapArrayReplaceMarkersInQuotes,
+} from './utils';
 
 export default {
   constants,
   deploy,
   keywordReplace,
   loadFileAndReplaceKeywords,
+  wrapArrayReplaceMarkersInQuotes,
   Auth0,
 };
 
-export { constants, deploy, keywordReplace, loadFileAndReplaceKeywords, Auth0 };
+export {
+  constants,
+  deploy,
+  keywordReplace,
+  loadFileAndReplaceKeywords,
+  wrapArrayReplaceMarkersInQuotes,
+  Auth0,
+};
diff --git a/src/tools/utils.ts b/src/tools/utils.ts
@@ -8,9 +8,11 @@ import constants from './constants';
 
 export const keywordReplaceArrayRegExp = (key) => {
   const pattern = `@@${key}@@`;
-  const patternWithQuotes = `"${pattern}"`;
+  //YAML format supports both single and double quotes for strings
+  const patternWithSingleQuotes = `'${pattern}'`;
+  const patternWithDoubleQuotes = `"${pattern}"`;
 
-  return new RegExp(`${patternWithQuotes}|${pattern}`, 'g');
+  return new RegExp(`${patternWithSingleQuotes}|${patternWithDoubleQuotes}|${pattern}`, 'g');
 };
 
 export const keywordReplaceStringRegExp = (key) => {
@@ -45,6 +47,20 @@ export function keywordReplace(input: string, mappings: KeywordMappings): string
   return input;
 }
 
+// wrapArrayReplaceMarkersInQuotes will wrap array replacement markers in quotes.
+// This is necessary for YAML format in the context of keyword replacement
+// to preserve the keyword markers while also maintaining valid YAML syntax.
+export function wrapArrayReplaceMarkersInQuotes(body: string, mappings: KeywordMappings): string {
+  let newBody = body;
+  Object.keys(mappings).forEach((keyword) => {
+    newBody = newBody.replace(
+      new RegExp('(?<![\'"])@@' + keyword + '@@(?![\'"])', 'g'),
+      `"@@${keyword}@@"`
+    );
+  });
+  return newBody;
+}
+
 export function convertClientNameToId(name: string, clients: Asset[]): string {
   const found = clients.find((c) => c.name === name);
   return (found && found.client_id) || name;

diff --git a/test/context/yaml/context.test.js b/test/context/yaml/context.test.js
@@ -541,23 +541,18 @@ describe('#YAML context validation', () => {
     cleanThenMkdir(dir);
     const tenantFile = path.join(dir, 'tenant.yml');
 
-    const localAssets = {
-      tenant: {
-        friendly_name: '##ENV## Tenant',
-        enabled_locales: '@@LANGUAGES@@',
-      },
-      connections: [
-        {
-          name: 'connection-1',
-          strategy: 'waad',
-          options: {
-            tenant_domain: '##DOMAIN##',
-          },
-        },
-      ],
-    };
-
-    fs.writeFileSync(tenantFile, jsYaml.dump(localAssets));
+    const localAssets = `
+    tenant:
+      friendly_name: "##ENV## Tenant"
+      enabled_locales: @@LANGUAGES@@
+    connections:
+      - name: connection-1
+        strategy: waad
+        options:
+          tenant_domain: "##DOMAIN##"
+    `;
+
+    fs.writeFileSync(tenantFile, localAssets);
     const context = new Context(
       {
         AUTH0_INPUT_FILE: tenantFile,
@@ -597,6 +592,20 @@ describe('#YAML context validation', () => {
     await context.dump();
     const yaml = jsYaml.load(fs.readFileSync(tenantFile));
 
-    expect(yaml).to.deep.equal(localAssets);
+    expect(yaml).to.deep.equal({
+      tenant: {
+        enabled_locales: '@@LANGUAGES@@',
+        friendly_name: '##ENV## Tenant',
+      },
+      connections: [
+        {
+          name: 'connection-1',
+          strategy: 'waad',
+          options: {
+            tenant_domain: '##DOMAIN##',
+          },
+        },
+      ],
+    });
   });
 });
diff --git a/test/e2e/e2e.test.ts b/test/e2e/e2e.test.ts
@@ -320,7 +320,7 @@ describe('#end-to-end keyword replacement', function () {
 
     const keywordMapping = {
       COMPANY_NAME: 'Travel0',
-      //LANGUAGES: ['en', 'es'], //TODO: support array replacement for directory format
+      LANGUAGES: ['en', 'es'],
     };
 
     await deploy({
@@ -424,7 +424,7 @@ describe('keyword preservation', () => {
       yaml.emailTemplates.find(({ template }) => template === 'welcome_email').resultUrl
     ).to.equal('https://##DOMAIN##/welcome');
 
-    // expect(yaml.tenant.enabled_locales).to.equal('@@LANGUAGES@@'); TODO: enable @@ARRAY@@ keyword preservation in yaml formats
+    expect(yaml.tenant.enabled_locales).to.equal('@@LANGUAGES@@');
 
     const emailTemplateHTML = fs
       .readFileSync(path.join(workDirectory, 'emailTemplates', 'welcome_email.html'))
@@ -458,20 +458,30 @@ describe('keyword preservation', () => {
       fs.readFileSync(path.join(workDirectory, 'tenant.json')).toString()
     );
 
-    expect(jsonWithoutPreservation.friendly_name).to.equal('This tenant name should be preserved');
-    expect(jsonWithoutPreservation.enabled_locales).to.deep.equal(['en', 'es']);
-    expect(jsonWithoutPreservation.support_email).to.equal('[email protected]');
-    expect(jsonWithoutPreservation.support_url).to.equal('https://travel0.com/support');
+    expect(jsonWithoutPreservation.friendly_name).to.equal(
+      config.AUTH0_KEYWORD_REPLACE_MAPPINGS.TENANT_NAME
+    );
+    expect(jsonWithoutPreservation.enabled_locales).to.deep.equal(
+      config.AUTH0_KEYWORD_REPLACE_MAPPINGS.LANGUAGES
+    );
+    expect(jsonWithoutPreservation.support_email).to.equal(
+      `support@${config.AUTH0_KEYWORD_REPLACE_MAPPINGS.DOMAIN}`
+    );
+    expect(jsonWithoutPreservation.support_url).to.equal(
+      `https://${config.AUTH0_KEYWORD_REPLACE_MAPPINGS.DOMAIN}/support`
+    );
 
     const emailTemplateJsonWithoutPreservation = JSON.parse(
       fs.readFileSync(path.join(workDirectory, 'emails', 'welcome_email.json')).toString()
     );
 
-    expect(emailTemplateJsonWithoutPreservation.resultUrl).to.equal('https://travel0.com/welcome');
+    expect(emailTemplateJsonWithoutPreservation.resultUrl).to.equal(
+      `https://${config.AUTH0_KEYWORD_REPLACE_MAPPINGS.DOMAIN}/welcome`
+    );
 
     expect(
       fs.readFileSync(path.join(workDirectory, 'emails', 'welcome_email.html')).toString()
-    ).to.contain('This tenant name should be preserved');
+    ).to.contain(config.AUTH0_KEYWORD_REPLACE_MAPPINGS.TENANT_NAME);
 
     emptyDirSync(workDirectory);
     copySync(`${__dirname}/testdata/should-preserve-keywords/directory`, workDirectory); //It is necessary to copy directory contents to work directory to prevent overwriting of Git-committed files

diff --git a/test/e2e/recordings/should-preserve-keywords-for-yaml-format.json b/test/e2e/recordings/should-preserve-keywords-for-yaml-format.json
diff --git a/test/e2e/testdata/should-preserve-keywords/yaml/tenant.yaml b/test/e2e/testdata/should-preserve-keywords/yaml/tenant.yaml
@@ -1,7 +1,7 @@
 tenant:
   allowed_logout_urls:
     - https://travel0.com/logoutCallback
-  # enabled_locales: @@LANGUAGES@@ TODO: enable @@ARRAY@@ keyword preservation in yaml
+  enabled_locales: @@LANGUAGES@@
   flags:
     allow_legacy_delegation_grant_types: true
     allow_legacy_ro_grant_types: true

diff --git a/test/tools/utils.test.js b/test/tools/utils.test.js
@@ -58,6 +58,63 @@ describe('#utils', function () {
     }).to.throw(/Unable to load file.*/);
   });
 
+  describe('wrapArrayReplaceMarkersInQuotes', () => {
+    it('should wrap @@ARRAY_KEYWORD@@ markers in quotes', () => {
+      const yaml = `
+      property:
+      - name: some-item
+        value: @@VALID_ARRAY_KEYWORD@@
+      - name: another-item
+        value: @@VALID_ARRAY_KEYWORD@@`;
+
+      const escapedYaml = utils.wrapArrayReplaceMarkersInQuotes(yaml, {
+        VALID_ARRAY_KEYWORD: [],
+      });
+      expect(escapedYaml).to.equal(`
+      property:
+      - name: some-item
+        value: "@@VALID_ARRAY_KEYWORD@@"
+      - name: another-item
+        value: "@@VALID_ARRAY_KEYWORD@@"`);
+      expect(() => jsYaml.load(escapedYaml)).to.not.throw();
+    });
+
+    it('should not wrap @@ARRAY_KEYWORD@@ markers in quotes if it is already wrapped', () => {
+      const yaml = `
+      property:
+      - name: some-item
+        value: "@@VALID_ARRAY_KEYWORD@@"
+      - name: some-single-quoted-item
+        value: '@@VALID_ARRAY_KEYWORD@@'
+      - name: another-item
+        value: "@@VALID_ARRAY_KEYWORD@@"`;
+
+      const escapedYaml = utils.wrapArrayReplaceMarkersInQuotes(yaml, {
+        VALID_ARRAY_KEYWORD: [],
+      });
+      expect(escapedYaml).to.equal(`
+      property:
+      - name: some-item
+        value: "@@VALID_ARRAY_KEYWORD@@"
+      - name: some-single-quoted-item
+        value: '@@VALID_ARRAY_KEYWORD@@'
+      - name: another-item
+        value: "@@VALID_ARRAY_KEYWORD@@"`);
+      expect(() => jsYaml.load(escapedYaml)).to.not.throw();
+    });
+
+    it('should not wrap @@ARRAY_KEYWORD@@ markers in quotes if keyword does not exist in mapping', () => {
+      const yaml = `
+      property:
+      - name: some-item
+        value: @@NOT_IN_KEYWORD_MAPPINGS@@`;
+
+      const escapedYaml = utils.wrapArrayReplaceMarkersInQuotes(yaml, {});
+      expect(escapedYaml).to.equal(yaml);
+      expect(() => jsYaml.load(escapedYaml)).to.throw(); // Because it is invalid yaml
+    });
+  });
+
   it('should do keyword replacements', (done) => {
     const kwContents =
       '{ "a": 1, "string_key": @@string@@, "array_key": @@array@@, "object_key": @@object@@,' +
@@ -349,7 +406,7 @@ describe('#keywordReplacement', () => {
       });
     });
 
-    it('should replace @@ wrapped values, even when wrapped with quotes', () => {
+    it('should replace @@ wrapped values, even when wrapped with quotes in JSON format', () => {
       const inputWrappedInQuotes = '{ "foo": "@@ARRAY_REPLACEMENT@@", "bar": "OTHER_REPLACEMENT"}';
       const output = utils.keywordArrayReplace(inputWrappedInQuotes, mapping);
       const parsedOutput = JSON.parse(output);
@@ -358,6 +415,19 @@ describe('#keywordReplacement', () => {
         bar: 'OTHER_REPLACEMENT',
       });
     });
+
+    it('should replace @@ wrapped values, even when wrapped with quotes in YAML format', () => {
+      const inputWrappedInQuotes = `
+        singleQuotes: '@@ARRAY_REPLACEMENT@@'
+        doubleQuotes: "@@ARRAY_REPLACEMENT@@"`;
+
+      const output = utils.keywordArrayReplace(inputWrappedInQuotes, mapping);
+      const parsedOutput = jsYaml.load(output);
+      expect(parsedOutput).to.deep.equal({
+        singleQuotes: mapping.ARRAY_REPLACEMENT,
+        doubleQuotes: mapping.ARRAY_REPLACEMENT,
+      });
+    });
   });
 });