Mintlify migration script (#437)

* Mintlify migration

* Fix function call error

* Fix index pages in menu

* Fix malformed styles bug

* Add version to groups in config

* Move pages to docs folder

* Make assets paths absolute

* Update index paths

* Update image sizes

* Refactor remark-mintlify-update-frontmatter

* Fix test suite name

* Refactor remark-mintlify-update-images

* Refactor remark-mintlify-update-links

* Refactor remark-mintlify-update-tags

* Clean up main index file

* Update tags plugin import

* Add innkeep

* Remove controlled fields from innkeep

* Update top menu config

.gitignore

@@ -1,5 +1,8 @@
 # See https://help.github.com/articles/ignoring-files/ for more about ignoring files.
 
+# mintlify
+/migration-result
+
 # dependencies
 /node_modules
 

migration/base/script.js

@@ -0,0 +1,110 @@
+"use client";
+
+const INKEEP_API_KEY = "b4a9d9f13d7d1540b8b200a2a3374353ec50b068a9bf9bd0";
+const INKEEP_INTEGRATION_ID = "88e39f4c03457df277d83807";
+const INKEEP_ORGANIZATION_ID = "teleport";
+
+// Get the Mintlify search containers, going to reuse them as the triggers for Inkeep
+const searchButtonContainerIds = [
+  "search-bar-entry",
+  "search-bar-entry-mobile",
+];
+
+// Clone and replace, needed to remove existing event listeners
+const clonedSearchButtonContainers = searchButtonContainerIds.map((id) => {
+  const originalElement = document.getElementById(id);
+  const clonedElement = originalElement.cloneNode(true);
+  originalElement.parentNode.replaceChild(clonedElement, originalElement);
+  return clonedElement;
+});
+
+// Load the Inkeep script
+const inkeepScript = document.createElement("script");
+inkeepScript.type = "module";
+inkeepScript.src =
+  "https://unpkg.com/@inkeep/[email protected]/dist/embed.js";
+document.body.appendChild(inkeepScript);
+// Once the Inkeep script has loaded, load the Inkeep chat components
+inkeepScript.addEventListener("load", function () {
+  // Customization settings
+  const sharedConfig = {
+    baseSettings: {
+      apiKey: INKEEP_API_KEY,
+      integrationId: INKEEP_INTEGRATION_ID,
+      organizationId: INKEEP_ORGANIZATION_ID,
+      organizationDisplayName: "Teleport",
+      primaryBrandColor: "#512FC9",
+      chatApiProxyDomain: "goteleport.com/inkeep-proxy",
+      remoteErrorLogsLevel: 1,
+      optOutAllAnalytics: true,
+      consoleDebugLevel: 0,
+      customCardSettings: [
+        {
+          filters: {
+            ContentType: "docs",
+          },
+          searchTabLabel: "Docs",
+          icon: { builtIn: "IoDocumentTextOutline" },
+        },
+      ],
+    },
+    aiChatSettings: {
+      botName: "Teleport",
+      botAvatarSrcUrl: "https://goteleport.com/static/pam-standing.svg",
+      defaultChatMode: "AUTO",
+    },
+    searchSettings: {
+      placeholder: "Search Docs",
+      tabSettings: {
+        isAllTabEnabled: false,
+        rootBreadcrumbsToUseAsTabs: ["Docs", "GitHub"],
+        tabOrderByLabel: ["Docs", "GitHub"],
+        alwaysDisplayedTabs: ["Docs", "GitHub"],
+        disabledDefaultTabs: undefined,
+      },
+      shouldOpenLinksInNewTab: true,
+    },
+    modalSettings: {
+      closeOnBlur: true,
+    },
+  };
+
+  // for syncing with dark mode
+  const colorModeSettings = {
+    observedElement: document.documentElement,
+    isDarkModeCallback: (el) => {
+      return el.classList.contains("dark");
+    },
+    colorModeAttribute: "class",
+  };
+  // add the "Ask AI" pill chat button
+  Inkeep().embed({
+    componentType: "ChatButton",
+    colorModeSync: colorModeSettings,
+    properties: sharedConfig,
+  });
+  // instantiate Inkeep "custom trigger" component
+  const inkeepSearchModal = Inkeep({
+    ...sharedConfig.baseSettings,
+  }).embed({
+    componentType: "CustomTrigger",
+    colorModeSync: colorModeSettings,
+    properties: {
+      ...sharedConfig,
+      isOpen: false,
+      onClose: () => {
+        inkeepSearchModal.render({
+          isOpen: false,
+        });
+      },
+    },
+  });
+  // When the Mintlify search bar clone is clicked, open the Inkeep search modal
+  clonedSearchButtonContainers.forEach((trigger) => {
+    trigger.addEventListener("click", function () {
+      inkeepSearchModal.render({
+        isOpen: true,
+      });
+    });
+  });
+});

migration/index.mjs

@@ -0,0 +1,197 @@
+/*
+ * List of plugins and settings for them used on the docs pages.
+ */
+
+import { resolve } from "path";
+import { readFileSync, writeFileSync, rmSync, existsSync, cpSync } from "fs";
+import { ensureFileSync } from "fs-extra/esm";
+import { readSync } from "to-vfile";
+import { unified } from "unified";
+import remarkParse from "remark-parse";
+import remarkStringify from "remark-stringify";
+import remarkMDX from "remark-mdx";
+import remarkGFM from "remark-gfm";
+import remarkFrontmatter from "remark-frontmatter";
+import remarkCopyLinkedFiles from "remark-copy-linked-files";
+
+import remarkIncludes from "../.build/server/remark-includes.mjs";
+import remarkVariables from "../.build/server/remark-variables.mjs";
+import remarkMintlifyUpdateImages from "../.build/server/remark-mintlify-update-images.mjs";
+import remarkMintlifyUpdateFrontmatter from "../.build/server/remark-mintlify-update-frontmatter.mjs";
+import remarkMintlifyUpdateLinks from "../.build/server/remark-mintlify-update-links.mjs";
+import remarkMintlifyUpdateMTags from "../.build/server/remark-mintlify-update-tags.mjs";
+
+import {
+  getVersion,
+  getVersionRootPath,
+} from "../.build/server/docs-helpers.mjs";
+import { loadConfig as loadConfigSite } from "../.build/server/config-site.mjs";
+import { loadConfig } from "../.build/server/config-docs.mjs";
+import { getDocsPagesMap } from "../.build/server/paths.mjs";
+import { getDocsPaths } from "../.build/server/docs-helpers.mjs";
+
+const RESULT_DIR = resolve("migration-result"); // Name of the result folder
+const ASSETS_DIR = `${RESULT_DIR}/assets`; // Name of the assets folder
+
+// Base mint.json fields
+const initialMintJson = JSON.parse(
+  readFileSync(resolve("migration/initial-mint.json"), "utf-8")
+);
+
+const processFile = async (vfile, { slug }) => {
+  return unified()
+    .use(remarkParse) // Parse to AST
+    .use(remarkMDX) // Add mdx parser
+    .use(remarkGFM) // Add tables parser
+    .use(remarkFrontmatter) // Add frontmatter support
+    .use(remarkMintlifyUpdateFrontmatter, { version: getVersion(vfile.path) }) // Add version to the frontmatter
+    .use(remarkIncludes, {
+      rootDir: getVersionRootPath(vfile.path),
+    }) // Resolves (!include.ext!) syntax
+    .use(remarkVariables, {
+      variables: loadConfig(getVersion(vfile.path)).variables || {},
+    }) // Resolves (=variable=) syntax
+    .use(remarkMintlifyUpdateLinks, { slug }) // Make links absolute and remove mdx extension
+    .use(remarkCopyLinkedFiles, {
+      destinationDir: ASSETS_DIR,
+      buildUrl: ({ filename }) => `/assets/${filename}`,
+    }) // Move all assets to public dir, add hashed to filenams and removes duplicates
+    .use(remarkMintlifyUpdateImages, {
+      staticPath: "/assets",
+      destinationDir: ASSETS_DIR,
+    }) // Convert markdown images to mdx images and add correct width and height
+    .use(remarkMintlifyUpdateMTags) // Migrate tags to Mintlify analogues
+    .use(remarkStringify, {
+      bullet: "-",
+      ruleRepetition: 3,
+      fences: true,
+      incrementListMarker: true,
+      checkBlanks: true,
+      resourceLink: true,
+      emphasis: "*",
+      tablePipeAlign: false,
+      tableCellPadding: true,
+      listItemIndent: 1,
+    }) // Stringify AST to string with correct syntax options
+    .process(vfile);
+};
+
+const docsPageMap = getDocsPagesMap(); // Slugs' hash in { slug: filepath } format
+
+const processFiles = () => {
+  // Get list of slugs that needs to be build
+  getDocsPaths()
+    // Convert from nextjs slug format to strings
+    .map(({ params }) =>
+      params && Array.isArray(params.slug) ? `/${params.slug.join("/")}/` : "/"
+    )
+    .forEach(async (slug) => {
+      const filePath = docsPageMap[slug]; // get filepath for slug
+
+      const file = readSync(filePath, "utf-8");
+
+      // Generates slug for the page in mintlify format
+      const newBasePath = `/docs${slug.replace(/\/$/, "")}.mdx`;
+
+      // Location for the generated mdx page
+      const newFilePath = resolve(`${RESULT_DIR}${newBasePath}`);
+
+      const result = await processFile(file, {
+        slug: `/docs${slug.replace(/\/$/, "")}`,
+      });
+
+      // Create folder recursively
+      ensureFileSync(newFilePath);
+
+      writeFileSync(
+        newFilePath,
+        result.value
+          .replaceAll(/\<!----\>\n\n/g, "") // HACK: Fixes bug with includes'stringifying
+          .replaceAll(
+            "project\\_path:{group}/{project}:ref\\_type:{type}:ref:{branch_name}",
+            "`project\\_path:{group}/{project}:ref\\_type:{type}:ref:{branch_name}`"
+          ) // HACK: Fixes bug with non-existing variables
+      );
+    });
+};
+
+// Process navigation entry
+const processEntry = ({ title, slug, entries }, version) => {
+  let newSlug = `docs${slug.replace(/\/\s*$/, "")}`;
+
+  // Mintlyfy does not allows categories to be links themselves so we need to move current link
+  // to the pages array to
+  if (entries && slug) {
+    return {
+      group: title,
+      pages: [newSlug, ...entries.map((entry) => processEntry(entry))],
+      version,
+    };
+  } else if (entries) {
+    return {
+      group: title,
+      pages: entries.map((entry) => processEntry(entry)),
+      version,
+    };
+  }
+
+  // Mintlify does not allow navigation items to have separate text from the pages,
+  // navigation item takes title from the page itself
+  return newSlug;
+};
+
+const generateMintJson = () => {
+  const siteConfig = loadConfigSite();
+
+  // Add list of versions
+  initialMintJson.versions = siteConfig.versions;
+
+  siteConfig.versions.forEach((version) => {
+    // Load processed and normalized config (urls has version prefixes added, etc)
+    const versionConfig = loadConfig(version);
+
+    // Add all pages from all versions to navigation
+    initialMintJson.navigation.push(
+      ...versionConfig.navigation.map((category) => {
+        return {
+          group: category.title,
+          pages: category.entries.map((entry) => processEntry(entry, version)),
+          version,
+        };
+      })
+    );
+
+    // Make hash from redirects to remove duplicated (Mintlify throws error on them)
+    const redirectsHash = versionConfig.redirects.reduce(
+      (result, { source, destination }) => {
+        return { ...result, [`/docs${source}`]: `/docs${destination}` };
+      },
+      {}
+    );
+
+    // Add all redirects to config
+    initialMintJson.redirects.push(
+      ...Object.entries(redirectsHash).map(([source, destination]) => ({
+        source,
+        destination,
+      }))
+    );
+  });
+
+  // Write config to file
+  writeFileSync(
+    resolve(`${RESULT_DIR}/mint.json`),
+    JSON.stringify(initialMintJson)
+  );
+};
+
+// Remove previous build results
+if (existsSync(RESULT_DIR)) {
+  rmSync(RESULT_DIR, { recursive: true });
+}
+
+// Copy assets and favicons to result folder
+cpSync(resolve("migration/base"), RESULT_DIR, { recursive: true });
+
+processFiles(); // Process, move and rename mdx files
+generateMintJson(); // Generate one config from separate configs

migration/initial-mint.json

@@ -0,0 +1,30 @@
+{
+  "$schema": "https://mintlify.com/schema.json",
+  "name": "Teleport",
+  "logo": {
+    "light": "/logo/light.png",
+    "dark": "/logo/dark.png"
+  },
+  "favicon": "/favicon.png",
+  "colors": {
+    "primary": "#512FC9",
+    "light": "#7956F5",
+    "dark": "#512FC9"
+  },
+  "feedback": {
+    "thumbsRating": true
+  },
+  "topbarCtaButton": {
+    "name": "Contact Sales",
+    "url": "https://goteleport.com/signup/enterprise/"
+  },
+  "topbarLinks": [
+    {
+      "name": "Try for Free",
+      "url": "https://goteleport.com/signup/"
+    }
+  ],
+  "versions": [],
+  "navigation": [],
+  "redirects": []
+}

package.json

@@ -31,7 +31,8 @@
     "build-storybook": "storybook build",
     "test-storybook": "test-storybook",
     "storybook:test-ci": "yarn build-storybook --quiet && npx concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"npx http-server storybook-static --port 6006 --silent\" \"npx wait-on tcp:6006 && yarn test-storybook\"",
-    "storybook:test-local": "yarn build-storybook --quiet && npx concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"npx http-server storybook-static --port 6006 --silent\" \"yarn test-storybook\""
+    "storybook:test-local": "yarn build-storybook --quiet && npx concurrently -k -s first -n \"SB,TEST\" -c \"magenta,blue\" \"npx http-server storybook-static --port 6006 --silent\" \"yarn test-storybook\"",
+    "mintlify": "node migration/index.mjs"
   },
   "lint-staged": {
     "*.{js,jsx,ts,tsx}": [

server/fixtures/mintlify-frontmatter-expected.mdx

@@ -0,0 +1,4 @@
+---
+title: Title
+version: "10.x"
+---

server/fixtures/mintlify-frontmatter-source.mdx

@@ -0,0 +1,3 @@
+---
+title: Title
+---

server/fixtures/mintlify-tags/code-result.mdx

@@ -0,0 +1,3 @@
+```bash
+$ bash command
+```

server/fixtures/mintlify-tags/code-source.mdx

@@ -0,0 +1,3 @@
+```code
+$ bash command
+```

server/fixtures/mintlify-tags/details-result.mdx

@@ -0,0 +1 @@
+<Accordion title="Title">Content</Accordion>

server/fixtures/mintlify-tags/details-source.mdx

@@ -0,0 +1 @@
+<Details title="Title">Content</Details>

server/fixtures/mintlify-tags/note-result.mdx

@@ -0,0 +1,13 @@
+<Note><p>
+  **Preview**
+</p><p>
+  Admonition
+</p></Note>
+<Tip>Admonition Tip</Tip>
+<Warning>Admonition Warning</Warning>
+<Note>Admonition Note</Note>
+
+<Note>Notice</Note>
+<Tip>Notice Tip</Tip>
+<Warning>Notice Warning</Warning>
+<Note>Notice Note</Note>