Add DocumentReflection to represent markdown documents in generated…

… site

This is a very large change and needs a significant amount of testing still. It
seems to work correctly for TypeDoc's changelog, but there's still several
TODO items remaining before this goes in a full release.

Closes #247
Closes #1870
Closes #2288
Closes #2565

.config/typedoc.json

@@ -13,6 +13,7 @@
     ],
     "sort": ["kind", "instance-first", "required-first", "alphabetical"],
     "entryPoints": ["../src/index.ts"],
+    "projectDocuments": ["../CHANGELOG.md"],
     "excludeExternals": true,
     "excludeInternal": false,
     "excludePrivate": true,

CHANGELOG.md

@@ -1,11 +1,17 @@
-# Beta
+# Beta (full release: 2024-06-18)
 
-Docs needing updating:
-@license, @import
-sitemapBaseUrl
-markedOptions -> markdownItOptions, markdownItLoader, navigation
+### To Do
 
-## Breaking Changes
+-   Handle YAML (ick, don't want to add a dependency for that, YAML-like?) frontmatter at the top of documents to set the category/group/title
+-   Handle `@document` tag to add documents to the tree anywhere
+-   Handle image and relative markdown links within documents
+-   Update website docs - consider if reworking website to just be a TypeDoc generated site is a good idea
+    `@license`, `@import`, sitemapBaseUrl, markedOptions -> markdownItOptions, markdownItLoader, navigation
+    sort - documents-first, documents-last, alphabetical-ignoring-documents
+    searchInDocuments
+-   Correctly handle the `html` being set/not set in markdown-it (currently hardcoded to `true`)
+
+### Breaking Changes
 
 -   Drop support for Node 16.
 -   Moved from `marked` to `markdown-it` for parsing as marked has moved to an async model which supporting would significantly complicate TypeDoc's rendering code.
@@ -28,9 +34,11 @@ markedOptions -> markdownItOptions, markdownItLoader, navigation
 ### Features
 
 -   Added support for TypeScript 5.5.
+-   Added new `--projectDocuments` option to specify additional Markdown documents to be included in the generated site #247, #1870, #2288, #2565.
 -   TypeDoc now has the architecture in place to support localization. No languages besides English
     are currently shipped in the package, but it is now possible to add support for additional languages, #2475.
 -   `--hostedBaseUrl` will now be used to generate a `<link rel="canonical">` element in the project root page, #2550.
+-   Added three new sort strategies `documents-first`, `documents-last`, and `alphabetical-ignoring-documents` to order markdown documents.
 
 ### Bug Fixes
 

src/lib/application.ts

@@ -271,7 +271,7 @@ export class Application extends ChildableComponent<
             } catch (error) {
                 ok(error instanceof Error);
                 if (reportErrors) {
-                    this.logger.error(error.message as TranslatedString); // GERRIT review
+                    this.logger.error(error.message as TranslatedString);
                 }
             }
         }
@@ -724,6 +724,13 @@ export class Application extends ChildableComponent<
         );
         this.logger.verbose(`Reviving projects took ${Date.now() - start}ms`);
 
+        // If we only revived one project, the project documents were set for
+        // it when it was created. If we revived more than one project then
+        // it's convenient to be able to add more documents now.
+        if (jsonProjects.length > 1) {
+            this.converter.addProjectDocuments(result);
+        }
+
         this.trigger(ApplicationEvents.REVIVE, result);
         return result;
     }

src/lib/converter/comments/discovery.ts

@@ -102,6 +102,8 @@ const wantedKinds: Record<ReflectionKind, ts.SyntaxKind[]> = {
         ts.SyntaxKind.NamespaceExport,
         ts.SyntaxKind.ExportSpecifier,
     ],
+    // Non-TS kind, will never have comments.
+    [ReflectionKind.Document]: [],
 };
 
 export interface DiscoveredComment {

src/lib/converter/comments/parser.ts

@@ -1,4 +1,4 @@
-import { ok } from "assert";
+import assert, { ok } from "assert";
 import type { CommentParserConfig } from ".";
 import {
     Comment,
@@ -107,6 +107,75 @@ export function parseComment(
     }
 }
 
+/**
+ * Intended for parsing markdown documents. This only parses code blocks and
+ * inline tags outside of code blocks, everything else is text.
+ *
+ * If you change this, also look at blockContent, as it likely needs similar
+ * modifications to ensure parsing is consistent.
+ */
+export function parseCommentString(
+    tokens: Generator<Token, undefined, undefined>,
+    config: CommentParserConfig,
+    file: MinimalSourceFile,
+    logger: Logger,
+) {
+    const suppressWarningsConfig: CommentParserConfig = {
+        ...config,
+        jsDocCompatibility: {
+            defaultTag: true,
+            exampleTag: true,
+            ignoreUnescapedBraces: true,
+            inheritDocTag: true,
+        },
+    };
+
+    const content: CommentDisplayPart[] = [];
+    const lexer = makeLookaheadGenerator(tokens);
+
+    while (!lexer.done()) {
+        let consume = true;
+        const next = lexer.peek();
+
+        switch (next.kind) {
+            case TokenSyntaxKind.TypeAnnotation:
+                // Shouldn't have been produced by our lexer
+                assert(false, "Should be unreachable");
+                break;
+            case TokenSyntaxKind.NewLine:
+            case TokenSyntaxKind.Text:
+            case TokenSyntaxKind.Tag:
+            case TokenSyntaxKind.CloseBrace:
+                content.push({ kind: "text", text: next.text });
+                break;
+
+            case TokenSyntaxKind.Code:
+                content.push({ kind: "code", text: next.text });
+                break;
+
+            case TokenSyntaxKind.OpenBrace:
+                inlineTag(
+                    lexer,
+                    content,
+                    suppressWarningsConfig,
+                    logger.i18n,
+                    (message, token) => logger.warn(message, token.pos, file),
+                );
+                consume = false;
+                break;
+
+            default:
+                assertNever(next.kind);
+        }
+
+        if (consume) {
+            lexer.take();
+        }
+    }
+
+    return content;
+}
+
 const HAS_USER_IDENTIFIER: `@${string}`[] = [
     "@callback",
     "@param",
@@ -391,6 +460,10 @@ function exampleBlock(
     }
 }
 
+/**
+ * If you change this, also look at parseCommentString as it
+ * likely needs similar modifications to ensure parsing is consistent.
+ */
 function blockContent(
     comment: Comment,
     lexer: LookaheadGenerator<Token>,

src/lib/converter/comments/rawLexer.ts

@@ -1,5 +1,11 @@
 import { type Token, TokenSyntaxKind } from "./lexer";
 
+/**
+ * Note: This lexer intentionally *only* recognizes inline tags and code blocks.
+ * This is because it is intended for use on markdown documents, and we shouldn't
+ * take some stray `@user` mention within a "Thanks" section of someone's changelog
+ * as starting a block!
+ */
 export function* lexCommentString(
     file: string,
 ): Generator<Token, undefined, undefined> {
@@ -44,7 +50,7 @@ function* lexCommentString2(
     }
 
     let lineStart = true;
-    let braceStartsType = false;
+    let expectingTag = false;
 
     for (;;) {
         if (pos >= end) {
@@ -59,23 +65,17 @@ function* lexCommentString2(
             case "\n":
                 yield makeToken(TokenSyntaxKind.NewLine, 1);
                 lineStart = true;
+                expectingTag = false;
                 break;
 
             case "{":
-                if (braceStartsType && nextNonWs(pos + 1) !== "@") {
-                    yield makeToken(
-                        TokenSyntaxKind.TypeAnnotation,
-                        findEndOfType(pos) - pos,
-                    );
-                    braceStartsType = false;
-                } else {
-                    yield makeToken(TokenSyntaxKind.OpenBrace, 1);
-                }
+                yield makeToken(TokenSyntaxKind.OpenBrace, 1);
+                expectingTag = true;
                 break;
 
             case "}":
                 yield makeToken(TokenSyntaxKind.CloseBrace, 1);
-                braceStartsType = false;
+                expectingTag = false;
                 break;
 
             case "`": {
@@ -84,7 +84,6 @@ function* lexCommentString2(
                 // 2. Code block: <3 ticks><language, no ticks>\n<text>\n<3 ticks>\n
                 // 3. Unmatched tick(s), not code, but part of some text.
                 // We don't quite handle #2 correctly yet. PR welcome!
-                braceStartsType = false;
                 let tickCount = 1;
                 let lookahead = pos;
 
@@ -107,6 +106,7 @@ function* lexCommentString2(
                             text: codeText.join(""),
                             pos,
                         };
+                        expectingTag = false;
                         pos = lookahead;
                         break;
                     } else if (file[lookahead] === "`") {
@@ -141,9 +141,11 @@ function* lexCommentString2(
                             text: codeText.join(""),
                             pos,
                         };
+                        expectingTag = false;
                         pos = lookahead;
                     } else {
                         yield makeToken(TokenSyntaxKind.Text, tickCount);
+                        expectingTag = false;
                     }
                 }
 
@@ -166,10 +168,10 @@ function* lexCommentString2(
                 }
 
                 if (
+                    expectingTag &&
                     lookahead !== pos + 1 &&
                     (lookahead === end || /[\s}]/.test(file[lookahead]))
                 ) {
-                    braceStartsType = true;
                     yield makeToken(TokenSyntaxKind.Tag, lookahead - pos);
                     break;
                 }
@@ -212,7 +214,7 @@ function* lexCommentString2(
                 textParts.push(file.substring(lookaheadStart, lookahead));
 
                 if (textParts.some((part) => /\S/.test(part))) {
-                    braceStartsType = false;
+                    expectingTag = false;
                 }
 
                 // This piece of text had line continuations or escaped text
@@ -245,64 +247,4 @@ function* lexCommentString2(
 
         return file.startsWith("`".repeat(n), pos) && file[pos + n] !== "`";
     }
-
-    function findEndOfType(pos: number): number {
-        let openBraces = 0;
-
-        while (pos < end) {
-            if (file[pos] === "{") {
-                openBraces++;
-            } else if (file[pos] === "}") {
-                if (--openBraces === 0) {
-                    break;
-                }
-            } else if ("`'\"".includes(file[pos])) {
-                pos = findEndOfString(pos);
-            }
-
-            pos++;
-        }
-
-        if (pos < end && file[pos] === "}") {
-            pos++;
-        }
-
-        return pos;
-    }
-
-    function findEndOfString(pos: number): number {
-        const endOfString = file[pos];
-        pos++;
-        while (pos < end) {
-            if (file[pos] === endOfString) {
-                break;
-            } else if (file[pos] === "\\") {
-                pos++; // Skip escaped character
-            } else if (
-                endOfString === "`" &&
-                file[pos] === "$" &&
-                file[pos + 1] === "{"
-            ) {
-                // Template literal with data inside a ${}
-                while (pos < end && file[pos] !== "}") {
-                    if ("`'\"".includes(file[pos])) {
-                        pos = findEndOfString(pos) + 1;
-                    } else {
-                        pos++;
-                    }
-                }
-            }
-
-            pos++;
-        }
-
-        return pos;
-    }
-
-    function nextNonWs(pos: number): string | undefined {
-        while (pos < end && /\s/.test(file[pos])) {
-            pos++;
-        }
-        return file[pos];
-    }
 }

src/lib/converter/context.ts

@@ -8,6 +8,7 @@ import {
     DeclarationReflection,
     ReflectionKind,
     ReflectionFlag,
+    type DocumentReflection,
 } from "../models/index";
 
 import type { Converter } from "./converter";
@@ -228,10 +229,9 @@ export class Context {
         );
     }
 
-    addChild(reflection: DeclarationReflection) {
+    addChild(reflection: DeclarationReflection | DocumentReflection) {
         if (this.scope instanceof ContainerReflection) {
-            this.scope.children ??= [];
-            this.scope.children.push(reflection);
+            this.scope.addChild(reflection);
         }
     }