Grammar: Fix comments in template params not tokenized (#3018)

fix #3017 --------- Co-authored-by: Mark Cowlishaw <[email protected]>
microsoft · Mar 22, 2024 · e0dfe6f · e0dfe6f
1 parent 06f514d
commit e0dfe6f
Show file tree

Hide file tree

Showing 4 changed files with 64 additions and 2 deletions.
diff --git a/.chronus/changes/fix-comment-in-template-params-2024-2-13-16-45-3.md b/.chronus/changes/fix-comment-in-template-params-2024-2-13-16-45-3.md
@@ -0,0 +1,8 @@
+---
+# Change versionKind to one of: internal, fix, dependencies, feature, deprecation, breaking
+changeKind: fix
+packages:
+  - "@typespec/compiler"
+---
+
+Grammar: Fix comments in template params not tokenized
diff --git a/grammars/typespec.json b/grammars/typespec.json
@@ -457,6 +457,9 @@
         {
           "include": "#token"
         },
+        {
+          "include": "#type-parameters"
+        },
         {
           "include": "#interface-heritage"
         },
@@ -1155,6 +1158,9 @@
       },
       "end": "(?=>)|(?=,|;|@|\\)|\\}|\\b(?:extern)\\b|\\b(?:namespace|model|op|using|import|enum|alias|union|interface|dec|fn)\\b)",
       "patterns": [
+        {
+          "include": "#token"
+        },
         {
           "include": "#type-parameter-constraint"
         },
@@ -1229,6 +1235,9 @@
         {
           "include": "#token"
         },
+        {
+          "include": "#type-parameters"
+        },
         {
           "include": "#expression"
         }

diff --git a/packages/compiler/src/server/tmlanguage.ts b/packages/compiler/src/server/tmlanguage.ts
@@ -348,7 +348,7 @@ const typeParameter: BeginEndRule = {
     "1": { scope: "entity.name.type.tsp" },
   },
   end: `(?=>)|${universalEnd}`,
-  patterns: [typeParameterConstraint, typeParameterDefault],
+  patterns: [token, typeParameterConstraint, typeParameterDefault],
 };
 
 const typeParameters: BeginEndRule = {
@@ -522,7 +522,7 @@ const unionStatement: BeginEndRule = {
     "1": { scope: "keyword.other.tsp" },
   },
   end: `(?<=\\})|${universalEnd}`,
-  patterns: [token, expression],
+  patterns: [token, typeParameters, expression],
 };
 
 const aliasStatement: BeginEndRule = {
@@ -663,6 +663,7 @@ const interfaceStatement: BeginEndRule = {
   end: `(?<=\\})|${universalEnd}`,
   patterns: [
     token,
+    typeParameters,
     interfaceHeritage, // before expression or extends will look like type name
     interfaceBody, // before expression or { will match model expression
     expression, // enough to match name and type parameters

diff --git a/packages/compiler/test/server/colorization.test.ts b/packages/compiler/test/server/colorization.test.ts
@@ -10,6 +10,7 @@ import { TypeSpecScope } from "../../src/server/tmlanguage.js";
 import { SemanticToken, SemanticTokenKind } from "../../src/server/types.js";
 import { createTestServerHost } from "../../src/testing/test-server-host.js";
 import { findTestPackageRoot } from "../../src/testing/test-utils.js";
+import { deepEquals } from "../../src/utils/index.js";
 
 const { parseRawGrammar, Registry } = vscode_textmate;
 const { createOnigScanner, createOnigString, loadWASM } = vscode_oniguruma;
@@ -1029,6 +1030,49 @@ function testColorization(description: string, tokenize: Tokenize) {
             Token.comment.block("*/"),
           ]);
         });
+
+        describe("in template parameters", () => {
+          it.each([
+            [
+              "alias",
+              `alias Foo<T // comment
+                > = T`,
+            ],
+            [
+              "model",
+              `model Foo<T // comment
+                > {}`,
+            ],
+            [
+              "union",
+              `union Foo<T // comment
+                > {}`,
+            ],
+            [
+              "interface",
+              `interface Foo<T // comment
+                > {}`,
+            ],
+            [
+              "operation",
+              `op foo<T // comment
+                >(): void;`,
+            ],
+          ])("%s", async () => {
+            const tokens = await tokenize(`alias Foo<T // comment
+            > = T`);
+
+            const index = tokens.findIndex((x) =>
+              deepEquals(x, Token.punctuation.typeParameters.begin)
+            );
+            deepStrictEqual(tokens.slice(index, index + 4), [
+              Token.punctuation.typeParameters.begin,
+              Token.identifiers.type("T"),
+              Token.comment.line("// comment"),
+              Token.punctuation.typeParameters.end,
+            ]);
+          });
+        });
       });
     }