fix: trie lookup performance (#5985)

.github/workflows/integration-test.yml

@@ -18,6 +18,7 @@ on:
       - 'packages/**/package.json'
       - 'packages/**/*-lock.yaml'
       - 'packages/**/*.ts'
+      - 'packages/**/*.mts'
       - 'integration-tests/**'
       - '!integration-tests/perf/**'
       - 'package.json'
@@ -149,9 +150,14 @@ jobs:
               'integration-tests/config/repositories/${{matrix.repo}}/**',
               'integration-tests/snapshots/${{ matrix.repo }}/*',
               'integration-tests/repositories/*',
-              'integration-tests/src/**/*.ts', 'integration-tests/tsconfig.json',
-              'packages/*/src/**/*.ts', 'packages/*/tsconfig.json',
+              'integration-tests/src/**/*.ts',
+              'integration-tests/src/**/*.mts',
+              'integration-tests/tsconfig.json',
+              'packages/*/src/**/*.ts',
+              'packages/*/src/**/*.mts',
+              'packages/*/tsconfig.json',
               'packages/*/*.ts',
+              'packages/*/*.mts',
               'tools/perf-chart/lib/app.cjs',
               '*-lock.yaml'
             ) }}

packages/cspell-dictionary/src/SpellingDictionary/SpellingDictionaryFromTrie.ts

@@ -211,18 +211,25 @@ function* outerWordForms(word: string, mapWord: (word: string) => string[]): Ite
     // Only generate the needed forms.
     const sent = new Set<string>();
     let w = word;
+    const ww = w;
     yield w;
     sent.add(w);
     w = word.normalize('NFC');
-    if (!sent.has(w)) yield w;
-    sent.add(w);
+    if (w !== ww) {
+        yield w;
+        sent.add(w);
+    }
     w = word.normalize('NFD');
-    if (!sent.has(w)) yield w;
-    sent.add(w);
-    for (const f of [...sent]) {
+    if (w !== ww && !sent.has(w)) {
+        yield w;
+        sent.add(w);
+    }
+    for (const f of sent) {
         for (const m of mapWord(f)) {
-            if (!sent.has(m)) yield m;
-            sent.add(m);
+            if (m !== ww && !sent.has(m)) {
+                yield m;
+                sent.add(m);
+            }
         }
     }
     return;

packages/cspell-dictionary/src/perf/has.perf.ts

@@ -18,19 +18,20 @@ suite('dictionary has', async (test) => {
     const dict3 = createSpellingDictionary(words3, 'test3', import.meta.url);
 
     const dictCol = createCollection([dict, dict2, dict3], 'test-collection');
+    const dictColRev = createCollection([dict3, dict2, dict], 'test-collection-reverse');
 
     test('dictionary has 100k words', () => {
         checkWords(dict, words);
     });
 
-    test('dictionary has 100k words (2nd time)', () => {
-        checkWords(dict, words);
-    });
-
     test('collection has 100k words', () => {
         checkWords(dictCol, words);
     });
 
+    test('collection reverse has 100k words', () => {
+        checkWords(dictColRev, words);
+    });
+
     test('iTrie has 100k words', () => {
         checkWords(iTrie, words);
     });
@@ -61,10 +62,6 @@ suite('dictionary has Not', async (test) => {
         checkWords(dict, missingWords, false);
     });
 
-    test('dictionary has not 100k words (2nd time)', () => {
-        checkWords(dict, missingWords, false);
-    });
-
     test('collection has not 100k words', () => {
         checkWords(dictCol, missingWords, false);
     });

packages/cspell-lib/src/lib/Models/Suggestion.ts

@@ -11,4 +11,9 @@ export interface ExtendedSuggestion {
      * The suggested word adjusted to match the original case.
      */
     wordAdjustedToMatchCase?: string;
+    /**
+     * The cost of using this word.
+     * The lower the cost, the better the suggestion.
+     */
+    cost?: number;
 }

packages/cspell-lib/src/lib/suggestions.test.ts

@@ -1,6 +1,6 @@
 import { describe, expect, test } from 'vitest';
 
-import type { SuggestionOptions } from './suggestions.js';
+import type { SuggestedWord, SuggestionOptions } from './suggestions.js';
 import { SuggestionError, suggestionsForWord, suggestionsForWords } from './suggestions.js';
 import { asyncIterableToArray } from './util/util.js';
 
@@ -22,6 +22,7 @@ describe('suggestions', () => {
         ${'apple'} | ${undefined}                            | ${{ dictionaries: ['en-gb'] }} | ${ac([sug('apple', 0, ['en_us', 'en-gb']), sug('Apple', 1, ['en_us', 'companies'])])}
     `(
         'suggestionsForWord default settings word: "$word", opts: $options, settings: $settings',
+        { timeout },
         async ({ word, options, settings, expected }) => {
             const results = await suggestionsForWord(word, options, settings);
             expect(results.word).toEqual(word);
@@ -32,26 +33,49 @@ describe('suggestions', () => {
             expect(resultsAsync[0].word).toEqual(word);
             expect(resultsAsync[0].suggestions).toEqual(expected);
         },
-        { timeout },
     );
 
     test.each`
         word       | options                               | settings
         ${'apple'} | ${opt({ dictionaries: ['unknown'] })} | ${undefined}
     `(
         'suggestionsForWord ERRORS word: "$word", opts: $options, settings: $settings',
+        { timeout },
         async ({ word, options, settings }) => {
             await expect(suggestionsForWord(word, options, settings)).rejects.toThrow(SuggestionError);
         },
-        { timeout },
     );
+});
 
-    function opt(opt: Partial<SuggestionOptions>): SuggestionOptions {
-        return opt;
-    }
+describe('Suggestions English', async () => {
+    // const configLoader = getDefaultConfigLoaderInternal();
+    // const settings = await configLoader.getGlobalSettingsAsync();
 
-    function sug(word: string, cost: number, dicts: string[]) {
-        const dictionaries = [...dicts].sort();
-        return oc({ word, cost, dictionaries });
-    }
+    // cspell:ignore orangges
+    test('Orangges', async () => {
+        const results = await suggestionsForWord('orangges', { languageId: 'typescript', locale: 'en-US' }, {});
+        expect(results.suggestions).toEqual([
+            sug('oranges', 100),
+            sug('ranges', 185),
+            sug('orangs', 190),
+            sug('orange', 200),
+            sug('orangey', 200),
+            sug('orangier', 200),
+            sug('orangiest'),
+            sug('Orange', 201),
+        ]);
+    });
 });
+
+function opt(opt: Partial<SuggestionOptions>): SuggestionOptions {
+    return opt;
+}
+
+function sug(word: string, cost?: number, dicts?: string[]) {
+    const suggestedWord: Partial<SuggestedWord> = { word };
+    if (cost !== undefined) suggestedWord.cost = cost;
+    if (dicts) {
+        suggestedWord.dictionaries = [...dicts].sort();
+    }
+    return oc(suggestedWord);
+}

packages/cspell-lib/src/lib/textValidation/__snapshots__/docValidator.test.ts.snap

@@ -0,0 +1,34 @@
+// Vitest Snapshot v1, https://vitest.dev/guide/snapshot.html
+
+exports[`docValidator suggestions > suggestions 1`] = `
+[
+  {
+    "word": "oranges",
+    "wordAdjustedToMatchCase": "Oranges",
+  },
+  {
+    "word": "orange",
+  },
+  {
+    "word": "Orange",
+  },
+  {
+    "word": "orangs",
+    "wordAdjustedToMatchCase": "Orangs",
+  },
+  {
+    "word": "orange's",
+  },
+  {
+    "word": "Orange's",
+  },
+  {
+    "word": "ranges",
+    "wordAdjustedToMatchCase": "Ranges",
+  },
+  {
+    "word": "orangier",
+    "wordAdjustedToMatchCase": "Orangier",
+  },
+]
+`;

packages/cspell-lib/src/lib/textValidation/docValidator.test.ts

@@ -283,6 +283,27 @@ describe('docValidator trace', () => {
     });
 });
 
+describe('docValidator suggestions', () => {
+    test('suggestions', async () => {
+        const doc = td(__filename, sampleCode());
+        const dVal = new DocumentValidator(doc, { generateSuggestions: true, numSuggestions: 8 }, {});
+        await dVal.prepare();
+        const issues = dVal.checkDocument();
+        expect(issues).toHaveLength(1);
+        expect(issues[0].suggestionsEx).toMatchSnapshot();
+    });
+});
+
+function sampleCode() {
+    // cspell:ignore Orangges
+    const text = `
+export function remainingOrangges(count: number): number {
+    return count % 42;
+}
+`;
+    return text;
+}
+
 function extractRawText(text: string, issues: ValidationIssue[]): string[] {
     return issues.map((issue) => {
         const start = issue.offset;

packages/cspell-tools/package.json

@@ -63,7 +63,6 @@
     "node": ">=18"
   },
   "devDependencies": {
-    "@types/glob": "^8.1.0",
     "lorem-ipsum": "^2.0.8",
     "ts-json-schema-generator": "^2.3.0"
   },