Pull request 799: Buffer-based tsurlfilter Engine

The idea is to replace String-based StringRuleList with a Uint8Array-based version.

Benefits of this appoarch:
* Immediately saves ~10-20MB of RAM due to the fact that the strings are now encoded in UTF-8 (javascript String is UTF-16 so it takes 2 bytes for every character, UTF-8 will use 1 byte for 99.9% of characters in filter lists).
* Fully solves the "leak" issues as there are no links to original filter list strings anymore.

Testing shows that the buffer-based approach does not affect performance and actually improves memory usage of the extension.

Squashed commit of the following:

commit b50b15c
Merge: 210ee8e 3f2e0d7
Author: Andrey Meshkov <[email protected]>
Date:   Tue Feb 13 14:36:32 2024 +0300

    Merge branch 'master' into buffer-rule-list

commit 210ee8e
Author: Andrey Meshkov <[email protected]>
Date:   Tue Feb 13 13:24:02 2024 +0300

    fix build, resolve review comments

commit ced9492
Author: Andrey Meshkov <[email protected]>
Date:   Tue Feb 13 13:13:30 2024 +0300

    fix review comment

commit a3ebdb2
Author: Andrey Meshkov <[email protected]>
Date:   Tue Feb 13 12:54:26 2024 +0300

    Fix comment, fix linter run

commit b27721b
Author: Andrey Meshkov <[email protected]>
Date:   Tue Feb 13 12:42:48 2024 +0300

    Resolve review comments

commit 85f095c
Author: Andrey Meshkov <[email protected]>
Date:   Mon Feb 12 15:12:23 2024 +0300

    Added comments

commit 9226ed8
Author: Andrey Meshkov <[email protected]>
Date:   Mon Feb 12 15:05:36 2024 +0300

    fix imports

commit f34428f
Merge: a2927eb aeb4336
Author: Andrey Meshkov <[email protected]>
Date:   Mon Feb 12 15:04:05 2024 +0300

    merge with master

commit a2927eb
Author: Andrey Meshkov <[email protected]>
Date:   Mon Feb 12 14:52:44 2024 +0300

    Added info about VS Code workspace

commit a990bed
Author: Andrey Meshkov <[email protected]>
Date:   Mon Feb 12 14:40:24 2024 +0300

    initial implementation of BufferRuleList

Author: ameshkov

.gitignore

@@ -1,9 +1,14 @@
+.DS_Store
 node_modules
 .nyc_output
-.DS_Store
 *.log
-.vscode
-.idea
 .awcache
 .rpt2_cache
 .nx
+
+# IDEA files
+.idea
+
+# VS code settings and workspace project
+.vscode
+tsurlfilter.code-workspace

README.md

@@ -97,3 +97,49 @@ Test pages:
 
 [testcasessimplerules]: https://testcases.agrd.dev/Filters/simple-rules/test-simple-rules.html
 [testcasesscriptrules]: https://testcases.agrd.dev/Filters/script-rules/test-script-rules.html
+
+### Visual Studio Code Workspace
+
+If you're using Visual Studio Code for development, it may be easier to work
+with the monorepo if you use the workspace functionality. To do this, create a
+`tsurlfilter.code-workspace` file in the monorepo root directory.
+
+`jest.runMode` and `jest.enable` would be useful to those that use
+[Jest][jestplugin] plugin.
+
+```json
+{
+    "folders": [
+        {
+            "path": "packages/tsurlfilter",
+        },
+        {
+            "path": "packages/tswebextension",
+        },
+        {
+            "path": "packages/agtree",
+        },
+        {
+            "path": "packages/css-tokenizer",
+        },
+        {
+            "path": "packages/adguard-api",
+        },
+        {
+            "path": "packages/examples/adguard-api",
+        },
+        {
+            "path": "packages/examples/tswebextension-mv2",
+        },
+        {
+            "path": "packages/examples/tswebextension-mv3",
+        }
+    ],
+    "settings": {
+        "jest.runMode": "on-demand",
+        "jest.enable": true,
+    }
+}
+```
+
+[jestplugin]: https://marketplace.visualstudio.com/items?itemName=Orta.vscode-jest

packages/tsurlfilter/src/filterlist/buffer-rule-list.ts

@@ -0,0 +1,130 @@
+import { BufferLineReader } from './reader/buffer-line-reader';
+import { type IRuleList, LIST_ID_MAX_VALUE } from './rule-list';
+import { RuleScanner } from './scanner/rule-scanner';
+import { ScannerType } from './scanner/scanner-type';
+
+/**
+ * BufferRuleList represents a string-based rule list. It keeps the original
+ * rule list as a byte array with UTF-8 encoded characters. This approach
+ * allows saving on the memory used by tsurlfilter compared to StringRuleList.
+ */
+export class BufferRuleList implements IRuleList {
+    /**
+     * Rule list ID.
+     */
+    private readonly id: number;
+
+    /**
+     * String with filtering rules (one per line) encoded as a
+     * UTF-8 array.
+     */
+    private readonly rulesBuffer: Uint8Array;
+
+    /**
+     * Whether to ignore cosmetic rules or not.
+     */
+    private readonly ignoreCosmetic: boolean;
+
+    /**
+     * Whether to ignore javascript cosmetic rules or not.
+     */
+    private readonly ignoreJS: boolean;
+
+    /**
+     * Whether to ignore unsafe rules or not.
+     */
+    private readonly ignoreUnsafe: boolean;
+
+    /**
+     * Text decoder that is used to read strings from the internal buffer of
+     * UTF-8 encoded characters.
+     */
+    private static readonly decoder = new TextDecoder('utf-8');
+
+    /**
+     * Constructor of BufferRuleList.
+     *
+     * @param listId - List identifier.
+     * @param rulesText - String with filtering rules (one per line).
+     * @param ignoreCosmetic - (Optional) True to ignore cosmetic rules.
+     * @param ignoreJS - (Optional) True to ignore JS rules.
+     * @param ignoreUnsafe - (Optional) True to ignore unsafe rules.
+     */
+    constructor(
+        listId: number,
+        rulesText: string,
+        ignoreCosmetic?: boolean,
+        ignoreJS?: boolean,
+        ignoreUnsafe?: boolean,
+    ) {
+        if (listId >= LIST_ID_MAX_VALUE) {
+            throw new Error(`Invalid list identifier, it must be less than ${LIST_ID_MAX_VALUE}`);
+        }
+
+        this.id = listId;
+        const encoder = new TextEncoder();
+        this.rulesBuffer = encoder.encode(rulesText);
+        this.ignoreCosmetic = !!ignoreCosmetic;
+        this.ignoreJS = !!ignoreJS;
+        this.ignoreUnsafe = !!ignoreUnsafe;
+    }
+
+    /**
+     * Close does nothing as here's nothing to close in the BufferRuleList.
+     */
+    // eslint-disable-next-line class-methods-use-this
+    public close(): void {
+        // Empty
+    }
+
+    /**
+     * @return - The rule list identifier
+     */
+    getId(): number {
+        return this.id;
+    }
+
+    /**
+     * Creates a new rules scanner that reads the list contents.
+     *
+     * @return - Scanner object.
+     */
+    newScanner(scannerType: ScannerType): RuleScanner {
+        const reader = new BufferLineReader(this.rulesBuffer);
+        return new RuleScanner(reader, this.id, {
+            scannerType,
+            ignoreCosmetic: this.ignoreCosmetic,
+            ignoreJS: this.ignoreJS,
+            ignoreUnsafe: this.ignoreUnsafe,
+        });
+    }
+
+    /**
+     * Finds rule text by its index.
+     *
+     * If there's no rule by that index or the rule is invalid, it will return
+     * null.
+     *
+     * @param ruleIdx - rule index.
+     * @return - rule text or null.
+     */
+    retrieveRuleText(ruleIdx: number): string | null {
+        if (ruleIdx < 0 || ruleIdx >= this.rulesBuffer.length) {
+            return null;
+        }
+
+        let endOfLine = this.rulesBuffer.indexOf(BufferLineReader.EOL, ruleIdx);
+        if (endOfLine === -1) {
+            endOfLine = this.rulesBuffer.length;
+        }
+
+        const lineBuffer = this.rulesBuffer.subarray(ruleIdx, endOfLine);
+        const line = BufferRuleList.decoder.decode(lineBuffer).trim();
+
+        if (!line) {
+            return null;
+        }
+
+        return line;
+    }
+}

packages/tsurlfilter/src/filterlist/list-cache.ts

@@ -1,36 +1,43 @@
 import { IRule } from '../rules/rule';
 
 /**
- * Rule list's cache
+ * Cache of an individual filter list.
  */
 export class ListCache {
     /**
-     * Cache with the rules which were retrieved.
+     * Cache with the rules which are stored inside this cache instance..
      */
     private readonly cache: Map<number, IRule>;
 
     /**
-     * Constructor
+     * ListCache constructor.
      */
     constructor() {
         this.cache = new Map();
     }
 
     /**
-     * @param key
-     * @return rule for specified key
+     * @param key - Cache key.
+     * @return - Rule found for specified key or undefined if nothing found.
      */
     public get(key: number): IRule | undefined {
         return this.cache.get(key);
     }
 
     /**
-     * Sets rule for specified key
+     * Stores the rule for the specified key in the cache.
      *
-     * @param key
-     * @param rule
+     * @param key - Cache key.
+     * @param rule - Cached value.
      */
     public set(key: number, rule: IRule): void {
         this.cache.set(key, rule);
     }
+
+    /**
+     * @returns - The list cache size.
+     */
+    public getSize() {
+        return this.cache.size;
+    }
 }

packages/tsurlfilter/src/filterlist/reader/buffer-line-reader.ts

@@ -0,0 +1,74 @@
+import { type ILineReader } from './line-reader';
+
+/**
+ * BufferLineReader is a class responsible for reading content line by line
+ * from a bytes buffer with a UTF-8 encoded string.
+ */
+export class BufferLineReader implements ILineReader {
+    /**
+     * EOL is a new line character that is used to detect line endings. We only
+     * rely on \n and not \r so the lines need to be trimmed after processing.
+     */
+    public static readonly EOL = '\n'.charCodeAt(0);
+
+    /**
+     * Byte buffer with a UTF-8 encoded string.
+     */
+    private readonly buffer: Uint8Array;
+
+    /**
+     * Current position of the reader.
+     */
+    private currentIndex = 0;
+
+    /**
+     * Text decoder that is used to read strings from the internal buffer of
+     * UTF-8 encoded characters.
+     */
+    private static readonly decoder = new TextDecoder('utf-8');
+
+    /**
+     * Constructor of a BufferLineReader.
+     *
+     * @param buffer - Uint8Array that contains a UTF-8 encoded string.
+     */
+    constructor(buffer: Uint8Array) {
+        this.buffer = buffer;
+    }
+
+    /**
+     * Reads the next line in the buffer
+     *
+     * @return text or null on end
+     */
+    public readLine(): string | null {
+        if (this.currentIndex === -1) {
+            return null;
+        }
+
+        const startIndex = this.currentIndex;
+        this.currentIndex = this.buffer.indexOf(BufferLineReader.EOL, startIndex);
+
+        if (this.currentIndex === -1) {
+            return BufferLineReader.decoder.decode(this.buffer.subarray(startIndex));
+        }
+
+        const lineBytes = this.buffer.subarray(startIndex, this.currentIndex);
+        const line = BufferLineReader.decoder.decode(lineBytes);
+
+        // Increment to not include the EOL character.
+        this.currentIndex += 1;
+
+        return line;
+    }
+
+    /**
+     * Returns the current position of this reader or -1 if there's nothing to
+     * read.
+     *
+     * @returns - The current position or -1 if there's nothing to read.
+     */
+    public getCurrentPos(): number {
+        return this.currentIndex;
+    }
+}

packages/tsurlfilter/src/filterlist/reader/file-line-reader.ts

@@ -1,31 +1,39 @@
 import fs from 'fs';
-import { ILineReader } from './line-reader';
-import { StringLineReader } from './string-line-reader';
+import { type ILineReader } from './line-reader';
+import { BufferLineReader } from './buffer-line-reader';
 
 /**
- * Reads file line by line
+ * FileLineReader is a class responsible for reading file contents line by line.
  */
 export class FileLineReader implements ILineReader {
     /**
-     * Temp implementation inner reader
+     * FileLineReader relies on an internal BufferLineReader to provide the line
+     * reading functionality.
      */
-    private readonly innerReader: StringLineReader;
+    private readonly innerReader: BufferLineReader;
 
     /**
-     * Constructor
-     * @param path
+     * Constructor of the FileLineReader.
      *
-     * @throws
+     * @param path - Path to the file to read.
+     * @throws Error if the file cannot be read.
      */
     constructor(path: string) {
-        const text = fs.readFileSync(path, 'utf8');
-        this.innerReader = new StringLineReader(text);
+        const buffer = fs.readFileSync(path);
+        this.innerReader = new BufferLineReader(buffer);
     }
 
     /**
-     * Reads next line
+     * Reads next line in the reader.
      */
     public readLine(): string | null {
         return this.innerReader.readLine();
     }
+
+    /**
+     * Returns the current position of this line reader.
+     */
+    getCurrentPos(): number {
+        return this.innerReader.getCurrentPos();
+    }
 }

packages/tsurlfilter/src/filterlist/reader/line-reader.ts

@@ -3,9 +3,14 @@
  */
 export interface ILineReader {
     /**
-     * Reads the next line
+     * Reads the next line.
      *
-     * @return line string or null
+     * @return line string or null.
      */
     readLine(): string | null;
+
+    /**
+     * Returns the current position of this line reader.
+     */
+    getCurrentPos(): number;
 }