AG-38480 Add stop functionality to CSS tokenizer callback and implement hasToken function

Squashed commit of the following:

commit c7c02e5
Merge: 0984c52 be39a69
Author: jellizaveta <[email protected]>
Date:   Mon Feb 3 17:42:56 2025 +0300

    Merge branch 'master' into fix/AG-38480_2

commit 0984c52
Author: jellizaveta <[email protected]>
Date:   Mon Feb 3 16:25:34 2025 +0300

    update changelog and readme

commit 3dab213
Author: jellizaveta <[email protected]>
Date:   Mon Feb 3 16:23:49 2025 +0300

    add test

commit 9cd8541
Author: jellizaveta <[email protected]>
Date:   Fri Jan 31 13:05:55 2025 +0300

    add jsDoc

commit 5a62369
Author: jellizaveta <[email protected]>
Date:   Fri Jan 31 12:15:59 2025 +0300

    update changelog, package.json

commit 29da501
Author: Dávid Tóta <[email protected]>
Date:   Fri Jan 31 12:03:47 2025 +0300

    Applied suggestion

commit f8d2593
Author: Dávid Tóta <[email protected]>
Date:   Fri Jan 31 12:03:39 2025 +0300

    Applied suggestion

commit 5063cc0
Merge: 80e3e1e 193e35c
Author: jellizaveta <[email protected]>
Date:   Fri Jan 31 10:22:51 2025 +0300

    merge master, resolve conflicts

commit 80e3e1e
Author: jellizaveta <[email protected]>
Date:   Fri Jan 31 10:16:07 2025 +0300

    fix date in changelog

commit 8dd7ada
Author: jellizaveta <[email protected]>
Date:   Fri Jan 31 10:15:22 2025 +0300

    fix arrow function

commit 25fc5c9
Author: jellizaveta <[email protected]>
Date:   Fri Jan 31 10:08:06 2025 +0300

    remove indent

commit add1b6b
Author: jellizaveta <[email protected]>
Date:   Fri Jan 31 10:07:09 2025 +0300

    update stop function

commit 4502a69
Author: jellizaveta <[email protected]>
Date:   Thu Jan 30 19:11:53 2025 +0300

    bind stop function to the context

commit 5699351
Author: Dávid Tóta <[email protected]>
Date:   Thu Jan 30 18:00:21 2025 +0300

    Applied suggestion

commit 2ed879d
Author: jellizaveta <[email protected]>
Date:   Thu Jan 30 15:44:16 2025 +0300

    fix order with balance

commit 8eff1ed
Author: jellizaveta <[email protected]>
Date:   Thu Jan 30 14:19:04 2025 +0300

    remove import

commit 71fd503
Author: jellizaveta <[email protected]>
Date:   Thu Jan 30 14:10:16 2025 +0300

    add indent

commit 3c2eeba
Author: jellizaveta <[email protected]>
Date:   Thu Jan 30 14:09:17 2025 +0300

    update tests

commit 7e691c6
Author: jellizaveta <[email protected]>
Date:   Thu Jan 30 13:48:30 2025 +0300

    update changelog date

commit d5937cd
Author: jellizaveta <[email protected]>
Date:   Thu Jan 30 13:46:55 2025 +0300

    update function-token.test.ts

... and 21 more commits

Author: jellizaveta

packages/agtree/CHANGELOG.md

@@ -8,6 +8,14 @@ The format is based on [Keep a Changelog], and this project adheres to [Semantic
 [Keep a Changelog]: https://keepachangelog.com/en/1.0.0/
 [Semantic Versioning]: https://semver.org/spec/v2.0.0.html
 
+## [3.0.0-alpha.4] - 2024-02-03
+
+### Changed
+
+- Updated [@adguard/css-tokenizer] to `v1.2.0` which introduces the `hasToken` function.
+
+[3.0.0-alpha.4]: https://github.com/AdguardTeam/tsurlfilter/releases/tag/agtree-3.0.0-alpha.4
+
 ## [3.0.0-alpha.3] - 2025-01-30
 
 ### Changed

packages/agtree/package.json

@@ -1,6 +1,6 @@
 {
     "name": "@adguard/agtree",
-    "version": "3.0.0-alpha.3",
+    "version": "3.0.0-alpha.4",
     "description": "Tool set for working with adblock filter lists",
     "keywords": [
         "adblock",

packages/agtree/src/parser/cosmetic/cosmetic-rule-parser.ts

@@ -1,5 +1,5 @@
 import { sprintf } from 'sprintf-js';
-import { TokenType } from '@adguard/css-tokenizer';
+import { hasToken, TokenType } from '@adguard/css-tokenizer';
 
 import { CosmeticRuleSeparatorUtils } from '../../utils/cosmetic-rule-separator';
 import { AdblockSyntax } from '../../utils/adblockers';
@@ -44,7 +44,6 @@ import { UboScriptletInjectionBodyParser } from './body/ubo-scriptlet-injection-
 import { AdgScriptletInjectionBodyParser } from './body/adg-scriptlet-injection-body-parser';
 import { BaseParser } from '../base-parser';
 import { UboPseudoName } from '../../common/ubo-selector-common';
-import { hasToken } from '../css/has-token';
 
 /**
  * Possible error messages for uBO selectors. Formatted with {@link sprintf}.

packages/agtree/src/parser/css/balancing.ts

@@ -15,6 +15,38 @@ import { sprintf } from 'sprintf-js';
 import { AdblockSyntaxError } from '../../errors/adblock-syntax-error';
 import { END_OF_INPUT, ERROR_MESSAGES } from './constants';
 
+/**
+ * Utility type to get the last element from a tuple, handling optional last elements correctly.
+ *
+ * @param T - The tuple to extract the last element from.
+ * @returns The last element of the tuple if present; `L | undefined` if the last element is optional.
+ */
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+type Last<T extends unknown[]> = T extends [...infer _I, infer L]
+    ? L
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    : T extends [...infer _I, (infer L)?] ? L | undefined : never;
+
+/**
+ * Utility type to remove the last element from a tuple, handling optional last elements correctly.
+ *
+ * @param T - The tuple to remove the last element from.
+ * @returns A tuple without the last element. If the last element is optional, it is also removed.
+ */
+// eslint-disable-next-line @typescript-eslint/no-unused-vars
+type OmitLast<T extends unknown[]> = T extends [...infer Rest, infer _Last]
+    ? Rest
+    // Handles cases where the last element is optional
+    // eslint-disable-next-line @typescript-eslint/no-unused-vars
+    : T extends [...infer Rest, (infer _Last)?]
+        ? Rest
+        : never;
+
+/**
+ * Extracts the parameters of `OnTokenCallback` as a tuple.
+ */
+type OnTokenCallbackParameters = Parameters<OnTokenCallback>;
+
 /**
  * Extended version of `OnTokenCallback` which also receives a `balance` parameter.
  *
@@ -23,11 +55,12 @@ import { END_OF_INPUT, ERROR_MESSAGES } from './constants';
  * @param end End index in the source string.
  * @param props Additional properties of the token (if any - can be `undefined`, depending on the token type).
  * @param balance Calculated balance level of the token.
+ * @param stop Function to halt tokenization.
  * @note This function is keeping the same signature as the original `OnTokenCallback` to avoid breaking changes,
  * just adding the `balance` parameter at the end.
  */
 export type OnBalancedTokenCallback = (
-    ...args: [...Parameters<OnTokenCallback>, ...[balance: number]]
+    ...args: [...OmitLast<OnTokenCallbackParameters>, balance: number, Last<OnTokenCallbackParameters>]
 ) => ReturnType<OnTokenCallback>;
 
 /**
@@ -72,7 +105,7 @@ const tokenizeWithBalancedPairs = (
 
     tokenizeExtended(
         raw,
-        (type: TokenType, start, end, props) => {
+        (type: TokenType, start, end, props, stop) => {
             if (tokenPairs.has(type)) {
                 // If the token is an opening token, push its corresponding closing token to the stack.
                 // It is safe to use non-null assertion here, because we have checked that the token exists in the map.
@@ -95,7 +128,7 @@ const tokenizeWithBalancedPairs = (
                 }
             }
 
-            onToken(type, start, end, props, stack.length);
+            onToken(type, start, end, props, stack.length, stop);
         },
         onError,
         functionHandlers,

packages/agtree/src/parser/css/has-token.ts

@@ -7,6 +7,14 @@ The format is based on [Keep a Changelog][keepachangelog], and this project adhe
 [keepachangelog]: https://keepachangelog.com/en/1.0.0/
 [semver]: https://semver.org/spec/v2.0.0.html
 
+## [1.2.0] - 2025-01-31
+
+### Added
+
+- Stop functionality to `CSS tokenizer` callback and implement `hasToken` function.
+
+[1.2.0]: https://github.com/AdguardTeam/tsurlfilter/releases/tag/css-tokenizer-v1.2.0
+
 ## [1.1.1] - 2024-09-19
 
 ### Fixed

packages/css-tokenizer/CHANGELOG.md

@@ -24,6 +24,7 @@ Table of contents:
         - [`tokenize`](#tokenize)
         - [`tokenizeExtended`](#tokenizeextended)
     - [Utilities](#utilities)
+        - [`hasToken`](#hastoken)
         - [`TokenizerContext`](#tokenizercontext)
         - [`decodeIdent`](#decodeident)
     - [`CSS_TOKENIZER_VERSION`](#css_tokenizer_version)
@@ -176,10 +177,17 @@ where
  * @param start Token start offset
  * @param end Token end offset
  * @param props Other token properties (if any)
+ * @param stop Function to halt the tokenization process
  * @note Hash tokens have a type flag set to either "id" or "unrestricted". The type flag defaults to "unrestricted" if
  * not otherwise set
  */
-type OnTokenCallback = (type: TokenType, start: number, end: number, props?: Record<string, unknown>) => void;
+type OnTokenCallback = (
+    type: TokenType,
+    start: number,
+    end: number,
+    props: Record<string, unknown> | undefined,
+    stop: () => void
+);
 ```
 
 ```ts
@@ -232,6 +240,27 @@ function tokenizeExtended(
 
 ### Utilities
 
+#### `hasToken`
+
+```ts
+/**
+ * Checks if the given raw string contains any of the specified tokens.
+ *
+ * @param raw - The raw string to be tokenized and checked.
+ * @param tokens - A set of token types to check for in the raw string.
+ * @param tokenizer - The tokenizer function to use. Defaults to `tokenizeExtended`.
+ *
+ * @example hasToken('div:contains("foo")', new Set([TokenType.Function]), tokenizeExtended); // true
+ *
+ * @returns `true` if any of the specified tokens are found in the raw string, otherwise `false`.
+ */
+function hasToken = (
+    raw: string,
+    tokens: Set<TokenType>,
+    tokenizer: TokenizerFunction = tokenizeExtended,
+): boolean
+```
+
 #### `TokenizerContext`
 
 A class that represents the tokenizer context. It is used to manage the tokenizer state and provides access to the