Replace TokenParser with shlex.split

news/453.removal.md

@@ -0,0 +1 @@
+Remove `TokenParser` and replace it with `helpers.tokenize` that maps directly to `shlex.split`.

src/cleo/helpers.py

@@ -40,3 +40,12 @@ def option(
         description=description,
         default=default,
     )
+
+
+def tokenize(string: str) -> list[str]:  # pragma: no cover
+    """
+    Split the string s using shell-like syntax. Maps directly to using `shlex.split`
+    """
+    import shlex
+
+    return shlex.split(string)

src/cleo/io/inputs/string_input.py

@@ -1,7 +1,7 @@
 from __future__ import annotations
 
+from cleo.helpers import tokenize
 from cleo.io.inputs.argv_input import ArgvInput
-from cleo.io.inputs.token_parser import TokenParser
 
 
 class StringInput(ArgvInput):
@@ -15,4 +15,4 @@ def __init__(self, input: str) -> None:
         self._set_tokens(self._tokenize(input))
 
     def _tokenize(self, input: str) -> list[str]:
-        return TokenParser().parse(input)
+        return tokenize(input)