Fixed Ignored objects being included in the output

CHANGELOG.md

@@ -5,6 +5,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [1.2.2] - 2022-11-18
+
+### Fixed
+- Fixed `Ignored` objects being included in the output
+
 ## [1.2.1] - 2022-11-18
 
 ### Fixed
@@ -35,3 +40,4 @@ Initial release 🎉
 [1.1.0]: https://github.com/samarium-lang/Samarium/compare/1.0.0...1.1.0
 [1.2.0]: https://github.com/samarium-lang/Samarium/compare/1.1.0...1.2.0
 [1.2.1]: https://github.com/samarium-lang/Samarium/compare/1.2.0...1.2.1
+[1.2.2]: https://github.com/samarium-lang/Samarium/compare/1.2.1...1.2.2

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "crossandra"
-version = "1.2.1"
+version = "1.2.2"
 description = "A simple tokenizer operating on enums with a decent amount of configuration"
 authors = ["trag1c <[email protected]>"]
 license = "MIT"

src/crossandra/lib.py

@@ -76,7 +76,7 @@ def tokenize(self, code: str) -> list[Enum | Any]:
                 tok = rule.apply(code)
                 if not isinstance(tok, NotApplied):
                     token_, length = tok
-                    if not isinstance(tok, Ignored):
+                    if not isinstance(token_, Ignored):
                         t_append(token_)
                     code = code[length:]
                     break