diff --git a/crates/biome_css_parser/src/lib.rs b/crates/biome_css_parser/src/lib.rs index c18459fc3cd9..6d820fe6b1f6 100644 --- a/crates/biome_css_parser/src/lib.rs +++ b/crates/biome_css_parser/src/lib.rs @@ -13,6 +13,7 @@ pub use parser::CssParserOptions; mod lexer; mod parser; mod prelude; +mod state; mod syntax; mod token_source; diff --git a/crates/biome_css_parser/src/parser.rs b/crates/biome_css_parser/src/parser.rs index 32f793969a9b..4daf37df363a 100644 --- a/crates/biome_css_parser/src/parser.rs +++ b/crates/biome_css_parser/src/parser.rs @@ -1,15 +1,17 @@ use crate::lexer::CssReLexContext; -use crate::token_source::CssTokenSource; +use crate::state::CssParserState; +use crate::token_source::{CssTokenSource, CssTokenSourceCheckpoint}; use biome_css_syntax::CssSyntaxKind; use biome_parser::diagnostic::merge_diagnostics; use biome_parser::event::Event; -use biome_parser::prelude::*; use biome_parser::token_source::Trivia; use biome_parser::ParserContext; +use biome_parser::{prelude::*, ParserContextCheckpoint}; pub(crate) struct CssParser<'source> { context: ParserContext, source: CssTokenSource<'source>, + state: CssParserState, } #[derive(Default, Debug, Clone, Copy)] @@ -29,6 +31,7 @@ impl<'source> CssParser<'source> { Self { context: ParserContext::default(), source: CssTokenSource::from_str(source, config), + state: CssParserState::new(), } } @@ -39,6 +42,35 @@ impl<'source> CssParser<'source> { self.source_mut().re_lex(context) } + #[allow(dead_code)] //TODO remove this allow once we actually use it + pub(crate) fn state(&self) -> &CssParserState { + &self.state + } + + pub(crate) fn state_mut(&mut self) -> &mut CssParserState { + &mut self.state + } + + pub fn checkpoint(&self) -> CssParserCheckpoint { + CssParserCheckpoint { + context: self.context.checkpoint(), + source: self.source.checkpoint(), + // `state` is not checkpointed because it (currently) only contains + // scoped properties that aren't only dependent on checkpoints and + // should be reset manually when the scope of their use is exited. + } + } + + pub fn rewind(&mut self, checkpoint: CssParserCheckpoint) { + let CssParserCheckpoint { context, source } = checkpoint; + + self.context.rewind(context); + self.source.rewind(source); + // `state` is not checkpointed because it (currently) only contains + // scoped properties that aren't only dependent on checkpoints and + // should be reset manually when the scope of their use is exited. + } + pub fn finish(self) -> (Vec>, Vec, Vec) { let (trivia, lexer_diagnostics) = self.source.finish(); let (events, parse_diagnostics) = self.context.finish(); @@ -68,4 +100,16 @@ impl<'source> Parser for CssParser<'source> { fn source_mut(&mut self) -> &mut Self::Source { &mut self.source } + + fn is_speculative_parsing(&self) -> bool { + self.state.speculative_parsing + } +} + +pub struct CssParserCheckpoint { + pub(super) context: ParserContextCheckpoint, + pub(super) source: CssTokenSourceCheckpoint, + // `state` is not checkpointed because it (currently) only contains + // scoped properties that aren't only dependent on checkpoints and + // should be reset manually when the scope of their use is exited. } diff --git a/crates/biome_css_parser/src/state.rs b/crates/biome_css_parser/src/state.rs new file mode 100644 index 000000000000..098f104d7061 --- /dev/null +++ b/crates/biome_css_parser/src/state.rs @@ -0,0 +1,22 @@ +pub(crate) struct CssParserState { + /// Indicates that the parser is speculatively parsing a syntax. Speculative parsing means that the + /// parser tries to parse a syntax as one kind and determines at the end if the assumption was right + /// by testing if the parser is at a specific token (or has no errors). For this approach to work, + /// the parser isn't allowed to skip any tokens while doing error recovery because it may then successfully + /// skip over all invalid tokens, so that it appears as if it was able to parse the syntax correctly. + /// + /// Speculative parsing is useful if a syntax is ambiguous and no amount of lookahead (except parsing the whole syntax) + /// is sufficient to determine what syntax it is. For example, the syntax `(a, b) ...` + /// in JavaScript is either a parenthesized expression or an arrow expression if `...` is a `=>`. + /// The challenge is, that it isn't possible to tell which of the two kinds it is until the parser + /// processed all of `(a, b)`. + pub(crate) speculative_parsing: bool, +} + +impl CssParserState { + pub fn new() -> Self { + Self { + speculative_parsing: false, + } + } +} diff --git a/crates/biome_css_parser/src/syntax/mod.rs b/crates/biome_css_parser/src/syntax/mod.rs index b87b1b07da64..06fb38df14ca 100644 --- a/crates/biome_css_parser/src/syntax/mod.rs +++ b/crates/biome_css_parser/src/syntax/mod.rs @@ -601,3 +601,86 @@ pub(crate) fn parse_string(p: &mut CssParser) -> ParsedSyntax { fn is_at_string(p: &mut CssParser) -> bool { p.at(CSS_STRING_LITERAL) } + +/// Attempt to parse some input with the given parsing function. If parsing +/// succeeds, `Ok` is returned with the result of the parse and the state is +/// preserved. If parsing fails, this function rewinds the parser back to +/// where it was before attempting the parse and the `Err` value is returned. +#[allow(dead_code)] // TODO: Remove this allow once it's actually used +pub(crate) fn try_parse( + p: &mut CssParser, + func: impl FnOnce(&mut CssParser) -> Result, +) -> Result { + let checkpoint = p.checkpoint(); + let old_speculative_parsing = std::mem::replace(&mut p.state_mut().speculative_parsing, true); + + let res = func(p); + p.state_mut().speculative_parsing = old_speculative_parsing; + + if res.is_err() { + p.rewind(checkpoint); + } + + res +} + +#[cfg(test)] +mod tests { + use crate::{parser::CssParser, CssParserOptions}; + use biome_css_syntax::{CssSyntaxKind, T}; + use biome_parser::prelude::ParsedSyntax::{Absent, Present}; + use biome_parser::Parser; + + use super::{parse_regular_identifier, parse_regular_number, try_parse}; + + #[test] + fn try_parse_rewinds_to_checkpoint() { + let mut p = CssParser::new("width: blue;", CssParserOptions::default()); + + let pre_try_range = p.cur_range(); + let result = try_parse(&mut p, |p| { + // advance the parser within the attempt + // parse `width` + parse_regular_identifier(p).ok(); + // parse `:` + p.expect(T![:]); + + // attempt to parse a number, but fail because the input has `blue`. + match parse_regular_number(p) { + Present(marker) => Ok(Present(marker)), + Absent => Err(()), + } + }); + + assert!(result.is_err()); + // The parser should've rewound back to the start. + assert_eq!(p.cur_range(), pre_try_range); + assert_eq!(p.cur_text(), "width"); + } + + #[test] + fn try_parse_preserves_position_on_success() { + let mut p = CssParser::new("width: 100;", CssParserOptions::default()); + + let pre_try_range = p.cur_range(); + let result = try_parse(&mut p, |p| { + // advance the parser within the attempt + // parse `width` + parse_regular_identifier(p).ok(); + // parse `:` + p.expect(T![:]); + + // attempt to parse a number, and succeed because the input has `100`. + match parse_regular_number(p) { + Present(marker) => Ok(Present(marker)), + Absent => Err(()), + } + }); + + assert!(result.is_ok()); + assert_eq!(result.unwrap().kind(&p), Some(CssSyntaxKind::CSS_NUMBER)); + // The parser should not have rewound and is now at the semicolon + assert_ne!(p.cur_range(), pre_try_range); + assert_eq!(p.cur_text(), ";"); + } +} diff --git a/crates/biome_css_parser/src/token_source.rs b/crates/biome_css_parser/src/token_source.rs index 71f5ecbbbb0a..61d0db8162a2 100644 --- a/crates/biome_css_parser/src/token_source.rs +++ b/crates/biome_css_parser/src/token_source.rs @@ -5,7 +5,7 @@ use biome_css_syntax::{CssSyntaxKind, TextRange}; use biome_parser::diagnostic::ParseDiagnostic; use biome_parser::lexer::{BufferedLexer, LexContext}; use biome_parser::prelude::{BumpWithContext, NthToken, TokenSource}; -use biome_parser::token_source::Trivia; +use biome_parser::token_source::{TokenSourceCheckpoint, Trivia}; use biome_rowan::TriviaPieceKind; use std::collections::VecDeque; @@ -34,6 +34,8 @@ struct Lookahead { after_newline: bool, } +pub(crate) type CssTokenSourceCheckpoint = TokenSourceCheckpoint; + impl<'src> CssTokenSource<'src> { /// Creates a new token source. pub(crate) fn new(lexer: BufferedLexer<'src, CssLexer<'src>>) -> CssTokenSource<'src> { @@ -139,6 +141,23 @@ impl<'src> CssTokenSource<'src> { None } + + /// Creates a checkpoint to which it can later return using [Self::rewind]. + pub fn checkpoint(&self) -> CssTokenSourceCheckpoint { + CssTokenSourceCheckpoint { + trivia_len: self.trivia_list.len() as u32, + lexer_checkpoint: self.lexer.checkpoint(), + } + } + + /// Restores the token source to a previous state + pub fn rewind(&mut self, checkpoint: CssTokenSourceCheckpoint) { + assert!(self.trivia_list.len() >= checkpoint.trivia_len as usize); + self.trivia_list.truncate(checkpoint.trivia_len as usize); + self.lexer.rewind(checkpoint.lexer_checkpoint); + self.non_trivia_lookahead.clear(); + self.lookahead_offset = 0; + } } impl<'source> TokenSource for CssTokenSource<'source> { diff --git a/crates/biome_js_parser/src/lib.rs b/crates/biome_js_parser/src/lib.rs index 776d6e060f9f..55b81fc41f05 100644 --- a/crates/biome_js_parser/src/lib.rs +++ b/crates/biome_js_parser/src/lib.rs @@ -139,7 +139,7 @@ use biome_js_factory::JsSyntaxFactory; use biome_js_syntax::{JsLanguage, JsSyntaxKind, LanguageVariant}; use biome_parser::tree_sink::LosslessTreeSink; pub(crate) use parser::{JsParser, ParseRecoveryTokenSet}; -pub(crate) use state::{ParserState, StrictMode}; +pub(crate) use state::{JsParserState, StrictMode}; use std::fmt::Debug; pub enum JsSyntaxFeature { diff --git a/crates/biome_js_parser/src/parser.rs b/crates/biome_js_parser/src/parser.rs index 5a01e0276efe..faa1c5ae1109 100644 --- a/crates/biome_js_parser/src/parser.rs +++ b/crates/biome_js_parser/src/parser.rs @@ -13,11 +13,9 @@ pub(crate) use crate::parser::parse_recovery::{ }; use crate::prelude::*; use crate::state::{ChangeParserState, ParserStateGuard}; +use crate::token_source::JsTokenSourceCheckpoint; use crate::*; -use crate::{ - state::ParserStateCheckpoint, - token_source::{JsTokenSource, TokenSourceCheckpoint}, -}; +use crate::{state::JsParserStateCheckpoint, token_source::JsTokenSource}; use biome_js_syntax::{ JsFileSource, JsSyntaxKind::{self}, @@ -33,7 +31,7 @@ pub(crate) use parsed_syntax::ParsedSyntax; /// The Parser yields lower level events instead of nodes. /// These events are then processed into a syntax tree through a [`TreeSink`] implementation. pub struct JsParser<'source> { - pub(super) state: ParserState, + pub(super) state: JsParserState, pub source_type: JsFileSource, context: ParserContext, source: JsTokenSource<'source>, @@ -46,7 +44,7 @@ impl<'source> JsParser<'source> { let source = JsTokenSource::from_str(source); JsParser { - state: ParserState::new(&source_type), + state: JsParserState::new(&source_type), source_type, context: ParserContext::default(), source, @@ -54,7 +52,7 @@ impl<'source> JsParser<'source> { } } - pub(crate) fn state(&self) -> &ParserState { + pub(crate) fn state(&self) -> &JsParserState { &self.state } @@ -62,7 +60,7 @@ impl<'source> JsParser<'source> { &self.options } - pub(crate) fn state_mut(&mut self) -> &mut ParserState { + pub(crate) fn state_mut(&mut self) -> &mut JsParserState { &mut self.state } @@ -213,8 +211,8 @@ impl<'source> Parser for JsParser<'source> { pub struct JsParserCheckpoint { pub(super) context: ParserContextCheckpoint, - pub(super) source: TokenSourceCheckpoint, - state: ParserStateCheckpoint, + pub(super) source: JsTokenSourceCheckpoint, + state: JsParserStateCheckpoint, } #[cfg(test)] diff --git a/crates/biome_js_parser/src/parser/rewrite_parser.rs b/crates/biome_js_parser/src/parser/rewrite_parser.rs index 8b80e8b44bd0..bb473479a305 100644 --- a/crates/biome_js_parser/src/parser/rewrite_parser.rs +++ b/crates/biome_js_parser/src/parser/rewrite_parser.rs @@ -1,5 +1,4 @@ -use crate::parser::JsParser; -use crate::token_source::TokenSourceCheckpoint; +use crate::{parser::JsParser, token_source::JsTokenSourceCheckpoint}; use crate::prelude::*; use biome_console::fmt::Display; @@ -34,7 +33,7 @@ pub(crate) struct RewriteParser<'parser, 'source> { } impl<'parser, 'source> RewriteParser<'parser, 'source> { - pub fn new(p: &'parser mut JsParser<'source>, checkpoint: TokenSourceCheckpoint) -> Self { + pub fn new(p: &'parser mut JsParser<'source>, checkpoint: JsTokenSourceCheckpoint) -> Self { Self { inner: p, offset: checkpoint.current_start(), diff --git a/crates/biome_js_parser/src/state.rs b/crates/biome_js_parser/src/state.rs index 54614ab0e254..3995cdbb3a8b 100644 --- a/crates/biome_js_parser/src/state.rs +++ b/crates/biome_js_parser/src/state.rs @@ -85,7 +85,7 @@ pub(crate) struct ExportDefaultItem { /// State kept by the parser while parsing. /// It is required for things such as strict mode or async functions #[derive(Debug)] -pub(crate) struct ParserState { +pub(crate) struct JsParserState { parsing_context: ParsingContextFlags, /// A list of labels for labelled statements used to report undefined label errors /// for break and continue, as well as duplicate labels. @@ -126,9 +126,9 @@ pub(crate) enum StrictMode { Class(TextRange), } -impl ParserState { +impl JsParserState { pub fn new(source_type: &JsFileSource) -> Self { - let mut state = ParserState { + let mut state = JsParserState { parsing_context: ParsingContextFlags::TOP_LEVEL, label_set: IndexMap::new(), strict: source_type @@ -201,58 +201,58 @@ impl ParserState { self.label_set.get(label) } - pub(super) fn checkpoint(&self) -> ParserStateCheckpoint { - ParserStateCheckpoint::snapshot(self) + pub(super) fn checkpoint(&self) -> JsParserStateCheckpoint { + JsParserStateCheckpoint::snapshot(self) } - pub(super) fn restore(&mut self, checkpoint: ParserStateCheckpoint) { + pub(super) fn restore(&mut self, checkpoint: JsParserStateCheckpoint) { checkpoint.rewind(self); } } -/// Stores a checkpoint of the [ParserState]. +/// Stores a checkpoint of the [JsParserState]. /// Allows rewinding the state to its previous state. /// /// It's important that creating and rewinding a snapshot is cheap. Consider the performance implications /// before adding new unscoped state. #[derive(Debug)] -pub(super) struct ParserStateCheckpoint { +pub(super) struct JsParserStateCheckpoint { /// Additional data that we only want to store in debug mode #[cfg(debug_assertions)] - debug_checkpoint: DebugParserStateCheckpoint, + debug_checkpoint: JsDebugParserStateCheckpoint, } -impl ParserStateCheckpoint { +impl JsParserStateCheckpoint { /// Creates a snapshot of the passed in state. #[cfg(debug_assertions)] - fn snapshot(state: &ParserState) -> Self { + fn snapshot(state: &JsParserState) -> Self { Self { - debug_checkpoint: DebugParserStateCheckpoint::snapshot(state), + debug_checkpoint: JsDebugParserStateCheckpoint::snapshot(state), } } #[cfg(not(debug_assertions))] - fn snapshot(_: &ParserState) -> Self { + fn snapshot(_: &JsParserState) -> Self { Self {} } /// Restores the `state values` to the time when this snapshot was created. #[cfg(debug_assertions)] - fn rewind(self, state: &mut ParserState) { + fn rewind(self, state: &mut JsParserState) { self.debug_checkpoint.rewind(state); } #[cfg(not(debug_assertions))] - fn rewind(self, _: &ParserState) {} + fn rewind(self, _: &JsParserState) {} } -/// Most of the [ParserState] is scoped state. It should, therefore, not be necessary to rewind +/// Most of the [JsParserState] is scoped state. It should, therefore, not be necessary to rewind /// that state because that's already taken care of by `with_state` and `with_scoped_state`. /// But, you can never no and better be safe than sorry. That's why we use some heuristics /// to verify that non of the scoped state did change and assert for it when rewinding. #[derive(Debug, Clone)] #[cfg(debug_assertions)] -pub(super) struct DebugParserStateCheckpoint { +pub(super) struct JsDebugParserStateCheckpoint { parsing_context: ParsingContextFlags, label_set_len: usize, strict: Option, @@ -262,8 +262,8 @@ pub(super) struct DebugParserStateCheckpoint { } #[cfg(debug_assertions)] -impl DebugParserStateCheckpoint { - fn snapshot(state: &ParserState) -> Self { +impl JsDebugParserStateCheckpoint { + fn snapshot(state: &JsParserState) -> Self { Self { parsing_context: state.parsing_context, label_set_len: state.label_set.len(), @@ -274,7 +274,7 @@ impl DebugParserStateCheckpoint { } } - fn rewind(self, state: &mut ParserState) { + fn rewind(self, state: &mut JsParserState) { assert_eq!(state.parsing_context, self.parsing_context); assert_eq!(state.label_set.len(), self.label_set_len); assert_eq!(state.strict, self.strict); @@ -333,10 +333,10 @@ pub(crate) trait ChangeParserState { type Snapshot: Default; /// Applies the change to the passed in state and returns snapshot that allows restoring the previous state. - fn apply(self, state: &mut ParserState) -> Self::Snapshot; + fn apply(self, state: &mut JsParserState) -> Self::Snapshot; /// Restores the state to its previous value - fn restore(state: &mut ParserState, value: Self::Snapshot); + fn restore(state: &mut JsParserState, value: Self::Snapshot); } #[derive(Default, Debug)] @@ -349,12 +349,12 @@ impl ChangeParserState for EnableStrictMode { type Snapshot = EnableStrictModeSnapshot; #[inline] - fn apply(self, state: &mut ParserState) -> Self::Snapshot { + fn apply(self, state: &mut JsParserState) -> Self::Snapshot { EnableStrictModeSnapshot(std::mem::replace(&mut state.strict, Some(self.0))) } #[inline] - fn restore(state: &mut ParserState, value: Self::Snapshot) { + fn restore(state: &mut JsParserState, value: Self::Snapshot) { state.strict = value.0 } } @@ -448,12 +448,12 @@ pub(crate) trait ChangeParserStateFlags { impl ChangeParserState for T { type Snapshot = ParsingContextFlagsSnapshot; - fn apply(self, state: &mut ParserState) -> Self::Snapshot { + fn apply(self, state: &mut JsParserState) -> Self::Snapshot { let new_flags = self.compute_new_flags(state.parsing_context); ParsingContextFlagsSnapshot(std::mem::replace(&mut state.parsing_context, new_flags)) } - fn restore(state: &mut ParserState, value: Self::Snapshot) { + fn restore(state: &mut JsParserState, value: Self::Snapshot) { state.parsing_context = value.0 } } @@ -507,7 +507,7 @@ impl ChangeParserState for EnterFunction { type Snapshot = EnterFunctionSnapshot; #[inline] - fn apply(self, state: &mut ParserState) -> Self::Snapshot { + fn apply(self, state: &mut JsParserState) -> Self::Snapshot { let new_flags = (state.parsing_context - ParsingContextFlags::FUNCTION_RESET_MASK) | ParsingContextFlags::IN_FUNCTION | ParsingContextFlags::from(self.0); @@ -519,7 +519,7 @@ impl ChangeParserState for EnterFunction { } #[inline] - fn restore(state: &mut ParserState, value: Self::Snapshot) { + fn restore(state: &mut JsParserState, value: Self::Snapshot) { state.parsing_context = value.parsing_context; state.label_set = value.label_set; } @@ -547,7 +547,7 @@ pub(crate) struct EnterClassStaticInitializationBlock; impl ChangeParserState for EnterClassStaticInitializationBlock { type Snapshot = EnterClassStaticInitializationBlockSnapshot; - fn apply(self, state: &mut ParserState) -> Self::Snapshot { + fn apply(self, state: &mut JsParserState) -> Self::Snapshot { let flags = (state.parsing_context - ParsingContextFlags::FUNCTION_RESET_MASK - ParsingContextFlags::IN_FUNCTION) @@ -558,7 +558,7 @@ impl ChangeParserState for EnterClassStaticInitializationBlock { } } - fn restore(state: &mut ParserState, value: Self::Snapshot) { + fn restore(state: &mut JsParserState, value: Self::Snapshot) { state.parsing_context = value.flags; state.label_set = value.label_set; } @@ -577,7 +577,7 @@ pub(crate) struct WithLabel(pub String, pub LabelledItem); impl ChangeParserState for WithLabel { type Snapshot = WithLabelSnapshot; - fn apply(self, state: &mut ParserState) -> Self::Snapshot { + fn apply(self, state: &mut JsParserState) -> Self::Snapshot { #[cfg(debug_assertions)] let previous_len = state.label_set.len(); state.label_set.insert(self.0, self.1); @@ -591,12 +591,12 @@ impl ChangeParserState for WithLabel { } #[cfg(not(debug_assertions))] - fn restore(state: &mut ParserState, _: Self::Snapshot) { + fn restore(state: &mut JsParserState, _: Self::Snapshot) { state.label_set.pop(); } #[cfg(debug_assertions)] - fn restore(state: &mut ParserState, value: Self::Snapshot) { + fn restore(state: &mut JsParserState, value: Self::Snapshot) { assert_eq!(state.label_set.len(), value.label_set_len + 1); state.label_set.pop(); } @@ -623,7 +623,7 @@ pub(crate) struct EnterAmbientContext; impl ChangeParserState for EnterAmbientContext { type Snapshot = EnterAmbientContextSnapshot; - fn apply(self, state: &mut ParserState) -> Self::Snapshot { + fn apply(self, state: &mut JsParserState) -> Self::Snapshot { let new_flags = state.parsing_context | ParsingContextFlags::AMBIENT_CONTEXT; EnterAmbientContextSnapshot { flags: std::mem::replace(&mut state.parsing_context, new_flags), @@ -632,7 +632,7 @@ impl ChangeParserState for EnterAmbientContext { } } - fn restore(state: &mut ParserState, value: Self::Snapshot) { + fn restore(state: &mut JsParserState, value: Self::Snapshot) { state.parsing_context = value.flags; state.default_item = value.default_item; state.strict = value.strict_mode; diff --git a/crates/biome_js_parser/src/token_source.rs b/crates/biome_js_parser/src/token_source.rs index 186df2abb5a0..de7bbf30da9a 100644 --- a/crates/biome_js_parser/src/token_source.rs +++ b/crates/biome_js_parser/src/token_source.rs @@ -2,8 +2,8 @@ use crate::lexer::{JsLexContext, JsLexer, JsReLexContext, TextRange}; use crate::prelude::*; use biome_js_syntax::JsSyntaxKind; use biome_js_syntax::JsSyntaxKind::EOF; -use biome_parser::lexer::{BufferedLexer, LexContext, LexerCheckpoint}; -use biome_parser::token_source::Trivia; +use biome_parser::lexer::{BufferedLexer, LexContext}; +use biome_parser::token_source::{TokenSourceCheckpoint, Trivia}; use biome_rowan::{TextSize, TriviaPieceKind}; use std::collections::VecDeque; @@ -34,6 +34,8 @@ struct Lookahead { after_newline: bool, } +pub(crate) type JsTokenSourceCheckpoint = TokenSourceCheckpoint; + impl<'l> JsTokenSource<'l> { /// Creates a new token source. pub(crate) fn new(lexer: BufferedLexer<'l, JsLexer<'l>>) -> JsTokenSource<'l> { @@ -155,18 +157,18 @@ impl<'l> JsTokenSource<'l> { } /// Creates a checkpoint to which it can later return using [Self::rewind]. - pub fn checkpoint(&self) -> TokenSourceCheckpoint { - TokenSourceCheckpoint { + pub fn checkpoint(&self) -> JsTokenSourceCheckpoint { + JsTokenSourceCheckpoint { trivia_len: self.trivia_list.len() as u32, - lexer: self.lexer.checkpoint(), + lexer_checkpoint: self.lexer.checkpoint(), } } /// Restores the token source to a previous state - pub fn rewind(&mut self, checkpoint: TokenSourceCheckpoint) { + pub fn rewind(&mut self, checkpoint: JsTokenSourceCheckpoint) { assert!(self.trivia_list.len() >= checkpoint.trivia_len as usize); self.trivia_list.truncate(checkpoint.trivia_len as usize); - self.lexer.rewind(checkpoint.lexer); + self.lexer.rewind(checkpoint.lexer_checkpoint); self.non_trivia_lookahead.clear(); self.lookahead_offset = 0; } @@ -273,23 +275,3 @@ impl<'source> NthToken for JsTokenSource<'source> { } } } - -#[derive(Debug)] -pub struct TokenSourceCheckpoint { - lexer: LexerCheckpoint, - /// A `u32` should be enough because `TextSize` is also limited to `u32`. - /// The worst case is a document where every character is its own token. This would - /// result in `u32::MAX` tokens - trivia_len: u32, -} - -impl TokenSourceCheckpoint { - /// byte offset in the source text - pub(crate) fn current_start(&self) -> TextSize { - self.lexer.current_start() - } - - pub(crate) fn trivia_position(&self) -> usize { - self.trivia_len as usize - } -} diff --git a/crates/biome_parser/src/token_source.rs b/crates/biome_parser/src/token_source.rs index 8dc8f69e6a0c..d9d6cc45d5d3 100644 --- a/crates/biome_parser/src/token_source.rs +++ b/crates/biome_parser/src/token_source.rs @@ -1,4 +1,4 @@ -use crate::diagnostic::ParseDiagnostic; +use crate::{diagnostic::ParseDiagnostic, lexer::LexerCheckpoint}; use biome_rowan::{SyntaxKind, TextRange, TextSize, TriviaPieceKind}; /// A comment or a whitespace trivia in the source code. @@ -93,3 +93,29 @@ pub trait NthToken: TokenSource { /// Returns true if the nth non-trivia token is preceded by a line break fn has_nth_preceding_line_break(&mut self, n: usize) -> bool; } + +#[derive(Debug)] +pub struct TokenSourceCheckpoint +where + K: SyntaxKind, +{ + pub lexer_checkpoint: LexerCheckpoint, + /// A `u32` should be enough because `TextSize` is also limited to `u32`. + /// The worst case is a document where every character is its own token. This would + /// result in `u32::MAX` tokens + pub trivia_len: u32, +} + +impl TokenSourceCheckpoint +where + K: SyntaxKind, +{ + /// byte offset in the source text + pub fn current_start(&self) -> TextSize { + self.lexer_checkpoint.current_start() + } + + pub fn trivia_position(&self) -> usize { + self.trivia_len as usize + } +}