diff --git a/src/parser.rs b/src/parser.rs index 2ca1b4dc..6d388597 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -13,15 +13,33 @@ use std::ops::BitOr; /// /// Can be used with the `Parser::reset` method to restore that state. /// Should only be used with the `Parser` instance it came from. -#[derive(Debug, Clone, Default)] +#[derive(Clone, Default)] pub struct ParserState { pub(crate) position: usize, - pub(crate) current_line_start_position: usize, - pub(crate) current_line_number: u32, - pub(crate) at_start_of: Option, + pub(crate) current_line_start_position: u32, + /// Current line number shifted by `Self::BLOCK_TYPE_BITS`, with the low 2 bits holding the + /// `BlockType` discriminant of the block the last returned token opened (if any). + line_number_and_block_type: u32, +} + +impl fmt::Debug for ParserState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ParserState") + .field("position", &self.position) + .field( + "current_line_start_position", + &self.current_line_start_position, + ) + .field("current_line_number", &self.line_number()) + .field("at_start_of", &self.at_start_of()) + .finish() + } } impl ParserState { + const BLOCK_TYPE_BITS: u32 = 2; + const BLOCK_TYPE_MASK: u32 = (1 << Self::BLOCK_TYPE_BITS) - 1; + /// The position from the start of the input, counted in UTF-8 bytes. #[inline] pub fn position(&self) -> SourcePosition { @@ -32,10 +50,48 @@ impl ParserState { #[inline] pub fn source_location(&self) -> SourceLocation { SourceLocation { - line: self.current_line_number, - column: (self.position - self.current_line_start_position + 1) as u32, + line: self.line_number(), + column: (self.position as u32) + .wrapping_sub(self.current_line_start_position) + .wrapping_add(1), + } + } + + #[inline] + pub(crate) fn line_number(&self) -> u32 { + self.line_number_and_block_type >> Self::BLOCK_TYPE_BITS + } + + #[inline] + pub(crate) fn advance_line_number(&mut self, count: u32) { + self.line_number_and_block_type += count << Self::BLOCK_TYPE_BITS; + } + + #[inline] + pub(crate) fn at_start_of(&self) -> Option { + let block_type = (self.line_number_and_block_type & Self::BLOCK_TYPE_MASK) as u8; + if block_type == 0 { + None + } else { + // SAFETY: All bit patterns that are non-zero are covered by the enum, and the value of + // this mask comes from a valid enum variant, see set_at_start_of. + Some(unsafe { std::mem::transmute::(block_type) }) } } + + #[inline] + pub(crate) fn set_at_start_of(&mut self, block_type: BlockType) { + debug_assert!(self.at_start_of().is_none()); + self.line_number_and_block_type |= block_type as u32; + debug_assert_eq!(self.at_start_of(), Some(block_type)); + } + + #[inline] + pub(crate) fn take_at_start_of(&mut self) -> Option { + let block_type = self.at_start_of(); + self.line_number_and_block_type &= !Self::BLOCK_TYPE_MASK; + block_type + } } /// When parsing until a given token, sometimes the caller knows that parsing is going to restart @@ -241,8 +297,10 @@ struct CachedToken<'i> { } #[derive(Copy, Clone, PartialEq, Eq, Debug)] +#[repr(u8)] pub(crate) enum BlockType { - Parenthesis, + // NOTE: ParserState::line_number_and_block_type relies on discriminants being non-zero. + Parenthesis = 1, SquareBracket, CurlyBracket, } @@ -939,7 +997,7 @@ pub fn parse_nested_block<'i, F, T, E>( where F: FnOnce(&mut Parser<'i>) -> Result>, { - let block_type = parser.state.at_start_of.take().expect( + let block_type = parser.state.take_at_start_of().expect( "\ A nested parser can only be created when a Function, \ ParenthesisBlock, SquareBracketBlock, or CurlyBracketBlock \ @@ -980,7 +1038,7 @@ impl Parser<'_> { // FIXME: have a special-purpose tokenizer method for this that does less work. while !self.is_eof() { let token = self.next_unchecked(); - if let Some(nested_block_type) = self.state.at_start_of.take() { + if let Some(nested_block_type) = self.state.take_at_start_of() { stack.push(nested_block_type); continue; } diff --git a/src/size_of_tests.rs b/src/size_of_tests.rs index 9d58dee1..f7567767 100644 --- a/src/size_of_tests.rs +++ b/src/size_of_tests.rs @@ -42,9 +42,9 @@ size_of_test!(token, Token, 32); size_of_test!(std_cow_str, std::borrow::Cow<'static, str>, 24, 32); size_of_test!(cow_rc_str, CowRcStr, 16); -size_of_test!(parser, crate::parser::Parser, 168); +size_of_test!(parser, crate::parser::Parser, 152); size_of_test!(source_position, crate::SourcePosition, 8); -size_of_test!(parser_state, crate::ParserState, 24); +size_of_test!(parser_state, crate::ParserState, 16); size_of_test!(basic_parse_error, crate::BasicParseError, 1); size_of_test!(parse_error_lower_bound, crate::ParseError<()>, 1); diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 7b3ef09f..64a7798f 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -238,14 +238,14 @@ impl<'a> Parser<'a> { /// Assumes non-EOF. #[inline] pub(crate) fn next_unchecked(&mut self) -> Token<'a> { - debug_assert!(self.state.at_start_of.is_none()); + debug_assert!(self.state.at_start_of().is_none()); next_token_unchecked(self) } /// If the last token returned opened a block, skip until after the end of that block. #[inline] pub(crate) fn skip_block_at_start(&mut self) { - if let Some(block_type) = self.state.at_start_of.take() { + if let Some(block_type) = self.state.take_at_start_of() { self.consume_until_end_of_block(block_type); } } @@ -456,8 +456,8 @@ impl<'a> Parser<'a> { if byte == b'\r' && self.next_byte() == Some(b'\n') { self.state.position += 1; } - self.state.current_line_start_position = self.state.position; - self.state.current_line_number += 1; + self.state.current_line_start_position = self.state.position as u32; + self.state.advance_line_number(1); } #[inline] @@ -476,7 +476,7 @@ impl<'a> Parser<'a> { self.state.current_line_start_position = self .state .current_line_start_position - .wrapping_add(len_utf8 - c.len_utf16()); + .wrapping_add((len_utf8 - c.len_utf16()) as u32); c } @@ -602,7 +602,7 @@ fn next_token_unchecked<'a>(parser: &mut Parser<'a>) -> Token<'a> { b'\'' => consume_string(parser, true), b'(' => { parser.advance(1); - parser.state.at_start_of = Some(BlockType::Parenthesis); + parser.state.set_at_start_of(BlockType::Parenthesis); ParenthesisBlock }, b')' => { parser.advance(1); CloseParenthesis }, @@ -683,7 +683,7 @@ fn next_token_unchecked<'a>(parser: &mut Parser<'a>) -> Token<'a> { b'a'..=b'z' | b'A'..=b'Z' | b'_' | b'\0' => consume_ident_like(parser), b'[' => { parser.advance(1); - parser.state.at_start_of = Some(BlockType::SquareBracket); + parser.state.set_at_start_of(BlockType::SquareBracket); SquareBracketBlock }, b'\\' => { @@ -697,7 +697,7 @@ fn next_token_unchecked<'a>(parser: &mut Parser<'a>) -> Token<'a> { }, b'{' => { parser.advance(1); - parser.state.at_start_of = Some(BlockType::CurlyBracket); + parser.state.set_at_start_of(BlockType::CurlyBracket); CurlyBracketBlock }, b'|' => { @@ -945,7 +945,7 @@ fn consume_ident_like<'a>(parser: &mut Parser<'a>) -> Token<'a> { return url; } } - parser.state.at_start_of = Some(BlockType::Parenthesis); + parser.state.set_at_start_of(BlockType::Parenthesis); parser.arbitrary_substitution_functions.see_function(&value); Function(value) } else { @@ -1215,10 +1215,10 @@ fn consume_unquoted_url<'a>(parser: &mut Parser<'a>) -> Result, ()> { } if newlines > 0 { - parser.state.current_line_number += newlines; + parser.state.advance_line_number(newlines); // No need for wrapping_add here, because there's no possible // way to wrap. - parser.state.current_line_start_position = start_position + last_newline + 1; + parser.state.current_line_start_position = (start_position + last_newline + 1) as u32; } if found_printable_char {