From 228c4092b1061c90743904fa619b1b1a1635227d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emilio=20Cobos=20=C3=81lvarez?= Date: Sun, 20 Sep 2026 17:59:15 +0200 Subject: [PATCH 1/2] tokenizer: Shrink current_line_start_position. We already truncate column numbers to u32, so this gives the same answer it would otherwise give, and saves some storage (not in this patch because ParserState is still 24 bytes, but we can improve on that in a bit). --- src/parser.rs | 6 ++++-- src/tokenizer.rs | 6 +++--- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 2ca1b4dc..72eda4f5 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -16,7 +16,7 @@ use std::ops::BitOr; #[derive(Debug, Clone, Default)] pub struct ParserState { pub(crate) position: usize, - pub(crate) current_line_start_position: usize, + pub(crate) current_line_start_position: u32, pub(crate) current_line_number: u32, pub(crate) at_start_of: Option, } @@ -33,7 +33,9 @@ impl ParserState { pub fn source_location(&self) -> SourceLocation { SourceLocation { line: self.current_line_number, - column: (self.position - self.current_line_start_position + 1) as u32, + column: (self.position as u32) + .wrapping_sub(self.current_line_start_position) + .wrapping_add(1), } } } diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 7b3ef09f..2bb19d47 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -456,7 +456,7 @@ impl<'a> Parser<'a> { if byte == b'\r' && self.next_byte() == Some(b'\n') { self.state.position += 1; } - self.state.current_line_start_position = self.state.position; + self.state.current_line_start_position = self.state.position as u32; self.state.current_line_number += 1; } @@ -476,7 +476,7 @@ impl<'a> Parser<'a> { self.state.current_line_start_position = self .state .current_line_start_position - .wrapping_add(len_utf8 - c.len_utf16()); + .wrapping_add((len_utf8 - c.len_utf16()) as u32); c } @@ -1218,7 +1218,7 @@ fn consume_unquoted_url<'a>(parser: &mut Parser<'a>) -> Result, ()> { parser.state.current_line_number += newlines; // No need for wrapping_add here, because there's no possible // way to wrap. - parser.state.current_line_start_position = start_position + last_newline + 1; + parser.state.current_line_start_position = (start_position + last_newline + 1) as u32; } if found_printable_char { From f3ad62b6857cac4d5e4f84e514739f684940df3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Emilio=20Cobos=20=C3=81lvarez?= Date: Sun, 20 Sep 2026 18:06:37 +0200 Subject: [PATCH 2/2] parser: Shrink ParserState by collapsing line number and opening block type. The line number range gets a bit smaller, but it pays off by making state restoration faster, which is a lot hotter. --- src/parser.rs | 70 +++++++++++++++++++++++++++++++++++++++----- src/size_of_tests.rs | 4 +-- src/tokenizer.rs | 16 +++++----- 3 files changed, 73 insertions(+), 17 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 72eda4f5..6d388597 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -13,15 +13,33 @@ use std::ops::BitOr; /// /// Can be used with the `Parser::reset` method to restore that state. /// Should only be used with the `Parser` instance it came from. -#[derive(Debug, Clone, Default)] +#[derive(Clone, Default)] pub struct ParserState { pub(crate) position: usize, pub(crate) current_line_start_position: u32, - pub(crate) current_line_number: u32, - pub(crate) at_start_of: Option, + /// Current line number shifted by `Self::BLOCK_TYPE_BITS`, with the low 2 bits holding the + /// `BlockType` discriminant of the block the last returned token opened (if any). + line_number_and_block_type: u32, +} + +impl fmt::Debug for ParserState { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_struct("ParserState") + .field("position", &self.position) + .field( + "current_line_start_position", + &self.current_line_start_position, + ) + .field("current_line_number", &self.line_number()) + .field("at_start_of", &self.at_start_of()) + .finish() + } } impl ParserState { + const BLOCK_TYPE_BITS: u32 = 2; + const BLOCK_TYPE_MASK: u32 = (1 << Self::BLOCK_TYPE_BITS) - 1; + /// The position from the start of the input, counted in UTF-8 bytes. #[inline] pub fn position(&self) -> SourcePosition { @@ -32,12 +50,48 @@ impl ParserState { #[inline] pub fn source_location(&self) -> SourceLocation { SourceLocation { - line: self.current_line_number, + line: self.line_number(), column: (self.position as u32) .wrapping_sub(self.current_line_start_position) .wrapping_add(1), } } + + #[inline] + pub(crate) fn line_number(&self) -> u32 { + self.line_number_and_block_type >> Self::BLOCK_TYPE_BITS + } + + #[inline] + pub(crate) fn advance_line_number(&mut self, count: u32) { + self.line_number_and_block_type += count << Self::BLOCK_TYPE_BITS; + } + + #[inline] + pub(crate) fn at_start_of(&self) -> Option { + let block_type = (self.line_number_and_block_type & Self::BLOCK_TYPE_MASK) as u8; + if block_type == 0 { + None + } else { + // SAFETY: All bit patterns that are non-zero are covered by the enum, and the value of + // this mask comes from a valid enum variant, see set_at_start_of. + Some(unsafe { std::mem::transmute::(block_type) }) + } + } + + #[inline] + pub(crate) fn set_at_start_of(&mut self, block_type: BlockType) { + debug_assert!(self.at_start_of().is_none()); + self.line_number_and_block_type |= block_type as u32; + debug_assert_eq!(self.at_start_of(), Some(block_type)); + } + + #[inline] + pub(crate) fn take_at_start_of(&mut self) -> Option { + let block_type = self.at_start_of(); + self.line_number_and_block_type &= !Self::BLOCK_TYPE_MASK; + block_type + } } /// When parsing until a given token, sometimes the caller knows that parsing is going to restart @@ -243,8 +297,10 @@ struct CachedToken<'i> { } #[derive(Copy, Clone, PartialEq, Eq, Debug)] +#[repr(u8)] pub(crate) enum BlockType { - Parenthesis, + // NOTE: ParserState::line_number_and_block_type relies on discriminants being non-zero. + Parenthesis = 1, SquareBracket, CurlyBracket, } @@ -941,7 +997,7 @@ pub fn parse_nested_block<'i, F, T, E>( where F: FnOnce(&mut Parser<'i>) -> Result>, { - let block_type = parser.state.at_start_of.take().expect( + let block_type = parser.state.take_at_start_of().expect( "\ A nested parser can only be created when a Function, \ ParenthesisBlock, SquareBracketBlock, or CurlyBracketBlock \ @@ -982,7 +1038,7 @@ impl Parser<'_> { // FIXME: have a special-purpose tokenizer method for this that does less work. while !self.is_eof() { let token = self.next_unchecked(); - if let Some(nested_block_type) = self.state.at_start_of.take() { + if let Some(nested_block_type) = self.state.take_at_start_of() { stack.push(nested_block_type); continue; } diff --git a/src/size_of_tests.rs b/src/size_of_tests.rs index 9d58dee1..f7567767 100644 --- a/src/size_of_tests.rs +++ b/src/size_of_tests.rs @@ -42,9 +42,9 @@ size_of_test!(token, Token, 32); size_of_test!(std_cow_str, std::borrow::Cow<'static, str>, 24, 32); size_of_test!(cow_rc_str, CowRcStr, 16); -size_of_test!(parser, crate::parser::Parser, 168); +size_of_test!(parser, crate::parser::Parser, 152); size_of_test!(source_position, crate::SourcePosition, 8); -size_of_test!(parser_state, crate::ParserState, 24); +size_of_test!(parser_state, crate::ParserState, 16); size_of_test!(basic_parse_error, crate::BasicParseError, 1); size_of_test!(parse_error_lower_bound, crate::ParseError<()>, 1); diff --git a/src/tokenizer.rs b/src/tokenizer.rs index 2bb19d47..64a7798f 100644 --- a/src/tokenizer.rs +++ b/src/tokenizer.rs @@ -238,14 +238,14 @@ impl<'a> Parser<'a> { /// Assumes non-EOF. #[inline] pub(crate) fn next_unchecked(&mut self) -> Token<'a> { - debug_assert!(self.state.at_start_of.is_none()); + debug_assert!(self.state.at_start_of().is_none()); next_token_unchecked(self) } /// If the last token returned opened a block, skip until after the end of that block. #[inline] pub(crate) fn skip_block_at_start(&mut self) { - if let Some(block_type) = self.state.at_start_of.take() { + if let Some(block_type) = self.state.take_at_start_of() { self.consume_until_end_of_block(block_type); } } @@ -457,7 +457,7 @@ impl<'a> Parser<'a> { self.state.position += 1; } self.state.current_line_start_position = self.state.position as u32; - self.state.current_line_number += 1; + self.state.advance_line_number(1); } #[inline] @@ -602,7 +602,7 @@ fn next_token_unchecked<'a>(parser: &mut Parser<'a>) -> Token<'a> { b'\'' => consume_string(parser, true), b'(' => { parser.advance(1); - parser.state.at_start_of = Some(BlockType::Parenthesis); + parser.state.set_at_start_of(BlockType::Parenthesis); ParenthesisBlock }, b')' => { parser.advance(1); CloseParenthesis }, @@ -683,7 +683,7 @@ fn next_token_unchecked<'a>(parser: &mut Parser<'a>) -> Token<'a> { b'a'..=b'z' | b'A'..=b'Z' | b'_' | b'\0' => consume_ident_like(parser), b'[' => { parser.advance(1); - parser.state.at_start_of = Some(BlockType::SquareBracket); + parser.state.set_at_start_of(BlockType::SquareBracket); SquareBracketBlock }, b'\\' => { @@ -697,7 +697,7 @@ fn next_token_unchecked<'a>(parser: &mut Parser<'a>) -> Token<'a> { }, b'{' => { parser.advance(1); - parser.state.at_start_of = Some(BlockType::CurlyBracket); + parser.state.set_at_start_of(BlockType::CurlyBracket); CurlyBracketBlock }, b'|' => { @@ -945,7 +945,7 @@ fn consume_ident_like<'a>(parser: &mut Parser<'a>) -> Token<'a> { return url; } } - parser.state.at_start_of = Some(BlockType::Parenthesis); + parser.state.set_at_start_of(BlockType::Parenthesis); parser.arbitrary_substitution_functions.see_function(&value); Function(value) } else { @@ -1215,7 +1215,7 @@ fn consume_unquoted_url<'a>(parser: &mut Parser<'a>) -> Result, ()> { } if newlines > 0 { - parser.state.current_line_number += newlines; + parser.state.advance_line_number(newlines); // No need for wrapping_add here, because there's no possible // way to wrap. parser.state.current_line_start_position = (start_position + last_newline + 1) as u32;