From 680d2829116d27c1fe55dafb68e15137bb291a26 Mon Sep 17 00:00:00 2001 From: Dmitri Plotnikov Date: Fri, 18 Sep 2026 14:08:16 -0700 Subject: [PATCH] [Pratt Parser] Cache source content size in Lexer to avoid repeated variant dispatch in character loops. `SourceContentView::size()` is defined out-of-line and dispatches via `absl::visit` over a 4-element `absl::variant`. Because the character scanning loops in `Lexer` also invoke out-of-line methods on `content_`, the compiler cannot hoist `content_.size()` out of loop conditions, causing it to be re-evaluated on every character scanned. Caching `content_size_` in `Lexer` improves Pratt parser CPU time across `pratt_parser_benchmark` by ~5.2% geomean (~6.7% geomean on valid expressions). Measured with `pratt_parser_benchmark` (built `-c opt --dynamic_mode=off`). "Pratt before" is this CL's parent (`p4head`), so the last column is this CL's own contribution. The ANTLR column is the series baseline and is unaffected by this change. CPU time, median across 36 interleaved repetitions: | Case | ANTLR | Pratt before | Pratt after | Pratt vs ANTLR | Delta this CL | | :--- | ---: | ---: | ---: | ---: | ---: | | ParseCommon | 210,982 ns | 11,606 ns | 11,003 ns | 19.2x faster | -5.2% | | ParseArithmeticChain/10 | 55,289 ns | 2,781 ns | 2,608 ns | 21.2x faster | -6.2% | | ParseArithmeticChain/50 | 291,952 ns | 13,794 ns | 13,265 ns | 22.0x faster | -3.8% | | ParseArithmeticChain/100 | 559,317 ns | 27,506 ns | 25,724 ns | 21.7x faster | -6.5% | | ParseLogicalChain/10 | 43,318 ns | 3,306 ns | 2,986 ns | 14.5x faster | -9.7% | | ParseLogicalChain/50 | 207,612 ns | 15,118 ns | 13,940 ns | 14.9x faster | -7.8% | | ParseLogicalChain/100 | 442,381 ns | 30,089 ns | 28,675 ns | 15.4x faster | -4.7% | | ParseMemberChain/10 | 34,693 ns | 1,819 ns | 1,705 ns | 20.4x faster | -6.3% | | ParseMemberChain/50 | 161,104 ns | 8,633 ns | 7,889 ns | 20.4x faster | -8.6% | | ParseMemberChain/100 | 335,750 ns | 16,917 ns | 15,724 ns | 21.4x faster | -7.1% | | ParseNestedParentheses/10 | 28,084 ns | 803 ns | 752 ns | 37.3x faster | -6.3% | | ParseNestedParentheses/50 | 123,579 ns | 2,841 ns | 2,620 ns | 47.2x faster | -7.8% | | ParseCommonSyntaxErrors | 429,515 ns | 18,317 ns | 17,561 ns | 24.5x faster | -4.1% | | ParseArithmeticChainSyntaxError/10 | 72,212 ns | 3,256 ns | 3,078 ns | 23.5x faster | -5.5% | | ParseArithmeticChainSyntaxError/50 | 295,287 ns | 14,321 ns | 13,419 ns | 22.0x faster | -6.3% | | ParseArithmeticChainSyntaxError/100 | 628,385 ns | 28,006 ns | 26,414 ns | 23.8x faster | -5.7% | | ParseLogicalChainSyntaxError/10 | 58,855 ns | 3,776 ns | 3,553 ns | 16.6x faster | -5.9% | | ParseLogicalChainSyntaxError/50 | 235,134 ns | 16,104 ns | 15,147 ns | 15.5x faster | -5.9% | | ParseLogicalChainSyntaxError/100 | 458,219 ns | 31,111 ns | 29,472 ns | 15.5x faster | -5.3% | | ParseMemberChainSyntaxError/10 | 50,335 ns | 2,144 ns | 2,051 ns | 24.5x faster | -4.3% | | ParseMemberChainSyntaxError/50 | 182,649 ns | 8,962 ns | 8,516 ns | 21.4x faster | -5.0% | | ParseMemberChainSyntaxError/100 | 346,654 ns | 17,161 ns | 16,230 ns | 21.4x faster | -5.4% | | ParseNestedParenthesesSyntaxError/10 | 241,350 ns | 6,098 ns | 5,982 ns | 40.3x faster | -1.9% | | ParseNestedParenthesesSyntaxError/50 | 1,561,548 ns | 28,725 ns | 27,390 ns | 57.0x faster | -4.6% | | ParseRepeatedSyntaxErrors/10 | 170,609 ns | 6,218 ns | 6,044 ns | 28.2x faster | -2.8% | | ParseRepeatedSyntaxErrors/50 | 172,554 ns | 8,602 ns | 8,620 ns | 20.0x faster | +0.2% | | ParseRepeatedSyntaxErrors/100 | 180,960 ns | 9,324 ns | 9,532 ns | 19.0x faster | +2.2% | | **Geomean** | | | | **22.5x faster** | **-5.2%** | PiperOrigin-RevId: 984058754 --- parser/internal/lexer.cc | 43 ++++++++++++++++++++-------------------- parser/internal/lexer.h | 15 ++++++++------ 2 files changed, 30 insertions(+), 28 deletions(-) diff --git a/parser/internal/lexer.cc b/parser/internal/lexer.cc index 4163833be..dfc78e922 100644 --- a/parser/internal/lexer.cc +++ b/parser/internal/lexer.cc @@ -162,7 +162,7 @@ std::string_view TokenTypeToString(TokenType type) { Token Lexer::Lex() { int32_t start = GetPosition(); - if (ABSL_PREDICT_FALSE(position_ >= content_.size())) { + if (ABSL_PREDICT_FALSE(position_ >= content_size_)) { return MakeToken(TokenType::kEnd, start, start); } char32_t c = content_.at(position_); @@ -182,8 +182,7 @@ Token Lexer::Lex() { return MakeToken(TokenType::kWhitespace, start, GetPosition()); } case '.': { - if (position_ + 1 < content_.size() && - content_.at(position_ + 1) <= 0x7f && + if (position_ + 1 < content_size_ && content_.at(position_ + 1) <= 0x7f && absl::ascii_isdigit(static_cast(content_.at(position_ + 1)))) { return ConsumeNumericLiteral(); } @@ -342,7 +341,7 @@ bool Lexer::ConsumeUntilAfter(char32_t c, bool is_raw) { ABSL_DCHECK_NE(c, '\r'); int32_t pos = position_; bool escaped = false; - while (pos < content_.size()) { + while (pos < content_size_) { char32_t cc = content_.at(pos); if (cc == '\n' || cc == '\r') { AdvanceProcessingNewLines(pos); @@ -359,7 +358,7 @@ bool Lexer::ConsumeUntilAfter(char32_t c, bool is_raw) { } ++pos; } - AdvanceProcessingNewLines(content_.size()); + AdvanceProcessingNewLines(content_size_); return false; } @@ -370,7 +369,7 @@ bool Lexer::ConsumeUntilAfter(char32_t c, bool is_raw) { bool Lexer::ConsumeUntilAfterString(std::u32string_view s) { ABSL_DCHECK(s.find(U'\n') == std::u32string_view::npos); int32_t pos = position_; - while (pos + static_cast(s.size()) <= content_.size()) { + while (pos + static_cast(s.size()) <= content_size_) { bool match = true; for (size_t i = 0; i < s.size(); ++i) { if (content_.at(pos + static_cast(i)) != s[i]) { @@ -384,7 +383,7 @@ bool Lexer::ConsumeUntilAfterString(std::u32string_view s) { } ++pos; } - AdvanceProcessingNewLines(content_.size()); + AdvanceProcessingNewLines(content_size_); return false; } @@ -396,12 +395,12 @@ bool Lexer::ConsumeUntilAfterUnescapedString(std::u32string_view s) { ABSL_DCHECK(s.find(U'\n') == std::u32string_view::npos); int32_t pos = position_; bool escaped = false; - while (pos < content_.size()) { + while (pos < content_size_) { char32_t cc = content_.at(pos); if (cc == '\\') { escaped = !escaped; } else { - if (!escaped && pos + static_cast(s.size()) <= content_.size()) { + if (!escaped && pos + static_cast(s.size()) <= content_size_) { bool match = true; for (size_t j = 0; j < s.size(); ++j) { if (content_.at(pos + static_cast(j)) != s[j]) { @@ -418,12 +417,12 @@ bool Lexer::ConsumeUntilAfterUnescapedString(std::u32string_view s) { } ++pos; } - AdvanceProcessingNewLines(content_.size()); + AdvanceProcessingNewLines(content_size_); return false; } bool Lexer::MatchString(std::u32string_view s) const { - if (position_ + static_cast(s.size()) > content_.size()) { + if (position_ + static_cast(s.size()) > content_size_) { return false; } for (size_t i = 0; i < s.size(); ++i) { @@ -436,7 +435,7 @@ bool Lexer::MatchString(std::u32string_view s) const { std::optional Lexer::MatchIf( absl::FunctionRef predicate) const { - if (position_ < content_.size()) { + if (position_ < content_size_) { char32_t cp = content_.at(position_); if (predicate(cp)) { return cp; @@ -446,7 +445,7 @@ std::optional Lexer::MatchIf( } void Lexer::ConsumeLine() { - while (position_ < content_.size()) { + while (position_ < content_size_) { if (content_.at(position_) == '\n') { Advance(1); return; @@ -456,7 +455,7 @@ void Lexer::ConsumeLine() { } void Lexer::ConsumeWhitespace() { - while (position_ < content_.size()) { + while (position_ < content_size_) { char32_t c = content_.at(position_); switch (c) { case '\f': @@ -517,7 +516,7 @@ std::optional Lexer::ConsumeIf( bool Lexer::ConsumeDigits() { bool advanced = false; - while (position_ < content_.size()) { + while (position_ < content_size_) { char32_t c = content_.at(position_); if (c > 0x7f || !absl::ascii_isdigit(static_cast(c))) { break; @@ -530,7 +529,7 @@ bool Lexer::ConsumeDigits() { bool Lexer::ConsumeHexDigits() { bool advanced = false; - while (position_ < content_.size()) { + while (position_ < content_size_) { char32_t c = content_.at(position_); if (c > 0x7f || !absl::ascii_isxdigit(static_cast(c))) { break; @@ -588,12 +587,12 @@ Token Lexer::ConsumeStringLiteral(int32_t start, char32_t quote, bool is_bytes, // rb"""...""", rb'''...''' std::optional Lexer::ConsumePrefixedStringLiteral() { int32_t start = GetPosition(); - if (position_ >= content_.size()) return std::nullopt; + if (position_ >= content_size_) return std::nullopt; char32_t c = content_.at(position_); bool is_bytes = (c == 'b' || c == 'B'); bool is_raw = (c == 'r' || c == 'R'); size_t lookahead = 1; - if (position_ + 1 < content_.size()) { + if (position_ + 1 < content_size_) { char32_t c2 = content_.at(position_ + 1); if ((is_bytes && (c2 == 'r' || c2 == 'R')) || (!is_bytes && (c2 == 'b' || c2 == 'B'))) { @@ -602,7 +601,7 @@ std::optional Lexer::ConsumePrefixedStringLiteral() { lookahead = 2; } } - if (position_ + static_cast(lookahead) < content_.size()) { + if (position_ + static_cast(lookahead) < content_size_) { char32_t quote = content_.at(position_ + static_cast(lookahead)); if (quote == '"' || quote == '\'') { Advance(lookahead); @@ -649,8 +648,8 @@ Token Lexer::ConsumeNumericLiteral() { } } static_cast(ConsumeDigits()); - if (position_ < content_.size() && content_.at(position_) == '.' && - position_ + 1 < content_.size() && content_.at(position_ + 1) <= 0x7f && + if (position_ < content_size_ && content_.at(position_) == '.' && + position_ + 1 < content_size_ && content_.at(position_ + 1) <= 0x7f && absl::ascii_isdigit(static_cast(content_.at(position_ + 1)))) { floating_point = true; Advance(1); @@ -679,7 +678,7 @@ Token Lexer::ConsumeNumericLiteral() { Token Lexer::ConsumeIdent() { int32_t start = GetPosition(); - while (position_ < content_.size()) { + while (position_ < content_size_) { char32_t c = content_.at(position_); if (!IsIdentTrailing(c)) { break; diff --git a/parser/internal/lexer.h b/parser/internal/lexer.h index 70ac703cb..217bc7c41 100644 --- a/parser/internal/lexer.h +++ b/parser/internal/lexer.h @@ -135,7 +135,9 @@ struct LexerError final { class Lexer final { public: explicit Lexer(const cel::Source& source) - : content_(source.content()), position_(0) { + : content_(source.content()), + content_size_(static_cast(content_.size())), + position_(0) { ABSL_DCHECK_LE(content_.size(), static_cast( std::numeric_limits::max())); } @@ -162,18 +164,18 @@ class Lexer final { void RestorePosition(int32_t position) { ABSL_DCHECK_GE(position, 0); - ABSL_DCHECK_LE(position, static_cast(content_.size())); + ABSL_DCHECK_LE(position, content_size_); position_ = position; error_ = LexerError{}; } private: [[nodiscard]] bool Match(char32_t c) const { - return position_ < content_.size() && content_.at(position_) == c; + return position_ < content_size_ && content_.at(position_) == c; } [[nodiscard]] bool MatchIgnoreCase(char32_t c) const { - if (position_ >= content_.size()) return false; + if (position_ >= content_size_) return false; char32_t cp = content_.at(position_); return cp <= 0x7f && c <= 0x7f && absl::ascii_tolower(static_cast(cp)) == @@ -181,12 +183,12 @@ class Lexer final { } void Advance(size_t n) { - ABSL_DCHECK_LE(n, static_cast(content_.size() - position_)); + ABSL_DCHECK_LE(n, static_cast(content_size_ - position_)); position_ += static_cast(n); } void AdvanceProcessingNewLines(int32_t end_position) { - ABSL_DCHECK_LE(end_position, content_.size()); + ABSL_DCHECK_LE(end_position, content_size_); ABSL_DCHECK_GE(end_position, position_); Advance(static_cast(end_position - position_)); } @@ -275,6 +277,7 @@ class Lexer final { [[nodiscard]] Token ConsumeIdent(); cel::SourceContentView content_; + int32_t content_size_ = 0; int32_t position_ = 0; LexerError error_; };