diff --git a/parser/internal/lexer.cc b/parser/internal/lexer.cc index 4163833be..dfc78e922 100644 --- a/parser/internal/lexer.cc +++ b/parser/internal/lexer.cc @@ -162,7 +162,7 @@ std::string_view TokenTypeToString(TokenType type) { Token Lexer::Lex() { int32_t start = GetPosition(); - if (ABSL_PREDICT_FALSE(position_ >= content_.size())) { + if (ABSL_PREDICT_FALSE(position_ >= content_size_)) { return MakeToken(TokenType::kEnd, start, start); } char32_t c = content_.at(position_); @@ -182,8 +182,7 @@ Token Lexer::Lex() { return MakeToken(TokenType::kWhitespace, start, GetPosition()); } case '.': { - if (position_ + 1 < content_.size() && - content_.at(position_ + 1) <= 0x7f && + if (position_ + 1 < content_size_ && content_.at(position_ + 1) <= 0x7f && absl::ascii_isdigit(static_cast(content_.at(position_ + 1)))) { return ConsumeNumericLiteral(); } @@ -342,7 +341,7 @@ bool Lexer::ConsumeUntilAfter(char32_t c, bool is_raw) { ABSL_DCHECK_NE(c, '\r'); int32_t pos = position_; bool escaped = false; - while (pos < content_.size()) { + while (pos < content_size_) { char32_t cc = content_.at(pos); if (cc == '\n' || cc == '\r') { AdvanceProcessingNewLines(pos); @@ -359,7 +358,7 @@ bool Lexer::ConsumeUntilAfter(char32_t c, bool is_raw) { } ++pos; } - AdvanceProcessingNewLines(content_.size()); + AdvanceProcessingNewLines(content_size_); return false; } @@ -370,7 +369,7 @@ bool Lexer::ConsumeUntilAfter(char32_t c, bool is_raw) { bool Lexer::ConsumeUntilAfterString(std::u32string_view s) { ABSL_DCHECK(s.find(U'\n') == std::u32string_view::npos); int32_t pos = position_; - while (pos + static_cast(s.size()) <= content_.size()) { + while (pos + static_cast(s.size()) <= content_size_) { bool match = true; for (size_t i = 0; i < s.size(); ++i) { if (content_.at(pos + static_cast(i)) != s[i]) { @@ -384,7 +383,7 @@ bool Lexer::ConsumeUntilAfterString(std::u32string_view s) { } ++pos; } - AdvanceProcessingNewLines(content_.size()); + AdvanceProcessingNewLines(content_size_); return false; } @@ -396,12 +395,12 @@ bool Lexer::ConsumeUntilAfterUnescapedString(std::u32string_view s) { ABSL_DCHECK(s.find(U'\n') == std::u32string_view::npos); int32_t pos = position_; bool escaped = false; - while (pos < content_.size()) { + while (pos < content_size_) { char32_t cc = content_.at(pos); if (cc == '\\') { escaped = !escaped; } else { - if (!escaped && pos + static_cast(s.size()) <= content_.size()) { + if (!escaped && pos + static_cast(s.size()) <= content_size_) { bool match = true; for (size_t j = 0; j < s.size(); ++j) { if (content_.at(pos + static_cast(j)) != s[j]) { @@ -418,12 +417,12 @@ bool Lexer::ConsumeUntilAfterUnescapedString(std::u32string_view s) { } ++pos; } - AdvanceProcessingNewLines(content_.size()); + AdvanceProcessingNewLines(content_size_); return false; } bool Lexer::MatchString(std::u32string_view s) const { - if (position_ + static_cast(s.size()) > content_.size()) { + if (position_ + static_cast(s.size()) > content_size_) { return false; } for (size_t i = 0; i < s.size(); ++i) { @@ -436,7 +435,7 @@ bool Lexer::MatchString(std::u32string_view s) const { std::optional Lexer::MatchIf( absl::FunctionRef predicate) const { - if (position_ < content_.size()) { + if (position_ < content_size_) { char32_t cp = content_.at(position_); if (predicate(cp)) { return cp; @@ -446,7 +445,7 @@ std::optional Lexer::MatchIf( } void Lexer::ConsumeLine() { - while (position_ < content_.size()) { + while (position_ < content_size_) { if (content_.at(position_) == '\n') { Advance(1); return; @@ -456,7 +455,7 @@ void Lexer::ConsumeLine() { } void Lexer::ConsumeWhitespace() { - while (position_ < content_.size()) { + while (position_ < content_size_) { char32_t c = content_.at(position_); switch (c) { case '\f': @@ -517,7 +516,7 @@ std::optional Lexer::ConsumeIf( bool Lexer::ConsumeDigits() { bool advanced = false; - while (position_ < content_.size()) { + while (position_ < content_size_) { char32_t c = content_.at(position_); if (c > 0x7f || !absl::ascii_isdigit(static_cast(c))) { break; @@ -530,7 +529,7 @@ bool Lexer::ConsumeDigits() { bool Lexer::ConsumeHexDigits() { bool advanced = false; - while (position_ < content_.size()) { + while (position_ < content_size_) { char32_t c = content_.at(position_); if (c > 0x7f || !absl::ascii_isxdigit(static_cast(c))) { break; @@ -588,12 +587,12 @@ Token Lexer::ConsumeStringLiteral(int32_t start, char32_t quote, bool is_bytes, // rb"""...""", rb'''...''' std::optional Lexer::ConsumePrefixedStringLiteral() { int32_t start = GetPosition(); - if (position_ >= content_.size()) return std::nullopt; + if (position_ >= content_size_) return std::nullopt; char32_t c = content_.at(position_); bool is_bytes = (c == 'b' || c == 'B'); bool is_raw = (c == 'r' || c == 'R'); size_t lookahead = 1; - if (position_ + 1 < content_.size()) { + if (position_ + 1 < content_size_) { char32_t c2 = content_.at(position_ + 1); if ((is_bytes && (c2 == 'r' || c2 == 'R')) || (!is_bytes && (c2 == 'b' || c2 == 'B'))) { @@ -602,7 +601,7 @@ std::optional Lexer::ConsumePrefixedStringLiteral() { lookahead = 2; } } - if (position_ + static_cast(lookahead) < content_.size()) { + if (position_ + static_cast(lookahead) < content_size_) { char32_t quote = content_.at(position_ + static_cast(lookahead)); if (quote == '"' || quote == '\'') { Advance(lookahead); @@ -649,8 +648,8 @@ Token Lexer::ConsumeNumericLiteral() { } } static_cast(ConsumeDigits()); - if (position_ < content_.size() && content_.at(position_) == '.' && - position_ + 1 < content_.size() && content_.at(position_ + 1) <= 0x7f && + if (position_ < content_size_ && content_.at(position_) == '.' && + position_ + 1 < content_size_ && content_.at(position_ + 1) <= 0x7f && absl::ascii_isdigit(static_cast(content_.at(position_ + 1)))) { floating_point = true; Advance(1); @@ -679,7 +678,7 @@ Token Lexer::ConsumeNumericLiteral() { Token Lexer::ConsumeIdent() { int32_t start = GetPosition(); - while (position_ < content_.size()) { + while (position_ < content_size_) { char32_t c = content_.at(position_); if (!IsIdentTrailing(c)) { break; diff --git a/parser/internal/lexer.h b/parser/internal/lexer.h index 70ac703cb..217bc7c41 100644 --- a/parser/internal/lexer.h +++ b/parser/internal/lexer.h @@ -135,7 +135,9 @@ struct LexerError final { class Lexer final { public: explicit Lexer(const cel::Source& source) - : content_(source.content()), position_(0) { + : content_(source.content()), + content_size_(static_cast(content_.size())), + position_(0) { ABSL_DCHECK_LE(content_.size(), static_cast( std::numeric_limits::max())); } @@ -162,18 +164,18 @@ class Lexer final { void RestorePosition(int32_t position) { ABSL_DCHECK_GE(position, 0); - ABSL_DCHECK_LE(position, static_cast(content_.size())); + ABSL_DCHECK_LE(position, content_size_); position_ = position; error_ = LexerError{}; } private: [[nodiscard]] bool Match(char32_t c) const { - return position_ < content_.size() && content_.at(position_) == c; + return position_ < content_size_ && content_.at(position_) == c; } [[nodiscard]] bool MatchIgnoreCase(char32_t c) const { - if (position_ >= content_.size()) return false; + if (position_ >= content_size_) return false; char32_t cp = content_.at(position_); return cp <= 0x7f && c <= 0x7f && absl::ascii_tolower(static_cast(cp)) == @@ -181,12 +183,12 @@ class Lexer final { } void Advance(size_t n) { - ABSL_DCHECK_LE(n, static_cast(content_.size() - position_)); + ABSL_DCHECK_LE(n, static_cast(content_size_ - position_)); position_ += static_cast(n); } void AdvanceProcessingNewLines(int32_t end_position) { - ABSL_DCHECK_LE(end_position, content_.size()); + ABSL_DCHECK_LE(end_position, content_size_); ABSL_DCHECK_GE(end_position, position_); Advance(static_cast(end_position - position_)); } @@ -275,6 +277,7 @@ class Lexer final { [[nodiscard]] Token ConsumeIdent(); cel::SourceContentView content_; + int32_t content_size_ = 0; int32_t position_ = 0; LexerError error_; };