diff --git a/lib/lrama/lexer.rb b/lib/lrama/lexer.rb index c70b91ce..852db5e5 100644 --- a/lib/lrama/lexer.rb +++ b/lib/lrama/lexer.rb @@ -133,7 +133,7 @@ def lex_token return [:CHARACTER, Lrama::Lexer::Token::Char.new(s_value: @scanner.matched, location: location)] when @scanner.scan(/'\\\\'|'\\b'|'\\t'|'\\f'|'\\r'|'\\n'|'\\v'|'\\13'/) return [:CHARACTER, Lrama::Lexer::Token::Char.new(s_value: @scanner.matched, location: location)] - when @scanner.scan(/".*?"/) + when @scanner.scan(/"(?:\\.|[^"\\\n])*"/) return [:STRING, Lrama::Lexer::Token::Str.new(s_value: %Q(#{@scanner.matched}), location: location)] when @scanner.scan(/\d+/) return [:INTEGER, Lrama::Lexer::Token::Int.new(s_value: Integer(@scanner.matched), location: location)] diff --git a/spec/lrama/lexer_spec.rb b/spec/lrama/lexer_spec.rb index f06fd6cc..2f7e7aea 100644 --- a/spec/lrama/lexer_spec.rb +++ b/spec/lrama/lexer_spec.rb @@ -431,6 +431,31 @@ MSG end + [%q("\""), %q("a\"b"), %q("\\\\"), %q("\n")].each do |literal| + it "lexes the escaped string alias #{literal.inspect}" do + grammar_file = Lrama::Lexer::GrammarFile.new("alias.y", "%token QUOTE #{literal} NEXT") + lexer = Lrama::Lexer.new(grammar_file) + + expect(lexer.next_token).to eq(['%token', token_class::Token.new(s_value: '%token')]) + expect(lexer.next_token).to eq([:IDENTIFIER, token_class::Ident.new(s_value: 'QUOTE')]) + token = lexer.next_token + + expect(token).to eq([:STRING, token_class::Str.new(s_value: literal)]) + expect(token[1].location).to eq Lrama::Lexer::Location.new(grammar_file: grammar_file, first_line: 1, first_column: 13, last_line: 1, last_column: 13 + literal.length) + expect(lexer.next_token).to eq([:IDENTIFIER, token_class::Ident.new(s_value: 'NEXT')]) + expect(lexer.next_token).to be_nil + end + end + + it 'rejects unterminated or multiline string aliases' do + [%q("\"), "\"a\nb\"", "\"a\\\nb\""].each do |literal| + grammar_file = Lrama::Lexer::GrammarFile.new("alias.y", literal) + lexer = Lrama::Lexer.new(grammar_file) + + expect { lexer.next_token }.to raise_error(ParseError, /Unexpected token/) + end + end + context 'unexpected_c_code.y' do it do grammar_file = Lrama::Lexer::GrammarFile.new("invalid.y", "@invalid") diff --git a/spec/lrama/parser_spec.rb b/spec/lrama/parser_spec.rb index cbccdc13..8a20ea24 100644 --- a/spec/lrama/parser_spec.rb +++ b/spec/lrama/parser_spec.rb @@ -118,6 +118,21 @@ end describe '#parse' do + it 'resolves an escaped string alias to its declared token' do + y = <<~'GRAMMAR' + %token QUOTE "\"" + %% + start: "\""; + GRAMMAR + grammar = Lrama::Parser.new(y, "alias.y").parse + grammar.prepare + grammar.validate! + + quote = grammar.find_symbol_by_s_value!("QUOTE") + expect(quote.alias_name).to eq(%q("\"")) + expect(grammar.rules.last.rhs).to eq([quote]) + end + it "no_union" do path = "common/no_union.y" y = File.read(fixture_path(path))