parser: return correct errors for unterminated triple quoted strings

Author: ncw

parser/grammar_data_test.go

@@ -29,7 +29,7 @@ var grammarTestData = []struct {
 	{"b'abc' b'''123'''", "eval", "Expression(body=Bytes(s=b'abc123'))", nil, ""},
 	{"1234", "eval", "Expression(body=Num(n=1234))", nil, ""},
 	{"01234", "eval", "", py.SyntaxError, "illegal decimal with leading zero"},
-	{"1234d", "eval", "", py.SyntaxError, "unexpected EOF while parsing"},
+	{"1234d", "eval", "", py.SyntaxError, "invalid syntax"},
 	{"1234d", "exec", "", py.SyntaxError, "invalid syntax"},
 	{"1234d", "single", "", py.SyntaxError, "unexpected EOF while parsing"},
 	{"0x1234", "eval", "Expression(body=Num(n=4660))", nil, ""},
@@ -317,4 +317,10 @@ var grammarTestData = []struct {
 	{"pass\n", "single", "Interactive(body=[Pass()])", nil, ""},
 	{"if True:\n   pass\n\n", "single", "Interactive(body=[If(test=NameConstant(value=True), body=[Pass()], orelse=[])])", nil, ""},
 	{"while True:\n pass\nelse:\n return\n", "single", "Interactive(body=[While(test=NameConstant(value=True), body=[Pass()], orelse=[Return(value=None)])])", nil, ""},
+	{"a='potato", "eval", "", py.SyntaxError, "invalid syntax"},
+	{"a='potato", "exec", "", py.SyntaxError, "EOL while scanning string literal"},
+	{"a='potato", "single", "", py.SyntaxError, "EOL while scanning string literal"},
+	{"a='''potato", "eval", "", py.SyntaxError, "invalid syntax"},
+	{"a='''potato", "exec", "", py.SyntaxError, "EOF while scanning triple-quoted string literal"},
+	{"a='''potato", "single", "", py.SyntaxError, "EOF while scanning triple-quoted string literal"},
 }

parser/lexer.go

@@ -639,12 +639,14 @@ func (x *yyLex) readOperator() int {
 
 const pointFloat = `([0-9]*\.[0-9]+|[0-9]+\.)`
 
-var decimalInteger = regexp.MustCompile(`^[0-9]+[jJ]?`)
-var illegalDecimalInteger = regexp.MustCompile(`^0[0-9]*[1-9][0-9]*$`)
-var octalInteger = regexp.MustCompile(`^0[oO][0-7]+`)
-var hexInteger = regexp.MustCompile(`^0[xX][0-9a-fA-F]+`)
-var binaryInteger = regexp.MustCompile(`^0[bB][01]+`)
-var floatNumber = regexp.MustCompile(`^(([0-9]+|` + pointFloat + `)[eE][+-]?[0-9]+|` + pointFloat + `)[jJ]?`)
+var (
+	decimalInteger        = regexp.MustCompile(`^[0-9]+[jJ]?`)
+	illegalDecimalInteger = regexp.MustCompile(`^0[0-9]*[1-9][0-9]*$`)
+	octalInteger          = regexp.MustCompile(`^0[oO][0-7]+`)
+	hexInteger            = regexp.MustCompile(`^0[xX][0-9a-fA-F]+`)
+	binaryInteger         = regexp.MustCompile(`^0[bB][01]+`)
+	floatNumber           = regexp.MustCompile(`^(([0-9]+|` + pointFloat + `)[eE][+-]?[0-9]+|` + pointFloat + `)[jJ]?`)
+)
 
 // Read one of the many types of python number
 //
@@ -837,12 +839,16 @@ found:
 			}
 		}
 		if !multiLineString {
-			x.SyntaxErrorf("Unterminated %sx%s string", stringEnd, stringEnd)
+			x.SyntaxErrorf("EOL while scanning string literal")
 			return eofError, nil
 		}
 	readMore:
 		if x.eof {
-			x.SyntaxErrorf("Unterminated %sx%s string", stringEnd, stringEnd)
+			if multiLineString {
+				x.SyntaxErrorf("EOF while scanning triple-quoted string literal")
+			} else {
+				x.SyntaxErrorf("EOL while scanning string literal")
+			}
 			return eofError, nil
 		}
 		x.refill()
@@ -887,7 +893,7 @@ func (x *yyLex) SyntaxErrorf(format string, a ...interface{}) {
 func (x *yyLex) ErrorReturn() error {
 	if x.error {
 		if x.errorString == "" {
-			if x.eof && !x.exec {
+			if x.eof && x.interactive {
 				x.errorString = "unexpected EOF while parsing"
 			} else {
 				x.errorString = "invalid syntax"

parser/lexer_test.go

@@ -342,7 +342,7 @@ func TestLex(t *testing.T) {
 			{STRING, py.String("1\n2\n")},
 			{ENDMARKER, nil},
 		}},
-		{"\"hello\n", "Unterminated \"x\" string", "eval", LexTokens{
+		{"\"hello\n", "EOL while scanning string literal", "eval", LexTokens{
 			{EVAL_INPUT, nil},
 		}},
 		{"1 >>-3\na <<=+12", "", "eval", LexTokens{

parser/make_grammar_test.py

@@ -25,7 +25,7 @@
     ("b'abc' b'''123'''", "eval"),
     ("1234", "eval"),
     ("01234", "eval", SyntaxError, "illegal decimal with leading zero"),
-    ("1234d", "eval", SyntaxError),
+    ("1234d", "eval", SyntaxError, "invalid syntax"),
     ("1234d", "exec", SyntaxError),
     ("1234d", "single", SyntaxError),
     ("0x1234", "eval"),
@@ -467,6 +467,13 @@ class A(B):
     ("pass\n", "single"),
     ("if True:\n   pass\n\n", "single"),
     ("while True:\n pass\nelse:\n return\n", "single"),
+    # unfinished strings
+    ("a='potato", "eval", SyntaxError),
+    ("a='potato", "exec", SyntaxError),
+    ("a='potato", "single", SyntaxError),
+    ("a='''potato", "eval", SyntaxError),
+    ("a='''potato", "exec", SyntaxError),
+    ("a='''potato", "single", SyntaxError),
 ]
 
 def dump(source, mode):