gh-159465: Preserve escape errors in unfinished f-strings and t-strings (#159669)

Author: pablogsal

Lib/codeop.py

@@ -57,17 +57,19 @@ def _maybe_compile(compiler, source, filename, symbol, flags):
         if symbol != "eval":
             source = "pass"     # Replace it with a 'pass' statement
 
+    incomplete_input = False
     # Disable compiler warnings when checking for incomplete input.
     with warnings.catch_warnings():
         warnings.simplefilter("ignore", (SyntaxWarning, DeprecationWarning))
         try:
             compiler(source, filename, symbol, flags=flags)
         except _IncompleteInputError:
             return None
-        except SyntaxError:  # Let other compile() errors propagate.
-            pass
+        except SyntaxError as error:  # Let other compile() errors propagate.
+            # Preserve escape errors that a missing closing quote can hide.
+            incomplete_input = error.msg.startswith("(unicode error)")
 
-    return compiler(source, filename, symbol, incomplete_input=False)
+    return compiler(source, filename, symbol, incomplete_input=incomplete_input)
 
 def _compile(source, filename, symbol, incomplete_input=True, *, flags=0):
     if incomplete_input:

Lib/test/test_codeop.py

@@ -176,6 +176,100 @@ def test_string_continuation(self, symbol, prefix):
         with self.assertRaises(_IncompleteInputError):
             Compile()(source, '<input>', symbol)
 
+    @subTests('symbol', ('single', 'exec', 'eval'))
+    @subTests('prefix', ('f', 't'))
+    @subTests('quote', ('"', '"""'))
+    def test_invalid_escape_in_incomplete_string(self, symbol, prefix, quote):
+        for escape in (r'\xZZ', r'\uZZZZ', r'\U00110000', r'\N{NOPE}',
+                       r'\x', r'\u12'):
+            for body in (escape, escape + '{0}', '{0}' + escape,
+                         '{0:' + escape):
+                source = prefix + quote + body + '\\'
+                for compiler in COMPILERS:
+                    with self.subTest(source=source, compiler=compiler):
+                        with self.assertRaisesRegex(SyntaxError, r'\(unicode error\)') as cm:
+                            compiler(source, '<input>', symbol)
+                        self.assertGreater(cm.exception.end_offset, 0)
+
+    @subTests('symbol', ('single', 'exec', 'eval'))
+    @subTests('prefix', ('f', 't'))
+    @subTests('newline', ('\n', '\r', '\r\n'))
+    def test_incomplete_escape_locations(self, symbol, prefix, newline):
+        cases = (
+            (r'\N{NOPE}', (1, 5, 1, 13)),
+            ('é\n{0}é' + r'\xZZ', (2, 4, 2, 9)),
+            ('éé\n中' + r'\xZZ', (1, 5, 2, 6)),
+            ('\n{0:' + r'\xZZ', (2, 4, 2, 8)),
+        )
+        for body, location in cases:
+            source = (prefix + '"""' + body).replace('\n', newline)
+            for compiler in COMPILERS:
+                with self.subTest(source=source, compiler=compiler):
+                    with self.assertRaisesRegex(SyntaxError, r'\(unicode error\)') as cm:
+                        compiler(source, '<input>', symbol)
+                    error = cm.exception
+                    self.assertEqual((error.lineno, error.offset,
+                                      error.end_lineno, error.end_offset), location)
+                    self.assertEqual(error.text.rstrip('\n'),
+                                     source.splitlines()[error.lineno - 1])
+
+    @subTests('compiler', WRAPPING_COMPILERS)
+    def test_non_decode_error_diagnostic(self, compiler):
+        source = '+\\\n'
+        with self.assertRaises(SyntaxError) as expected:
+            compile(source, '<input>', 'single')
+        with self.assertRaises(SyntaxError) as actual:
+            compiler(source, '<input>', 'single')
+        self.assertEqual(actual.exception.args, expected.exception.args)
+
+    @subTests('compiler', COMPILERS)
+    @subTests('prefix', ('f', 't'))
+    def test_incomplete_escape_warning(self, compiler, prefix):
+        source = r"'\e'; " + prefix + '"""' + r'\N{NOPE}'
+        with warnings.catch_warnings(record=True, action='always') as caught:
+            with self.assertRaisesRegex(SyntaxError, 'unknown Unicode character name'):
+                compiler(source, '<input>', 'single')
+        self.assertEqual([warning.category for warning in caught], [SyntaxWarning])
+        with warnings.catch_warnings(action='error', category=SyntaxWarning):
+            with self.assertRaisesRegex(SyntaxError, 'invalid escape sequence'):
+                compiler(source, '<input>', 'single')
+
+    @subTests('symbol', ('single', 'exec', 'eval'))
+    @subTests('prefix', ('f', 't'))
+    @subTests('quote', ('"', '"""'))
+    def test_valid_escape_in_incomplete_string(self, symbol, prefix, quote):
+        for escape in (r'\x41', r'\u00e9', r'\U0001f600', r'\\xZZ'):
+            source = prefix + quote + escape + '\\'
+            with self.subTest(escape=escape):
+                self.assertIsNone(compile_command(source, symbol=symbol))
+                with self.assertRaises(_IncompleteInputError):
+                    Compile()(source, '<input>', symbol)
+                for compiler in COMPILERS:
+                    self.assertValid(source + '\n' + quote, symbol,
+                                     compiler=compiler)
+
+        source = 'r' + prefix + quote + r'\xZZ' + '\\'
+        self.assertIsNone(compile_command(source, symbol=symbol))
+        with self.assertRaises(_IncompleteInputError):
+            Compile()(source, '<input>', symbol)
+        for compiler in COMPILERS:
+            self.assertValid(source + '\n' + quote, symbol, compiler=compiler)
+
+    @subTests('symbol', ('single', 'exec', 'eval'))
+    @subTests('prefix', ('f', 't'))
+    def test_unfinished_escape_in_incomplete_string(self, symbol, prefix):
+        for escape, suffix in ((r'\x', '41'), (r'\u12', '34'),
+                               (r'\U0001', 'f600'),
+                               (r'\N{LATIN', ' SMALL LETTER A}')):
+            source = prefix + '"""' + escape
+            with self.subTest(escape=escape):
+                self.assertIsNone(compile_command(source, symbol=symbol))
+                with self.assertRaises(_IncompleteInputError):
+                    Compile()(source, '<input>', symbol)
+                for compiler in COMPILERS:
+                    self.assertValid(source + suffix + '"""', symbol,
+                                     compiler=compiler)
+
     @subTests('symbol', ('single', 'exec', 'eval'))
     @subTests('newline', ('\n', '\r', '\r\n'))
     def test_continued_final_line(self, symbol, newline):

Misc/NEWS.d/next/Library/2026-10-18-12-20-00.gh-issue-159465.partial-escapes.rst

@@ -0,0 +1,2 @@
+Preserve :exc:`SyntaxError` for malformed escapes in unfinished f-strings and
+t-strings passed to :func:`codeop.compile_command`.

Parser/action_helpers.c

@@ -1507,8 +1507,15 @@ expr_ty _PyPegen_decoded_constant_from_token(Parser* p, Token* tok) {
 
     PyObject* str = _PyPegen_decode_string(p, is_raw, bstr, bsize, tok);
     if (str == NULL) {
+        if ((p->flags & PyPARSE_ALLOW_INCOMPLETE_INPUT) || tok->is_incomplete) {
+            _Pypegen_raise_decode_error(p);
+        }
         return NULL;
     }
+    if (tok->is_incomplete) {
+        Py_DECREF(str);
+        return _PyPegen_raise_incomplete_input(p);
+    }
     if (_PyArena_AddPyObject(p->arena, str) < 0) {
         Py_DECREF(str);
         return NULL;
@@ -1523,6 +1530,21 @@ expr_ty _PyPegen_constant_from_token(Parser* p, Token* tok) {
         return _PyPegen_raise_incomplete_input(p);
     }
     const char *bstr = PyBytes_AS_STRING(tok->bytes);
+    Py_ssize_t bsize = PyBytes_GET_SIZE(tok->bytes);
+    if (((p->flags & PyPARSE_ALLOW_INCOMPLETE_INPUT) || tok->is_incomplete) &&
+        !tok->is_raw && memchr(bstr, '\\', bsize) != NULL) {
+        // Check escapes before the string is complete, without emitting warnings.
+        PyObject *decoded = _PyPegen_decode_string(
+            p, 0, bstr, bsize, tok->is_incomplete ? tok : NULL);
+        if (decoded == NULL) {
+            _Pypegen_raise_decode_error(p);
+            return NULL;
+        }
+        Py_DECREF(decoded);
+    }
+    if (tok->is_incomplete) {
+        return _PyPegen_raise_incomplete_input(p);
+    }
     PyObject* str = PyUnicode_FromString(bstr);
     if (str == NULL) {
         return NULL;

Parser/lexer/state.c

@@ -86,6 +86,7 @@ _PyLexer_token_setup(struct tok_state *tok, struct token *token, int type, _PyTo
 {
     token->level = tok->level;
     token->is_raw = 0;
+    token->is_incomplete = 0;
     assert((start == -1 && end == -1) || (start >= 0 && end >= start));
     token->span = (_PyTok_Span){start, end};
     if (start >= 0) {

Parser/lexer/string.c

@@ -27,6 +27,7 @@ incomplete_string_token(struct tok_state *tok, struct token *token,
 {
     string_error_token(tok, token, start, location);
     tok->done = E_INCOMPLETE;
+    token->is_incomplete = 1;
     return type;
 }
 
@@ -464,8 +465,18 @@ _PyLexer_get_ftstring(struct tok_state *tok, ftstring_state *current, struct tok
                 return MAKE_TOKEN(ERRORTOKEN);
             }
             if (c == EOF && tok->allow_incomplete_input) {
-                return incomplete_string_token(tok, token,
+                int type = incomplete_string_token(tok, token,
                     FTSTRING_MIDDLE(current), current->start, current->start_loc);
+                // Preserve the actual prefix for incremental escape decoding.
+                _PyTok_Off end = tok->cur;
+                if (end > tok->start && tok->implicit_newline) {
+                    end--;
+                }
+                token->span = (_PyTok_Span){tok->start, end};
+                token->start_loc = tok->start_loc;
+                token->end_loc = (_PyTok_Loc){tok->lineno, (int)(end - tok->line_start)};
+                token->is_raw = raw;
+                return type;
             }
 
             if (in_format_spec && c == '\n') {

Parser/pegen.c

@@ -279,6 +279,7 @@ initialize_token(Parser *p, Token *parser_token, struct token *new_token, int to
 
     parser_token->level = new_token->level;
     parser_token->is_raw = new_token->is_raw;
+    parser_token->is_incomplete = new_token->is_incomplete;
     parser_token->lineno = new_token->start_loc.lineno;
     parser_token->col_offset = new_token->end_loc.lineno == p->starting_lineno
         ? p->starting_col_offset + new_token->start_loc.byte_col

Parser/pegen.h

@@ -39,11 +39,12 @@ typedef struct _memo {
 typedef struct {
     int type;
     // Text required by grammar actions or helpers; otherwise NULL.
-    // An unfinished string token also has NULL text. Its grammar action
-    // requests more input only if the token is valid at that position.
+    // Unfinished f-string and t-string middle tokens retain their text so
+    // their grammar action can check escapes before requesting more input.
     PyObject *bytes;
     int level;
-    int is_raw;
+    unsigned char is_raw;
+    unsigned char is_incomplete;
     int lineno, col_offset, end_lineno, end_col_offset;
     Memo *memo;
     // Filter over the rule types present in `memo` (bit `type & 63` is set

Parser/pegen_errors.c

@@ -278,10 +278,17 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
     PyObject *tmp = NULL;
     p->error_indicator = 1;
     _PyTokenizer_Info info = _PyTokenizer_GetInfo(p->tok);
-    _PyTok_Loc location = info.diagnostic.location.lineno != 0
-        ? info.diagnostic.location : info.location;
-    _PyTok_Span text_span = info.diagnostic.location.lineno != 0
-        ? info.diagnostic.text_span : info.line_span;
+    _PyTok_Loc location = info.location;
+    _PyTok_Span text_span = info.line_span;
+
+    if (errtype == PyExc_IncompleteInputError &&
+        info.diagnostic.location.lineno != 0) {
+        location = info.diagnostic.location;
+        text_span = info.diagnostic.text_span;
+        lineno = end_lineno = location.lineno;
+        col_offset = location.byte_col;
+        end_col_offset = -1;
+    }
 
     if (end_lineno == CURRENT_POS) {
         end_lineno = location.lineno;
@@ -346,7 +353,13 @@ _PyPegen_raise_error_known_location(Parser *p, PyObject *errtype,
     }
 
     if (end_col_offset > 0) {
-        end_col_number = _PyPegen_byte_offset_to_character_offset(error_line, end_col_offset);
+        PyObject *end_line = end_lineno != lineno && _PyTokenizer_RetainedSource(p->tok) != NULL
+            ? get_error_line_from_source(p, end_lineno) : Py_NewRef(error_line);
+        if (end_line == NULL) {
+            goto error;
+        }
+        end_col_number = _PyPegen_byte_offset_to_character_offset(end_line, end_col_offset);
+        Py_DECREF(end_line);
         if (end_col_number < 0) {
             goto error;
         }

Parser/string_parser.c

@@ -194,13 +194,17 @@ decode_unicode_with_escapes(Parser *parser, const char *s, size_t len, Token *t)
 
     int first_invalid_escape_char;
     const char *first_invalid_escape_ptr;
-    v = _PyUnicode_DecodeUnicodeEscapeInternal2(buf, (Py_ssize_t)len, NULL, NULL,
+    // Allow a trailing unfinished escape in a partial token.
+    int incomplete = t != NULL && t->is_incomplete;
+    Py_ssize_t consumed;
+    v = _PyUnicode_DecodeUnicodeEscapeInternal2(buf, (Py_ssize_t)len, NULL,
+                                                incomplete ? &consumed : NULL,
                                                 &first_invalid_escape_char,
                                                 &first_invalid_escape_ptr);
 
     // HACK: later we can simply pass the line no, since we don't preserve the tokens
     // when we are decoding the string but we preserve the line numbers.
-    if (v != NULL && first_invalid_escape_ptr != NULL && t != NULL) {
+    if (v != NULL && first_invalid_escape_ptr != NULL && t != NULL && !incomplete) {
         if (warn_invalid_escape_sequence(parser, buf, first_invalid_escape_ptr, t) < 0) {
             /* We have not deallocated the buffer before because
              * first_invalid_escape_ptr points inside buf. */