gh-154682: Fix corruption of locale-encoded text on FreeBSD, NetBSD, DragonFly BSD and macOS

Include/internal/pycore_fileutils.h

@@ -225,6 +225,29 @@ extern int _Py_GetForceASCII(void);
    encoding. */
 extern void _Py_ResetForceASCII(void);
 
+#ifdef HAVE_LANGINFO_H
+#  include <langinfo.h>           // CODESET
+#endif
+
+/* On FreeBSD, NetBSD, DragonFly BSD and macOS, wchar_t values are not
+   Unicode code points in non-UTF-8 locales.  Locale-encoded text is converted
+   with iconv() (see Python/fileutils.c), and the wide character
+   functions which use the locale (such as wcscoll()) cannot be used
+   with Unicode wchar_t. */
+#if (defined(__FreeBSD__) || defined(__NetBSD__) || defined(__DragonFly__) \
+     || defined(__APPLE__)) \
+    && defined(HAVE_ICONV) \
+    && defined(HAVE_LANGINFO_H) && defined(CODESET) && SIZEOF_WCHAR_T == 4
+#  define _Py_NON_UNICODE_WCHAR_T
+
+// Convert between Unicode code points and the native wchar_t form for
+// C library functions which use the latter (such as the curses library).
+// Export for '_curses'.
+PyAPI_FUNC(int) _Py_LocaleNeedsWcharConversion(void);
+PyAPI_FUNC(int) _Py_UnicodeToLocaleWchar_InPlace(wchar_t *str, Py_ssize_t size);
+PyAPI_FUNC(void) _Py_LocaleWcharToUnicode_InPlace(wchar_t *str, Py_ssize_t size);
+#endif
+
 
 extern int _Py_GetLocaleconvNumeric(
     struct lconv *lc,

Include/internal/pycore_runtime_structs.h

@@ -52,6 +52,12 @@ struct pyhash_runtime_state {
 
 struct _fileutils_state {
     int force_ascii;
+    /* Memoized result of _Py_LocaleNeedsWcharConversion() with the locale
+       encoding it was computed for (see Python/fileutils.c). */
+    struct _Py_wchar_conversion_state {
+        char codeset[32];
+        int needed;
+    } wchar;
 };
 
 #include "pycore_interpframe_structs.h" // _PyInterpreterFrame

Lib/test/test_locale.py

@@ -350,6 +350,53 @@ def test_strxfrm(self):
         self.assertRaises(ValueError, locale.strxfrm, 'a\0')
 
 
+class TestCollationAcrossEncodings(unittest.TestCase):
+    # gh-154682: collation must give the same results in locales which
+    # only differ in the encoding.
+
+    @support.thread_unsafe('setlocale is not thread-safe')
+    @unittest.skipIf(sys.platform.startswith("netbsd"),
+                     "gh-124108: NetBSD doesn't support UTF-8 for LC_COLLATE")
+    @unittest.skipIf(sys.platform.startswith("dragonfly") or
+                     sys.platform == "darwin",
+                     "no collation data for non-UTF-8 locales")
+    def test_across_encodings(self):
+        def sign(x):
+            return (x > 0) - (x < 0)
+        cases = [
+            ('uk_UA.UTF-8', 'uk_UA.KOI8-U', 'гґдиіїя'),
+            ('uk_UA.UTF-8', 'uk_UA.CP1251', 'гґдиіїя'),
+            ('el_GR.UTF-8', 'el_GR.ISO8859-7', 'αβδω'),
+            ('de_DE.UTF-8', 'de_DE.ISO8859-1', 'aäbößz'),
+            ('tr_TR.UTF-8', 'tr_TR.ISO8859-9', 'cçdıisşz'),
+            ('ca_ES.UTF-8', 'ca_ES.ISO8859-1', 'cçde'),
+        ]
+        oldloc = locale.setlocale(locale.LC_ALL)
+        self.addCleanup(locale.setlocale, locale.LC_ALL, oldloc)
+        tested = False
+        for utf8_loc, legacy_loc, chars in cases:
+            try:
+                locale.setlocale(locale.LC_ALL, utf8_loc)
+            except locale.Error:
+                continue
+            expected_order = sorted(chars, key=locale.strxfrm)
+            expected_signs = [[sign(locale.strcoll(a, b)) for b in chars]
+                              for a in chars]
+            try:
+                locale.setlocale(locale.LC_ALL, legacy_loc)
+            except locale.Error:
+                continue
+            with self.subTest(locale=legacy_loc):
+                self.assertEqual(sorted(chars, key=locale.strxfrm),
+                                 expected_order)
+                signs = [[sign(locale.strcoll(a, b)) for b in chars]
+                         for a in chars]
+                self.assertEqual(signs, expected_signs)
+            tested = True
+        if not tested:
+            self.skipTest('no suitable locales')
+
+
 class TestEnUSCollation(BaseLocalizedTest, TestCollation):
     # Test string collation functions with a real English locale
 

Lib/test/test_readline.py

@@ -125,6 +125,10 @@ def test_write_read_append(self):
             readline.append_history_file(-2147483648, hfilename)
 
     def test_nonascii_history(self):
+        try:
+            "entrée".encode(locale.getencoding())
+        except UnicodeEncodeError as err:
+            self.skipTest("Locale cannot encode test data: " + format(err))
         readline.clear_history()
         try:
             readline.add_history("entrée 1")

Lib/test/test_time.py

@@ -3,6 +3,7 @@
 import decimal
 import enum
 import fractions
+import locale
 import math
 import platform
 import sys
@@ -232,6 +233,41 @@ def test_strftime(self):
 
         self.assertRaises(TypeError, time.strftime, b'%S', tt)
 
+    @support.thread_unsafe('setlocale is not thread-safe')
+    def test_strftime_locale_encodings(self):
+        # gh-154682: the result must be the same in locales which only
+        # differ in the encoding.
+        pairs = [
+            ('uk_UA.UTF-8', 'uk_UA.KOI8-U'),
+            ('uk_UA.UTF-8', 'uk_UA.CP1251'),
+            ('el_GR.UTF-8', 'el_GR.ISO8859-7'),
+            ('ja_JP.UTF-8', 'ja_JP.eucJP'),
+            ('de_DE.UTF-8', 'de_DE.ISO8859-1'),
+            ('tr_TR.UTF-8', 'tr_TR.ISO8859-9'),
+            ('ca_ES.UTF-8', 'ca_ES.ISO8859-1'),
+        ]
+        oldloc = locale.setlocale(locale.LC_ALL)
+        self.addCleanup(locale.setlocale, locale.LC_ALL, oldloc)
+        # Wednesday, March: non-ASCII in all tested locales
+        # (including Turkish and German)
+        tt = (2026, 3, 4, 12, 34, 56, 2, 63, 0)
+        tested = False
+        for utf8_loc, legacy_loc in pairs:
+            try:
+                locale.setlocale(locale.LC_ALL, utf8_loc)
+            except locale.Error:
+                continue
+            expected = time.strftime('%A, %B', tt)
+            try:
+                locale.setlocale(locale.LC_ALL, legacy_loc)
+            except locale.Error:
+                continue
+            with self.subTest(locale=legacy_loc):
+                self.assertEqual(time.strftime('%A, %B', tt), expected)
+            tested = True
+        if not tested:
+            self.skipTest('no suitable locales')
+
     def test_strftime_invalid_format(self):
         tt = time.gmtime(self.t)
         with SuppressCrashReport():

Misc/NEWS.d/next/Core_and_Builtins/2026-07-25-13-50-00.gh-issue-154682.wChBSD.rst

@@ -0,0 +1,10 @@
+Fix corruption of locale-encoded text on FreeBSD, NetBSD, DragonFly BSD
+and macOS
+in non-UTF-8 locales,
+e.g. in the results of :func:`time.strftime` and :func:`locale.nl_langinfo`,
+in localized error messages,
+and in decoding of the command line arguments,
+fix wrong collation in :func:`locale.strcoll` and :func:`locale.strxfrm`,
+and fix display and input of non-ASCII characters in :mod:`curses`.
+``wchar_t`` values are not Unicode code points on these platforms,
+so ``iconv()`` is now used instead of ``mbstowcs()`` and ``wcstombs()``.

Modules/_cursesmodule.c

@@ -527,6 +527,11 @@ PyCurses_ConvertToWideCell(PyObject *obj, wchar_t *wch)
                      (int)CCHARW_MAX, PyUnicode_GET_LENGTH(obj));
         return -1;
     }
+#ifdef _Py_NON_UNICODE_WCHAR_T
+    if (_Py_UnicodeToLocaleWchar_InPlace(wch, nch) < 0) {
+        return -1;
+    }
+#endif
     /* A lone control character is allowed (like addch(ord('\n'))), but in a
        multi-character cell the base must be a printable spacing character and
        the rest zero-width combining characters.  Check explicitly: otherwise
@@ -622,6 +627,13 @@ PyCurses_ConvertToString(PyCursesWindowObject *win, PyObject *obj,
         *wstr = PyUnicode_AsWideCharString(obj, NULL);
         if (*wstr == NULL)
             return 0;
+#ifdef _Py_NON_UNICODE_WCHAR_T
+        if (_Py_UnicodeToLocaleWchar_InPlace(*wstr, wcslen(*wstr)) < 0) {
+            PyMem_Free(*wstr);
+            *wstr = NULL;
+            return 0;
+        }
+#endif
         return 2;
 #else
         assert (wstr == NULL);
@@ -880,6 +892,9 @@ curses_cell_text(cursesmodule_state *state, const curses_cell_t *cell)
         PyErr_SetString(state->error, "getcchar() returned ERR");
         return NULL;
     }
+#ifdef _Py_NON_UNICODE_WCHAR_T
+    _Py_LocaleWcharToUnicode_InPlace(wstr, wcslen(wstr));
+#endif
     return PyUnicode_FromWideChar(wstr, -1);
 #else
     char ch = (char)(*cell & A_CHARTEXT);
@@ -1296,6 +1311,12 @@ complexstr_from_string(cursesmodule_state *state, PyObject *str,
     if (wbuf == NULL) {
         return NULL;
     }
+#ifdef _Py_NON_UNICODE_WCHAR_T
+    if (_Py_UnicodeToLocaleWchar_InPlace(wbuf, n) < 0) {
+        PyMem_Free(wbuf);
+        return NULL;
+    }
+#endif
     curses_cell_t *cells = n > 0 ? PyMem_New(curses_cell_t, n) : NULL;
     if (n > 0 && cells == NULL) {
         PyMem_Free(wbuf);
@@ -1585,6 +1606,9 @@ complexstr_str(PyObject *self)
         }
         pos += wcslen(buf + pos);
     }
+#ifdef _Py_NON_UNICODE_WCHAR_T
+    _Py_LocaleWcharToUnicode_InPlace(buf, pos);
+#endif
     PyObject *res = PyUnicode_FromWideChar(buf, pos);
     PyMem_Free(buf);
     return res;
@@ -3275,6 +3299,13 @@ _curses_window_getkey_impl(PyCursesWindowObject *self, int group_right_1,
             rtn += 256;
         }
 #endif
+#endif
+#ifdef _Py_NON_UNICODE_WCHAR_T
+        {
+            wchar_t wch = (wchar_t)rtn;
+            _Py_LocaleWcharToUnicode_InPlace(&wch, 1);
+            return PyUnicode_FromOrdinal(wch);
+        }
 #endif
         return PyUnicode_FromOrdinal(rtn);
     } else {
@@ -3325,8 +3356,16 @@ _curses_window_get_wch_impl(PyCursesWindowObject *self, int group_right_1,
     }
     if (ct == KEY_CODE_YES)
         return PyLong_FromLong(rtn);
+#ifdef _Py_NON_UNICODE_WCHAR_T
+    else {
+        wchar_t wch = (wchar_t)rtn;
+        _Py_LocaleWcharToUnicode_InPlace(&wch, 1);
+        return PyUnicode_FromOrdinal(wch);
+    }
+#else
     else
         return PyUnicode_FromOrdinal(rtn);
+#endif
 #else
     /* Without the wide library, read one key with wgetch(): a value above 255
        is a function key (returned as an int); a byte is decoded with the
@@ -3801,6 +3840,9 @@ PyCursesWindow_get_wstr(PyObject *op, PyObject *args)
     for (Py_ssize_t i = 0; i < len; i++) {
         wbuf[i] = (wchar_t)buf[i];
     }
+#ifdef _Py_NON_UNICODE_WCHAR_T
+    _Py_LocaleWcharToUnicode_InPlace(wbuf, len);
+#endif
     PyObject *res = PyUnicode_FromWideChar(wbuf, len);
     PyMem_Free(wbuf);
     PyMem_Free(buf);
@@ -3866,6 +3908,9 @@ PyCursesWindow_in_wstr(PyObject *op, PyObject *args)
         PyMem_Free(buf);
         return Py_GetConstant(Py_CONSTANT_EMPTY_STR);
     }
+#ifdef _Py_NON_UNICODE_WCHAR_T
+    _Py_LocaleWcharToUnicode_InPlace(buf, wcslen(buf));
+#endif
     PyObject *res = PyUnicode_FromWideChar(buf, -1);
     PyMem_Free(buf);
     return res;
@@ -5854,6 +5899,9 @@ _curses_erasewchar_impl(PyObject *module)
         curses_set_error(module, "erasewchar", NULL);
         return NULL;
     }
+#ifdef _Py_NON_UNICODE_WCHAR_T
+    _Py_LocaleWcharToUnicode_InPlace(&ch, 1);
+#endif
     return PyUnicode_FromWideChar(&ch, 1);
 #else
     /* Without the wide library, decode the single-byte erase character
@@ -7076,6 +7124,9 @@ _curses_killwchar_impl(PyObject *module)
         curses_set_error(module, "killwchar", NULL);
         return NULL;
     }
+#ifdef _Py_NON_UNICODE_WCHAR_T
+    _Py_LocaleWcharToUnicode_InPlace(&ch, 1);
+#endif
     return PyUnicode_FromWideChar(&ch, 1);
 #else
     /* Without the wide library, decode the single-byte kill character
@@ -8018,6 +8069,19 @@ _curses_wunctrl(PyObject *module, PyObject *ch)
         curses_set_null_error(module, "wunctrl", NULL);
         return NULL;
     }
+#ifdef _Py_NON_UNICODE_WCHAR_T
+    {
+        /* wunctrl() returns a pointer to a static buffer; make a copy
+           before converting in place. */
+        wchar_t wbuf[8];
+        size_t len = wcslen(res);
+        if (len < Py_ARRAY_LENGTH(wbuf)) {
+            wcscpy(wbuf, res);
+            _Py_LocaleWcharToUnicode_InPlace(wbuf, len);
+            return PyUnicode_FromWideChar(wbuf, len);
+        }
+    }
+#endif
     return PyUnicode_FromWideChar(res, -1);
 #else
     /* Without the wide library, fall back to the single-byte unctrl() and
@@ -8107,6 +8171,10 @@ _curses_ungetch(PyObject *module, PyObject *ch)
         wchar_t wch;
         if (!PyCurses_ConvertToWchar_t(ch, &wch))
             return NULL;
+#ifdef _Py_NON_UNICODE_WCHAR_T
+        if (_Py_UnicodeToLocaleWchar_InPlace(&wch, 1) < 0)
+            return NULL;
+#endif
         return curses_check_err(module, unget_wch(wch), "unget_wch", "ungetch");
     }
 #endif
@@ -8138,6 +8206,10 @@ _curses_unget_wch(PyObject *module, PyObject *ch)
     if (!PyCurses_ConvertToWchar_t(ch, &wch))
         return NULL;
 #ifdef HAVE_NCURSESW
+#ifdef _Py_NON_UNICODE_WCHAR_T
+    if (_Py_UnicodeToLocaleWchar_InPlace(&wch, 1) < 0)
+        return NULL;
+#endif
     return curses_check_err(module, unget_wch(wch), "unget_wch", NULL);
 #else
     /* Without the wide library there is no unget_wch(): encode the character as
@@ -8226,6 +8298,12 @@ _curses_slk_set_impl(PyObject *module, int labnum, PyObject *label,
     if (wstr == NULL) {
         return NULL;
     }
+#ifdef _Py_NON_UNICODE_WCHAR_T
+    if (_Py_UnicodeToLocaleWchar_InPlace(wstr, wcslen(wstr)) < 0) {
+        PyMem_Free(wstr);
+        return NULL;
+    }
+#endif
     rtn = slk_wset(labnum, wstr, justify);
     PyMem_Free(wstr);
 #else

Modules/_decimal/_decimal.c

@@ -3657,20 +3657,20 @@ dotsep_as_utf8(const char *s)
 {
     PyObject *utf8;
     PyObject *tmp;
-    wchar_t buf[2];
-    size_t n;
 
-    n = mbstowcs(buf, s, 2);
-    if (n != 1) { /* Issue #7442 */
+    /* Do not use mbstowcs(): wchar_t values are not Unicode code points
+       in non-UTF-8 locales on some platforms (e.g. the BSDs and macOS). */
+    tmp = PyUnicode_DecodeLocale(s, NULL);
+    if (tmp == NULL) {
+        return NULL;
+    }
+    if (PyUnicode_GET_LENGTH(tmp) != 1) { /* Issue #7442 */
+        Py_DECREF(tmp);
         PyErr_SetString(PyExc_ValueError,
             "invalid decimal point or unsupported "
             "combination of LC_CTYPE and LC_NUMERIC");
         return NULL;
     }
-    tmp = PyUnicode_FromWideChar(buf, n);
-    if (tmp == NULL) {
-        return NULL;
-    }
     utf8 = PyUnicode_AsUTF8String(tmp);
     Py_DECREF(tmp);
     return utf8;