From d4dd5e5a34cc6c9dbc27b8a4e50d0469667e90f4 Mon Sep 17 00:00:00 2001 From: Chris Speciale <26172437+dimensionscape@users.noreply.github.com> Date: Fri, 6 Nov 2026 18:48:32 -0500 Subject: [PATCH 1/3] Fix three cpp.encoding.Utf8.encode faults at API level 500 encode(String) returned 0 for the empty string, which made a null array (libhxcpp #406). It returns an empty one. encode(char32_t, View) chose a byte count by range alone, so a surrogate went out as three bytes and anything past U+10FFFF as four, where the UTF-16 encoder refused both (libhxcpp #407). Both encoders, and both getByteCount for a code point, throw for one that is no scalar value, before writing anything. A String's lone surrogate still becomes U+FFFD before it gets there. The array encode(String) allocated three bytes for each unit of a string that was not all ASCII and resize only shortened it, so Haxe 5's Bytes.ofString kept 3 MB for a megabyte of ASCII and one other character (libhxcpp #405). It counts the bytes first and allocates them exactly. The fixtures do the level 500 calls in native code that passes below that level, as Haxe 4 and 5.0.0-preview.1 build against the 430 runtime, and Utf8EncodeCapacity checks Haxe 4's Bytes.ofString too. --- Changes.md | 3 ++ src/cpp/encoding/Encodings.cpp | 36 +++++++++++-- test/regression/Utf8EmptyArray/Main.hx | 40 ++++++++++++++ test/regression/Utf8EmptyArray/build.hxml | 2 + test/regression/Utf8EmptyArray/stdout.txt | 3 ++ test/regression/Utf8EncodeCapacity/Main.hx | 29 +++++++++++ test/regression/Utf8EncodeCapacity/build.hxml | 2 + test/regression/Utf8EncodeCapacity/stdout.txt | 3 ++ test/regression/Utf8InvalidScalars/Main.hx | 52 +++++++++++++++++++ test/regression/Utf8InvalidScalars/build.hxml | 2 + test/regression/Utf8InvalidScalars/stdout.txt | 4 ++ 11 files changed, 171 insertions(+), 5 deletions(-) create mode 100644 test/regression/Utf8EmptyArray/Main.hx create mode 100644 test/regression/Utf8EmptyArray/build.hxml create mode 100644 test/regression/Utf8EmptyArray/stdout.txt create mode 100644 test/regression/Utf8EncodeCapacity/Main.hx create mode 100644 test/regression/Utf8EncodeCapacity/build.hxml create mode 100644 test/regression/Utf8EncodeCapacity/stdout.txt create mode 100644 test/regression/Utf8InvalidScalars/Main.hx create mode 100644 test/regression/Utf8InvalidScalars/build.hxml create mode 100644 test/regression/Utf8InvalidScalars/stdout.txt diff --git a/Changes.md b/Changes.md index d76013cc0..c9e9b75d0 100644 --- a/Changes.md +++ b/Changes.md @@ -2,6 +2,9 @@ libhxcpp 6.0.0 (unreleased) ------------------------------------------------------------ +* Fixed cpp.encoding.Utf8.encode returning a null array for the empty string, at API level 500 (libhxcpp #406) +* Fixed cpp.encoding.Utf8.encode writing bytes for a code point that is no Unicode scalar value, a surrogate or one past U+10FFFF, which UTF-8 cannot hold and the UTF-16 encoder refused. Both encoders, and both getByteCount, throw for one before writing anything, at API level 500 (libhxcpp #407) +* Fixed Haxe 5's Bytes.ofString keeping an array of three bytes for each unit of a string that was not all ASCII: cpp.encoding.Utf8.encode allocated for the worst case and only shortened it, so a megabyte of ASCII with one other character held 3 MB. It counts the bytes and allocates them exactly (libhxcpp #405) * Fixed an exception from blit, setSize or resize under the cppia JIT ending the process instead of reaching the script's catch, on typed arrays and on Array: their native calls had no exception bridge. The script catches it, as it does interpreted and in compiled code (libhxcpp #404) * Fixed resize() and cpp.NativeArray.setSize() on an Array under the cppia JIT evaluating the array expression twice, so a call in it ran twice and the array it returned the second time was the one resized. The JIT evaluates it once (libhxcpp #411) * Fixed indexOf and lastIndexOf on an Array under the cppia JIT ignoring fromIndex: the JIT never generated it, so a call or exception in it was lost too. The search starts from it, as it does interpreted and in compiled code (libhxcpp #410) diff --git a/src/cpp/encoding/Encodings.cpp b/src/cpp/encoding/Encodings.cpp index b377a2aad..986baf936 100644 --- a/src/cpp/encoding/Encodings.cpp +++ b/src/cpp/encoding/Encodings.cpp @@ -142,6 +142,17 @@ namespace return 0xFFFD; } #endif + + // A code point that is not a Unicode scalar value, a surrogate or past + // U+10FFFF, has no UTF-8 or UTF-16 form. The UTF-16 encoder refused them + // and the UTF-8 one wrote invalid bytes + void checkScalar(char32_t codepoint) + { + if ((codepoint >= 0xD800 && codepoint < 0xE000) || codepoint > 0x10FFFF) + { + hx::Throw(HX_CSTRING("Invalid codepoint")); + } + } } int cpp::encoding::Utf8::getByteCount(const null&) @@ -152,6 +163,8 @@ int cpp::encoding::Utf8::getByteCount(const null&) int cpp::encoding::Utf8::getByteCount(char32_t codepoint) { + checkScalar(codepoint); + if (codepoint <= 0x7F) { return 1; @@ -281,7 +294,8 @@ Array cpp::encoding::Utf8::encode(const String& string) if (0 == string.length) { - return 0; + // An empty array: 0 would be a null one + return Array(0, 0); } if (string.isAsciiEncoded()) @@ -297,8 +311,17 @@ Array cpp::encoding::Utf8::encode(const String& string) } #if defined(HX_SMART_STRINGS) - // estimate the utf8 length with an upper bound - Array out(string.length * 3, 0); + // Counted first: an array of three bytes a unit, which resize only made + // shorter, kept that capacity as long as the bytes lived, three times + // what a mostly ASCII string needs (libhxcpp #405) + auto bytes = getByteCount(string); + + if (bytes > 0x7fffffff) + { + return hx::Throw(HX_CSTRING("String too long")); + } + + Array out(static_cast(bytes), 0); View buffer(out->Pointer(), out->length); auto units = string.raw_wptr(); @@ -310,8 +333,6 @@ Array cpp::encoding::Utf8::encode(const String& string) k += encode(stringCodepoint(units, string.length, i), buffer.slice(k)); } - out->resize(k); - return out; #else return hx::Throw(HX_CSTRING("Unexpected encoding error")); @@ -320,6 +341,9 @@ Array cpp::encoding::Utf8::encode(const String& string) int cpp::encoding::Utf8::encode(char32_t codepoint, const cpp::marshal::View& buffer) { + // Before anything is written + checkScalar(codepoint); + if (codepoint <= 0x7F) { buffer[0] = static_cast(codepoint); @@ -557,6 +581,8 @@ int cpp::encoding::Utf16::getByteCount(const null&) int cpp::encoding::Utf16::getByteCount(char32_t codepoint) { + checkScalar(codepoint); + return codepoint <= 0xFFFF ? 2 : 4; } diff --git a/test/regression/Utf8EmptyArray/Main.hx b/test/regression/Utf8EmptyArray/Main.hx new file mode 100644 index 000000000..467dee0f8 --- /dev/null +++ b/test/regression/Utf8EmptyArray/Main.hx @@ -0,0 +1,40 @@ +// cpp.encoding.Utf8.encode gave a null array for the empty string +// (openfl/libhxcpp#406). It is API level 500, which Haxe 4 and Haxe +// 5.0.0-preview.1 do not build against, so the checks are native code +// that passes there. +@:cppFileCode(' +#if (HXCPP_API_LEVEL>=500) +static bool encodes(String inText, const unsigned char *inBytes, int inLength) +{ + Array bytes = cpp::encoding::Utf8::encode(inText); + if (!bytes.mPtr || bytes->length != inLength) + return false; + for (int i = 0; i < inLength; i++) + if (bytes[i] != inBytes[i]) + return false; + return true; +} +static bool emptyEncoding() { return encodes(HX_CSTRING(""), nullptr, 0); } +static bool asciiEncoding() +{ + static const unsigned char abc[] = { 0x61, 0x62, 0x63 }; + return encodes(HX_CSTRING("abc"), abc, 3); +} +static bool unicodeEncoding() +{ + static const unsigned char wide[] = { 0xC4, 0x80 }; + return encodes(String::fromCharCode(0x100), wide, 2); +} +#else +static bool emptyEncoding() { return true; } +static bool asciiEncoding() { return true; } +static bool unicodeEncoding() { return true; } +#endif +') +class Main { + static function main() { + Sys.println("empty UTF8 encoding returns an empty array: " + (untyped __cpp__("emptyEncoding()") : Bool)); + Sys.println("ASCII UTF8 encoding remains intact: " + (untyped __cpp__("asciiEncoding()") : Bool)); + Sys.println("Unicode UTF8 encoding remains intact: " + (untyped __cpp__("unicodeEncoding()") : Bool)); + } +} diff --git a/test/regression/Utf8EmptyArray/build.hxml b/test/regression/Utf8EmptyArray/build.hxml new file mode 100644 index 000000000..b484e67e7 --- /dev/null +++ b/test/regression/Utf8EmptyArray/build.hxml @@ -0,0 +1,2 @@ +-m Main +--cpp bin diff --git a/test/regression/Utf8EmptyArray/stdout.txt b/test/regression/Utf8EmptyArray/stdout.txt new file mode 100644 index 000000000..9e3eda8c7 --- /dev/null +++ b/test/regression/Utf8EmptyArray/stdout.txt @@ -0,0 +1,3 @@ +empty UTF8 encoding returns an empty array: true +ASCII UTF8 encoding remains intact: true +Unicode UTF8 encoding remains intact: true diff --git a/test/regression/Utf8EncodeCapacity/Main.hx b/test/regression/Utf8EncodeCapacity/Main.hx new file mode 100644 index 000000000..1590f60a1 --- /dev/null +++ b/test/regression/Utf8EncodeCapacity/Main.hx @@ -0,0 +1,29 @@ +// Haxe 5's Bytes.ofString kept an array of three bytes for each unit of a +// string that was not all ASCII (openfl/libhxcpp#405). Haxe 4's allocates what +// it needs, which this checks too. +class Main { + static function main() { + var correct = true; + var bounded = true; + var asciiExact = true; + var plain = haxe.io.Bytes.alloc(1 << 20); + plain.fill(0, plain.length, 97); + var ascii = plain.toString(); + var text = ascii + "\u0100"; + var encoded = haxe.io.Bytes.ofString(text); + var data = encoded.getData(); + correct = encoded.length == plain.length + 2 && encoded.toString() == text; + for (_ in 0...3) cpp.vm.Gc.run(true); + var capacity:Int = untyped __cpp__("{0}->capacity()", data); + bounded = capacity <= encoded.length + 65536; + var control = haxe.io.Bytes.ofString(ascii); + var controlData = control.getData(); + var controlCapacity:Int = untyped __cpp__("{0}->capacity()", controlData); + asciiExact = controlCapacity <= control.length + 65536 && control.toString() == ascii; + if (Sys.args().length > 0) + Sys.println(haxe.Json.stringify({length: encoded.length, capacity: capacity, asciiLength: control.length, asciiCapacity: controlCapacity})); + Sys.println("UTF8 encoding preserves the complete payload: " + correct); + Sys.println("retained UTF8 storage stays close to the output length: " + bounded); + Sys.println("ASCII control storage stays close to the output length: " + asciiExact); + } +} diff --git a/test/regression/Utf8EncodeCapacity/build.hxml b/test/regression/Utf8EncodeCapacity/build.hxml new file mode 100644 index 000000000..b484e67e7 --- /dev/null +++ b/test/regression/Utf8EncodeCapacity/build.hxml @@ -0,0 +1,2 @@ +-m Main +--cpp bin diff --git a/test/regression/Utf8EncodeCapacity/stdout.txt b/test/regression/Utf8EncodeCapacity/stdout.txt new file mode 100644 index 000000000..db5e7a814 --- /dev/null +++ b/test/regression/Utf8EncodeCapacity/stdout.txt @@ -0,0 +1,3 @@ +UTF8 encoding preserves the complete payload: true +retained UTF8 storage stays close to the output length: true +ASCII control storage stays close to the output length: true diff --git a/test/regression/Utf8InvalidScalars/Main.hx b/test/regression/Utf8InvalidScalars/Main.hx new file mode 100644 index 000000000..a6f08a644 --- /dev/null +++ b/test/regression/Utf8InvalidScalars/Main.hx @@ -0,0 +1,52 @@ +// cpp.encoding.Utf8.encode wrote bytes for a surrogate or a value past +// U+10FFFF, which the UTF-16 encoder refuses (openfl/libhxcpp#407). It is API +// level 500, which Haxe 4 and Haxe 5.0.0-preview.1 do not build against, so +// the checks are native code that passes there. The result has a bit set for +// each check that failed. +@:cppFileCode(' +#if (HXCPP_API_LEVEL>=500) +static int scalarFailures() +{ + using cpp::marshal::View; + int failed = 0; + const char32_t invalid[] = { 0xD800, 0xDFFF, 0x110000, (char32_t)-1 }; + for (char32_t point : invalid) + { + unsigned char data[4] = { 0x55, 0x55, 0x55, 0x55 }; + View view(cpp::Pointer(data), 4); + bool refused = false; + try { cpp::encoding::Utf8::encode(point, view); } catch (Dynamic &) { refused = true; } + if (!refused) + failed |= 1; + for (unsigned char byte : data) + if (byte != 0x55) + failed |= 2; + refused = false; + try { cpp::encoding::Utf16::encode(point, view); } catch (Dynamic &) { refused = true; } + if (!refused) + failed |= 4; + } + const char32_t valid[] = { 0, 0x7F, 0x80, 0xD7FF, 0xE000, 0x10FFFF }; + for (char32_t point : valid) + { + unsigned char data[4]; + View view(cpp::Pointer(data), 4); + int length = cpp::encoding::Utf8::encode(point, view); + if (cpp::encoding::Utf8::codepoint(view.slice(0, length)) != point) + failed |= 8; + } + return failed; +} +#else +static int scalarFailures() { return 0; } +#endif +') +class Main { + static function main() { + var failed:Int = untyped __cpp__("scalarFailures()"); + Sys.println("invalid scalars are rejected by UTF8 encoding: " + (failed & 1 == 0)); + Sys.println("invalid scalars leave the output buffer unchanged: " + (failed & 2 == 0)); + Sys.println("UTF16 encoding rejects the same invalid scalars: " + (failed & 4 == 0)); + Sys.println("valid boundary scalars round trip: " + (failed & 8 == 0)); + } +} diff --git a/test/regression/Utf8InvalidScalars/build.hxml b/test/regression/Utf8InvalidScalars/build.hxml new file mode 100644 index 000000000..b484e67e7 --- /dev/null +++ b/test/regression/Utf8InvalidScalars/build.hxml @@ -0,0 +1,2 @@ +-m Main +--cpp bin diff --git a/test/regression/Utf8InvalidScalars/stdout.txt b/test/regression/Utf8InvalidScalars/stdout.txt new file mode 100644 index 000000000..1e0a2e2c3 --- /dev/null +++ b/test/regression/Utf8InvalidScalars/stdout.txt @@ -0,0 +1,4 @@ +invalid scalars are rejected by UTF8 encoding: true +invalid scalars leave the output buffer unchanged: true +UTF16 encoding rejects the same invalid scalars: true +valid boundary scalars round trip: true From 6a8fa7e72b656901c73c028a7d85fc814e37e0b0 Mon Sep 17 00:00:00 2001 From: Chris Speciale <26172437+dimensionscape@users.noreply.github.com> Date: Fri, 6 Nov 2026 18:48:35 -0500 Subject: [PATCH 2/3] Keep String's view conversions inside their views at API level 500 utf8_str into a View checked that the bytes fit but not the terminator, so a view with room for the bytes alone took them and then threw, and an empty view threw. It returns false before writing. For a UTF-16 string it also scanned to the first NUL, a character in a Haxe string, so the rest was dropped and the length was short. It converts the string's whole length (libhxcpp #408). create from a View scanned with Char16Advance, which completes a high surrogate with the next unit wherever it is, so a pair split by the end of the view took the unit past it (libhxcpp #412). The scan stops at the view's end or a terminator, and a pair the view does not complete throws Invalid UTF16, as a broken pair inside it did. --- Changes.md | 2 + src/String.cpp | 39 ++++++------ test/regression/StringUtf16ViewBounds/Main.hx | 43 ++++++++++++++ .../StringUtf16ViewBounds/build.hxml | 2 + .../StringUtf16ViewBounds/stdout.txt | 3 + test/regression/StringUtf8Buffer/Main.hx | 59 +++++++++++++++++++ test/regression/StringUtf8Buffer/build.hxml | 2 + test/regression/StringUtf8Buffer/stdout.txt | 4 ++ 8 files changed, 135 insertions(+), 19 deletions(-) create mode 100644 test/regression/StringUtf16ViewBounds/Main.hx create mode 100644 test/regression/StringUtf16ViewBounds/build.hxml create mode 100644 test/regression/StringUtf16ViewBounds/stdout.txt create mode 100644 test/regression/StringUtf8Buffer/Main.hx create mode 100644 test/regression/StringUtf8Buffer/build.hxml create mode 100644 test/regression/StringUtf8Buffer/stdout.txt diff --git a/Changes.md b/Changes.md index c9e9b75d0..e5b2676a6 100644 --- a/Changes.md +++ b/Changes.md @@ -2,6 +2,8 @@ libhxcpp 6.0.0 (unreleased) ------------------------------------------------------------ +* Fixed String::utf8_str into a view, at API level 500, writing the bytes and then throwing when the view had room for them but not the terminator. It returns false before writing anything. It also stopped a UTF-16 string at its first NUL, which is a character, and reported that length (libhxcpp #408) +* Fixed String::create from a UTF-16 view, at API level 500, completing a high surrogate at the end of the view with the unit after it. The view bounds it: a high surrogate the view does not complete throws Invalid UTF16, as a broken pair inside it did (libhxcpp #412) * Fixed cpp.encoding.Utf8.encode returning a null array for the empty string, at API level 500 (libhxcpp #406) * Fixed cpp.encoding.Utf8.encode writing bytes for a code point that is no Unicode scalar value, a surrogate or one past U+10FFFF, which UTF-8 cannot hold and the UTF-16 encoder refused. Both encoders, and both getByteCount, throw for one before writing anything, at API level 500 (libhxcpp #407) * Fixed Haxe 5's Bytes.ofString keeping an array of three bytes for each unit of a string that was not all ASCII: cpp.encoding.Utf8.encode allocated for the worst case and only shortened it, so a megabyte of ASCII with one other character held 3 MB. It counts the bytes and allocates them exactly (libhxcpp #405) diff --git a/src/String.cpp b/src/String.cpp index 04b02756e..9112ae0fb 100644 --- a/src/String.cpp +++ b/src/String.cpp @@ -965,21 +965,27 @@ String String::create(const::cpp::marshal::View& buffer) String String::create(const cpp::marshal::View& buffer) { - auto start = reinterpret_cast(buffer.ptr.ptr); - auto end = start + buffer.length; - auto extra = 0; + auto units = reinterpret_cast(buffer.ptr.ptr); + auto length = int64_t{ 0 }; - while (start < end) { - if (Char16Advance(start) == false) + // Up to a terminator, inside the view: a high surrogate is completed only + // by a low one the view holds, and Char16Advance would read past its end + while (length < buffer.length && units[length] != 0) + { + auto unit = units[length++]; + + if (IsUtf16Surrogate(unit)) { - // set extra to 1 so we don't include the null terminating character in the calculated length. - extra = 1; + if (IsUtf16LowSurrogate(unit) || length >= buffer.length || !IsUtf16LowSurrogate(units[length])) + { + hx::Throw(HX_CSTRING("Invalid UTF16")); + } - break; + length++; } } - return String::create(buffer.ptr.ptr, buffer.length - (end - start) - extra); + return String::create(buffer.ptr.ptr, length); } #endif @@ -2886,16 +2892,10 @@ bool String::utf8_str(::cpp::marshal::View buffer, int* outByteLength) con #ifdef HX_SMART_STRINGS if (isUTF16Encoded()) { + // All of it: a NUL in the string is a character, not its end auto cursor = __w; - - while (Char16Advance(cursor,false)) {} - - auto calculated = cursor - __w - 1; - - cursor = __w; - - auto end = cursor + calculated; - auto chars = 0; + auto end = __w + length; + auto chars = 0; while (cursor < end) { @@ -2926,7 +2926,8 @@ bool String::utf8_str(::cpp::marshal::View buffer, int* outByteLength) con } #endif - if (buffer.length < length) + // The terminator too, before anything is written + if (buffer.length < length + 1) { return false; } diff --git a/test/regression/StringUtf16ViewBounds/Main.hx b/test/regression/StringUtf16ViewBounds/Main.hx new file mode 100644 index 000000000..50577b94c --- /dev/null +++ b/test/regression/StringUtf16ViewBounds/Main.hx @@ -0,0 +1,43 @@ +// String::create from a UTF-16 view completed a high surrogate at the end of +// the view with the unit past it (openfl/libhxcpp#412). It is API level 500, +// so the checks are native code that passes below it. +@:cppFileCode(' +#if (HXCPP_API_LEVEL>=500) +#include +static bool boundedUtf16Prefix() { + char16_t data[3] = {0xD83D, 0xDE00, 0}; + try { + String value = String::create(cpp::marshal::View(data, 1)); + return value.length <= 1; + } catch (...) { return true; } +} +static bool completeUtf16Pair() { + char16_t data[3] = {0xD83D, 0xDE00, 0}; + String value = String::create(cpp::marshal::View(data, 2)); + // fromCharCode gives the string either way, smart strings or not + return value == String::fromCharCode(0x1F600); +} +static bool terminatedUtf16View() { + char16_t data[4] = {0x0100, 0, 65, 0}; + String value = String::create(cpp::marshal::View(data, 3)); + return value == String::fromCharCode(0x0100); +} +#else +static bool boundedUtf16Prefix() { return true; } +static bool completeUtf16Pair() { return true; } +static bool terminatedUtf16View() { return true; } +#endif +') +class Main { + static function main() { + var bounded = true; + var pair = true; + var terminated = true; + bounded = untyped __cpp__("boundedUtf16Prefix()"); + pair = untyped __cpp__("completeUtf16Pair()"); + terminated = untyped __cpp__("terminatedUtf16View()"); + Sys.println("UTF16 construction stays inside the supplied view: " + bounded); + Sys.println("a complete surrogate pair is preserved: " + pair); + Sys.println("a NUL terminator ends native string construction: " + terminated); + } +} diff --git a/test/regression/StringUtf16ViewBounds/build.hxml b/test/regression/StringUtf16ViewBounds/build.hxml new file mode 100644 index 000000000..b484e67e7 --- /dev/null +++ b/test/regression/StringUtf16ViewBounds/build.hxml @@ -0,0 +1,2 @@ +-m Main +--cpp bin diff --git a/test/regression/StringUtf16ViewBounds/stdout.txt b/test/regression/StringUtf16ViewBounds/stdout.txt new file mode 100644 index 000000000..fed8c1e56 --- /dev/null +++ b/test/regression/StringUtf16ViewBounds/stdout.txt @@ -0,0 +1,3 @@ +UTF16 construction stays inside the supplied view: true +a complete surrogate pair is preserved: true +a NUL terminator ends native string construction: true diff --git a/test/regression/StringUtf8Buffer/Main.hx b/test/regression/StringUtf8Buffer/Main.hx new file mode 100644 index 000000000..ddaf652e7 --- /dev/null +++ b/test/regression/StringUtf8Buffer/Main.hx @@ -0,0 +1,59 @@ +// String::utf8_str into a view checked for the bytes but not the terminator, +// and stopped a UTF-16 string at its first NUL (openfl/libhxcpp#408). It is +// API level 500, so the checks are native code that passes below it. +@:cppFileCode(' +#if (HXCPP_API_LEVEL>=500) +#include +static int shortUtf8Buffer(bool empty) { + char buffer[4] = {63, 63, 63, 88}; + bool result = true; + bool threw = false; + try { + String value = empty ? String::emptyString : HX_CSTRING("abc"); + result = value.utf8_str(cpp::marshal::View(buffer, empty ? 0 : 3)); + } catch (...) { threw = true; } + int flags = (!result && !threw) ? 1 : 0; + if (buffer[0] == 63 && buffer[1] == 63 && buffer[2] == 63 && buffer[3] == 88) flags |= 2; + if (threw) flags |= 4; + return flags; +} +static bool utf8BufferMatches(String value, Array expected) { + char buffer[64]; + memset(buffer, 0x55, sizeof(buffer)); + int length = -1; + bool result = value.utf8_str(cpp::marshal::View(buffer, sizeof(buffer)), &length); + return result && length == expected->length + 1 + && memcmp(buffer, expected->GetBase(), expected->length) == 0 + && buffer[expected->length] == 0; +} +#else +static int shortUtf8Buffer(bool) { return 3; } +static bool utf8BufferMatches(String, Array) { return true; } +#endif +') +class Main { + static function main() { + var shortAscii = true; + var empty = true; + var embedded = true; + var ordinary = true; + shortAscii = untyped __cpp__("shortUtf8Buffer(false) == 3"); + empty = untyped __cpp__("shortUtf8Buffer(true) == 3"); + for (value in ["\u0100\x00tail", "\x00\u0100tail", "\u0100\x00"]) + if (!matches(value)) embedded = false; + for (value in ["abc", "a\x00b", "\u0100tail", String.fromCharCode(0x1F600)]) + if (!matches(value)) ordinary = false; + if (Sys.args().length > 0) { + Sys.println("short buffer flags: " + untyped __cpp__("shortUtf8Buffer(false)")); + Sys.println("empty buffer flags: " + untyped __cpp__("shortUtf8Buffer(true)")); + } + Sys.println("ASCII buffer without terminator space is rejected without writes: " + shortAscii); + Sys.println("empty buffer without terminator space is rejected without writes: " + empty); + Sys.println("UTF16 strings retain embedded NULs in UTF8 buffers: " + embedded); + Sys.println("buffers with room preserve ordinary text: " + ordinary); + } + static function matches(value:String):Bool { + var expected = haxe.io.Bytes.ofString(value).getData(); + return untyped __cpp__("utf8BufferMatches({0}, {1})", value, expected); + } +} diff --git a/test/regression/StringUtf8Buffer/build.hxml b/test/regression/StringUtf8Buffer/build.hxml new file mode 100644 index 000000000..b484e67e7 --- /dev/null +++ b/test/regression/StringUtf8Buffer/build.hxml @@ -0,0 +1,2 @@ +-m Main +--cpp bin diff --git a/test/regression/StringUtf8Buffer/stdout.txt b/test/regression/StringUtf8Buffer/stdout.txt new file mode 100644 index 000000000..4026fce4f --- /dev/null +++ b/test/regression/StringUtf8Buffer/stdout.txt @@ -0,0 +1,4 @@ +ASCII buffer without terminator space is rejected without writes: true +empty buffer without terminator space is rejected without writes: true +UTF16 strings retain embedded NULs in UTF8 buffers: true +buffers with room preserve ordinary text: true From 5511123ab93554e3847493f46a024cc830c44bd8 Mon Sep 17 00:00:00 2001 From: Chris Speciale <26172437+dimensionscape@users.noreply.github.com> Date: Fri, 6 Nov 2026 18:57:34 -0500 Subject: [PATCH 3/3] Replace malformed UTF-8 in Utf8.decode instead of throwing Haxe 5's Bytes.toString and getString decode with cpp.encoding.Utf8.decode, which threw for malformed input, where Haxe 4's go through String::create and give U+FFFD for each malformed sequence (48fe31a6). The nightly workflow's Utf8Malformed failed on it. decode now gives the same text as Haxe 4: valid input still takes the strict decoder that counts the UTF-16 units first, and input it refuses goes to _hx_utf8_to_utf16, the decoder String::create uses. Without smart strings, where a String keeps UTF-8, valid bytes are still copied as they are and malformed ones are rewritten by the same decoder, in _hx_utf8_replace_malformed. Utf8.codepoint, which reads one character, still throws. The native suite's test_decode_malformed_string expected the throw. It checks the replacement text now, which is the same in both string modes. --- Changes.md | 1 + src/String.cpp | 21 ++++++++ src/cpp/encoding/Encodings.cpp | 73 +++++++++++++++++--------- test/native/tests/encoding/TestUtf8.hx | 14 ++++- 4 files changed, 81 insertions(+), 28 deletions(-) diff --git a/Changes.md b/Changes.md index e5b2676a6..ac5107a79 100644 --- a/Changes.md +++ b/Changes.md @@ -2,6 +2,7 @@ libhxcpp 6.0.0 (unreleased) ------------------------------------------------------------ +* cpp.encoding.Utf8.decode replaces each malformed UTF-8 sequence with U+FFFD instead of throwing, with smart strings or without. Haxe 5's Bytes.toString and getString decode with it, so they threw for bytes that Haxe 4's, which go through String::create, decode with U+FFFD in the same places. Valid input still takes the decoder that counts first; only malformed input takes the replacing one. Utf8.codepoint still throws * Fixed String::utf8_str into a view, at API level 500, writing the bytes and then throwing when the view had room for them but not the terminator. It returns false before writing anything. It also stopped a UTF-16 string at its first NUL, which is a character, and reported that length (libhxcpp #408) * Fixed String::create from a UTF-16 view, at API level 500, completing a high surrogate at the end of the view with the unit after it. The view bounds it: a high surrogate the view does not complete throws Invalid UTF16, as a broken pair inside it did (libhxcpp #412) * Fixed cpp.encoding.Utf8.encode returning a null array for the empty string, at API level 500 (libhxcpp #406) diff --git a/src/String.cpp b/src/String.cpp index 9112ae0fb..ceddb9f91 100644 --- a/src/String.cpp +++ b/src/String.cpp @@ -2598,6 +2598,27 @@ String _hx_utf8_to_utf16(const unsigned char *ptr, int inUtf8Len, bool addHash) return String(str, char16Count); } +#elif (HXCPP_API_LEVEL>=500) +// For cpp.encoding.Utf8.decode without smart strings, where a String keeps +// UTF-8: the bytes, with each malformed sequence replaced by U+FFFD as +// _hx_utf8_to_utf16 replaces it, so both give the same text +String _hx_utf8_replace_malformed(const unsigned char *ptr, int inUtf8Len) +{ + int bytes = 0; + const unsigned char *u = ptr; + const unsigned char *end = ptr + inUtf8Len; + while(u& buffer) { return Ascii::decode(buffer); } -#if defined(HX_SMART_STRINGS) - auto bytes = utf16Units(buffer) * sizeof(char16_t); - auto exact = static_cast(::hx::InternalNew(bytes + sizeof(char16_t), false)); - auto output = View(reinterpret_cast(exact), bytes); - int64_t k = 0; - int64_t i = 0; + if (buffer.length > 0x7fffffff) + { + return hx::Throw(HX_CSTRING("String too long")); + } - while (i < buffer.length) + // Malformed input decodes as Haxe 4's String::create and Bytes.toString + // decode it, each malformed sequence as one U+FFFD, since Haxe 5's + // Bytes.toString and getString come here. The strict decoder below + // throws for one, and is kept for valid input, which it decodes faster + auto ptr = static_cast(buffer.ptr.ptr); + auto len = static_cast(buffer.length); +#if defined(HX_SMART_STRINGS) + try { - auto p = codepoint(buffer.slice(i)); + auto bytes = utf16Units(buffer) * sizeof(char16_t); + auto exact = static_cast(::hx::InternalNew(bytes + sizeof(char16_t), false)); + auto output = View(reinterpret_cast(exact), bytes); + int64_t k = 0; + int64_t i = 0; - i += getByteCount(p); - k += Utf16::encode(p, output.slice(k)); - } + while (i < buffer.length) + { + auto p = codepoint(buffer.slice(i)); + + i += getByteCount(p); + k += Utf16::encode(p, output.slice(k)); + } - auto chars = static_cast(k / sizeof(char16_t)); + auto chars = static_cast(k / sizeof(char16_t)); - ((unsigned int*)exact)[-1] |= HX_GC_STRING_CHAR16_T; - exact[chars] = 0; + ((unsigned int*)exact)[-1] |= HX_GC_STRING_CHAR16_T; + exact[chars] = 0; - return String(exact, chars); + return String(exact, chars); + } + catch (Dynamic&) + { + return _hx_utf8_to_utf16(ptr, len, false); + } #else - // The bytes are kept as they are, but malformed input still throws, as - // it does with smart strings. codepoint takes an encoded surrogate - // (ED A0 80 to ED BF BF), which Utf16::encode rejects there, so it is - // rejected here too - for (int64_t i = 0; i < buffer.length;) + // The bytes are kept as they are when they are valid. codepoint takes an + // encoded surrogate (ED A0 80 to ED BF BF), which getByteCount refuses + try { - auto p = codepoint(buffer.slice(i)); - - if (0xD800 <= p && p < 0xE000) + for (int64_t i = 0; i < buffer.length;) { - hx::Throw(HX_CSTRING("Failed to read codepoint")); + i += getByteCount(codepoint(buffer.slice(i))); } - - i += getByteCount(p); + } + catch (Dynamic&) + { + return _hx_utf8_replace_malformed(ptr, len); } // +1 for the terminator - the allocation is not zeroed and hxcpp diff --git a/test/native/tests/encoding/TestUtf8.hx b/test/native/tests/encoding/TestUtf8.hx index 4fcc5a36c..da4cc0055 100644 --- a/test/native/tests/encoding/TestUtf8.hx +++ b/test/native/tests/encoding/TestUtf8.hx @@ -157,8 +157,18 @@ class TestUtf8 extends Test { } public function test_decode_malformed_string() { - for (hex in ['61626364656667f09d84', '6162636465666768c3', 'e3833061', '616263646566676880', 'eda080', 'c0af']) { - Assert.raises(() -> Utf8.decode(Bytes.ofHex(hex).asView())); + // Each malformed sequence becomes one U+FFFD, as Haxe 4's Bytes.toString + // gives, since Haxe 5's decodes here + var expected = [ + '61626364656667f09d84' => 'abcdefg\u{FFFD}', + '6162636465666768c3' => 'abcdefgh\u{FFFD}', + 'e3833061' => '\u{FFFD}0a', + '616263646566676880' => 'abcdefgh\u{FFFD}', + 'eda080' => '\u{FFFD}\u{FFFD}\u{FFFD}', + 'c0af' => '\u{FFFD}\u{FFFD}' + ]; + for (hex => text in expected) { + Assert.equals(text, Utf8.decode(Bytes.ofHex(hex).asView())); } } } \ No newline at end of file