diff --git a/Changes.md b/Changes.md index d3372b98b..cf45d8cb9 100644 --- a/Changes.md +++ b/Changes.md @@ -2,6 +2,12 @@ libhxcpp 6.0.0 (unreleased) ------------------------------------------------------------ +* cpp.encoding.Utf8.decode replaces each malformed UTF-8 sequence with U+FFFD instead of throwing, with smart strings or without. Haxe 5's Bytes.toString and getString decode with it, so they threw for bytes that Haxe 4's, which go through String::create, decode with U+FFFD in the same places. Valid input still takes the decoder that counts first; only malformed input takes the replacing one. Utf8.codepoint still throws +* Fixed String::utf8_str into a view, at API level 500, writing the bytes and then throwing when the view had room for them but not the terminator. It returns false before writing anything. It also stopped a UTF-16 string at its first NUL, which is a character, and reported that length (libhxcpp #408) +* Fixed String::create from a UTF-16 view, at API level 500, completing a high surrogate at the end of the view with the unit after it. The view bounds it: a high surrogate the view does not complete throws Invalid UTF16, as a broken pair inside it did (libhxcpp #412) +* Fixed cpp.encoding.Utf8.encode returning a null array for the empty string, at API level 500 (libhxcpp #406) +* Fixed cpp.encoding.Utf8.encode writing bytes for a code point that is no Unicode scalar value, a surrogate or one past U+10FFFF, which UTF-8 cannot hold and the UTF-16 encoder refused. Both encoders, and both getByteCount, throw for one before writing anything, at API level 500 (libhxcpp #407) +* Fixed Haxe 5's Bytes.ofString keeping an array of three bytes for each unit of a string that was not all ASCII: cpp.encoding.Utf8.encode allocated for the worst case and only shortened it, so a megabyte of ASCII with one other character held 3 MB. It counts the bytes and allocates them exactly (libhxcpp #405) * Added timedWait to hx::thread::ConditionVariable, which Haxe 5's sys.thread.Condition.timedWait calls: with the Haxe nightly, every program that used Condition failed to compile. It returns true once a signal or broadcast came and false when the time ran out, and it waits out the wakeups MSVC's timed waits give just before the deadline with nothing signalled * Fixed an exception from blit, setSize or resize under the cppia JIT ending the process instead of reaching the script's catch, on typed arrays and on Array: their native calls had no exception bridge. The script catches it, as it does interpreted and in compiled code (libhxcpp #404) * Fixed resize() and cpp.NativeArray.setSize() on an Array under the cppia JIT evaluating the array expression twice, so a call in it ran twice and the array it returned the second time was the one resized. The JIT evaluates it once (libhxcpp #411) diff --git a/src/String.cpp b/src/String.cpp index 04b02756e..ceddb9f91 100644 --- a/src/String.cpp +++ b/src/String.cpp @@ -965,21 +965,27 @@ String String::create(const::cpp::marshal::View& buffer) String String::create(const cpp::marshal::View& buffer) { - auto start = reinterpret_cast(buffer.ptr.ptr); - auto end = start + buffer.length; - auto extra = 0; + auto units = reinterpret_cast(buffer.ptr.ptr); + auto length = int64_t{ 0 }; - while (start < end) { - if (Char16Advance(start) == false) + // Up to a terminator, inside the view: a high surrogate is completed only + // by a low one the view holds, and Char16Advance would read past its end + while (length < buffer.length && units[length] != 0) + { + auto unit = units[length++]; + + if (IsUtf16Surrogate(unit)) { - // set extra to 1 so we don't include the null terminating character in the calculated length. - extra = 1; + if (IsUtf16LowSurrogate(unit) || length >= buffer.length || !IsUtf16LowSurrogate(units[length])) + { + hx::Throw(HX_CSTRING("Invalid UTF16")); + } - break; + length++; } } - return String::create(buffer.ptr.ptr, buffer.length - (end - start) - extra); + return String::create(buffer.ptr.ptr, length); } #endif @@ -2592,6 +2598,27 @@ String _hx_utf8_to_utf16(const unsigned char *ptr, int inUtf8Len, bool addHash) return String(str, char16Count); } +#elif (HXCPP_API_LEVEL>=500) +// For cpp.encoding.Utf8.decode without smart strings, where a String keeps +// UTF-8: the bytes, with each malformed sequence replaced by U+FFFD as +// _hx_utf8_to_utf16 replaces it, so both give the same text +String _hx_utf8_replace_malformed(const unsigned char *ptr, int inUtf8Len) +{ + int bytes = 0; + const unsigned char *u = ptr; + const unsigned char *end = ptr + inUtf8Len; + while(u buffer, int* outByteLength) con #ifdef HX_SMART_STRINGS if (isUTF16Encoded()) { + // All of it: a NUL in the string is a character, not its end auto cursor = __w; - - while (Char16Advance(cursor,false)) {} - - auto calculated = cursor - __w - 1; - - cursor = __w; - - auto end = cursor + calculated; - auto chars = 0; + auto end = __w + length; + auto chars = 0; while (cursor < end) { @@ -2926,7 +2947,8 @@ bool String::utf8_str(::cpp::marshal::View buffer, int* outByteLength) con } #endif - if (buffer.length < length) + // The terminator too, before anything is written + if (buffer.length < length + 1) { return false; } diff --git a/src/cpp/encoding/Encodings.cpp b/src/cpp/encoding/Encodings.cpp index b377a2aad..cf1a06d0a 100644 --- a/src/cpp/encoding/Encodings.cpp +++ b/src/cpp/encoding/Encodings.cpp @@ -5,6 +5,11 @@ using namespace cpp::marshal; +#if !defined(HX_SMART_STRINGS) +// In String.cpp, beside _hx_utf8_to_utf16 +String _hx_utf8_replace_malformed(const unsigned char *ptr, int inUtf8Len); +#endif + bool cpp::encoding::Ascii::isEncoded(const String& string) { if (null() == string) @@ -142,6 +147,17 @@ namespace return 0xFFFD; } #endif + + // A code point that is not a Unicode scalar value, a surrogate or past + // U+10FFFF, has no UTF-8 or UTF-16 form. The UTF-16 encoder refused them + // and the UTF-8 one wrote invalid bytes + void checkScalar(char32_t codepoint) + { + if ((codepoint >= 0xD800 && codepoint < 0xE000) || codepoint > 0x10FFFF) + { + hx::Throw(HX_CSTRING("Invalid codepoint")); + } + } } int cpp::encoding::Utf8::getByteCount(const null&) @@ -152,6 +168,8 @@ int cpp::encoding::Utf8::getByteCount(const null&) int cpp::encoding::Utf8::getByteCount(char32_t codepoint) { + checkScalar(codepoint); + if (codepoint <= 0x7F) { return 1; @@ -281,7 +299,8 @@ Array cpp::encoding::Utf8::encode(const String& string) if (0 == string.length) { - return 0; + // An empty array: 0 would be a null one + return Array(0, 0); } if (string.isAsciiEncoded()) @@ -297,8 +316,17 @@ Array cpp::encoding::Utf8::encode(const String& string) } #if defined(HX_SMART_STRINGS) - // estimate the utf8 length with an upper bound - Array out(string.length * 3, 0); + // Counted first: an array of three bytes a unit, which resize only made + // shorter, kept that capacity as long as the bytes lived, three times + // what a mostly ASCII string needs (libhxcpp #405) + auto bytes = getByteCount(string); + + if (bytes > 0x7fffffff) + { + return hx::Throw(HX_CSTRING("String too long")); + } + + Array out(static_cast(bytes), 0); View buffer(out->Pointer(), out->length); auto units = string.raw_wptr(); @@ -310,8 +338,6 @@ Array cpp::encoding::Utf8::encode(const String& string) k += encode(stringCodepoint(units, string.length, i), buffer.slice(k)); } - out->resize(k); - return out; #else return hx::Throw(HX_CSTRING("Unexpected encoding error")); @@ -320,6 +346,9 @@ Array cpp::encoding::Utf8::encode(const String& string) int cpp::encoding::Utf8::encode(char32_t codepoint, const cpp::marshal::View& buffer) { + // Before anything is written + checkScalar(codepoint); + if (codepoint <= 0x7F) { buffer[0] = static_cast(codepoint); @@ -383,42 +412,58 @@ String cpp::encoding::Utf8::decode(const cpp::marshal::View& buffer) { return Ascii::decode(buffer); } -#if defined(HX_SMART_STRINGS) - auto bytes = utf16Units(buffer) * sizeof(char16_t); - auto exact = static_cast(::hx::InternalNew(bytes + sizeof(char16_t), false)); - auto output = View(reinterpret_cast(exact), bytes); - int64_t k = 0; - int64_t i = 0; + if (buffer.length > 0x7fffffff) + { + return hx::Throw(HX_CSTRING("String too long")); + } - while (i < buffer.length) + // Malformed input decodes as Haxe 4's String::create and Bytes.toString + // decode it, each malformed sequence as one U+FFFD, since Haxe 5's + // Bytes.toString and getString come here. The strict decoder below + // throws for one, and is kept for valid input, which it decodes faster + auto ptr = static_cast(buffer.ptr.ptr); + auto len = static_cast(buffer.length); +#if defined(HX_SMART_STRINGS) + try { - auto p = codepoint(buffer.slice(i)); + auto bytes = utf16Units(buffer) * sizeof(char16_t); + auto exact = static_cast(::hx::InternalNew(bytes + sizeof(char16_t), false)); + auto output = View(reinterpret_cast(exact), bytes); + int64_t k = 0; + int64_t i = 0; - i += getByteCount(p); - k += Utf16::encode(p, output.slice(k)); - } + while (i < buffer.length) + { + auto p = codepoint(buffer.slice(i)); + + i += getByteCount(p); + k += Utf16::encode(p, output.slice(k)); + } - auto chars = static_cast(k / sizeof(char16_t)); + auto chars = static_cast(k / sizeof(char16_t)); - ((unsigned int*)exact)[-1] |= HX_GC_STRING_CHAR16_T; - exact[chars] = 0; + ((unsigned int*)exact)[-1] |= HX_GC_STRING_CHAR16_T; + exact[chars] = 0; - return String(exact, chars); + return String(exact, chars); + } + catch (Dynamic&) + { + return _hx_utf8_to_utf16(ptr, len, false); + } #else - // The bytes are kept as they are, but malformed input still throws, as - // it does with smart strings. codepoint takes an encoded surrogate - // (ED A0 80 to ED BF BF), which Utf16::encode rejects there, so it is - // rejected here too - for (int64_t i = 0; i < buffer.length;) + // The bytes are kept as they are when they are valid. codepoint takes an + // encoded surrogate (ED A0 80 to ED BF BF), which getByteCount refuses + try { - auto p = codepoint(buffer.slice(i)); - - if (0xD800 <= p && p < 0xE000) + for (int64_t i = 0; i < buffer.length;) { - hx::Throw(HX_CSTRING("Failed to read codepoint")); + i += getByteCount(codepoint(buffer.slice(i))); } - - i += getByteCount(p); + } + catch (Dynamic&) + { + return _hx_utf8_replace_malformed(ptr, len); } // +1 for the terminator - the allocation is not zeroed and hxcpp @@ -557,6 +602,8 @@ int cpp::encoding::Utf16::getByteCount(const null&) int cpp::encoding::Utf16::getByteCount(char32_t codepoint) { + checkScalar(codepoint); + return codepoint <= 0xFFFF ? 2 : 4; } diff --git a/test/native/tests/encoding/TestUtf8.hx b/test/native/tests/encoding/TestUtf8.hx index 4fcc5a36c..da4cc0055 100644 --- a/test/native/tests/encoding/TestUtf8.hx +++ b/test/native/tests/encoding/TestUtf8.hx @@ -157,8 +157,18 @@ class TestUtf8 extends Test { } public function test_decode_malformed_string() { - for (hex in ['61626364656667f09d84', '6162636465666768c3', 'e3833061', '616263646566676880', 'eda080', 'c0af']) { - Assert.raises(() -> Utf8.decode(Bytes.ofHex(hex).asView())); + // Each malformed sequence becomes one U+FFFD, as Haxe 4's Bytes.toString + // gives, since Haxe 5's decodes here + var expected = [ + '61626364656667f09d84' => 'abcdefg\u{FFFD}', + '6162636465666768c3' => 'abcdefgh\u{FFFD}', + 'e3833061' => '\u{FFFD}0a', + '616263646566676880' => 'abcdefgh\u{FFFD}', + 'eda080' => '\u{FFFD}\u{FFFD}\u{FFFD}', + 'c0af' => '\u{FFFD}\u{FFFD}' + ]; + for (hex => text in expected) { + Assert.equals(text, Utf8.decode(Bytes.ofHex(hex).asView())); } } } \ No newline at end of file diff --git a/test/regression/StringUtf16ViewBounds/Main.hx b/test/regression/StringUtf16ViewBounds/Main.hx new file mode 100644 index 000000000..50577b94c --- /dev/null +++ b/test/regression/StringUtf16ViewBounds/Main.hx @@ -0,0 +1,43 @@ +// String::create from a UTF-16 view completed a high surrogate at the end of +// the view with the unit past it (openfl/libhxcpp#412). It is API level 500, +// so the checks are native code that passes below it. +@:cppFileCode(' +#if (HXCPP_API_LEVEL>=500) +#include +static bool boundedUtf16Prefix() { + char16_t data[3] = {0xD83D, 0xDE00, 0}; + try { + String value = String::create(cpp::marshal::View(data, 1)); + return value.length <= 1; + } catch (...) { return true; } +} +static bool completeUtf16Pair() { + char16_t data[3] = {0xD83D, 0xDE00, 0}; + String value = String::create(cpp::marshal::View(data, 2)); + // fromCharCode gives the string either way, smart strings or not + return value == String::fromCharCode(0x1F600); +} +static bool terminatedUtf16View() { + char16_t data[4] = {0x0100, 0, 65, 0}; + String value = String::create(cpp::marshal::View(data, 3)); + return value == String::fromCharCode(0x0100); +} +#else +static bool boundedUtf16Prefix() { return true; } +static bool completeUtf16Pair() { return true; } +static bool terminatedUtf16View() { return true; } +#endif +') +class Main { + static function main() { + var bounded = true; + var pair = true; + var terminated = true; + bounded = untyped __cpp__("boundedUtf16Prefix()"); + pair = untyped __cpp__("completeUtf16Pair()"); + terminated = untyped __cpp__("terminatedUtf16View()"); + Sys.println("UTF16 construction stays inside the supplied view: " + bounded); + Sys.println("a complete surrogate pair is preserved: " + pair); + Sys.println("a NUL terminator ends native string construction: " + terminated); + } +} diff --git a/test/regression/StringUtf16ViewBounds/build.hxml b/test/regression/StringUtf16ViewBounds/build.hxml new file mode 100644 index 000000000..b484e67e7 --- /dev/null +++ b/test/regression/StringUtf16ViewBounds/build.hxml @@ -0,0 +1,2 @@ +-m Main +--cpp bin diff --git a/test/regression/StringUtf16ViewBounds/stdout.txt b/test/regression/StringUtf16ViewBounds/stdout.txt new file mode 100644 index 000000000..fed8c1e56 --- /dev/null +++ b/test/regression/StringUtf16ViewBounds/stdout.txt @@ -0,0 +1,3 @@ +UTF16 construction stays inside the supplied view: true +a complete surrogate pair is preserved: true +a NUL terminator ends native string construction: true diff --git a/test/regression/StringUtf8Buffer/Main.hx b/test/regression/StringUtf8Buffer/Main.hx new file mode 100644 index 000000000..ddaf652e7 --- /dev/null +++ b/test/regression/StringUtf8Buffer/Main.hx @@ -0,0 +1,59 @@ +// String::utf8_str into a view checked for the bytes but not the terminator, +// and stopped a UTF-16 string at its first NUL (openfl/libhxcpp#408). It is +// API level 500, so the checks are native code that passes below it. +@:cppFileCode(' +#if (HXCPP_API_LEVEL>=500) +#include +static int shortUtf8Buffer(bool empty) { + char buffer[4] = {63, 63, 63, 88}; + bool result = true; + bool threw = false; + try { + String value = empty ? String::emptyString : HX_CSTRING("abc"); + result = value.utf8_str(cpp::marshal::View(buffer, empty ? 0 : 3)); + } catch (...) { threw = true; } + int flags = (!result && !threw) ? 1 : 0; + if (buffer[0] == 63 && buffer[1] == 63 && buffer[2] == 63 && buffer[3] == 88) flags |= 2; + if (threw) flags |= 4; + return flags; +} +static bool utf8BufferMatches(String value, Array expected) { + char buffer[64]; + memset(buffer, 0x55, sizeof(buffer)); + int length = -1; + bool result = value.utf8_str(cpp::marshal::View(buffer, sizeof(buffer)), &length); + return result && length == expected->length + 1 + && memcmp(buffer, expected->GetBase(), expected->length) == 0 + && buffer[expected->length] == 0; +} +#else +static int shortUtf8Buffer(bool) { return 3; } +static bool utf8BufferMatches(String, Array) { return true; } +#endif +') +class Main { + static function main() { + var shortAscii = true; + var empty = true; + var embedded = true; + var ordinary = true; + shortAscii = untyped __cpp__("shortUtf8Buffer(false) == 3"); + empty = untyped __cpp__("shortUtf8Buffer(true) == 3"); + for (value in ["\u0100\x00tail", "\x00\u0100tail", "\u0100\x00"]) + if (!matches(value)) embedded = false; + for (value in ["abc", "a\x00b", "\u0100tail", String.fromCharCode(0x1F600)]) + if (!matches(value)) ordinary = false; + if (Sys.args().length > 0) { + Sys.println("short buffer flags: " + untyped __cpp__("shortUtf8Buffer(false)")); + Sys.println("empty buffer flags: " + untyped __cpp__("shortUtf8Buffer(true)")); + } + Sys.println("ASCII buffer without terminator space is rejected without writes: " + shortAscii); + Sys.println("empty buffer without terminator space is rejected without writes: " + empty); + Sys.println("UTF16 strings retain embedded NULs in UTF8 buffers: " + embedded); + Sys.println("buffers with room preserve ordinary text: " + ordinary); + } + static function matches(value:String):Bool { + var expected = haxe.io.Bytes.ofString(value).getData(); + return untyped __cpp__("utf8BufferMatches({0}, {1})", value, expected); + } +} diff --git a/test/regression/StringUtf8Buffer/build.hxml b/test/regression/StringUtf8Buffer/build.hxml new file mode 100644 index 000000000..b484e67e7 --- /dev/null +++ b/test/regression/StringUtf8Buffer/build.hxml @@ -0,0 +1,2 @@ +-m Main +--cpp bin diff --git a/test/regression/StringUtf8Buffer/stdout.txt b/test/regression/StringUtf8Buffer/stdout.txt new file mode 100644 index 000000000..4026fce4f --- /dev/null +++ b/test/regression/StringUtf8Buffer/stdout.txt @@ -0,0 +1,4 @@ +ASCII buffer without terminator space is rejected without writes: true +empty buffer without terminator space is rejected without writes: true +UTF16 strings retain embedded NULs in UTF8 buffers: true +buffers with room preserve ordinary text: true diff --git a/test/regression/Utf8EmptyArray/Main.hx b/test/regression/Utf8EmptyArray/Main.hx new file mode 100644 index 000000000..467dee0f8 --- /dev/null +++ b/test/regression/Utf8EmptyArray/Main.hx @@ -0,0 +1,40 @@ +// cpp.encoding.Utf8.encode gave a null array for the empty string +// (openfl/libhxcpp#406). It is API level 500, which Haxe 4 and Haxe +// 5.0.0-preview.1 do not build against, so the checks are native code +// that passes there. +@:cppFileCode(' +#if (HXCPP_API_LEVEL>=500) +static bool encodes(String inText, const unsigned char *inBytes, int inLength) +{ + Array bytes = cpp::encoding::Utf8::encode(inText); + if (!bytes.mPtr || bytes->length != inLength) + return false; + for (int i = 0; i < inLength; i++) + if (bytes[i] != inBytes[i]) + return false; + return true; +} +static bool emptyEncoding() { return encodes(HX_CSTRING(""), nullptr, 0); } +static bool asciiEncoding() +{ + static const unsigned char abc[] = { 0x61, 0x62, 0x63 }; + return encodes(HX_CSTRING("abc"), abc, 3); +} +static bool unicodeEncoding() +{ + static const unsigned char wide[] = { 0xC4, 0x80 }; + return encodes(String::fromCharCode(0x100), wide, 2); +} +#else +static bool emptyEncoding() { return true; } +static bool asciiEncoding() { return true; } +static bool unicodeEncoding() { return true; } +#endif +') +class Main { + static function main() { + Sys.println("empty UTF8 encoding returns an empty array: " + (untyped __cpp__("emptyEncoding()") : Bool)); + Sys.println("ASCII UTF8 encoding remains intact: " + (untyped __cpp__("asciiEncoding()") : Bool)); + Sys.println("Unicode UTF8 encoding remains intact: " + (untyped __cpp__("unicodeEncoding()") : Bool)); + } +} diff --git a/test/regression/Utf8EmptyArray/build.hxml b/test/regression/Utf8EmptyArray/build.hxml new file mode 100644 index 000000000..b484e67e7 --- /dev/null +++ b/test/regression/Utf8EmptyArray/build.hxml @@ -0,0 +1,2 @@ +-m Main +--cpp bin diff --git a/test/regression/Utf8EmptyArray/stdout.txt b/test/regression/Utf8EmptyArray/stdout.txt new file mode 100644 index 000000000..9e3eda8c7 --- /dev/null +++ b/test/regression/Utf8EmptyArray/stdout.txt @@ -0,0 +1,3 @@ +empty UTF8 encoding returns an empty array: true +ASCII UTF8 encoding remains intact: true +Unicode UTF8 encoding remains intact: true diff --git a/test/regression/Utf8EncodeCapacity/Main.hx b/test/regression/Utf8EncodeCapacity/Main.hx new file mode 100644 index 000000000..1590f60a1 --- /dev/null +++ b/test/regression/Utf8EncodeCapacity/Main.hx @@ -0,0 +1,29 @@ +// Haxe 5's Bytes.ofString kept an array of three bytes for each unit of a +// string that was not all ASCII (openfl/libhxcpp#405). Haxe 4's allocates what +// it needs, which this checks too. +class Main { + static function main() { + var correct = true; + var bounded = true; + var asciiExact = true; + var plain = haxe.io.Bytes.alloc(1 << 20); + plain.fill(0, plain.length, 97); + var ascii = plain.toString(); + var text = ascii + "\u0100"; + var encoded = haxe.io.Bytes.ofString(text); + var data = encoded.getData(); + correct = encoded.length == plain.length + 2 && encoded.toString() == text; + for (_ in 0...3) cpp.vm.Gc.run(true); + var capacity:Int = untyped __cpp__("{0}->capacity()", data); + bounded = capacity <= encoded.length + 65536; + var control = haxe.io.Bytes.ofString(ascii); + var controlData = control.getData(); + var controlCapacity:Int = untyped __cpp__("{0}->capacity()", controlData); + asciiExact = controlCapacity <= control.length + 65536 && control.toString() == ascii; + if (Sys.args().length > 0) + Sys.println(haxe.Json.stringify({length: encoded.length, capacity: capacity, asciiLength: control.length, asciiCapacity: controlCapacity})); + Sys.println("UTF8 encoding preserves the complete payload: " + correct); + Sys.println("retained UTF8 storage stays close to the output length: " + bounded); + Sys.println("ASCII control storage stays close to the output length: " + asciiExact); + } +} diff --git a/test/regression/Utf8EncodeCapacity/build.hxml b/test/regression/Utf8EncodeCapacity/build.hxml new file mode 100644 index 000000000..b484e67e7 --- /dev/null +++ b/test/regression/Utf8EncodeCapacity/build.hxml @@ -0,0 +1,2 @@ +-m Main +--cpp bin diff --git a/test/regression/Utf8EncodeCapacity/stdout.txt b/test/regression/Utf8EncodeCapacity/stdout.txt new file mode 100644 index 000000000..db5e7a814 --- /dev/null +++ b/test/regression/Utf8EncodeCapacity/stdout.txt @@ -0,0 +1,3 @@ +UTF8 encoding preserves the complete payload: true +retained UTF8 storage stays close to the output length: true +ASCII control storage stays close to the output length: true diff --git a/test/regression/Utf8InvalidScalars/Main.hx b/test/regression/Utf8InvalidScalars/Main.hx new file mode 100644 index 000000000..a6f08a644 --- /dev/null +++ b/test/regression/Utf8InvalidScalars/Main.hx @@ -0,0 +1,52 @@ +// cpp.encoding.Utf8.encode wrote bytes for a surrogate or a value past +// U+10FFFF, which the UTF-16 encoder refuses (openfl/libhxcpp#407). It is API +// level 500, which Haxe 4 and Haxe 5.0.0-preview.1 do not build against, so +// the checks are native code that passes there. The result has a bit set for +// each check that failed. +@:cppFileCode(' +#if (HXCPP_API_LEVEL>=500) +static int scalarFailures() +{ + using cpp::marshal::View; + int failed = 0; + const char32_t invalid[] = { 0xD800, 0xDFFF, 0x110000, (char32_t)-1 }; + for (char32_t point : invalid) + { + unsigned char data[4] = { 0x55, 0x55, 0x55, 0x55 }; + View view(cpp::Pointer(data), 4); + bool refused = false; + try { cpp::encoding::Utf8::encode(point, view); } catch (Dynamic &) { refused = true; } + if (!refused) + failed |= 1; + for (unsigned char byte : data) + if (byte != 0x55) + failed |= 2; + refused = false; + try { cpp::encoding::Utf16::encode(point, view); } catch (Dynamic &) { refused = true; } + if (!refused) + failed |= 4; + } + const char32_t valid[] = { 0, 0x7F, 0x80, 0xD7FF, 0xE000, 0x10FFFF }; + for (char32_t point : valid) + { + unsigned char data[4]; + View view(cpp::Pointer(data), 4); + int length = cpp::encoding::Utf8::encode(point, view); + if (cpp::encoding::Utf8::codepoint(view.slice(0, length)) != point) + failed |= 8; + } + return failed; +} +#else +static int scalarFailures() { return 0; } +#endif +') +class Main { + static function main() { + var failed:Int = untyped __cpp__("scalarFailures()"); + Sys.println("invalid scalars are rejected by UTF8 encoding: " + (failed & 1 == 0)); + Sys.println("invalid scalars leave the output buffer unchanged: " + (failed & 2 == 0)); + Sys.println("UTF16 encoding rejects the same invalid scalars: " + (failed & 4 == 0)); + Sys.println("valid boundary scalars round trip: " + (failed & 8 == 0)); + } +} diff --git a/test/regression/Utf8InvalidScalars/build.hxml b/test/regression/Utf8InvalidScalars/build.hxml new file mode 100644 index 000000000..b484e67e7 --- /dev/null +++ b/test/regression/Utf8InvalidScalars/build.hxml @@ -0,0 +1,2 @@ +-m Main +--cpp bin diff --git a/test/regression/Utf8InvalidScalars/stdout.txt b/test/regression/Utf8InvalidScalars/stdout.txt new file mode 100644 index 000000000..1e0a2e2c3 --- /dev/null +++ b/test/regression/Utf8InvalidScalars/stdout.txt @@ -0,0 +1,4 @@ +invalid scalars are rejected by UTF8 encoding: true +invalid scalars leave the output buffer unchanged: true +UTF16 encoding rejects the same invalid scalars: true +valid boundary scalars round trip: true