From f33116583490e9c4425905ca935dc6cb4c129261 Mon Sep 17 00:00:00 2001 From: Artem Ukrainskiy Date: Mon, 12 Oct 2026 20:36:04 +0300 Subject: [PATCH] ext/standard: Add php_next_utf8_char_inline() json_encode(), utf8_decode() and the xml extension decode one character at a time with php_next_utf8_char(), and every call went through the generic charset decoder of html.c. The new static inline function in html.h decodes valid UTF-8 at the call site and leaves every invalid sequence to php_next_utf8_char(), which is unchanged. --- UPGRADING | 4 + UPGRADING.INTERNALS | 3 + ext/json/json_encoder.c | 2 +- .../tests/json_encode_utf8_sequences.phpt | 217 ++++++++++++++++++ ext/standard/html.h | 53 +++++ ext/standard/string.c | 2 +- .../tests/strings/utf8_decode_sequences.phpt | 83 +++++++ ext/xml/xml.c | 2 +- 8 files changed, 363 insertions(+), 3 deletions(-) create mode 100644 ext/json/tests/json_encode_utf8_sequences.phpt create mode 100644 ext/standard/tests/strings/utf8_decode_sequences.phpt diff --git a/UPGRADING b/UPGRADING index 46cf4cafab12..5e951d848c88 100644 --- a/UPGRADING +++ b/UPGRADING @@ -106,5 +106,9 @@ PHP 8.7 UPGRADE NOTES . Improved performance of pack() for large strings using the a, A and Z formats. +- JSON: + . Improved performance of json_encode() for strings with non-ASCII + characters. + - MBString: . Improved performance of mb_strlen for UTF-8 strings. diff --git a/UPGRADING.INTERNALS b/UPGRADING.INTERNALS index 2f6ea7d9ea84..566b65740d59 100644 --- a/UPGRADING.INTERNALS +++ b/UPGRADING.INTERNALS @@ -75,6 +75,9 @@ PHP 8.7 INTERNALS UPGRADE NOTES - Standard: . php_explode() now takes a HashTable* instead of a zval* to store the resulting parts. It already expected the zval* to be IS_ARRAY. + . Added php_next_utf8_char_inline() to ext/standard/html.h. It has the same + contract as php_next_utf8_char() and decodes valid UTF-8 at the call + site. ======================== 4. OpCode changes diff --git a/ext/json/json_encoder.c b/ext/json/json_encoder.c index b8ae31040c8b..18a23a0f2d4e 100644 --- a/ext/json/json_encoder.c +++ b/ext/json/json_encoder.c @@ -405,7 +405,7 @@ zend_result php_json_escape_string( us = (unsigned char)s[0]; if (UNEXPECTED(us >= 0x80)) { zend_result status; - us = php_next_utf8_char((unsigned char *)s, len, &pos, &status); + us = php_next_utf8_char_inline((const unsigned char *)s, len, &pos, &status); /* check whether UTF8 character is correct */ if (UNEXPECTED(status != SUCCESS)) { diff --git a/ext/json/tests/json_encode_utf8_sequences.phpt b/ext/json/tests/json_encode_utf8_sequences.phpt new file mode 100644 index 000000000000..57331d204250 --- /dev/null +++ b/ext/json/tests/json_encode_utf8_sequences.phpt @@ -0,0 +1,217 @@ +--TEST-- +json_encode() with valid and invalid UTF-8 sequences around the two-, three- and four-byte boundaries +--FILE-- + "\xC2\x80", + 'DF BF, largest two-byte' => "\xDF\xBF", + 'C0 AF, overlong' => "\xC0\xAF", + 'C1 BF, overlong' => "\xC1\xBF", + '80, lone continuation byte' => "\x80", + 'C2 at the end' => "a\xC2", + 'C2 then ASCII' => "\xC2a", + 'C2 then lead byte' => "\xC2\xC3\xA9", + 'E0 A0 80, smallest three-byte' => "\xE0\xA0\x80", + 'E0 9F BF, overlong' => "\xE0\x9F\xBF", + 'ED 9F BF, before surrogates' => "\xED\x9F\xBF", + 'ED A0 80, surrogate' => "\xED\xA0\x80", + 'ED BF BF, surrogate' => "\xED\xBF\xBF", + 'ED A0 at the end' => "a\xED\xA0", + 'EE 80 80, after surrogates' => "\xEE\x80\x80", + 'EF BF BF, largest three-byte' => "\xEF\xBF\xBF", + 'E2 82 at the end' => "a\xE2\x82", + 'E2 82 then ASCII' => "\xE2\x82A", + 'E2 then ASCII' => "\xE2ab", + 'E2 80 A8, U+2028' => "\xE2\x80\xA8", + 'F0 90 80 80, smallest four-byte' => "\xF0\x90\x80\x80", + 'F4 8F BF BF, largest four-byte' => "\xF4\x8F\xBF\xBF", + 'F0 8F BF BF, overlong' => "\xF0\x8F\xBF\xBF", + 'F4 90 80 80, above U+10FFFF' => "\xF4\x90\x80\x80", + 'F5 80 80 80' => "\xF5\x80\x80\x80", + 'F0 9F 98 at the end' => "a\xF0\x9F\x98", + 'F0 9F 98 then ASCII' => "\xF0\x9F\x98a", + 'F0 9F 98 then lead byte' => "\xF0\x9F\x98\xF0\x9F\x98\x80", + 'four-byte between ASCII' => "x\xF0\x9F\x98\x80y", + 'two-byte after ASCII and at the end' => "abc\xD0\xB6", + 'quote after two-byte' => "\xD0\xB6\"\xD0\xB6/", + 'three-byte between ASCII' => "x\xE6\xBC\xA2y", +]; +foreach ($cases as $name => $str) { + echo "$name:\n"; + var_dump(json_encode($str)); + var_dump(json_encode($str, JSON_INVALID_UTF8_IGNORE)); + var_dump(json_encode($str, JSON_INVALID_UTF8_SUBSTITUTE)); + var_dump(bin2hex(json_encode($str, JSON_UNESCAPED_UNICODE | JSON_INVALID_UTF8_SUBSTITUTE))); +} +echo "U+2028 with JSON_UNESCAPED_LINE_TERMINATORS:\n"; +var_dump(bin2hex(json_encode("a\xE2\x80\xA8b", JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_LINE_TERMINATORS))); +echo "JSON_PARTIAL_OUTPUT_ON_ERROR:\n"; +var_dump(json_encode(["ok\xD0\xB6", "bad\xC0\xAF", "\xE2\x82"], JSON_PARTIAL_OUTPUT_ON_ERROR)); +var_dump(json_last_error_msg()); +?> +--EXPECT-- +C2 80, smallest two-byte: +string(8) ""\u0080"" +string(8) ""\u0080"" +string(8) ""\u0080"" +string(8) "22c28022" +DF BF, largest two-byte: +string(8) ""\u07ff"" +string(8) ""\u07ff"" +string(8) ""\u07ff"" +string(8) "22dfbf22" +C0 AF, overlong: +bool(false) +string(2) """" +string(14) ""\ufffd\ufffd"" +string(16) "22efbfbdefbfbd22" +C1 BF, overlong: +bool(false) +string(2) """" +string(14) ""\ufffd\ufffd"" +string(16) "22efbfbdefbfbd22" +80, lone continuation byte: +bool(false) +string(2) """" +string(8) ""\ufffd"" +string(10) "22efbfbd22" +C2 at the end: +bool(false) +string(3) ""a"" +string(9) ""a\ufffd"" +string(12) "2261efbfbd22" +C2 then ASCII: +bool(false) +string(3) ""a"" +string(9) ""\ufffda"" +string(12) "22efbfbd6122" +C2 then lead byte: +bool(false) +string(8) ""\u00e9"" +string(14) ""\ufffd\u00e9"" +string(14) "22efbfbdc3a922" +E0 A0 80, smallest three-byte: +string(8) ""\u0800"" +string(8) ""\u0800"" +string(8) ""\u0800"" +string(10) "22e0a08022" +E0 9F BF, overlong: +bool(false) +string(2) """" +string(8) ""\ufffd"" +string(10) "22efbfbd22" +ED 9F BF, before surrogates: +string(8) ""\ud7ff"" +string(8) ""\ud7ff"" +string(8) ""\ud7ff"" +string(10) "22ed9fbf22" +ED A0 80, surrogate: +bool(false) +string(2) """" +string(8) ""\ufffd"" +string(10) "22efbfbd22" +ED BF BF, surrogate: +bool(false) +string(2) """" +string(8) ""\ufffd"" +string(10) "22efbfbd22" +ED A0 at the end: +bool(false) +string(3) ""a"" +string(9) ""a\ufffd"" +string(12) "2261efbfbd22" +EE 80 80, after surrogates: +string(8) ""\ue000"" +string(8) ""\ue000"" +string(8) ""\ue000"" +string(10) "22ee808022" +EF BF BF, largest three-byte: +string(8) ""\uffff"" +string(8) ""\uffff"" +string(8) ""\uffff"" +string(10) "22efbfbf22" +E2 82 at the end: +bool(false) +string(3) ""a"" +string(9) ""a\ufffd"" +string(12) "2261efbfbd22" +E2 82 then ASCII: +bool(false) +string(3) ""A"" +string(9) ""\ufffdA"" +string(12) "22efbfbd4122" +E2 then ASCII: +bool(false) +string(4) ""ab"" +string(10) ""\ufffdab"" +string(14) "22efbfbd616222" +E2 80 A8, U+2028: +string(8) ""\u2028"" +string(8) ""\u2028"" +string(8) ""\u2028"" +string(16) "225c753230323822" +F0 90 80 80, smallest four-byte: +string(14) ""\ud800\udc00"" +string(14) ""\ud800\udc00"" +string(14) ""\ud800\udc00"" +string(12) "22f090808022" +F4 8F BF BF, largest four-byte: +string(14) ""\udbff\udfff"" +string(14) ""\udbff\udfff"" +string(14) ""\udbff\udfff"" +string(12) "22f48fbfbf22" +F0 8F BF BF, overlong: +bool(false) +string(2) """" +string(8) ""\ufffd"" +string(10) "22efbfbd22" +F4 90 80 80, above U+10FFFF: +bool(false) +string(2) """" +string(8) ""\ufffd"" +string(10) "22efbfbd22" +F5 80 80 80: +bool(false) +string(2) """" +string(26) ""\ufffd\ufffd\ufffd\ufffd"" +string(28) "22efbfbdefbfbdefbfbdefbfbd22" +F0 9F 98 at the end: +bool(false) +string(3) ""a"" +string(9) ""a\ufffd"" +string(12) "2261efbfbd22" +F0 9F 98 then ASCII: +bool(false) +string(3) ""a"" +string(9) ""\ufffda"" +string(12) "22efbfbd6122" +F0 9F 98 then lead byte: +bool(false) +string(14) ""\ud83d\ude00"" +string(20) ""\ufffd\ud83d\ude00"" +string(18) "22efbfbdf09f988022" +four-byte between ASCII: +string(16) ""x\ud83d\ude00y"" +string(16) ""x\ud83d\ude00y"" +string(16) ""x\ud83d\ude00y"" +string(16) "2278f09f98807922" +two-byte after ASCII and at the end: +string(11) ""abc\u0436"" +string(11) ""abc\u0436"" +string(11) ""abc\u0436"" +string(14) "22616263d0b622" +quote after two-byte: +string(18) ""\u0436\"\u0436\/"" +string(18) ""\u0436\"\u0436\/"" +string(18) ""\u0436\"\u0436\/"" +string(20) "22d0b65c22d0b65c2f22" +three-byte between ASCII: +string(10) ""x\u6f22y"" +string(10) ""x\u6f22y"" +string(10) ""x\u6f22y"" +string(14) "2278e6bca27922" +U+2028 with JSON_UNESCAPED_LINE_TERMINATORS: +string(14) "2261e280a86222" +JSON_PARTIAL_OUTPUT_ON_ERROR: +string(22) "["ok\u0436",null,null]" +string(56) "Malformed UTF-8 characters, possibly incorrectly encoded" diff --git a/ext/standard/html.h b/ext/standard/html.h index 8576a5c8b9ce..ed67e7a91d91 100644 --- a/ext/standard/html.h +++ b/ext/standard/html.h @@ -47,4 +47,57 @@ PHPAPI zend_string *php_escape_html_entities_ex(const unsigned char *old, size_t PHPAPI zend_string *php_unescape_html_entities(zend_string *str, int all, int flags, const char *hint_charset); PHPAPI unsigned int php_next_utf8_char(const unsigned char *str, size_t str_len, size_t *cursor, zend_result *status); +/* Same contract as php_next_utf8_char(). Decodes valid UTF-8 inline; invalid input goes + * to php_next_utf8_char(). */ +static zend_always_inline unsigned int php_next_utf8_char_inline( + const unsigned char *str, size_t str_len, size_t *cursor, zend_result *status) +{ + size_t pos = *cursor; + size_t avail; + + ZEND_ASSERT(pos <= str_len); + avail = str_len - pos; + if (EXPECTED(avail >= 1)) { + unsigned int c = str[pos]; + + if (c < 0x80) { + *cursor = pos + 1; + *status = SUCCESS; + return c; + } + if (EXPECTED(c < 0xe0)) { + /* C2..DF + continuation byte: always a valid shortest form */ + if (c >= 0xc2 && avail >= 2 && (str[pos + 1] & 0xc0) == 0x80) { + *cursor = pos + 2; + *status = SUCCESS; + return ((c & 0x1f) << 6) | (str[pos + 1] & 0x3f); + } + } else if (c < 0xf0) { + if (avail >= 3 && (str[pos + 1] & 0xc0) == 0x80 && (str[pos + 2] & 0xc0) == 0x80) { + unsigned int cp = ((c & 0x0f) << 12) | ((str[pos + 1] & 0x3f) << 6) | (str[pos + 2] & 0x3f); + /* neither overlong nor a surrogate */ + if (cp >= 0x800 && (cp < 0xd800 || cp > 0xdfff)) { + *cursor = pos + 3; + *status = SUCCESS; + return cp; + } + } + } else if (c < 0xf5) { + if (avail >= 4 && (str[pos + 1] & 0xc0) == 0x80 && (str[pos + 2] & 0xc0) == 0x80 + && (str[pos + 3] & 0xc0) == 0x80) { + unsigned int cp = ((c & 0x07) << 18) | ((str[pos + 1] & 0x3f) << 12) + | ((str[pos + 2] & 0x3f) << 6) | (str[pos + 3] & 0x3f); + /* neither overlong nor above U+10FFFF */ + if (cp >= 0x10000 && cp <= 0x10ffff) { + *cursor = pos + 4; + *status = SUCCESS; + return cp; + } + } + } + } + + return php_next_utf8_char(str, str_len, cursor, status); +} + #endif /* HTML_H */ diff --git a/ext/standard/string.c b/ext/standard/string.c index 2e949da741dc..d5810de79fab 100644 --- a/ext/standard/string.c +++ b/ext/standard/string.c @@ -6431,7 +6431,7 @@ static zend_string *php_utf8_decode(const char *s, size_t len) ZSTR_LEN(str) = 0; while (pos < len) { zend_result status = FAILURE; - c = php_next_utf8_char((const unsigned char*)s, (size_t) len, &pos, &status); + c = php_next_utf8_char_inline((const unsigned char*)s, (size_t) len, &pos, &status); /* The lower 256 codepoints of Unicode are identical to Latin-1, * so we don't need to do any mapping here beyond replacing non-Latin-1 diff --git a/ext/standard/tests/strings/utf8_decode_sequences.phpt b/ext/standard/tests/strings/utf8_decode_sequences.phpt new file mode 100644 index 000000000000..63b72b59d5be --- /dev/null +++ b/ext/standard/tests/strings/utf8_decode_sequences.phpt @@ -0,0 +1,83 @@ +--TEST-- +utf8_decode() with valid and invalid UTF-8 sequences around the two-, three- and four-byte boundaries +--INI-- +error_reporting=E_ALL & ~E_DEPRECATED +--FILE-- + "\xC2\x80", + 'C3 A9, Latin-1' => "\xC3\xA9", + 'C3 BF, largest Latin-1' => "\xC3\xBF", + 'C4 80, above Latin-1' => "\xC4\x80", + 'DF BF, largest two-byte' => "\xDF\xBF", + 'C0 AF, overlong' => "\xC0\xAF", + 'C1 BF, overlong' => "\xC1\xBF", + '80, lone continuation byte' => "\x80", + 'C2 at the end' => "a\xC2", + 'C2 then ASCII' => "\xC2a", + 'C2 then lead byte' => "\xC2\xC3\xA9", + 'E0 A0 80, smallest three-byte' => "\xE0\xA0\x80", + 'E0 9F BF, overlong' => "\xE0\x9F\xBF", + 'E0 83 A9, overlong' => "\xE0\x83\xA9", + 'ED 9F BF, before surrogates' => "\xED\x9F\xBF", + 'ED A0 80, surrogate' => "\xED\xA0\x80", + 'ED BF BF, surrogate' => "\xED\xBF\xBF", + 'ED A0 at the end' => "a\xED\xA0", + 'EE 80 80, after surrogates' => "\xEE\x80\x80", + 'EF BF BF, largest three-byte' => "\xEF\xBF\xBF", + 'E2 82 at the end' => "a\xE2\x82", + 'E2 82 then ASCII' => "\xE2\x82A", + 'E2 then ASCII' => "\xE2ab", + 'F0 90 80 80, smallest four-byte' => "\xF0\x90\x80\x80", + 'F4 8F BF BF, largest four-byte' => "\xF4\x8F\xBF\xBF", + 'F0 8F BF BF, overlong' => "\xF0\x8F\xBF\xBF", + 'F0 80 83 A9, overlong' => "\xF0\x80\x83\xA9", + 'F4 90 80 80, above U+10FFFF' => "\xF4\x90\x80\x80", + 'F5 80 80 80' => "\xF5\x80\x80\x80", + 'F0 9F 98 at the end' => "a\xF0\x9F\x98", + 'F0 9F 98 then ASCII' => "\xF0\x9F\x98a", + 'F0 9F 98 then lead byte' => "\xF0\x9F\x98\xF0\x9F\x98\x80", + 'four-byte between ASCII' => "x\xF0\x9F\x98\x80y", + 'two-byte between ASCII' => "x\xC3\xA9y", + 'three-byte between ASCII' => "x\xE6\xBC\xA2y", +]; +foreach ($cases as $name => $str) { + echo "$name: ", bin2hex(utf8_decode($str)), "\n"; +} +?> +--EXPECT-- +C2 80, smallest two-byte: 80 +C3 A9, Latin-1: e9 +C3 BF, largest Latin-1: ff +C4 80, above Latin-1: 3f +DF BF, largest two-byte: 3f +C0 AF, overlong: 3f3f +C1 BF, overlong: 3f3f +80, lone continuation byte: 3f +C2 at the end: 613f +C2 then ASCII: 3f61 +C2 then lead byte: 3fe9 +E0 A0 80, smallest three-byte: 3f +E0 9F BF, overlong: 3f +E0 83 A9, overlong: 3f +ED 9F BF, before surrogates: 3f +ED A0 80, surrogate: 3f +ED BF BF, surrogate: 3f +ED A0 at the end: 613f +EE 80 80, after surrogates: 3f +EF BF BF, largest three-byte: 3f +E2 82 at the end: 613f +E2 82 then ASCII: 3f41 +E2 then ASCII: 3f6162 +F0 90 80 80, smallest four-byte: 3f +F4 8F BF BF, largest four-byte: 3f +F0 8F BF BF, overlong: 3f +F0 80 83 A9, overlong: 3f +F4 90 80 80, above U+10FFFF: 3f +F5 80 80 80: 3f3f3f3f +F0 9F 98 at the end: 613f +F0 9F 98 then ASCII: 3f61 +F0 9F 98 then lead byte: 3f3f +four-byte between ASCII: 783f79 +two-byte between ASCII: 78e979 +three-byte between ASCII: 783f79 diff --git a/ext/xml/xml.c b/ext/xml/xml.c index f300693f150a..e7531a2bed2f 100644 --- a/ext/xml/xml.c +++ b/ext/xml/xml.c @@ -513,7 +513,7 @@ static zend_string *xml_utf8_decode(const XML_Char *s, size_t len, const XML_Cha ZSTR_LEN(str) = 0; while (pos < len) { zend_result status = FAILURE; - c = php_next_utf8_char((const unsigned char*)s, len, &pos, &status); + c = php_next_utf8_char_inline((const unsigned char*)s, len, &pos, &status); if (status == FAILURE || c > 0xFFU) { c = '?';