Skip to content

Commit ca35c07

Browse files
ArtUkrainskiyArtUkrainskiy
committed
ext/standard: Add php_next_utf8_char_inline()
json_encode(), utf8_decode() and the xml extension decode one character at a time with php_next_utf8_char(), and every call went through the generic charset decoder of html.c. The new static inline function in html.h decodes ASCII and valid two- and three-byte sequences at the call site and leaves everything else, including every invalid sequence, to php_next_utf8_char(), which is unchanged.
1 parent bd6c637 commit ca35c07

7 files changed

Lines changed: 244 additions & 3 deletions

File tree

‎UPGRADING‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -106,5 +106,9 @@ PHP 8.7 UPGRADE NOTES
106106
. Improved performance of pack() for large strings using the a, A and Z
107107
formats.
108108

109+
- JSON:
110+
. Improved performance of json_encode() for strings with non-ASCII
111+
characters.
112+
109113
- MBString:
110114
. Improved performance of mb_strlen for UTF-8 strings.

‎UPGRADING.INTERNALS‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,9 @@ PHP 8.7 INTERNALS UPGRADE NOTES
7575
- Standard:
7676
. php_explode() now takes a HashTable* instead of a zval* to store the
7777
resulting parts. It already expected the zval* to be IS_ARRAY.
78+
. Added php_next_utf8_char_inline() to ext/standard/html.h. It has the same
79+
contract as php_next_utf8_char() and decodes ASCII and valid two- and
80+
three-byte sequences at the call site.
7881

7982
========================
8083
4. OpCode changes

‎ext/json/json_encoder.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -405,7 +405,7 @@ zend_result php_json_escape_string(
405405
us = (unsigned char)s[0];
406406
if (UNEXPECTED(us >= 0x80)) {
407407
zend_result status;
408-
us = php_next_utf8_char((unsigned char *)s, len, &pos, &status);
408+
us = php_next_utf8_char_inline((const unsigned char *)s, len, &pos, &status);
409409

410410
/* check whether UTF8 character is correct */
411411
if (UNEXPECTED(status != SUCCESS)) {

‎ext/json/tests/json_encode_utf8_sequences.phpt‎

Lines changed: 193 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,193 @@
1+
--TEST--
2+
json_encode() with valid and invalid UTF-8 sequences around the two-, three- and four-byte boundaries
3+
--FILE--
4+
<?php
5+
$cases = [
6+
'C2 80, smallest two-byte' => "\xC2\x80",
7+
'DF BF, largest two-byte' => "\xDF\xBF",
8+
'C0 AF, overlong' => "\xC0\xAF",
9+
'C1 BF, overlong' => "\xC1\xBF",
10+
'80, lone continuation byte' => "\x80",
11+
'C2 at the end' => "a\xC2",
12+
'C2 then ASCII' => "\xC2a",
13+
'C2 then lead byte' => "\xC2\xC3\xA9",
14+
'E0 A0 80, smallest three-byte' => "\xE0\xA0\x80",
15+
'E0 9F BF, overlong' => "\xE0\x9F\xBF",
16+
'ED 9F BF, before surrogates' => "\xED\x9F\xBF",
17+
'ED A0 80, surrogate' => "\xED\xA0\x80",
18+
'ED BF BF, surrogate' => "\xED\xBF\xBF",
19+
'ED A0 at the end' => "a\xED\xA0",
20+
'EE 80 80, after surrogates' => "\xEE\x80\x80",
21+
'EF BF BF, largest three-byte' => "\xEF\xBF\xBF",
22+
'E2 82 at the end' => "a\xE2\x82",
23+
'E2 82 then ASCII' => "\xE2\x82A",
24+
'E2 then ASCII' => "\xE2ab",
25+
'E2 80 A8, U+2028' => "\xE2\x80\xA8",
26+
'F0 90 80 80, smallest four-byte' => "\xF0\x90\x80\x80",
27+
'F4 8F BF BF, largest four-byte' => "\xF4\x8F\xBF\xBF",
28+
'F0 8F BF BF, overlong' => "\xF0\x8F\xBF\xBF",
29+
'F4 90 80 80, above U+10FFFF' => "\xF4\x90\x80\x80",
30+
'F5 80 80 80' => "\xF5\x80\x80\x80",
31+
'two-byte after ASCII and at the end' => "abc\xD0\xB6",
32+
'quote after two-byte' => "\xD0\xB6\"\xD0\xB6/",
33+
'three-byte between ASCII' => "x\xE6\xBC\xA2y",
34+
];
35+
foreach ($cases as $name => $str) {
36+
echo "$name:\n";
37+
var_dump(json_encode($str));
38+
var_dump(json_encode($str, JSON_INVALID_UTF8_IGNORE));
39+
var_dump(json_encode($str, JSON_INVALID_UTF8_SUBSTITUTE));
40+
var_dump(bin2hex(json_encode($str, JSON_UNESCAPED_UNICODE | JSON_INVALID_UTF8_SUBSTITUTE)));
41+
}
42+
echo "U+2028 with JSON_UNESCAPED_LINE_TERMINATORS:\n";
43+
var_dump(bin2hex(json_encode("a\xE2\x80\xA8b", JSON_UNESCAPED_UNICODE | JSON_UNESCAPED_LINE_TERMINATORS)));
44+
echo "JSON_PARTIAL_OUTPUT_ON_ERROR:\n";
45+
var_dump(json_encode(["ok\xD0\xB6", "bad\xC0\xAF", "\xE2\x82"], JSON_PARTIAL_OUTPUT_ON_ERROR));
46+
var_dump(json_last_error_msg());
47+
?>
48+
--EXPECT--
49+
C2 80, smallest two-byte:
50+
string(8) ""\u0080""
51+
string(8) ""\u0080""
52+
string(8) ""\u0080""
53+
string(8) "22c28022"
54+
DF BF, largest two-byte:
55+
string(8) ""\u07ff""
56+
string(8) ""\u07ff""
57+
string(8) ""\u07ff""
58+
string(8) "22dfbf22"
59+
C0 AF, overlong:
60+
bool(false)
61+
string(2) """"
62+
string(14) ""\ufffd\ufffd""
63+
string(16) "22efbfbdefbfbd22"
64+
C1 BF, overlong:
65+
bool(false)
66+
string(2) """"
67+
string(14) ""\ufffd\ufffd""
68+
string(16) "22efbfbdefbfbd22"
69+
80, lone continuation byte:
70+
bool(false)
71+
string(2) """"
72+
string(8) ""\ufffd""
73+
string(10) "22efbfbd22"
74+
C2 at the end:
75+
bool(false)
76+
string(3) ""a""
77+
string(9) ""a\ufffd""
78+
string(12) "2261efbfbd22"
79+
C2 then ASCII:
80+
bool(false)
81+
string(3) ""a""
82+
string(9) ""\ufffda""
83+
string(12) "22efbfbd6122"
84+
C2 then lead byte:
85+
bool(false)
86+
string(8) ""\u00e9""
87+
string(14) ""\ufffd\u00e9""
88+
string(14) "22efbfbdc3a922"
89+
E0 A0 80, smallest three-byte:
90+
string(8) ""\u0800""
91+
string(8) ""\u0800""
92+
string(8) ""\u0800""
93+
string(10) "22e0a08022"
94+
E0 9F BF, overlong:
95+
bool(false)
96+
string(2) """"
97+
string(8) ""\ufffd""
98+
string(10) "22efbfbd22"
99+
ED 9F BF, before surrogates:
100+
string(8) ""\ud7ff""
101+
string(8) ""\ud7ff""
102+
string(8) ""\ud7ff""
103+
string(10) "22ed9fbf22"
104+
ED A0 80, surrogate:
105+
bool(false)
106+
string(2) """"
107+
string(8) ""\ufffd""
108+
string(10) "22efbfbd22"
109+
ED BF BF, surrogate:
110+
bool(false)
111+
string(2) """"
112+
string(8) ""\ufffd""
113+
string(10) "22efbfbd22"
114+
ED A0 at the end:
115+
bool(false)
116+
string(3) ""a""
117+
string(9) ""a\ufffd""
118+
string(12) "2261efbfbd22"
119+
EE 80 80, after surrogates:
120+
string(8) ""\ue000""
121+
string(8) ""\ue000""
122+
string(8) ""\ue000""
123+
string(10) "22ee808022"
124+
EF BF BF, largest three-byte:
125+
string(8) ""\uffff""
126+
string(8) ""\uffff""
127+
string(8) ""\uffff""
128+
string(10) "22efbfbf22"
129+
E2 82 at the end:
130+
bool(false)
131+
string(3) ""a""
132+
string(9) ""a\ufffd""
133+
string(12) "2261efbfbd22"
134+
E2 82 then ASCII:
135+
bool(false)
136+
string(3) ""A""
137+
string(9) ""\ufffdA""
138+
string(12) "22efbfbd4122"
139+
E2 then ASCII:
140+
bool(false)
141+
string(4) ""ab""
142+
string(10) ""\ufffdab""
143+
string(14) "22efbfbd616222"
144+
E2 80 A8, U+2028:
145+
string(8) ""\u2028""
146+
string(8) ""\u2028""
147+
string(8) ""\u2028""
148+
string(16) "225c753230323822"
149+
F0 90 80 80, smallest four-byte:
150+
string(14) ""\ud800\udc00""
151+
string(14) ""\ud800\udc00""
152+
string(14) ""\ud800\udc00""
153+
string(12) "22f090808022"
154+
F4 8F BF BF, largest four-byte:
155+
string(14) ""\udbff\udfff""
156+
string(14) ""\udbff\udfff""
157+
string(14) ""\udbff\udfff""
158+
string(12) "22f48fbfbf22"
159+
F0 8F BF BF, overlong:
160+
bool(false)
161+
string(2) """"
162+
string(8) ""\ufffd""
163+
string(10) "22efbfbd22"
164+
F4 90 80 80, above U+10FFFF:
165+
bool(false)
166+
string(2) """"
167+
string(8) ""\ufffd""
168+
string(10) "22efbfbd22"
169+
F5 80 80 80:
170+
bool(false)
171+
string(2) """"
172+
string(26) ""\ufffd\ufffd\ufffd\ufffd""
173+
string(28) "22efbfbdefbfbdefbfbdefbfbd22"
174+
two-byte after ASCII and at the end:
175+
string(11) ""abc\u0436""
176+
string(11) ""abc\u0436""
177+
string(11) ""abc\u0436""
178+
string(14) "22616263d0b622"
179+
quote after two-byte:
180+
string(18) ""\u0436\"\u0436\/""
181+
string(18) ""\u0436\"\u0436\/""
182+
string(18) ""\u0436\"\u0436\/""
183+
string(20) "22d0b65c22d0b65c2f22"
184+
three-byte between ASCII:
185+
string(10) ""x\u6f22y""
186+
string(10) ""x\u6f22y""
187+
string(10) ""x\u6f22y""
188+
string(14) "2278e6bca27922"
189+
U+2028 with JSON_UNESCAPED_LINE_TERMINATORS:
190+
string(14) "2261e280a86222"
191+
JSON_PARTIAL_OUTPUT_ON_ERROR:
192+
string(22) "["ok\u0436",null,null]"
193+
string(56) "Malformed UTF-8 characters, possibly incorrectly encoded"

‎ext/standard/html.h‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,4 +47,45 @@ PHPAPI zend_string *php_escape_html_entities_ex(const unsigned char *old, size_t
4747
PHPAPI zend_string *php_unescape_html_entities(zend_string *str, int all, int flags, const char *hint_charset);
4848
PHPAPI unsigned int php_next_utf8_char(const unsigned char *str, size_t str_len, size_t *cursor, zend_result *status);
4949

50+
/* Same contract as php_next_utf8_char(). Decodes ASCII and valid two- and three-byte
51+
* sequences inline; any other input goes to php_next_utf8_char(). */
52+
static zend_always_inline unsigned int php_next_utf8_char_inline(
53+
const unsigned char *str, size_t str_len, size_t *cursor, zend_result *status)
54+
{
55+
size_t pos = *cursor;
56+
size_t avail;
57+
58+
ZEND_ASSERT(pos <= str_len);
59+
avail = str_len - pos;
60+
if (EXPECTED(avail >= 1)) {
61+
unsigned int c = str[pos];
62+
63+
if (c < 0x80) {
64+
*cursor = pos + 1;
65+
*status = SUCCESS;
66+
return c;
67+
}
68+
if (EXPECTED(c < 0xe0)) {
69+
/* C2..DF + continuation byte: always a valid shortest form */
70+
if (c >= 0xc2 && avail >= 2 && (str[pos + 1] & 0xc0) == 0x80) {
71+
*cursor = pos + 2;
72+
*status = SUCCESS;
73+
return ((c & 0x1f) << 6) | (str[pos + 1] & 0x3f);
74+
}
75+
} else if (c < 0xf0) {
76+
if (avail >= 3 && (str[pos + 1] & 0xc0) == 0x80 && (str[pos + 2] & 0xc0) == 0x80) {
77+
unsigned int cp = ((c & 0x0f) << 12) | ((str[pos + 1] & 0x3f) << 6) | (str[pos + 2] & 0x3f);
78+
/* neither overlong nor a surrogate */
79+
if (cp >= 0x800 && (cp < 0xd800 || cp > 0xdfff)) {
80+
*cursor = pos + 3;
81+
*status = SUCCESS;
82+
return cp;
83+
}
84+
}
85+
}
86+
}
87+
88+
return php_next_utf8_char(str, str_len, cursor, status);
89+
}
90+
5091
#endif /* HTML_H */

‎ext/standard/string.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6431,7 +6431,7 @@ static zend_string *php_utf8_decode(const char *s, size_t len)
64316431
ZSTR_LEN(str) = 0;
64326432
while (pos < len) {
64336433
zend_result status = FAILURE;
6434-
c = php_next_utf8_char((const unsigned char*)s, (size_t) len, &pos, &status);
6434+
c = php_next_utf8_char_inline((const unsigned char*)s, (size_t) len, &pos, &status);
64356435

64366436
/* The lower 256 codepoints of Unicode are identical to Latin-1,
64376437
* so we don't need to do any mapping here beyond replacing non-Latin-1

‎ext/xml/xml.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -513,7 +513,7 @@ static zend_string *xml_utf8_decode(const XML_Char *s, size_t len, const XML_Cha
513513
ZSTR_LEN(str) = 0;
514514
while (pos < len) {
515515
zend_result status = FAILURE;
516-
c = php_next_utf8_char((const unsigned char*)s, len, &pos, &status);
516+
c = php_next_utf8_char_inline((const unsigned char*)s, len, &pos, &status);
517517

518518
if (status == FAILURE || c > 0xFFU) {
519519
c = '?';

0 commit comments

Comments
 (0)