Skip to content

ext/standard: Check for UTF-8 first when resolving the charset of html functions #23961

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
arnaud-lb merged 1 commit into from
Oct 14, 2026
Merged

ext/standard: Check for UTF-8 first when resolving the charset of html functions #23961

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions NEWS
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,8 @@ PHP NEWS
in a php://filter URL. (Sjoerd Langkemper)
. Improved performance of array_chunk() when not preserving keys.
(mehmetcansahin)
. Improved performance of resolving the charset of HTML functions,
especially UTF-8. (Nicolas Grekas)

- URI:
. Fix casing of enum cases in UriHostType and UrlHostType to match the RFC
Expand Down
3 changes: 3 additions & 0 deletions UPGRADING
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -105,6 +105,9 @@ PHP 8.7 UPGRADE NOTES
. Improved performance of str_rot13().
. Improved performance of pack() for large strings using the a, A and Z
formats.
. Improved performance of htmlspecialchars(), htmlentities(),
html_entity_decode() and get_html_translation_table(), especially with
UTF-8.

- MBString:
. Improved performance of mb_strlen for UTF-8 strings.
62 changes: 47 additions & 15 deletions ext/standard/html.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -356,32 +356,64 @@ PHPAPI unsigned int php_next_utf8_char(
}
/* }}} */

/* Returns whether a charset name is "utf-8", in any case */
static zend_always_inline bool charset_is_utf8(const char *charset_hint)
{
return zend_tolower_ascii(charset_hint[0]) == 'u' && zend_tolower_ascii(charset_hint[1]) == 't'
&& zend_tolower_ascii(charset_hint[2]) == 'f' && charset_hint[3] == '-' && charset_hint[4] == '8'
&& charset_hint[5] == '\0';
}

/* Returns the charset identifier of a non-empty charset name, or UTF-8 if it is not supported. */
static zend_never_inline enum entity_charset lookup_charset(const char *charset_hint, bool quiet)
{
/* the codesets of the charset map are lowercase, so lowercase the hint once */
char lowercase_hint[LONGEST_CODESET_LENGTH];
size_t len;

for (len = 0; charset_hint[len]; len++) {
if (len == LONGEST_CODESET_LENGTH) {
/* longer than any codeset */
goto not_supported;
}
lowercase_hint[len] = zend_tolower_ascii(charset_hint[len]);
}

/* now walk the charset map and look for the codeset (UTF-8 is resolved by the caller) */
for (size_t i = 0; i < sizeof(charset_map)/sizeof(charset_map[0]); i++) {
if (charset_map[i].codeset_len == len && charset_map[i].codeset[0] == lowercase_hint[0]
&& memcmp(lowercase_hint, charset_map[i].codeset, len) == 0) {
return charset_map[i].charset;
}
}

not_supported:
if (!quiet) {
php_error_docref(NULL, E_WARNING, "Charset \"%s\" is not supported, assuming UTF-8",
charset_hint);
}

return cs_utf_8;
}

/* {{{ entity_charset determine_charset
* Returns the charset identifier based on an explicitly provided charset,
* the internal_encoding and default_charset ini settings, or UTF-8 by default. */
static enum entity_charset determine_charset(const char *charset_hint, bool quiet)
{
if (!charset_hint || !*charset_hint) {
charset_hint = get_default_charset();
}

if (charset_hint && *charset_hint) {
size_t len = strlen(charset_hint);
/* now walk the charset map and look for the codeset */
for (size_t i = 0; i < sizeof(charset_map)/sizeof(charset_map[0]); i++) {
if (len == charset_map[i].codeset_len &&
zend_binary_strcasecmp(charset_hint, len, charset_map[i].codeset, len) == 0) {
return charset_map[i].charset;
}
if (!charset_hint) {
return cs_utf_8;
}
}

if (!quiet) {
php_error_docref(NULL, E_WARNING, "Charset \"%s\" is not supported, assuming UTF-8",
charset_hint);
}
/* UTF-8 is the most used charset, and it is not in the charset map */
if (charset_is_utf8(charset_hint)) {
return cs_utf_8;
}

return cs_utf_8;
return lookup_charset(charset_hint, quiet);
}
/* }}} */

Expand Down
41 changes: 22 additions & 19 deletions ext/standard/html_tables.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,46 +31,49 @@ enum entity_charset { cs_utf_8, cs_8859_1, cs_cp1252, cs_8859_15, cs_cp1251,
#define CHARSET_SINGLE_BYTE(cs) ((cs) > cs_utf_8 && (cs) < cs_big5)
#define CHARSET_PARTIAL_SUPPORT(cs) ((cs) >= cs_big5)

/* codesets must be lowercase, since lookup_charset() compares them with a lowercase copy of the hint */
static const struct {
const char *codeset;
uint32_t codeset_len;
enum entity_charset charset;
} charset_map[] = {
{ "ISO-8859-1", sizeof("ISO-8859-1")-1, cs_8859_1 },
{ "ISO8859-1", sizeof("ISO8859-1")-1, cs_8859_1 },
{ "ISO-8859-15", sizeof("ISO-8859-15")-1, cs_8859_15 },
{ "ISO8859-15", sizeof("ISO8859-15")-1, cs_8859_15 },
{ "utf-8", sizeof("utf-8")-1, cs_utf_8 },
{ "iso-8859-1", sizeof("iso-8859-1")-1, cs_8859_1 },
{ "iso8859-1", sizeof("iso8859-1")-1, cs_8859_1 },
{ "iso-8859-15", sizeof("iso-8859-15")-1, cs_8859_15 },
{ "iso8859-15", sizeof("iso8859-15")-1, cs_8859_15 },
{ "cp1252", sizeof("cp1252")-1, cs_cp1252 },
{ "Windows-1252", sizeof("Windows-1252")-1, cs_cp1252 },
{ "windows-1252", sizeof("windows-1252")-1, cs_cp1252 },
{ "1252", sizeof("1252")-1, cs_cp1252 },
{ "BIG5", sizeof("BIG5")-1, cs_big5 },
{ "big5", sizeof("big5")-1, cs_big5 },
{ "950", sizeof("950")-1, cs_big5 },
{ "GB2312", sizeof("GB2312")-1, cs_gb2312 },
{ "gb2312", sizeof("gb2312")-1, cs_gb2312 },
{ "936", sizeof("936")-1, cs_gb2312 },
{ "BIG5-HKSCS", sizeof("BIG5-HKSCS")-1, cs_big5hkscs },
{ "Shift_JIS", sizeof("Shift_JIS")-1, cs_sjis },
{ "SJIS", sizeof("SJIS")-1, cs_sjis },
{ "big5-hkscs", sizeof("big5-hkscs")-1, cs_big5hkscs },
{ "shift_jis", sizeof("shift_jis")-1, cs_sjis },
{ "sjis", sizeof("sjis")-1, cs_sjis },
{ "932", sizeof("932")-1, cs_sjis },
{ "SJIS-win", sizeof("SJIS-win")-1, cs_sjis },
{ "CP932", sizeof("CP932")-1, cs_sjis },
{ "EUCJP", sizeof("EUCJP")-1, cs_eucjp },
{ "EUC-JP", sizeof("EUC-JP")-1, cs_eucjp },
{ "eucJP-win", sizeof("eucJP-win")-1, cs_eucjp },
{ "KOI8-R", sizeof("KOI8-R")-1, cs_koi8r },
{ "sjis-win", sizeof("sjis-win")-1, cs_sjis },
{ "cp932", sizeof("cp932")-1, cs_sjis },
{ "eucjp", sizeof("eucjp")-1, cs_eucjp },
{ "euc-jp", sizeof("euc-jp")-1, cs_eucjp },
{ "eucjp-win", sizeof("eucjp-win")-1, cs_eucjp },
{ "koi8-r", sizeof("koi8-r")-1, cs_koi8r },
{ "koi8-ru", sizeof("koi8-ru")-1, cs_koi8r },
{ "koi8r", sizeof("koi8r")-1, cs_koi8r },
{ "cp1251", sizeof("cp1251")-1, cs_cp1251 },
{ "Windows-1251", sizeof("Windows-1251")-1, cs_cp1251 },
{ "windows-1251", sizeof("windows-1251")-1, cs_cp1251 },
{ "win-1251", sizeof("win-1251")-1, cs_cp1251 },
{ "iso8859-5", sizeof("iso8859-5")-1, cs_8859_5 },
{ "iso-8859-5", sizeof("iso-8859-5")-1, cs_8859_5 },
{ "cp866", sizeof("cp866")-1, cs_cp866 },
{ "866", sizeof("866")-1, cs_cp866 },
{ "ibm866", sizeof("ibm866")-1, cs_cp866 },
{ "MacRoman", sizeof("MacRoman")-1, cs_macroman }
{ "macroman", sizeof("macroman")-1, cs_macroman }
};

/* length of the longest codeset */
#define LONGEST_CODESET_LENGTH 12

/* longest entity name length excluding & and ; */
#define LONGEST_ENTITY_LENGTH 31

Expand Down
41 changes: 22 additions & 19 deletions ext/standard/html_tables/html_table_gen.php
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,46 +51,49 @@ enum entity_charset { cs_utf_8, cs_8859_1, cs_cp1252, cs_8859_15, cs_cp1251,
#define CHARSET_SINGLE_BYTE(cs) ((cs) > cs_utf_8 && (cs) < cs_big5)
#define CHARSET_PARTIAL_SUPPORT(cs) ((cs) >= cs_big5)

/* codesets must be lowercase, since lookup_charset() compares them with a lowercase copy of the hint */
static const struct {
const char *codeset;
uint32_t codeset_len;
enum entity_charset charset;
} charset_map[] = {
{ "ISO-8859-1", sizeof("ISO-8859-1")-1, cs_8859_1 },
{ "ISO8859-1", sizeof("ISO8859-1")-1, cs_8859_1 },
{ "ISO-8859-15", sizeof("ISO-8859-15")-1, cs_8859_15 },
{ "ISO8859-15", sizeof("ISO8859-15")-1, cs_8859_15 },
{ "utf-8", sizeof("utf-8")-1, cs_utf_8 },
{ "iso-8859-1", sizeof("iso-8859-1")-1, cs_8859_1 },
{ "iso8859-1", sizeof("iso8859-1")-1, cs_8859_1 },
{ "iso-8859-15", sizeof("iso-8859-15")-1, cs_8859_15 },
{ "iso8859-15", sizeof("iso8859-15")-1, cs_8859_15 },
{ "cp1252", sizeof("cp1252")-1, cs_cp1252 },
{ "Windows-1252", sizeof("Windows-1252")-1, cs_cp1252 },
{ "windows-1252", sizeof("windows-1252")-1, cs_cp1252 },
{ "1252", sizeof("1252")-1, cs_cp1252 },
{ "BIG5", sizeof("BIG5")-1, cs_big5 },
{ "big5", sizeof("big5")-1, cs_big5 },
{ "950", sizeof("950")-1, cs_big5 },
{ "GB2312", sizeof("GB2312")-1, cs_gb2312 },
{ "gb2312", sizeof("gb2312")-1, cs_gb2312 },
{ "936", sizeof("936")-1, cs_gb2312 },
{ "BIG5-HKSCS", sizeof("BIG5-HKSCS")-1, cs_big5hkscs },
{ "Shift_JIS", sizeof("Shift_JIS")-1, cs_sjis },
{ "SJIS", sizeof("SJIS")-1, cs_sjis },
{ "big5-hkscs", sizeof("big5-hkscs")-1, cs_big5hkscs },
{ "shift_jis", sizeof("shift_jis")-1, cs_sjis },
{ "sjis", sizeof("sjis")-1, cs_sjis },
{ "932", sizeof("932")-1, cs_sjis },
{ "SJIS-win", sizeof("SJIS-win")-1, cs_sjis },
{ "CP932", sizeof("CP932")-1, cs_sjis },
{ "EUCJP", sizeof("EUCJP")-1, cs_eucjp },
{ "EUC-JP", sizeof("EUC-JP")-1, cs_eucjp },
{ "eucJP-win", sizeof("eucJP-win")-1, cs_eucjp },
{ "KOI8-R", sizeof("KOI8-R")-1, cs_koi8r },
{ "sjis-win", sizeof("sjis-win")-1, cs_sjis },
{ "cp932", sizeof("cp932")-1, cs_sjis },
{ "eucjp", sizeof("eucjp")-1, cs_eucjp },
{ "euc-jp", sizeof("euc-jp")-1, cs_eucjp },
{ "eucjp-win", sizeof("eucjp-win")-1, cs_eucjp },
{ "koi8-r", sizeof("koi8-r")-1, cs_koi8r },
{ "koi8-ru", sizeof("koi8-ru")-1, cs_koi8r },
{ "koi8r", sizeof("koi8r")-1, cs_koi8r },
{ "cp1251", sizeof("cp1251")-1, cs_cp1251 },
{ "Windows-1251", sizeof("Windows-1251")-1, cs_cp1251 },
{ "windows-1251", sizeof("windows-1251")-1, cs_cp1251 },
{ "win-1251", sizeof("win-1251")-1, cs_cp1251 },
{ "iso8859-5", sizeof("iso8859-5")-1, cs_8859_5 },
{ "iso-8859-5", sizeof("iso-8859-5")-1, cs_8859_5 },
{ "cp866", sizeof("cp866")-1, cs_cp866 },
{ "866", sizeof("866")-1, cs_cp866 },
{ "ibm866", sizeof("ibm866")-1, cs_cp866 },
{ "MacRoman", sizeof("MacRoman")-1, cs_macroman }
{ "macroman", sizeof("macroman")-1, cs_macroman }
};

/* length of the longest codeset */
#define LONGEST_CODESET_LENGTH 12

/* longest entity name length excluding & and ; */
#define LONGEST_ENTITY_LENGTH 31

Expand Down
19 changes: 19 additions & 0 deletions ext/standard/tests/strings/html_charset_lookup.phpt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
--TEST--
Charset names are case-insensitive, and names longer than the longest one are not supported
--FILE--
<?php
foreach (['WINDOWS-1252', 'wInDoWs-1252', 'MACROMAN', 'Windows-12520', 'MacRoman-but-longer'] as $charset) {
$decoded = html_entity_decode('&eacute;', ENT_QUOTES, $charset);
echo var_export($charset, true), ': ', bin2hex($decoded), "\n";
}
?>
--EXPECTF--
'WINDOWS-1252': e9
'wInDoWs-1252': e9
'MACROMAN': 8e

Warning: html_entity_decode(): Charset "Windows-12520" is not supported, assuming UTF-8 in %s on line %d
'Windows-12520': c3a9

Warning: html_entity_decode(): Charset "MacRoman-but-longer" is not supported, assuming UTF-8 in %s on line %d
'MacRoman-but-longer': c3a9
32 changes: 32 additions & 0 deletions ext/standard/tests/strings/html_charset_utf8.phpt
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
--TEST--
Charset names that are UTF-8 or close to it
--FILE--
<?php
foreach (['UTF-8', 'utf-8', 'uTf-8', 'UTF8', 'UTF-', 'UTF-8 ', 'UTF-88', 'UTF_8'] as $charset) {
$escaped = htmlspecialchars("\xC3\xA9<", ENT_QUOTES, $charset);
echo var_export($charset, true), ": $escaped\n";
}

ini_set('default_charset', 'uTF-8');
echo htmlspecialchars("\xC3\xA9<", ENT_QUOTES), "\n";
?>
--EXPECTF--
'UTF-8': é&lt;
'utf-8': é&lt;
'uTf-8': é&lt;

Warning: htmlspecialchars(): Charset "UTF8" is not supported, assuming UTF-8 in %s on line %d
'UTF8': é&lt;

Warning: htmlspecialchars(): Charset "UTF-" is not supported, assuming UTF-8 in %s on line %d
'UTF-': é&lt;

Warning: htmlspecialchars(): Charset "UTF-8 " is not supported, assuming UTF-8 in %s on line %d
'UTF-8 ': é&lt;

Warning: htmlspecialchars(): Charset "UTF-88" is not supported, assuming UTF-8 in %s on line %d
'UTF-88': é&lt;

Warning: htmlspecialchars(): Charset "UTF_8" is not supported, assuming UTF-8 in %s on line %d
'UTF_8': é&lt;
é&lt;
Loading