clones/php-src
1
0
Fork 0
mirror of https://github.com/php/php-src.git synced 2024-09-22 10:27:25 +00:00
Code Issues Packages Projects Releases Wiki Activity

mb_convert_encoding will not auto-detect input string as UUEncode, Base64, QPrint

Browse Source 
In a2bc57e0e5, mb_detect_encoding was modified to ensure it would never
return 'UUENCODE', 'QPrint', or other non-encodings as the "detected
text encoding". Before mb_detect_encoding was enhanced so that it could
detect any supported text encoding, those were never returned, and they
are not desired. Actually, we want to eventually remove them completely
from mbstring, since PHP already contains other implementations of
UUEncode, QPrint, Base64, and HTML entities.

For more clarity on why we need to suppress UUEncode, etc. from being
detected by mb_detect_encoding, the existing UUEncode implementation
in mbstring *never* treats any input as erroneous. It just accepts
everything. This means that it would *always* be treated as a valid
choice by mb_detect_encoding, and would be returned in many, many cases
where the input is obviously not UUEncoded.

It turns out that the form of mb_convert_encoding where the user passes
multiple candidate encodings (and mbstring auto-detects which one to
use) was also affected by the same issue. Apply the same fix.
This commit is contained in:
Alex Dowad 2021-12-05 19:29:00 +02:00
parent 069bbf3e80
commit f07c193583
2 changed files with 30 additions and 33 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

38
ext/mbstring/mbstring.c
View File

@ -2491,6 +2491,23 @@ try_again:
}
/* }}} */
static void remove_non_encodings_from_elist(const mbfl_encoding **elist, size_t *size)
{
/* mbstring supports some 'text encodings' which aren't really text encodings
* at all, but really 'byte encodings', like Base64, QPrint, and so on.
* These should never be returned by `mb_detect_encoding`. */
int shift = 0;
for (int i = 0; i < *size; i++) {
const mbfl_encoding *encoding = elist[i];
if (encoding->no_encoding <= mbfl_no_encoding_charset_min) {
shift++; /* Remove this encoding from the list */
} else if (shift) {
elist[i - shift] = encoding;
}
}
*size -= shift;
}
/* {{{ Returns converted string in desired encoding */
PHP_FUNCTION(mb_convert_encoding)
{
@ -2531,6 +2548,10 @@ PHP_FUNCTION(mb_convert_encoding)
free_from_encodings = 0;
}
if (num_from_encodings > 1) {
remove_non_encodings_from_elist(from_encodings, &num_from_encodings);
}
if (!num_from_encodings) {
efree(ZEND_VOIDP(from_encodings));
zend_argument_value_error(3, "must specify at least one encoding");
@ -2664,23 +2685,6 @@ PHP_FUNCTION(mb_strtolower)
}
/* }}} */
static void remove_non_encodings_from_elist(const mbfl_encoding **elist, size_t *size)
{
/* mbstring supports some 'text encodings' which aren't really text encodings
* at all, but really 'byte encodings', like Base64, QPrint, and so on.
* These should never be returned by `mb_detect_encoding`. */
int shift = 0;
for (int i = 0; i < *size; i++) {
const mbfl_encoding *encoding = elist[i];
if (encoding->no_encoding <= mbfl_no_encoding_charset_min) {
shift++; /* Remove this encoding from the list */
} else if (shift) {
elist[i - shift] = encoding;
}
}
*size -= shift;
}
/* {{{ Encodings of the given string is returned (as a string) */
PHP_FUNCTION(mb_detect_encoding)
{

25
ext/mbstring/tests/mb_convert_encoding.phpt
View File

@ -9,33 +9,24 @@ mbstring.language=Japanese
<?php
// TODO: Add more tests
// SJIS string (BASE64 encoded)
$sjis = base64_decode('k/qWe4zqg2WDTINYg2eCxYK3gUIwMTIzNIJUglWCVoJXgliBQg==');
// JIS string (BASE64 encoded)
$jis = base64_decode('GyRCRnxLXDhsJUYlLSU5JUgkRyQ5ISMbKEIwMTIzNBskQiM1IzYjNyM4IzkhIxsoQg==');
// EUC-JP string
$euc_jp = '日本語テキストです。01234５６７８９。';
$euc_jp = "\xC6\xFC\xCB\xDC\xB8\xEC\xA5\xC6\xA5\xAD\xA5\xB9\xA5\xC8\xA4\xC7\xA4\xB9\xA1\xA301234\xA3\xB5\xA3\xB6\xA3\xB7\xA3\xB8\xA3\xB9\xA1\xA3";
// Test with single "form encoding"
// Note: For some reason it complains, results are different. Not researched.
echo "== BASIC TEST ==\n";
$s = $sjis;
$s = bin2hex(mb_convert_encoding($s, 'EUC-JP', 'SJIS'));
$s = bin2hex(mb_convert_encoding($sjis, 'EUC-JP', 'SJIS'));
print("EUC-JP: $s\n"); // EUC-JP
$s = $jis;
$s = bin2hex(mb_convert_encoding($s, 'EUC-JP', 'JIS'));
$s = bin2hex(mb_convert_encoding($jis, 'EUC-JP', 'JIS'));
print("EUC-JP: $s\n"); // EUC-JP
$s = $euc_jp;
$s = mb_convert_encoding($s, 'SJIS', 'EUC-JP');
$s = mb_convert_encoding($euc_jp, 'SJIS', 'EUC-JP');
print("SJIS: ".base64_encode($s)."\n"); // SJIS
$s = $euc_jp;
$s = mb_convert_encoding($s, 'JIS', 'EUC-JP');
$s = mb_convert_encoding($euc_jp, 'JIS', 'EUC-JP');
print("JIS: ".base64_encode($s)."\n"); // JIS
// Using Encoding List Array
echo "== STRING ENCODING LIST ==\n";
@ -52,11 +43,10 @@ $s = $euc_jp;
$s = mb_convert_encoding($s, 'JIS', $a);
print("JIS: ".base64_encode($s)."\n"); // JIS
// Using Encoding List Array
echo "== ARRAY ENCODING LIST ==\n";
$a = array(0=>'JIS', 1=>'UTF-8', 2=>'EUC-JP', 3=>'SJIS');
$a = ['JIS', 'UTF-8', 'EUC-JP', 'SJIS'];
$s = $jis;
$s = bin2hex(mb_convert_encoding($s, 'EUC-JP', $a));
print("EUC-JP: $s\n"); // EUC-JP
@ -69,6 +59,8 @@ $s = $euc_jp;
$s = mb_convert_encoding($s, 'JIS', $a);
print("JIS: ".base64_encode($s)."\n"); // JIS
// Regression test for bug #81676
echo "UTF-8: " . mb_convert_encoding('test', 'UTF-8', mb_list_encodings()), "\n";
// Using Detect Order
echo "== DETECT ORDER ==\n";
@ -117,6 +109,7 @@ JIS: GyRCRnxLXDhsJUYlLSU5JUgkRyQ5ISMbKEIwMTIzNBskQiM1IzYjNyM4IzkhIxsoQg==
EUC-JP: c6fccbdcb8eca5c6a5ada5b9a5c8a4c7a4b9a1a33031323334a3b5a3b6a3b7a3b8a3b9a1a3
SJIS: k/qWe4zqg2WDTINYg2eCxYK3gUIwMTIzNIJUglWCVoJXgliBQg==
JIS: GyRCRnxLXDhsJUYlLSU5JUgkRyQ5ISMbKEIwMTIzNBskQiM1IzYjNyM4IzkhIxsoQg==
UTF-8: test
== DETECT ORDER ==
EUC-JP: c6fccbdcb8eca5c6a5ada5b9a5c8a4c7a4b9a1a33031323334a3b5a3b6a3b7a3b8a3b9a1a3
SJIS: k/qWe4zqg2WDTINYg2eCxYK3gUIwMTIzNIJUglWCVoJXgliBQg==