From 0aff15c2c9c66b71c9ff5e0b1bb28708d5ead767 Mon Sep 17 00:00:00 2001 From: Petr Skoda Date: Tue, 15 May 2012 23:07:10 +0200 Subject: [PATCH] MDL-33007 add workaround for broken iconv //IGNORE This patch adds mbstring utf-8 cleanup fallback and admin warning if no utf-8 cleanup possible in user submitted data. --- admin/index.php | 9 +++------ admin/renderer.php | 17 ++++++++++++++++- install.php | 5 ----- lang/en/admin.php | 1 + lib/moodlelib.php | 28 ++++++++++++++++++++++++++-- lib/tests/moodlelib_test.php | 3 ++- 6 files changed, 48 insertions(+), 15 deletions(-) diff --git a/admin/index.php b/admin/index.php index bf3eea52aa6..c868fce2197 100644 --- a/admin/index.php +++ b/admin/index.php @@ -44,11 +44,6 @@ if (!function_exists('iconv')) { echo 'Moodle requires the iconv PHP extension. Please install or enable the iconv extension.'; die(); } -if (iconv('UTF-8', 'UTF-8//IGNORE', 'abc') !== 'abc') { - // known to be broken in mid-2011 MAMP installations - echo 'Broken iconv PHP extension detected, installation/upgrade can not continue.'; - die(); -} define('NO_OUTPUT_BUFFERING', true); @@ -434,6 +429,8 @@ $availableupdates = $updateschecker->get_update_info('core', array('minmaturity' => $CFG->updateminmaturity, 'notifybuilds' => $CFG->updatenotifybuilds)); $availableupdatesfetch = $updateschecker->get_last_timefetched(); +$buggyiconvnomb = (!function_exists('mb_convert_encoding') and @iconv('UTF-8', 'UTF-8//IGNORE', '100'.chr(130).'€') !== '100€'); + admin_externalpage_setup('adminnotifications'); if ($fetchupdates) { @@ -444,4 +441,4 @@ if ($fetchupdates) { $output = $PAGE->get_renderer('core', 'admin'); echo $output->admin_notifications_page($maturity, $insecuredataroot, $errorsdisplayed, - $cronoverdue, $dbproblems, $maintenancemode, $availableupdates, $availableupdatesfetch); + $cronoverdue, $dbproblems, $maintenancemode, $availableupdates, $availableupdatesfetch, $buggyiconvnomb); diff --git a/admin/renderer.php b/admin/renderer.php index 9b689a6903e..b1286f9f944 100644 --- a/admin/renderer.php +++ b/admin/renderer.php @@ -242,13 +242,14 @@ class core_admin_renderer extends plugin_renderer_base { * @param bool $cronoverdue warn cron not running * @param bool $dbproblems warn db has problems * @param bool $maintenancemode warn in maintenance mode + * @param bool $buggyiconvnomb warn iconv problems * @param array|null $availableupdates array of available_update_info objects or null * @param int|null $availableupdatesfetch timestamp of the most recent updates fetch or null (unknown) * * @return string HTML to output. */ public function admin_notifications_page($maturity, $insecuredataroot, $errorsdisplayed, - $cronoverdue, $dbproblems, $maintenancemode, $availableupdates, $availableupdatesfetch) { + $cronoverdue, $dbproblems, $maintenancemode, $availableupdates, $availableupdatesfetch, $buggyiconvnomb) { global $CFG; $output = ''; @@ -257,6 +258,7 @@ class core_admin_renderer extends plugin_renderer_base { $output .= empty($CFG->disableupdatenotifications) ? $this->available_updates($availableupdates, $availableupdatesfetch) : ''; $output .= $this->insecure_dataroot_warning($insecuredataroot); $output .= $this->display_errors_warning($errorsdisplayed); + $output .= $this->buggy_iconv_warning($buggyiconvnomb); $output .= $this->cron_overdue_warning($cronoverdue); $output .= $this->db_problems($dbproblems); $output .= $this->maintenance_mode_warning($maintenancemode); @@ -381,6 +383,19 @@ class core_admin_renderer extends plugin_renderer_base { return $this->warning(get_string('displayerrorswarning', 'admin')); } + /** + * Render an appropriate message if iconv is buggy and mbstring missing. + * @param bool $buggyiconvnomb + * @return string HTML to output. + */ + protected function buggy_iconv_warning($buggyiconvnomb) { + if (!$buggyiconvnomb) { + return ''; + } + + return $this->warning(get_string('warningiconvbuggy', 'admin')); + } + /** * Render an appropriate message if cron has not been run recently. * @param bool $cronoverdue diff --git a/install.php b/install.php index 341a1472d27..e50e3bc8ccb 100644 --- a/install.php +++ b/install.php @@ -73,11 +73,6 @@ if (!function_exists('iconv')) { echo 'Moodle requires the iconv PHP extension. Please install or enable the iconv extension.'; die(); } -if (iconv('UTF-8', 'UTF-8//IGNORE', 'abc') !== 'abc') { - // known to be broken in mid-2011 MAMP installations - echo 'Broken iconv PHP extension detected, installation can not continue.'; - die; -} if (PHP_INT_SIZE > 4) { // most probably 64bit PHP - we need a lot more memory diff --git a/lang/en/admin.php b/lang/en/admin.php index 6dbb85977b9..847dd53d9e8 100644 --- a/lang/en/admin.php +++ b/lang/en/admin.php @@ -1031,6 +1031,7 @@ $string['usetags'] = 'Enable tags functionality'; $string['validateerror'] = 'This value was not valid:'; $string['verifychangedemail'] = 'Restrict domains when changing email'; $string['warningcurrentsetting'] = 'Invalid current value: {$a}'; +$string['warningiconvbuggy'] = 'Your version of the iconv library does not support the //IGNORE modifier. You should install the mbstring extension which can be used instead for cleaning strings containing invalid UTF-8 characters.'; $string['webproxy'] = 'Web proxy'; $string['webproxyinfo'] = 'Fill in following options if your Moodle server can not access internet directly. Internet access is required for download of environment data, language packs, RSS feeds, timezones, etc.
PHP cURL extension is highly recommended.'; $string['xmlrpcrecommended'] = 'The xmlrpc extension is needed for hub communication, and useful for web services and Moodle networking'; diff --git a/lib/moodlelib.php b/lib/moodlelib.php index 23ee2f5630c..9fdc96dd7dc 100644 --- a/lib/moodlelib.php +++ b/lib/moodlelib.php @@ -1127,15 +1127,39 @@ function fix_utf8($value) { // shortcut return $value; } - // lower error reporting because glibc throws bogus notices + + // Lower error reporting because glibc throws bogus notices. $olderror = error_reporting(); if ($olderror & E_NOTICE) { error_reporting($olderror ^ E_NOTICE); } - $result = iconv('UTF-8', 'UTF-8//IGNORE', $value); + + // Note: this duplicates min_fix_utf8() intentionally. + static $buggyiconv = null; + if ($buggyiconv === null) { + $buggyiconv = (!function_exists('iconv') or iconv('UTF-8', 'UTF-8//IGNORE', '100'.chr(130).'€') !== '100€'); + } + + if ($buggyiconv) { + if (function_exists('mb_convert_encoding')) { + $subst = mb_substitute_character(); + mb_substitute_character(''); + $result = mb_convert_encoding($value, 'utf-8', 'utf-8'); + mb_substitute_character($subst); + + } else { + // Warn admins on admin/index.php page. + $result = $value; + } + + } else { + $result = iconv('UTF-8', 'UTF-8//IGNORE', $value); + } + if ($olderror & E_NOTICE) { error_reporting($olderror); } + return $result; } else if (is_array($value)) { diff --git a/lib/tests/moodlelib_test.php b/lib/tests/moodlelib_test.php index 618991973b2..5ce3a2cedba 100644 --- a/lib/tests/moodlelib_test.php +++ b/lib/tests/moodlelib_test.php @@ -301,6 +301,7 @@ class moodlelib_testcase extends advanced_testcase { $this->assertSame(1.1, fix_utf8(1.1)); $this->assertSame(true, fix_utf8(true)); $this->assertSame('', fix_utf8('')); + $this->assertSame('abc', fix_utf8('abc')); $array = array('do', 're', 'mi'); $this->assertSame($array, fix_utf8($array)); $object = new stdClass(); @@ -312,7 +313,7 @@ class moodlelib_testcase extends advanced_testcase { $this->assertSame("žlutý koníček přeskočil potůček \n\t\r\0", fix_utf8("žlutý koníček přeskočil potůček \n\t\r\0")); // invalid utf8 string - $this->assertSame('aaabbb', fix_utf8('aaa'.chr(130).'bbb')); + $this->assertSame('aš', fix_utf8('a'.chr(130).'š'), 'This fails with buggy iconv() when mbstring extenstion is not available as fallback.'); } function test_optional_param() {