MDL-33007 add workaround for broken iconv //IGNORE

This patch adds mbstring utf-8 cleanup fallback and admin warning if no utf-8 cleanup possible in user submitted data.

admin/index.php

@@ -44,11 +44,6 @@ if (!function_exists('iconv')) {
     echo 'Moodle requires the iconv PHP extension. Please install or enable the iconv extension.';
     die();
 }
-if (iconv('UTF-8', 'UTF-8//IGNORE', 'abc') !== 'abc') {
-    // known to be broken in mid-2011 MAMP installations
-    echo 'Broken iconv PHP extension detected, installation/upgrade can not continue.';
-    die();
-}
 
 define('NO_OUTPUT_BUFFERING', true);
 
@@ -434,6 +429,8 @@ $availableupdates = $updateschecker->get_update_info('core',
     array('minmaturity' => $CFG->updateminmaturity, 'notifybuilds' => $CFG->updatenotifybuilds));
 $availableupdatesfetch = $updateschecker->get_last_timefetched();
 
+$buggyiconvnomb = (!function_exists('mb_convert_encoding') and @iconv('UTF-8', 'UTF-8//IGNORE', '100'.chr(130).'€') !== '100€');
+
 admin_externalpage_setup('adminnotifications');
 
 if ($fetchupdates) {
@@ -444,4 +441,4 @@ if ($fetchupdates) {
 
 $output = $PAGE->get_renderer('core', 'admin');
 echo $output->admin_notifications_page($maturity, $insecuredataroot, $errorsdisplayed,
-        $cronoverdue, $dbproblems, $maintenancemode, $availableupdates, $availableupdatesfetch);
+        $cronoverdue, $dbproblems, $maintenancemode, $availableupdates, $availableupdatesfetch, $buggyiconvnomb);

admin/renderer.php

@@ -242,13 +242,14 @@ class core_admin_renderer extends plugin_renderer_base {
      * @param bool $cronoverdue warn cron not running
      * @param bool $dbproblems warn db has problems
      * @param bool $maintenancemode warn in maintenance mode
+     * @param bool $buggyiconvnomb warn iconv problems
      * @param array|null $availableupdates array of available_update_info objects or null
      * @param int|null $availableupdatesfetch timestamp of the most recent updates fetch or null (unknown)
      *
      * @return string HTML to output.
      */
     public function admin_notifications_page($maturity, $insecuredataroot, $errorsdisplayed,
-            $cronoverdue, $dbproblems, $maintenancemode, $availableupdates, $availableupdatesfetch) {
+            $cronoverdue, $dbproblems, $maintenancemode, $availableupdates, $availableupdatesfetch, $buggyiconvnomb) {
         global $CFG;
         $output = '';
 
@@ -257,6 +258,7 @@ class core_admin_renderer extends plugin_renderer_base {
         $output .= empty($CFG->disableupdatenotifications) ? $this->available_updates($availableupdates, $availableupdatesfetch) : '';
         $output .= $this->insecure_dataroot_warning($insecuredataroot);
         $output .= $this->display_errors_warning($errorsdisplayed);
+        $output .= $this->buggy_iconv_warning($buggyiconvnomb);
         $output .= $this->cron_overdue_warning($cronoverdue);
         $output .= $this->db_problems($dbproblems);
         $output .= $this->maintenance_mode_warning($maintenancemode);
@@ -381,6 +383,19 @@ class core_admin_renderer extends plugin_renderer_base {
         return $this->warning(get_string('displayerrorswarning', 'admin'));
     }
 
+    /**
+     * Render an appropriate message if iconv is buggy and mbstring missing.
+     * @param bool $buggyiconvnomb
+     * @return string HTML to output.
+     */
+    protected function buggy_iconv_warning($buggyiconvnomb) {
+        if (!$buggyiconvnomb) {
+            return '';
+        }
+
+        return $this->warning(get_string('warningiconvbuggy', 'admin'));
+    }
+
     /**
      * Render an appropriate message if cron has not been run recently.
      * @param bool $cronoverdue

install.php

@@ -73,11 +73,6 @@ if (!function_exists('iconv')) {
     echo 'Moodle requires the iconv PHP extension. Please install or enable the iconv extension.';
     die();
 }
-if (iconv('UTF-8', 'UTF-8//IGNORE', 'abc') !== 'abc') {
-    // known to be broken in mid-2011 MAMP installations
-    echo 'Broken iconv PHP extension detected, installation can not continue.';
-    die;
-}
 
 if (PHP_INT_SIZE > 4) {
     // most probably 64bit PHP - we need a lot more memory

lang/en/admin.php

@@ -1031,6 +1031,7 @@ $string['usetags'] = 'Enable tags functionality';
 $string['validateerror'] = 'This value was not valid:';
 $string['verifychangedemail'] = 'Restrict domains when changing email';
 $string['warningcurrentsetting'] = 'Invalid current value: {$a}';
+$string['warningiconvbuggy'] = 'Your version of the iconv library does not support the //IGNORE modifier. You should install the mbstring extension which can be used instead for cleaning strings containing invalid UTF-8 characters.';
 $string['webproxy'] = 'Web proxy';
 $string['webproxyinfo'] = 'Fill in following options if your Moodle server can not access internet directly. Internet access is required for download of environment data, language packs, RSS feeds, timezones, etc.<br /><em>PHP cURL extension is highly recommended.</em>';
 $string['xmlrpcrecommended'] = 'The xmlrpc extension is needed for hub communication, and useful for web services and Moodle networking';

lib/moodlelib.php

@@ -1127,15 +1127,39 @@ function fix_utf8($value) {
             // shortcut
             return $value;
         }
-        // lower error reporting because glibc throws bogus notices
+
+        // Lower error reporting because glibc throws bogus notices.
         $olderror = error_reporting();
         if ($olderror & E_NOTICE) {
             error_reporting($olderror ^ E_NOTICE);
         }
-        $result = iconv('UTF-8', 'UTF-8//IGNORE', $value);
+
+        // Note: this duplicates min_fix_utf8() intentionally.
+        static $buggyiconv = null;
+        if ($buggyiconv === null) {
+            $buggyiconv = (!function_exists('iconv') or iconv('UTF-8', 'UTF-8//IGNORE', '100'.chr(130).'€') !== '100€');
+        }
+
+        if ($buggyiconv) {
+            if (function_exists('mb_convert_encoding')) {
+                $subst = mb_substitute_character();
+                mb_substitute_character('');
+                $result = mb_convert_encoding($value, 'utf-8', 'utf-8');
+                mb_substitute_character($subst);
+
+            } else {
+                // Warn admins on admin/index.php page.
+                $result = $value;
+            }
+
+        } else {
+            $result = iconv('UTF-8', 'UTF-8//IGNORE', $value);
+        }
+
         if ($olderror & E_NOTICE) {
             error_reporting($olderror);
         }
+
         return $result;
 
     } else if (is_array($value)) {

lib/tests/moodlelib_test.php

@@ -301,6 +301,7 @@ class moodlelib_testcase extends advanced_testcase {
         $this->assertSame(1.1, fix_utf8(1.1));
         $this->assertSame(true, fix_utf8(true));
         $this->assertSame('', fix_utf8(''));
+        $this->assertSame('abc', fix_utf8('abc'));
         $array = array('do', 're', 'mi');
         $this->assertSame($array, fix_utf8($array));
         $object = new stdClass();
@@ -312,7 +313,7 @@ class moodlelib_testcase extends advanced_testcase {
         $this->assertSame("žlutý koníček přeskočil potůček \n\t\r\0", fix_utf8("žlutý koníček přeskočil potůček \n\t\r\0"));
 
         // invalid utf8 string
-        $this->assertSame('aaabbb', fix_utf8('aaa'.chr(130).'bbb'));
+        $this->assertSame('aš', fix_utf8('a'.chr(130).'š'), 'This fails with buggy iconv() when mbstring extenstion is not available as fallback.');
     }
 
     function test_optional_param() {