Fix usages of json_encode which did not handle invalid UTF8 gracefully, fixes #1392

src/Monolog/Formatter/FluentdFormatter.php

@@ -11,6 +11,8 @@
 
 namespace Monolog\Formatter;
 
+use Monolog\Utils;
+
 /**
  * Class FluentdFormatter
  *
@@ -71,7 +73,7 @@ class FluentdFormatter implements FormatterInterface
             $message['level_name'] = $record['level_name'];
         }
 
-        return json_encode(array($tag, $record['datetime']->getTimestamp(), $message));
+        return Utils::jsonEncode(array($tag, $record['datetime']->getTimestamp(), $message));
     }
 
     public function formatBatch(array $records)

src/Monolog/Formatter/HtmlFormatter.php

@@ -11,6 +11,7 @@
 namespace Monolog\Formatter;
 
 use Monolog\Logger;
+use Monolog\Utils;
 
 /**
  * Formats incoming records into an HTML table
@@ -133,9 +134,9 @@ class HtmlFormatter extends NormalizerFormatter
 
         $data = $this->normalize($data);
         if (version_compare(PHP_VERSION, '5.4.0', '>=')) {
-            return json_encode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
+            return Utils::jsonEncode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE, true);
         }
 
-        return str_replace('\\/', '/', json_encode($data));
+        return str_replace('\\/', '/', Utils::jsonEncode($data, null, true));
     }
 }

src/Monolog/Formatter/LineFormatter.php

@@ -163,7 +163,7 @@ class LineFormatter extends NormalizerFormatter
             return $this->toJson($data, true);
         }
 
-        return str_replace('\\/', '/', @json_encode($data));
+        return str_replace('\\/', '/', $this->toJson($data, true));
     }
 
     protected function replaceNewlines($str)

src/Monolog/Formatter/NormalizerFormatter.php

@@ -171,127 +171,6 @@ class NormalizerFormatter implements FormatterInterface
      */
     protected function toJson($data, $ignoreErrors = false)
     {
-        // suppress json_encode errors since it's twitchy with some inputs
+        return Utils::jsonEncode($data, null, $ignoreErrors);
-        if ($ignoreErrors) {
-            return @$this->jsonEncode($data);
-        }
-
-        $json = $this->jsonEncode($data);
-
-        if ($json === false) {
-            $json = $this->handleJsonError(json_last_error(), $data);
-        }
-
-        return $json;
-    }
-
-    /**
-     * @param  mixed  $data
-     * @return string JSON encoded data or null on failure
-     */
-    private function jsonEncode($data)
-    {
-        if (version_compare(PHP_VERSION, '5.4.0', '>=')) {
-            return json_encode($data, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
-        }
-
-        return json_encode($data);
-    }
-
-    /**
-     * Handle a json_encode failure.
-     *
-     * If the failure is due to invalid string encoding, try to clean the
-     * input and encode again. If the second encoding attempt fails, the
-     * inital error is not encoding related or the input can't be cleaned then
-     * raise a descriptive exception.
-     *
-     * @param  int               $code return code of json_last_error function
-     * @param  mixed             $data data that was meant to be encoded
-     * @throws \RuntimeException if failure can't be corrected
-     * @return string            JSON encoded data after error correction
-     */
-    private function handleJsonError($code, $data)
-    {
-        if ($code !== JSON_ERROR_UTF8) {
-            $this->throwEncodeError($code, $data);
-        }
-
-        if (is_string($data)) {
-            $this->detectAndCleanUtf8($data);
-        } elseif (is_array($data)) {
-            array_walk_recursive($data, array($this, 'detectAndCleanUtf8'));
-        } else {
-            $this->throwEncodeError($code, $data);
-        }
-
-        $json = $this->jsonEncode($data);
-
-        if ($json === false) {
-            $this->throwEncodeError(json_last_error(), $data);
-        }
-
-        return $json;
-    }
-
-    /**
-     * Throws an exception according to a given code with a customized message
-     *
-     * @param  int               $code return code of json_last_error function
-     * @param  mixed             $data data that was meant to be encoded
-     * @throws \RuntimeException
-     */
-    private function throwEncodeError($code, $data)
-    {
-        switch ($code) {
-            case JSON_ERROR_DEPTH:
-                $msg = 'Maximum stack depth exceeded';
-                break;
-            case JSON_ERROR_STATE_MISMATCH:
-                $msg = 'Underflow or the modes mismatch';
-                break;
-            case JSON_ERROR_CTRL_CHAR:
-                $msg = 'Unexpected control character found';
-                break;
-            case JSON_ERROR_UTF8:
-                $msg = 'Malformed UTF-8 characters, possibly incorrectly encoded';
-                break;
-            default:
-                $msg = 'Unknown error';
-        }
-
-        throw new \RuntimeException('JSON encoding failed: '.$msg.'. Encoding: '.var_export($data, true));
-    }
-
-    /**
-     * Detect invalid UTF-8 string characters and convert to valid UTF-8.
-     *
-     * Valid UTF-8 input will be left unmodified, but strings containing
-     * invalid UTF-8 codepoints will be reencoded as UTF-8 with an assumed
-     * original encoding of ISO-8859-15. This conversion may result in
-     * incorrect output if the actual encoding was not ISO-8859-15, but it
-     * will be clean UTF-8 output and will not rely on expensive and fragile
-     * detection algorithms.
-     *
-     * Function converts the input in place in the passed variable so that it
-     * can be used as a callback for array_walk_recursive.
-     *
-     * @param mixed &$data Input to check and convert if needed
-     * @private
-     */
-    public function detectAndCleanUtf8(&$data)
-    {
-        if (is_string($data) && !preg_match('//u', $data)) {
-            $data = preg_replace_callback(
-                '/[\x80-\xFF]+/',
-                function ($m) { return utf8_encode($m[0]); },
-                $data
-            );
-            $data = str_replace(
-                array('¤', '¦', '¨', '´', '¸', '¼', '½', '¾'),
-                array('€', 'Š', 'š', 'Ž', 'ž', 'Œ', 'œ', 'Ÿ'),
-                $data
-            );
-        }
     }
 }

src/Monolog/Handler/ChromePHPHandler.php

@@ -13,6 +13,7 @@ namespace Monolog\Handler;
 
 use Monolog\Formatter\ChromePHPFormatter;
 use Monolog\Logger;
+use Monolog\Utils;
 
 /**
  * Handler sending logs to the ChromePHP extension (http://www.chromephp.com/)
@@ -134,7 +135,7 @@ class ChromePHPHandler extends AbstractProcessingHandler
             self::$json['request_uri'] = isset($_SERVER['REQUEST_URI']) ? $_SERVER['REQUEST_URI'] : '';
         }
 
-        $json = @json_encode(self::$json);
+        $json = Utils::jsonEncode(self::$json, null, true);
         $data = base64_encode(utf8_encode($json));
         if (strlen($data) > 3 * 1024) {
             self::$overflowed = true;
@@ -149,7 +150,7 @@ class ChromePHPHandler extends AbstractProcessingHandler
                 'extra' => array(),
             );
             self::$json['rows'][count(self::$json['rows']) - 1] = $this->getFormatter()->format($record);
-            $json = @json_encode(self::$json);
+            $json = Utils::jsonEncode(self::$json, null, true);
             $data = base64_encode(utf8_encode($json));
         }
 

src/Monolog/Handler/CubeHandler.php

@@ -12,6 +12,7 @@
 namespace Monolog\Handler;
 
 use Monolog\Logger;
+use Monolog\Utils;
 
 /**
  * Logs to Cube.
@@ -119,9 +120,9 @@ class CubeHandler extends AbstractProcessingHandler
         $data['data']['level'] = $record['level'];
 
         if ($this->scheme === 'http') {
-            $this->writeHttp(json_encode($data));
+            $this->writeHttp(Utils::jsonEncode($data));
         } else {
-            $this->writeUdp(json_encode($data));
+            $this->writeUdp(Utils::jsonEncode($data));
         }
     }
 

src/Monolog/Handler/FlowdockHandler.php

@@ -12,6 +12,7 @@
 namespace Monolog\Handler;
 
 use Monolog\Logger;
+use Monolog\Utils;
 use Monolog\Formatter\FlowdockFormatter;
 use Monolog\Formatter\FormatterInterface;
 
@@ -105,7 +106,7 @@ class FlowdockHandler extends SocketHandler
      */
     private function buildContent($record)
     {
-        return json_encode($record['formatted']['flowdock']);
+        return Utils::jsonEncode($record['formatted']['flowdock']);
     }
 
     /**

src/Monolog/Handler/IFTTTHandler.php

@@ -12,6 +12,7 @@
 namespace Monolog\Handler;
 
 use Monolog\Logger;
+use Monolog\Utils;
 
 /**
  * IFTTTHandler uses cURL to trigger IFTTT Maker actions
@@ -53,7 +54,7 @@ class IFTTTHandler extends AbstractProcessingHandler
             "value2" => $record["level_name"],
             "value3" => $record["message"],
         );
-        $postString = json_encode($postData);
+        $postString = Utils::jsonEncode($postData);
 
         $ch = curl_init();
         curl_setopt($ch, CURLOPT_URL, "https://maker.ifttt.com/trigger/" . $this->eventName . "/with/key/" . $this->secretKey);

src/Monolog/Handler/NewRelicHandler.php

@@ -12,6 +12,7 @@
 namespace Monolog\Handler;
 
 use Monolog\Logger;
+use Monolog\Utils;
 use Monolog\Formatter\NormalizerFormatter;
 
 /**
@@ -190,7 +191,7 @@ class NewRelicHandler extends AbstractProcessingHandler
         if (null === $value || is_scalar($value)) {
             newrelic_add_custom_parameter($key, $value);
         } else {
-            newrelic_add_custom_parameter($key, @json_encode($value));
+            newrelic_add_custom_parameter($key, Utils::jsonEncode($value, null, true));
         }
     }
 

src/Monolog/Handler/PHPConsoleHandler.php

@@ -14,6 +14,7 @@ namespace Monolog\Handler;
 use Exception;
 use Monolog\Formatter\LineFormatter;
 use Monolog\Logger;
+use Monolog\Utils;
 use PhpConsole\Connector;
 use PhpConsole\Handler;
 use PhpConsole\Helper;
@@ -188,7 +189,7 @@ class PHPConsoleHandler extends AbstractProcessingHandler
         $tags = $this->getRecordTags($record);
         $message = $record['message'];
         if ($record['context']) {
-            $message .= ' ' . json_encode($this->connector->getDumper()->dump(array_filter($record['context'])));
+            $message .= ' ' . Utils::jsonEncode($this->connector->getDumper()->dump(array_filter($record['context'])), null, true);
         }
         $this->connector->getDebugDispatcher()->dispatchDebug($message, $tags, $this->options['classesPartialsTraceIgnore']);
     }

src/Monolog/Handler/Slack/SlackRecord.php

@@ -12,6 +12,7 @@
 namespace Monolog\Handler\Slack;
 
 use Monolog\Logger;
+use Monolog\Utils;
 use Monolog\Formatter\NormalizerFormatter;
 use Monolog\Formatter\FormatterInterface;
 
@@ -212,8 +213,8 @@ class SlackRecord
         $hasNonNumericKeys = !count(array_filter(array_keys($normalized), 'is_numeric'));
 
         return $hasSecondDimension || $hasNonNumericKeys
-            ? json_encode($normalized, $prettyPrintFlag)
+            ? Utils::jsonEncode($normalized, $prettyPrintFlag)
-            : json_encode($normalized);
+            : Utils::jsonEncode($normalized);
     }
 
     /**

src/Monolog/Handler/SlackHandler.php

@@ -13,6 +13,7 @@ namespace Monolog\Handler;
 
 use Monolog\Formatter\FormatterInterface;
 use Monolog\Logger;
+use Monolog\Utils;
 use Monolog\Handler\Slack\SlackRecord;
 
 /**
@@ -118,7 +119,7 @@ class SlackHandler extends SocketHandler
         $dataArray['token'] = $this->token;
 
         if (!empty($dataArray['attachments'])) {
-            $dataArray['attachments'] = json_encode($dataArray['attachments']);
+            $dataArray['attachments'] = Utils::jsonEncode($dataArray['attachments']);
         }
 
         return $dataArray;

src/Monolog/Handler/SlackWebhookHandler.php

@@ -13,6 +13,7 @@ namespace Monolog\Handler;
 
 use Monolog\Formatter\FormatterInterface;
 use Monolog\Logger;
+use Monolog\Utils;
 use Monolog\Handler\Slack\SlackRecord;
 
 /**
@@ -83,7 +84,7 @@ class SlackWebhookHandler extends AbstractProcessingHandler
     protected function write(array $record)
     {
         $postData = $this->slackRecord->getSlackData($record);
-        $postString = json_encode($postData);
+        $postString = Utils::jsonEncode($postData);
 
         $ch = curl_init();
         $options = array(

src/Monolog/Utils.php

@@ -22,4 +22,134 @@ class Utils
 
         return 'c' === $class[0] && 0 === strpos($class, "class@anonymous\0") ? get_parent_class($class).'@anonymous' : $class;
     }
+
+    /**
+     * Return the JSON representation of a value
+     *
+     * @param  mixed             $data
+     * @param  int               $encodeFlags flags to pass to json encode, defaults to JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE
+     * @param  bool              $ignoreErrors whether to ignore encoding errors or to throw on error, when ignored and the encoding fails, "null" is returned which is valid json for null
+     * @throws \RuntimeException if encoding fails and errors are not ignored
+     * @return string
+     */
+    public static function jsonEncode($data, $encodeFlags = null, $ignoreErrors = false)
+    {
+        if (null === $encodeFlags && version_compare(PHP_VERSION, '5.4.0', '>=')) {
+            $encodeFlags = JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE;
+        }
+
+        $json = json_encode($data, $encodeFlags);
+
+        if (false === $json) {
+            if ($ignoreErrors) {
+                return 'null';
+            }
+
+            $json = self::handleJsonError(json_last_error(), $data);
+        }
+
+        return $json;
+    }
+
+    /**
+     * Handle a json_encode failure.
+     *
+     * If the failure is due to invalid string encoding, try to clean the
+     * input and encode again. If the second encoding attempt fails, the
+     * inital error is not encoding related or the input can't be cleaned then
+     * raise a descriptive exception.
+     *
+     * @param  int               $code return code of json_last_error function
+     * @param  mixed             $data data that was meant to be encoded
+     * @param  int               $encodeFlags flags to pass to json encode, defaults to JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE
+     * @throws \RuntimeException if failure can't be corrected
+     * @return string            JSON encoded data after error correction
+     */
+    public static function handleJsonError($code, $data, $encodeFlags = null)
+    {
+        if ($code !== JSON_ERROR_UTF8) {
+            self::throwEncodeError($code, $data);
+        }
+
+        if (is_string($data)) {
+            self::detectAndCleanUtf8($data);
+        } elseif (is_array($data)) {
+            array_walk_recursive($data, array('Monolog\Utils', 'detectAndCleanUtf8'));
+        } else {
+            self::throwEncodeError($code, $data);
+        }
+
+        if (null === $encodeFlags && version_compare(PHP_VERSION, '5.4.0', '>=')) {
+            $encodeFlags = JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE;
+        }
+
+        $json = json_encode($data, $encodeFlags);
+
+        if ($json === false) {
+            self::throwEncodeError(json_last_error(), $data);
+        }
+
+        return $json;
+    }
+
+    /**
+     * Throws an exception according to a given code with a customized message
+     *
+     * @param  int               $code return code of json_last_error function
+     * @param  mixed             $data data that was meant to be encoded
+     * @throws \RuntimeException
+     */
+    private static function throwEncodeError($code, $data)
+    {
+        switch ($code) {
+            case JSON_ERROR_DEPTH:
+                $msg = 'Maximum stack depth exceeded';
+                break;
+            case JSON_ERROR_STATE_MISMATCH:
+                $msg = 'Underflow or the modes mismatch';
+                break;
+            case JSON_ERROR_CTRL_CHAR:
+                $msg = 'Unexpected control character found';
+                break;
+            case JSON_ERROR_UTF8:
+                $msg = 'Malformed UTF-8 characters, possibly incorrectly encoded';
+                break;
+            default:
+                $msg = 'Unknown error';
+        }
+
+        throw new \RuntimeException('JSON encoding failed: '.$msg.'. Encoding: '.var_export($data, true));
+    }
+
+    /**
+     * Detect invalid UTF-8 string characters and convert to valid UTF-8.
+     *
+     * Valid UTF-8 input will be left unmodified, but strings containing
+     * invalid UTF-8 codepoints will be reencoded as UTF-8 with an assumed
+     * original encoding of ISO-8859-15. This conversion may result in
+     * incorrect output if the actual encoding was not ISO-8859-15, but it
+     * will be clean UTF-8 output and will not rely on expensive and fragile
+     * detection algorithms.
+     *
+     * Function converts the input in place in the passed variable so that it
+     * can be used as a callback for array_walk_recursive.
+     *
+     * @param mixed &$data Input to check and convert if needed
+     * @private
+     */
+    public static function detectAndCleanUtf8(&$data)
+    {
+        if (is_string($data) && !preg_match('//u', $data)) {
+            $data = preg_replace_callback(
+                '/[\x80-\xFF]+/',
+                function ($m) { return utf8_encode($m[0]); },
+                $data
+            );
+            $data = str_replace(
+                array('¤', '¦', '¨', '´', '¸', '¼', '½', '¾'),
+                array('€', 'Š', 'š', 'Ž', 'ž', 'Œ', 'œ', 'Ÿ'),
+                $data
+            );
+        }
+    }
 }

tests/Monolog/Formatter/NormalizerFormatterTest.php

@@ -190,7 +190,7 @@ class NormalizerFormatterTest extends \PHPUnit_Framework_TestCase
 
         restore_error_handler();
 
-        $this->assertEquals(@json_encode(array($foo, $bar)), $res);
+        $this->assertEquals('null', $res);
     }
 
     public function testCanNormalizeReferences()
@@ -223,7 +223,7 @@ class NormalizerFormatterTest extends \PHPUnit_Framework_TestCase
 
         restore_error_handler();
 
-        $this->assertEquals(@json_encode(array($resource)), $res);
+        $this->assertEquals('null', $res);
     }
 
     public function testNormalizeHandleLargeArraysWithExactly1000Items()
@@ -305,63 +305,6 @@ class NormalizerFormatterTest extends \PHPUnit_Framework_TestCase
         }
     }
 
-    /**
-     * @param mixed $in     Input
-     * @param mixed $expect Expected output
-     * @covers Monolog\Formatter\NormalizerFormatter::detectAndCleanUtf8
-     * @dataProvider providesDetectAndCleanUtf8
-     */
-    public function testDetectAndCleanUtf8($in, $expect)
-    {
-        $formatter = new NormalizerFormatter();
-        $formatter->detectAndCleanUtf8($in);
-        $this->assertSame($expect, $in);
-    }
-
-    public function providesDetectAndCleanUtf8()
-    {
-        $obj = new \stdClass;
-
-        return array(
-            'null' => array(null, null),
-            'int' => array(123, 123),
-            'float' => array(123.45, 123.45),
-            'bool false' => array(false, false),
-            'bool true' => array(true, true),
-            'ascii string' => array('abcdef', 'abcdef'),
-            'latin9 string' => array("\xB1\x31\xA4\xA6\xA8\xB4\xB8\xBC\xBD\xBE\xFF", '±1€ŠšŽžŒœŸÿ'),
-            'unicode string' => array('¤¦¨´¸¼½¾€ŠšŽžŒœŸ', '¤¦¨´¸¼½¾€ŠšŽžŒœŸ'),
-            'empty array' => array(array(), array()),
-            'array' => array(array('abcdef'), array('abcdef')),
-            'object' => array($obj, $obj),
-        );
-    }
-
-    /**
-     * @param int    $code
-     * @param string $msg
-     * @dataProvider providesHandleJsonErrorFailure
-     */
-    public function testHandleJsonErrorFailure($code, $msg)
-    {
-        $formatter = new NormalizerFormatter();
-        $reflMethod = new \ReflectionMethod($formatter, 'handleJsonError');
-        $reflMethod->setAccessible(true);
-
-        $this->setExpectedException('RuntimeException', $msg);
-        $reflMethod->invoke($formatter, $code, 'faked');
-    }
-
-    public function providesHandleJsonErrorFailure()
-    {
-        return array(
-            'depth' => array(JSON_ERROR_DEPTH, 'Maximum stack depth exceeded'),
-            'state' => array(JSON_ERROR_STATE_MISMATCH, 'Underflow or the modes mismatch'),
-            'ctrl' => array(JSON_ERROR_CTRL_CHAR, 'Unexpected control character found'),
-            'default' => array(-1, 'Unknown error'),
-        );
-    }
-
     public function testExceptionTraceWithArgs()
     {
         if (defined('HHVM_VERSION')) {

tests/Monolog/UtilsTest.php

@@ -0,0 +1,67 @@
+<?php
+
+/*
+ * This file is part of the Monolog package.
+ *
+ * (c) Jordi Boggiano <j.boggiano@seld.be>
+ *
+ * For the full copyright and license information, please view the LICENSE
+ * file that was distributed with this source code.
+ */
+
+namespace Monolog;
+
+class UtilsTest extends \PHPUnit_Framework_TestCase
+{
+    /**
+     * @param int    $code
+     * @param string $msg
+     * @dataProvider providesHandleJsonErrorFailure
+     */
+    public function testHandleJsonErrorFailure($code, $msg)
+    {
+        $this->setExpectedException('RuntimeException', $msg);
+        Utils::handleJsonError($code, 'faked');
+    }
+
+    public function providesHandleJsonErrorFailure()
+    {
+        return array(
+            'depth' => array(JSON_ERROR_DEPTH, 'Maximum stack depth exceeded'),
+            'state' => array(JSON_ERROR_STATE_MISMATCH, 'Underflow or the modes mismatch'),
+            'ctrl' => array(JSON_ERROR_CTRL_CHAR, 'Unexpected control character found'),
+            'default' => array(-1, 'Unknown error'),
+        );
+    }
+
+    /**
+     * @param mixed $in     Input
+     * @param mixed $expect Expected output
+     * @covers Monolog\Formatter\NormalizerFormatter::detectAndCleanUtf8
+     * @dataProvider providesDetectAndCleanUtf8
+     */
+    public function testDetectAndCleanUtf8($in, $expect)
+    {
+        Utils::detectAndCleanUtf8($in);
+        $this->assertSame($expect, $in);
+    }
+
+    public function providesDetectAndCleanUtf8()
+    {
+        $obj = new \stdClass;
+
+        return array(
+            'null' => array(null, null),
+            'int' => array(123, 123),
+            'float' => array(123.45, 123.45),
+            'bool false' => array(false, false),
+            'bool true' => array(true, true),
+            'ascii string' => array('abcdef', 'abcdef'),
+            'latin9 string' => array("\xB1\x31\xA4\xA6\xA8\xB4\xB8\xBC\xBD\xBE\xFF", '±1€ŠšŽžŒœŸÿ'),
+            'unicode string' => array('¤¦¨´¸¼½¾€ŠšŽžŒœŸ', '¤¦¨´¸¼½¾€ŠšŽžŒœŸ'),
+            'empty array' => array(array(), array()),
+            'array' => array(array('abcdef'), array('abcdef')),
+            'object' => array($obj, $obj),
+        );
+    }
+}