1
0
mirror of https://github.com/Seldaek/monolog.git synced 2025-07-30 18:00:17 +02:00

Fix usages of json_encode which did not handle invalid UTF8 gracefully, fixes #1392

This commit is contained in:
Jordi Boggiano
2019-11-12 21:24:23 +01:00
parent e1af546a55
commit 12a76ad61e
16 changed files with 228 additions and 197 deletions

View File

@@ -11,6 +11,8 @@
namespace Monolog\Formatter;
use Monolog\Utils;
/**
* Class FluentdFormatter
*
@@ -71,7 +73,7 @@ class FluentdFormatter implements FormatterInterface
$message['level_name'] = $record['level_name'];
}
return json_encode(array($tag, $record['datetime']->getTimestamp(), $message));
return Utils::jsonEncode(array($tag, $record['datetime']->getTimestamp(), $message));
}
public function formatBatch(array $records)

View File

@@ -11,6 +11,7 @@
namespace Monolog\Formatter;
use Monolog\Logger;
use Monolog\Utils;
/**
* Formats incoming records into an HTML table
@@ -133,9 +134,9 @@ class HtmlFormatter extends NormalizerFormatter
$data = $this->normalize($data);
if (version_compare(PHP_VERSION, '5.4.0', '>=')) {
return json_encode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
return Utils::jsonEncode($data, JSON_PRETTY_PRINT | JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE, true);
}
return str_replace('\\/', '/', json_encode($data));
return str_replace('\\/', '/', Utils::jsonEncode($data, null, true));
}
}

View File

@@ -163,7 +163,7 @@ class LineFormatter extends NormalizerFormatter
return $this->toJson($data, true);
}
return str_replace('\\/', '/', @json_encode($data));
return str_replace('\\/', '/', $this->toJson($data, true));
}
protected function replaceNewlines($str)

View File

@@ -171,127 +171,6 @@ class NormalizerFormatter implements FormatterInterface
*/
protected function toJson($data, $ignoreErrors = false)
{
// suppress json_encode errors since it's twitchy with some inputs
if ($ignoreErrors) {
return @$this->jsonEncode($data);
}
$json = $this->jsonEncode($data);
if ($json === false) {
$json = $this->handleJsonError(json_last_error(), $data);
}
return $json;
}
/**
* @param mixed $data
* @return string JSON encoded data or null on failure
*/
private function jsonEncode($data)
{
if (version_compare(PHP_VERSION, '5.4.0', '>=')) {
return json_encode($data, JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE);
}
return json_encode($data);
}
/**
* Handle a json_encode failure.
*
* If the failure is due to invalid string encoding, try to clean the
* input and encode again. If the second encoding attempt fails, the
* inital error is not encoding related or the input can't be cleaned then
* raise a descriptive exception.
*
* @param int $code return code of json_last_error function
* @param mixed $data data that was meant to be encoded
* @throws \RuntimeException if failure can't be corrected
* @return string JSON encoded data after error correction
*/
private function handleJsonError($code, $data)
{
if ($code !== JSON_ERROR_UTF8) {
$this->throwEncodeError($code, $data);
}
if (is_string($data)) {
$this->detectAndCleanUtf8($data);
} elseif (is_array($data)) {
array_walk_recursive($data, array($this, 'detectAndCleanUtf8'));
} else {
$this->throwEncodeError($code, $data);
}
$json = $this->jsonEncode($data);
if ($json === false) {
$this->throwEncodeError(json_last_error(), $data);
}
return $json;
}
/**
* Throws an exception according to a given code with a customized message
*
* @param int $code return code of json_last_error function
* @param mixed $data data that was meant to be encoded
* @throws \RuntimeException
*/
private function throwEncodeError($code, $data)
{
switch ($code) {
case JSON_ERROR_DEPTH:
$msg = 'Maximum stack depth exceeded';
break;
case JSON_ERROR_STATE_MISMATCH:
$msg = 'Underflow or the modes mismatch';
break;
case JSON_ERROR_CTRL_CHAR:
$msg = 'Unexpected control character found';
break;
case JSON_ERROR_UTF8:
$msg = 'Malformed UTF-8 characters, possibly incorrectly encoded';
break;
default:
$msg = 'Unknown error';
}
throw new \RuntimeException('JSON encoding failed: '.$msg.'. Encoding: '.var_export($data, true));
}
/**
* Detect invalid UTF-8 string characters and convert to valid UTF-8.
*
* Valid UTF-8 input will be left unmodified, but strings containing
* invalid UTF-8 codepoints will be reencoded as UTF-8 with an assumed
* original encoding of ISO-8859-15. This conversion may result in
* incorrect output if the actual encoding was not ISO-8859-15, but it
* will be clean UTF-8 output and will not rely on expensive and fragile
* detection algorithms.
*
* Function converts the input in place in the passed variable so that it
* can be used as a callback for array_walk_recursive.
*
* @param mixed &$data Input to check and convert if needed
* @private
*/
public function detectAndCleanUtf8(&$data)
{
if (is_string($data) && !preg_match('//u', $data)) {
$data = preg_replace_callback(
'/[\x80-\xFF]+/',
function ($m) { return utf8_encode($m[0]); },
$data
);
$data = str_replace(
array('¤', '¦', '¨', '´', '¸', '¼', '½', '¾'),
array('€', 'Š', 'š', 'Ž', 'ž', 'Œ', 'œ', 'Ÿ'),
$data
);
}
return Utils::jsonEncode($data, null, $ignoreErrors);
}
}

View File

@@ -13,6 +13,7 @@ namespace Monolog\Handler;
use Monolog\Formatter\ChromePHPFormatter;
use Monolog\Logger;
use Monolog\Utils;
/**
* Handler sending logs to the ChromePHP extension (http://www.chromephp.com/)
@@ -134,7 +135,7 @@ class ChromePHPHandler extends AbstractProcessingHandler
self::$json['request_uri'] = isset($_SERVER['REQUEST_URI']) ? $_SERVER['REQUEST_URI'] : '';
}
$json = @json_encode(self::$json);
$json = Utils::jsonEncode(self::$json, null, true);
$data = base64_encode(utf8_encode($json));
if (strlen($data) > 3 * 1024) {
self::$overflowed = true;
@@ -149,7 +150,7 @@ class ChromePHPHandler extends AbstractProcessingHandler
'extra' => array(),
);
self::$json['rows'][count(self::$json['rows']) - 1] = $this->getFormatter()->format($record);
$json = @json_encode(self::$json);
$json = Utils::jsonEncode(self::$json, null, true);
$data = base64_encode(utf8_encode($json));
}

View File

@@ -12,6 +12,7 @@
namespace Monolog\Handler;
use Monolog\Logger;
use Monolog\Utils;
/**
* Logs to Cube.
@@ -119,9 +120,9 @@ class CubeHandler extends AbstractProcessingHandler
$data['data']['level'] = $record['level'];
if ($this->scheme === 'http') {
$this->writeHttp(json_encode($data));
$this->writeHttp(Utils::jsonEncode($data));
} else {
$this->writeUdp(json_encode($data));
$this->writeUdp(Utils::jsonEncode($data));
}
}

View File

@@ -12,6 +12,7 @@
namespace Monolog\Handler;
use Monolog\Logger;
use Monolog\Utils;
use Monolog\Formatter\FlowdockFormatter;
use Monolog\Formatter\FormatterInterface;
@@ -105,7 +106,7 @@ class FlowdockHandler extends SocketHandler
*/
private function buildContent($record)
{
return json_encode($record['formatted']['flowdock']);
return Utils::jsonEncode($record['formatted']['flowdock']);
}
/**

View File

@@ -12,6 +12,7 @@
namespace Monolog\Handler;
use Monolog\Logger;
use Monolog\Utils;
/**
* IFTTTHandler uses cURL to trigger IFTTT Maker actions
@@ -53,7 +54,7 @@ class IFTTTHandler extends AbstractProcessingHandler
"value2" => $record["level_name"],
"value3" => $record["message"],
);
$postString = json_encode($postData);
$postString = Utils::jsonEncode($postData);
$ch = curl_init();
curl_setopt($ch, CURLOPT_URL, "https://maker.ifttt.com/trigger/" . $this->eventName . "/with/key/" . $this->secretKey);

View File

@@ -12,6 +12,7 @@
namespace Monolog\Handler;
use Monolog\Logger;
use Monolog\Utils;
use Monolog\Formatter\NormalizerFormatter;
/**
@@ -190,7 +191,7 @@ class NewRelicHandler extends AbstractProcessingHandler
if (null === $value || is_scalar($value)) {
newrelic_add_custom_parameter($key, $value);
} else {
newrelic_add_custom_parameter($key, @json_encode($value));
newrelic_add_custom_parameter($key, Utils::jsonEncode($value, null, true));
}
}

View File

@@ -14,6 +14,7 @@ namespace Monolog\Handler;
use Exception;
use Monolog\Formatter\LineFormatter;
use Monolog\Logger;
use Monolog\Utils;
use PhpConsole\Connector;
use PhpConsole\Handler;
use PhpConsole\Helper;
@@ -188,7 +189,7 @@ class PHPConsoleHandler extends AbstractProcessingHandler
$tags = $this->getRecordTags($record);
$message = $record['message'];
if ($record['context']) {
$message .= ' ' . json_encode($this->connector->getDumper()->dump(array_filter($record['context'])));
$message .= ' ' . Utils::jsonEncode($this->connector->getDumper()->dump(array_filter($record['context'])), null, true);
}
$this->connector->getDebugDispatcher()->dispatchDebug($message, $tags, $this->options['classesPartialsTraceIgnore']);
}

View File

@@ -12,6 +12,7 @@
namespace Monolog\Handler\Slack;
use Monolog\Logger;
use Monolog\Utils;
use Monolog\Formatter\NormalizerFormatter;
use Monolog\Formatter\FormatterInterface;
@@ -212,8 +213,8 @@ class SlackRecord
$hasNonNumericKeys = !count(array_filter(array_keys($normalized), 'is_numeric'));
return $hasSecondDimension || $hasNonNumericKeys
? json_encode($normalized, $prettyPrintFlag)
: json_encode($normalized);
? Utils::jsonEncode($normalized, $prettyPrintFlag)
: Utils::jsonEncode($normalized);
}
/**

View File

@@ -13,6 +13,7 @@ namespace Monolog\Handler;
use Monolog\Formatter\FormatterInterface;
use Monolog\Logger;
use Monolog\Utils;
use Monolog\Handler\Slack\SlackRecord;
/**
@@ -118,7 +119,7 @@ class SlackHandler extends SocketHandler
$dataArray['token'] = $this->token;
if (!empty($dataArray['attachments'])) {
$dataArray['attachments'] = json_encode($dataArray['attachments']);
$dataArray['attachments'] = Utils::jsonEncode($dataArray['attachments']);
}
return $dataArray;

View File

@@ -13,6 +13,7 @@ namespace Monolog\Handler;
use Monolog\Formatter\FormatterInterface;
use Monolog\Logger;
use Monolog\Utils;
use Monolog\Handler\Slack\SlackRecord;
/**
@@ -83,7 +84,7 @@ class SlackWebhookHandler extends AbstractProcessingHandler
protected function write(array $record)
{
$postData = $this->slackRecord->getSlackData($record);
$postString = json_encode($postData);
$postString = Utils::jsonEncode($postData);
$ch = curl_init();
$options = array(

View File

@@ -22,4 +22,134 @@ class Utils
return 'c' === $class[0] && 0 === strpos($class, "class@anonymous\0") ? get_parent_class($class).'@anonymous' : $class;
}
/**
* Return the JSON representation of a value
*
* @param mixed $data
* @param int $encodeFlags flags to pass to json encode, defaults to JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE
* @param bool $ignoreErrors whether to ignore encoding errors or to throw on error, when ignored and the encoding fails, "null" is returned which is valid json for null
* @throws \RuntimeException if encoding fails and errors are not ignored
* @return string
*/
public static function jsonEncode($data, $encodeFlags = null, $ignoreErrors = false)
{
if (null === $encodeFlags && version_compare(PHP_VERSION, '5.4.0', '>=')) {
$encodeFlags = JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE;
}
$json = json_encode($data, $encodeFlags);
if (false === $json) {
if ($ignoreErrors) {
return 'null';
}
$json = self::handleJsonError(json_last_error(), $data);
}
return $json;
}
/**
* Handle a json_encode failure.
*
* If the failure is due to invalid string encoding, try to clean the
* input and encode again. If the second encoding attempt fails, the
* inital error is not encoding related or the input can't be cleaned then
* raise a descriptive exception.
*
* @param int $code return code of json_last_error function
* @param mixed $data data that was meant to be encoded
* @param int $encodeFlags flags to pass to json encode, defaults to JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE
* @throws \RuntimeException if failure can't be corrected
* @return string JSON encoded data after error correction
*/
public static function handleJsonError($code, $data, $encodeFlags = null)
{
if ($code !== JSON_ERROR_UTF8) {
self::throwEncodeError($code, $data);
}
if (is_string($data)) {
self::detectAndCleanUtf8($data);
} elseif (is_array($data)) {
array_walk_recursive($data, array('Monolog\Utils', 'detectAndCleanUtf8'));
} else {
self::throwEncodeError($code, $data);
}
if (null === $encodeFlags && version_compare(PHP_VERSION, '5.4.0', '>=')) {
$encodeFlags = JSON_UNESCAPED_SLASHES | JSON_UNESCAPED_UNICODE;
}
$json = json_encode($data, $encodeFlags);
if ($json === false) {
self::throwEncodeError(json_last_error(), $data);
}
return $json;
}
/**
* Throws an exception according to a given code with a customized message
*
* @param int $code return code of json_last_error function
* @param mixed $data data that was meant to be encoded
* @throws \RuntimeException
*/
private static function throwEncodeError($code, $data)
{
switch ($code) {
case JSON_ERROR_DEPTH:
$msg = 'Maximum stack depth exceeded';
break;
case JSON_ERROR_STATE_MISMATCH:
$msg = 'Underflow or the modes mismatch';
break;
case JSON_ERROR_CTRL_CHAR:
$msg = 'Unexpected control character found';
break;
case JSON_ERROR_UTF8:
$msg = 'Malformed UTF-8 characters, possibly incorrectly encoded';
break;
default:
$msg = 'Unknown error';
}
throw new \RuntimeException('JSON encoding failed: '.$msg.'. Encoding: '.var_export($data, true));
}
/**
* Detect invalid UTF-8 string characters and convert to valid UTF-8.
*
* Valid UTF-8 input will be left unmodified, but strings containing
* invalid UTF-8 codepoints will be reencoded as UTF-8 with an assumed
* original encoding of ISO-8859-15. This conversion may result in
* incorrect output if the actual encoding was not ISO-8859-15, but it
* will be clean UTF-8 output and will not rely on expensive and fragile
* detection algorithms.
*
* Function converts the input in place in the passed variable so that it
* can be used as a callback for array_walk_recursive.
*
* @param mixed &$data Input to check and convert if needed
* @private
*/
public static function detectAndCleanUtf8(&$data)
{
if (is_string($data) && !preg_match('//u', $data)) {
$data = preg_replace_callback(
'/[\x80-\xFF]+/',
function ($m) { return utf8_encode($m[0]); },
$data
);
$data = str_replace(
array('¤', '¦', '¨', '´', '¸', '¼', '½', '¾'),
array('€', 'Š', 'š', 'Ž', 'ž', 'Œ', 'œ', 'Ÿ'),
$data
);
}
}
}

View File

@@ -190,7 +190,7 @@ class NormalizerFormatterTest extends \PHPUnit_Framework_TestCase
restore_error_handler();
$this->assertEquals(@json_encode(array($foo, $bar)), $res);
$this->assertEquals('null', $res);
}
public function testCanNormalizeReferences()
@@ -223,7 +223,7 @@ class NormalizerFormatterTest extends \PHPUnit_Framework_TestCase
restore_error_handler();
$this->assertEquals(@json_encode(array($resource)), $res);
$this->assertEquals('null', $res);
}
public function testNormalizeHandleLargeArraysWithExactly1000Items()
@@ -305,63 +305,6 @@ class NormalizerFormatterTest extends \PHPUnit_Framework_TestCase
}
}
/**
* @param mixed $in Input
* @param mixed $expect Expected output
* @covers Monolog\Formatter\NormalizerFormatter::detectAndCleanUtf8
* @dataProvider providesDetectAndCleanUtf8
*/
public function testDetectAndCleanUtf8($in, $expect)
{
$formatter = new NormalizerFormatter();
$formatter->detectAndCleanUtf8($in);
$this->assertSame($expect, $in);
}
public function providesDetectAndCleanUtf8()
{
$obj = new \stdClass;
return array(
'null' => array(null, null),
'int' => array(123, 123),
'float' => array(123.45, 123.45),
'bool false' => array(false, false),
'bool true' => array(true, true),
'ascii string' => array('abcdef', 'abcdef'),
'latin9 string' => array("\xB1\x31\xA4\xA6\xA8\xB4\xB8\xBC\xBD\xBE\xFF", '±1€ŠšŽžŒœŸÿ'),
'unicode string' => array('¤¦¨´¸¼½¾€ŠšŽžŒœŸ', '¤¦¨´¸¼½¾€ŠšŽžŒœŸ'),
'empty array' => array(array(), array()),
'array' => array(array('abcdef'), array('abcdef')),
'object' => array($obj, $obj),
);
}
/**
* @param int $code
* @param string $msg
* @dataProvider providesHandleJsonErrorFailure
*/
public function testHandleJsonErrorFailure($code, $msg)
{
$formatter = new NormalizerFormatter();
$reflMethod = new \ReflectionMethod($formatter, 'handleJsonError');
$reflMethod->setAccessible(true);
$this->setExpectedException('RuntimeException', $msg);
$reflMethod->invoke($formatter, $code, 'faked');
}
public function providesHandleJsonErrorFailure()
{
return array(
'depth' => array(JSON_ERROR_DEPTH, 'Maximum stack depth exceeded'),
'state' => array(JSON_ERROR_STATE_MISMATCH, 'Underflow or the modes mismatch'),
'ctrl' => array(JSON_ERROR_CTRL_CHAR, 'Unexpected control character found'),
'default' => array(-1, 'Unknown error'),
);
}
public function testExceptionTraceWithArgs()
{
if (defined('HHVM_VERSION')) {

View File

@@ -0,0 +1,67 @@
<?php
/*
* This file is part of the Monolog package.
*
* (c) Jordi Boggiano <j.boggiano@seld.be>
*
* For the full copyright and license information, please view the LICENSE
* file that was distributed with this source code.
*/
namespace Monolog;
class UtilsTest extends \PHPUnit_Framework_TestCase
{
/**
* @param int $code
* @param string $msg
* @dataProvider providesHandleJsonErrorFailure
*/
public function testHandleJsonErrorFailure($code, $msg)
{
$this->setExpectedException('RuntimeException', $msg);
Utils::handleJsonError($code, 'faked');
}
public function providesHandleJsonErrorFailure()
{
return array(
'depth' => array(JSON_ERROR_DEPTH, 'Maximum stack depth exceeded'),
'state' => array(JSON_ERROR_STATE_MISMATCH, 'Underflow or the modes mismatch'),
'ctrl' => array(JSON_ERROR_CTRL_CHAR, 'Unexpected control character found'),
'default' => array(-1, 'Unknown error'),
);
}
/**
* @param mixed $in Input
* @param mixed $expect Expected output
* @covers Monolog\Formatter\NormalizerFormatter::detectAndCleanUtf8
* @dataProvider providesDetectAndCleanUtf8
*/
public function testDetectAndCleanUtf8($in, $expect)
{
Utils::detectAndCleanUtf8($in);
$this->assertSame($expect, $in);
}
public function providesDetectAndCleanUtf8()
{
$obj = new \stdClass;
return array(
'null' => array(null, null),
'int' => array(123, 123),
'float' => array(123.45, 123.45),
'bool false' => array(false, false),
'bool true' => array(true, true),
'ascii string' => array('abcdef', 'abcdef'),
'latin9 string' => array("\xB1\x31\xA4\xA6\xA8\xB4\xB8\xBC\xBD\xBE\xFF", '±1€ŠšŽžŒœŸÿ'),
'unicode string' => array('¤¦¨´¸¼½¾€ŠšŽžŒœŸ', '¤¦¨´¸¼½¾€ŠšŽžŒœŸ'),
'empty array' => array(array(), array()),
'array' => array(array('abcdef'), array('abcdef')),
'object' => array($obj, $obj),
);
}
}