From e17aad1a99bb5e370c2094a6e93e994c150b7815 Mon Sep 17 00:00:00 2001 From: Adam Pancutt Date: Tue, 21 Jan 2014 10:28:27 +0000 Subject: [PATCH 1/6] Added support for Loggly batch uploads --- src/Monolog/Formatter/LogglyFormatter.php | 36 +++++++++++++++++++ src/Monolog/Handler/LogglyHandler.php | 36 +++++++++++++++---- .../Monolog/Formatter/LogglyFormatterTest.php | 36 +++++++++++++++++++ 3 files changed, 102 insertions(+), 6 deletions(-) create mode 100644 src/Monolog/Formatter/LogglyFormatter.php create mode 100644 tests/Monolog/Formatter/LogglyFormatterTest.php diff --git a/src/Monolog/Formatter/LogglyFormatter.php b/src/Monolog/Formatter/LogglyFormatter.php new file mode 100644 index 00000000..46062aa9 --- /dev/null +++ b/src/Monolog/Formatter/LogglyFormatter.php @@ -0,0 +1,36 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Monolog\Formatter; + +/** + * Extends the standard JsonFormatter to format messages compatible with Loggly. + * + * @author Adam Pancutt + */ +class LogglyFormatter extends JsonFormatter +{ + + /** + * {@inheritdoc} + */ + public function formatBatch(array $records) + { + $instance = $this; + + array_walk($records, function(&$value, $key) use ($instance) { + $value = $instance->format($value); + }); + + return implode("\n", $records); + } + +} diff --git a/src/Monolog/Handler/LogglyHandler.php b/src/Monolog/Handler/LogglyHandler.php index e0061a53..9a3de6e6 100644 --- a/src/Monolog/Handler/LogglyHandler.php +++ b/src/Monolog/Handler/LogglyHandler.php @@ -12,16 +12,19 @@ namespace Monolog\Handler; use Monolog\Logger; -use Monolog\Formatter\JsonFormatter; +use Monolog\Formatter\LogglyFormatter; /** * Sends errors to Loggly. * * @author Przemek Sobstel + * @author Adam Pancutt */ class LogglyHandler extends AbstractProcessingHandler { const HOST = 'logs-01.loggly.com'; + const ENDPOINT_SINGLE = 'inputs'; + const ENDPOINT_BATCH = 'bulk'; protected $token; @@ -45,17 +48,38 @@ class LogglyHandler extends AbstractProcessingHandler protected function write(array $record) { - $url = sprintf("http://%s/inputs/%s/", self::HOST, $this->token); + $this->send($record["formatted"], self::ENDPOINT_SINGLE); + } + + public function handleBatch(array $records) + { + $level = $this->level; + + $records = array_filter($records, function ($record) use ($level) { + return ($record['level'] >= $level); + }); + + if ($records) { + $this->send($this->getFormatter()->formatBatch($records), self::ENDPOINT_BATCH); + } + } + + protected function send($data, $endpoint) + { + $url = sprintf("https://%s/%s/%s/", self::HOST, $endpoint, $this->token); + + $headers = array('Content-Type: application/json'); + if ($this->tag) { - $url .= sprintf("tag/%s/", $this->tag); + $headers[] = "X-LOGGLY-TAG: {$this->tag}"; } $ch = curl_init(); curl_setopt($ch, CURLOPT_URL, $url); curl_setopt($ch, CURLOPT_POST, true); - curl_setopt($ch, CURLOPT_POSTFIELDS, $record["formatted"]); - curl_setopt($ch, CURLOPT_HTTPHEADER, array('Content-Type: application/json')); + curl_setopt($ch, CURLOPT_POSTFIELDS, $data); + curl_setopt($ch, CURLOPT_HTTPHEADER, $headers); curl_setopt($ch, CURLOPT_RETURNTRANSFER, true); curl_exec($ch); @@ -64,6 +88,6 @@ class LogglyHandler extends AbstractProcessingHandler protected function getDefaultFormatter() { - return new JsonFormatter(); + return new LogglyFormatter(); } } diff --git a/tests/Monolog/Formatter/LogglyFormatterTest.php b/tests/Monolog/Formatter/LogglyFormatterTest.php new file mode 100644 index 00000000..a6d8d57a --- /dev/null +++ b/tests/Monolog/Formatter/LogglyFormatterTest.php @@ -0,0 +1,36 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Monolog\Formatter; + +use Monolog\Logger; +use Monolog\TestCase; + +class LogglyFormatterTest extends TestCase +{ + + /** + * @covers Monolog\Formatter\LogglyFormatter::formatBatch + */ + public function testFormatBatch() + { + $formatter = new LogglyFormatter(); + $records = $expected = array( + $this->getRecord(Logger::WARNING), + $this->getRecord(Logger::DEBUG), + ); + array_walk($expected, function(&$value, $key) { + $value = json_encode($value); + }); + $this->assertEquals(implode("\n", $expected), $formatter->formatBatch($records)); + } + +} From 95f0649b5931bc8423319dad2517947005dcec05 Mon Sep 17 00:00:00 2001 From: Adam Pancutt Date: Tue, 21 Jan 2014 11:39:58 +0000 Subject: [PATCH 2/6] Replaced LogglyFormatter with option to change batch formatting in JsonFormatter --- src/Monolog/Formatter/JsonFormatter.php | 67 +++++++++++++++++++ src/Monolog/Formatter/LogglyFormatter.php | 36 ---------- src/Monolog/Handler/LogglyHandler.php | 4 +- .../Monolog/Formatter/LogglyFormatterTest.php | 36 ---------- 4 files changed, 69 insertions(+), 74 deletions(-) delete mode 100644 src/Monolog/Formatter/LogglyFormatter.php delete mode 100644 tests/Monolog/Formatter/LogglyFormatterTest.php diff --git a/src/Monolog/Formatter/JsonFormatter.php b/src/Monolog/Formatter/JsonFormatter.php index 822af0ea..62c70026 100644 --- a/src/Monolog/Formatter/JsonFormatter.php +++ b/src/Monolog/Formatter/JsonFormatter.php @@ -20,6 +20,34 @@ namespace Monolog\Formatter; */ class JsonFormatter implements FormatterInterface { + + protected $batch_mode; + + const BATCH_MODE_JSON = 1; + const BATCH_MODE_NEWLINES = 2; + + /** + * @param int $batch_mode + */ + public function __construct($batch_mode = self::BATCH_MODE_JSON) + { + $this->batch_mode = $batch_mode; + } + + /** + * The batch mode option configures the formatting style for + * multiple records. By default, multiple records will be + * formatted as a JSON-encoded array. However, for + * compatibility with some API endpoints, alternive styles + * are available. + * + * @return int + */ + public function getBatchMode() + { + return $this->batch_mode; + } + /** * {@inheritdoc} */ @@ -32,7 +60,46 @@ class JsonFormatter implements FormatterInterface * {@inheritdoc} */ public function formatBatch(array $records) + { + switch ($this->batch_mode) { + + case static::BATCH_MODE_NEWLINES: + return $this->formatBatchNewlines($records); + + case static::BATCH_MODE_JSON: + default: + return $this->formatBatchJson($records); + + } + } + + /** + * Return a JSON-encoded array of records. + * + * @param array $records + * @return string + */ + protected function formatBatchJson(array $records) { return json_encode($records); } + + /** + * Use new lines to separate records instead of a + * JSON-encoded array. + * + * @param array $records + * @return string + */ + protected function formatBatchNewlines(array $records) + { + $instance = $this; + + array_walk($records, function(&$value, $key) use ($instance) { + $value = $instance->format($value); + }); + + return implode("\n", $records); + } + } diff --git a/src/Monolog/Formatter/LogglyFormatter.php b/src/Monolog/Formatter/LogglyFormatter.php deleted file mode 100644 index 46062aa9..00000000 --- a/src/Monolog/Formatter/LogglyFormatter.php +++ /dev/null @@ -1,36 +0,0 @@ - - * - * For the full copyright and license information, please view the LICENSE - * file that was distributed with this source code. - */ - -namespace Monolog\Formatter; - -/** - * Extends the standard JsonFormatter to format messages compatible with Loggly. - * - * @author Adam Pancutt - */ -class LogglyFormatter extends JsonFormatter -{ - - /** - * {@inheritdoc} - */ - public function formatBatch(array $records) - { - $instance = $this; - - array_walk($records, function(&$value, $key) use ($instance) { - $value = $instance->format($value); - }); - - return implode("\n", $records); - } - -} diff --git a/src/Monolog/Handler/LogglyHandler.php b/src/Monolog/Handler/LogglyHandler.php index 9a3de6e6..ba2c7f11 100644 --- a/src/Monolog/Handler/LogglyHandler.php +++ b/src/Monolog/Handler/LogglyHandler.php @@ -12,7 +12,7 @@ namespace Monolog\Handler; use Monolog\Logger; -use Monolog\Formatter\LogglyFormatter; +use Monolog\Formatter\JsonFormatter; /** * Sends errors to Loggly. @@ -88,6 +88,6 @@ class LogglyHandler extends AbstractProcessingHandler protected function getDefaultFormatter() { - return new LogglyFormatter(); + return new JsonFormatter(JsonFormatter::BATCH_MODE_NEWLINES); } } diff --git a/tests/Monolog/Formatter/LogglyFormatterTest.php b/tests/Monolog/Formatter/LogglyFormatterTest.php deleted file mode 100644 index a6d8d57a..00000000 --- a/tests/Monolog/Formatter/LogglyFormatterTest.php +++ /dev/null @@ -1,36 +0,0 @@ - - * - * For the full copyright and license information, please view the LICENSE - * file that was distributed with this source code. - */ - -namespace Monolog\Formatter; - -use Monolog\Logger; -use Monolog\TestCase; - -class LogglyFormatterTest extends TestCase -{ - - /** - * @covers Monolog\Formatter\LogglyFormatter::formatBatch - */ - public function testFormatBatch() - { - $formatter = new LogglyFormatter(); - $records = $expected = array( - $this->getRecord(Logger::WARNING), - $this->getRecord(Logger::DEBUG), - ); - array_walk($expected, function(&$value, $key) { - $value = json_encode($value); - }); - $this->assertEquals(implode("\n", $expected), $formatter->formatBatch($records)); - } - -} From 70290e7087e1954f7c885682f5a6e9ea262a6739 Mon Sep 17 00:00:00 2001 From: Adam Pancutt Date: Tue, 21 Jan 2014 11:47:47 +0000 Subject: [PATCH 3/6] Added batch mode tests to JsonFormatter --- tests/Monolog/Formatter/JsonFormatterTest.php | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/Monolog/Formatter/JsonFormatterTest.php b/tests/Monolog/Formatter/JsonFormatterTest.php index ba6152c9..bad19f73 100644 --- a/tests/Monolog/Formatter/JsonFormatterTest.php +++ b/tests/Monolog/Formatter/JsonFormatterTest.php @@ -16,6 +16,18 @@ use Monolog\TestCase; class JsonFormatterTest extends TestCase { + /** + * @covers Monolog\Formatter\JsonFormatter::__construct + * @covers Monolog\Formatter\JsonFormatter::getBatchMode + */ + public function testConstruct() + { + $formatter = new JsonFormatter(); + $this->assertEquals(JsonFormatter::BATCH_MODE_JSON, $formatter->getBatchMode()); + $formatter = new JsonFormatter(JsonFormatter::BATCH_MODE_NEWLINES); + $this->assertEquals(JsonFormatter::BATCH_MODE_NEWLINES, $formatter->getBatchMode()); + } + /** * @covers Monolog\Formatter\JsonFormatter::format */ @@ -28,6 +40,7 @@ class JsonFormatterTest extends TestCase /** * @covers Monolog\Formatter\JsonFormatter::formatBatch + * @covers Monolog\Formatter\JsonFormatter::formatBatchJson */ public function testFormatBatch() { @@ -38,4 +51,22 @@ class JsonFormatterTest extends TestCase ); $this->assertEquals(json_encode($records), $formatter->formatBatch($records)); } + + /** + * @covers Monolog\Formatter\JsonFormatter::formatBatch + * @covers Monolog\Formatter\JsonFormatter::formatBatchNewlines + */ + public function testFormatBatchNewlines() + { + + $formatter = new JsonFormatter(JsonFormatter::BATCH_MODE_NEWLINES); + $records = $expected = array( + $this->getRecord(Logger::WARNING), + $this->getRecord(Logger::DEBUG), + ); + array_walk($expected, function(&$value, $key) { + $value = json_encode($value); + }); + $this->assertEquals(implode("\n", $expected), $formatter->formatBatch($records)); + } } From 4f91d1c58fe208d09ba4a8a14f75b0f02752e0d1 Mon Sep 17 00:00:00 2001 From: Adam Pancutt Date: Tue, 21 Jan 2014 17:03:50 +0000 Subject: [PATCH 4/6] Simplified formatBatchNewlines() --- src/Monolog/Formatter/JsonFormatter.php | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/src/Monolog/Formatter/JsonFormatter.php b/src/Monolog/Formatter/JsonFormatter.php index 62c70026..e3ce8ccf 100644 --- a/src/Monolog/Formatter/JsonFormatter.php +++ b/src/Monolog/Formatter/JsonFormatter.php @@ -93,13 +93,7 @@ class JsonFormatter implements FormatterInterface */ protected function formatBatchNewlines(array $records) { - $instance = $this; - - array_walk($records, function(&$value, $key) use ($instance) { - $value = $instance->format($value); - }); - - return implode("\n", $records); + return implode(PHP_EOL, array_map($records, array($this, 'format'))); } } From b748794bba4ed73c9a7b295175f1eca192526256 Mon Sep 17 00:00:00 2001 From: Adam Pancutt Date: Tue, 21 Jan 2014 18:21:50 +0000 Subject: [PATCH 5/6] Revert "Simplified formatBatchNewlines()" This reverts commit 4f91d1c58fe208d09ba4a8a14f75b0f02752e0d1. --- src/Monolog/Formatter/JsonFormatter.php | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Monolog/Formatter/JsonFormatter.php b/src/Monolog/Formatter/JsonFormatter.php index e3ce8ccf..62c70026 100644 --- a/src/Monolog/Formatter/JsonFormatter.php +++ b/src/Monolog/Formatter/JsonFormatter.php @@ -93,7 +93,13 @@ class JsonFormatter implements FormatterInterface */ protected function formatBatchNewlines(array $records) { - return implode(PHP_EOL, array_map($records, array($this, 'format'))); + $instance = $this; + + array_walk($records, function(&$value, $key) use ($instance) { + $value = $instance->format($value); + }); + + return implode("\n", $records); } } From a46413b15b9a42a9b90d851f943b73115c166cbe Mon Sep 17 00:00:00 2001 From: Adam Pancutt Date: Wed, 22 Jan 2014 14:31:31 +0000 Subject: [PATCH 6/6] Added 'timestamp' parameter to $record for indexing by Loggly --- src/Monolog/Formatter/LogglyFormatter.php | 48 +++++++++++++++++++ src/Monolog/Handler/LogglyHandler.php | 4 +- .../Monolog/Formatter/LogglyFormatterTest.php | 41 ++++++++++++++++ 3 files changed, 91 insertions(+), 2 deletions(-) create mode 100644 src/Monolog/Formatter/LogglyFormatter.php create mode 100644 tests/Monolog/Formatter/LogglyFormatterTest.php diff --git a/src/Monolog/Formatter/LogglyFormatter.php b/src/Monolog/Formatter/LogglyFormatter.php new file mode 100644 index 00000000..db9e7c3f --- /dev/null +++ b/src/Monolog/Formatter/LogglyFormatter.php @@ -0,0 +1,48 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Monolog\Formatter; + +/** + * Encodes message information into JSON in a format compatible with Loggly. + * + * @author Adam Pancutt + */ +class LogglyFormatter extends JsonFormatter +{ + + /** + * Overrides the default batch mode to new lines for compatibility with the + * Loggly bulk API. + * + * @param integer $batch_mode + */ + public function __construct($batch_mode = self::BATCH_MODE_NEWLINES) + { + parent::__construct($batch_mode); + } + + /** + * Appends the 'timestamp' parameter for indexing by Loggly. + * + * @see https://www.loggly.com/docs/automated-parsing/#json + * @see \Monolog\Formatter\JsonFormatter::format() + */ + public function format(array $record) + { + if (isset($record["datetime"]) && ($record["datetime"] instanceof \DateTime)) { + $record["timestamp"] = $record["datetime"]->format("c"); + // @todo unset the 'datetime' parameter, retained for BC + } + return parent::format($record); + } + +} diff --git a/src/Monolog/Handler/LogglyHandler.php b/src/Monolog/Handler/LogglyHandler.php index ba2c7f11..9a3de6e6 100644 --- a/src/Monolog/Handler/LogglyHandler.php +++ b/src/Monolog/Handler/LogglyHandler.php @@ -12,7 +12,7 @@ namespace Monolog\Handler; use Monolog\Logger; -use Monolog\Formatter\JsonFormatter; +use Monolog\Formatter\LogglyFormatter; /** * Sends errors to Loggly. @@ -88,6 +88,6 @@ class LogglyHandler extends AbstractProcessingHandler protected function getDefaultFormatter() { - return new JsonFormatter(JsonFormatter::BATCH_MODE_NEWLINES); + return new LogglyFormatter(); } } diff --git a/tests/Monolog/Formatter/LogglyFormatterTest.php b/tests/Monolog/Formatter/LogglyFormatterTest.php new file mode 100644 index 00000000..4ef0cb8a --- /dev/null +++ b/tests/Monolog/Formatter/LogglyFormatterTest.php @@ -0,0 +1,41 @@ + + * + * For the full copyright and license information, please view the LICENSE + * file that was distributed with this source code. + */ + +namespace Monolog\Formatter; + +use Monolog\Logger; +use Monolog\TestCase; + +class LogglyFormatterTest extends TestCase +{ + /** + * @covers Monolog\Formatter\LogglyFormatter::__construct + */ + public function testConstruct() + { + $formatter = new LogglyFormatter(); + $this->assertEquals(LogglyFormatter::BATCH_MODE_NEWLINES, $formatter->getBatchMode()); + $formatter = new LogglyFormatter(LogglyFormatter::BATCH_MODE_JSON); + $this->assertEquals(LogglyFormatter::BATCH_MODE_JSON, $formatter->getBatchMode()); + } + + /** + * @covers Monolog\Formatter\LogglyFormatter::format + */ + public function testFormat() + { + $formatter = new LogglyFormatter(); + $record = $this->getRecord(); + $formatted_decoded = json_decode($formatter->format($record), true); + $this->assertArrayHasKey("timestamp", $formatted_decoded); + $this->assertEquals(new \DateTime($formatted_decoded["timestamp"]), $record["datetime"]); + } +}