From 955a90f637bad912d44bb3fac26a3e72e1cca6d9 Mon Sep 17 00:00:00 2001
From: Filip Halaxa <filip@halaxa.cz>
Date: Sun, 9 Jan 2022 16:45:08 +0100
Subject: [PATCH] Stateful Lexer

---
 src/Lexer.php                               | 17 +++++++++------
 src/QueueChunks.php                         | 23 ++++++++++++++++++++
 test/JsonMachineTest/FollowUpParserTest.php | 15 +++++++++++++
 test/JsonMachineTest/LexerTest.php          | 24 +++++++++++++++++----
 4 files changed, 69 insertions(+), 10 deletions(-)
 create mode 100644 src/QueueChunks.php
 create mode 100644 test/JsonMachineTest/FollowUpParserTest.php

diff --git a/src/Lexer.php b/src/Lexer.php
index 22e9241..7681a50 100644
--- a/src/Lexer.php
+++ b/src/Lexer.php
@@ -9,6 +9,10 @@ class Lexer implements \IteratorAggregate, PositionAware
     /** @var iterable */
     private $jsonChunks;
 
+    private $inString = false;
+    private $tokenBuffer = '';
+    private $escaping = false;
+
     /**
      * @param iterable<string> $jsonChunks
      */
@@ -31,9 +35,9 @@ class Lexer implements \IteratorAggregate, PositionAware
         $tokenBoundaries = $this->tokenBoundaries();
         $colonCommaBracket = $this->colonCommaBracketTokenBoundaries();
 
-        $inString = false;
-        $tokenBuffer = '';
-        $escaping = false;
+        $inString = $this->inString;
+        $tokenBuffer = $this->tokenBuffer;
+        $escaping = $this->escaping;
 
         foreach ($this->jsonChunks as $jsonChunk) {
             $bytesLength = strlen($jsonChunk);
@@ -75,9 +79,10 @@ class Lexer implements \IteratorAggregate, PositionAware
                 }
             }
         }
-        if ($tokenBuffer != '') {
-            yield $tokenBuffer;
-        }
+
+        $this->inString = $inString;
+        $this->tokenBuffer = $tokenBuffer;
+        $this->escaping = $escaping;
     }
 
     private function tokenBoundaries()
diff --git a/src/QueueChunks.php b/src/QueueChunks.php
new file mode 100644
index 0000000..282e0e8
--- /dev/null
+++ b/src/QueueChunks.php
@@ -0,0 +1,23 @@
+<?php declare(strict_types=1);
+
+
+namespace JsonMachine;
+
+use Exception;
+use Traversable;
+
+class QueueChunks implements \IteratorAggregate
+{
+    private $queue = [];
+
+    public function push(string $jsonChunk)
+    {
+        $this->queue[] = $jsonChunk;
+    }
+
+    public function getIterator()
+    {
+        yield from $this->queue;
+        $this->queue = [];
+    }
+}
diff --git a/test/JsonMachineTest/FollowUpParserTest.php b/test/JsonMachineTest/FollowUpParserTest.php
new file mode 100644
index 0000000..abec9b0
--- /dev/null
+++ b/test/JsonMachineTest/FollowUpParserTest.php
@@ -0,0 +1,15 @@
+<?php
+
+namespace JsonMachineTest;
+
+use JsonMachine\Lexer;
+use JsonMachine\QueueChunks;
+use PHPUnit\Framework\TestCase;
+
+class FollowUpParserTest extends TestCase
+{
+    public function testParses()
+    {
+
+    }
+}
diff --git a/test/JsonMachineTest/LexerTest.php b/test/JsonMachineTest/LexerTest.php
index 3d9b305..f1220ac 100644
--- a/test/JsonMachineTest/LexerTest.php
+++ b/test/JsonMachineTest/LexerTest.php
@@ -5,6 +5,7 @@ namespace JsonMachineTest;
 use JsonMachine\DebugLexer;
 use JsonMachine\Lexer;
 use JsonMachine\Exception;
+use JsonMachine\QueueChunks;
 use JsonMachine\StreamChunks;
 use JsonMachine\StringChunks;
 
@@ -23,8 +24,8 @@ class LexerTest extends \PHPUnit_Framework_TestCase
      */
     public function testCorrectlyYieldsZeroToken($lexerClass)
     {
-        $data = ['0'];
-        $expected = ['0'];
+        $data = ['[0]'];
+        $expected = ['[', '0', ']'];
         $this->assertEquals($expected, iterator_to_array(new $lexerClass(new \ArrayIterator($data))));
 
         $stream = fopen('data://text/plain,{"value":0}', 'r');
@@ -37,8 +38,8 @@ class LexerTest extends \PHPUnit_Framework_TestCase
      */
     public function testGeneratesTokens($lexerClass)
     {
-        $data = ['{}[],:null,"string" false:', 'true,1,100000,1.555{-56]"","\\""'];
-        $expected = ['{','}','[',']',',',':','null',',','"string"','false',':','true',',','1',',','100000',',','1.555','{','-56',']','""',',','"\\""'];
+        $data = ['{}[],:null,"string" false:', 'true,1,100000,1.555{-56]"","\\""]'];
+        $expected = ['{','}','[',']',',',':','null',',','"string"','false',':','true',',','1',',','100000',',','1.555','{','-56',']','""',',','"\\""',']'];
         $this->assertEquals($expected, iterator_to_array(new $lexerClass(new \ArrayIterator($data))));
     }
 
@@ -144,6 +145,21 @@ class LexerTest extends \PHPUnit_Framework_TestCase
         }
     }
 
+    public function testRepeatedlyParsesModifiedQueue()
+    {
+        $queue = new QueueChunks();
+        $lexer = new Lexer($queue);
+
+        $queue->push('[1,2,4');
+        $this->assertEquals(['[', '1', ',', '2', ','], iterator_to_array($lexer));
+
+        $queue->push('2,"some ');
+        $this->assertEquals(['42', ','], iterator_to_array($lexer));
+
+        $queue->push('string"]');
+        $this->assertEquals(['"some string"', ']'], iterator_to_array($lexer));
+    }
+
     /**
      * @param string $formattedJsonFilePath
      * @dataProvider dataProvidesLocationalData