Keep track of expected tokens

src/Tokens/Colon.php

@@ -0,0 +1,23 @@
+<?php
+
+namespace Cerbero\JsonParser\Tokens;
+
+use Cerbero\JsonParser\State;
+
+/**
+ * The colon token.
+ *
+ */
+final class Colon extends Token
+{
+    /**
+     * Mutate the given state
+     *
+     * @param State $state
+     * @return void
+     */
+    public function mutateState(State $state): void
+    {
+        $state->expectedToken = Tokens::VALUE_ANY;
+    }
+}

src/Tokens/Comma.php

@@ -10,16 +10,6 @@ use Cerbero\JsonParser\State;
  */
 final class Comma extends Token
 {
-    /**
-     * Retrieve the token type
-     *
-     * @return int
-     */
-    public function type(): int
-    {
-        return Tokens::COMMA;
-    }
-
     /**
      * Mutate the given state
      *
@@ -28,6 +18,7 @@ final class Comma extends Token
      */
     public function mutateState(State $state): void
     {
-        $state->expectsKey = $state->inObject();
+        $state->expectsKey = $state->tree()->inObject();
+        $state->expectedToken = $state->expectsKey ? Tokens::SCALAR_STRING : Tokens::VALUE_ANY;
     }
 }

src/Tokens/CompoundBegin.php

@@ -10,16 +10,6 @@ use Cerbero\JsonParser\State;
  */
 final class CompoundBegin extends Token
 {
-    /**
-     * Retrieve the token type
-     *
-     * @return int
-     */
-    public function type(): int
-    {
-        return Tokens::COMPOUND_BEGIN;
-    }
-
     /**
      * Mutate the given state
      *
@@ -28,8 +18,8 @@ final class CompoundBegin extends Token
      */
     public function mutateState(State $state): void
     {
-        $state->tree()->deepen();
+        $state->expectsKey = $beginsObject = $this->value == '{';
-
+        $state->expectedToken = $beginsObject ? Tokens::AFTER_OBJECT_BEGIN : Tokens::AFTER_ARRAY_BEGIN;
-        $state->expectsKey = $this->value == '{';
+        $state->tree()->deepen($beginsObject);
     }
 }

src/Tokens/CompoundEnd.php

@@ -10,16 +10,6 @@ use Cerbero\JsonParser\State;
  */
 final class CompoundEnd extends Token
 {
-    /**
-     * Retrieve the token type
-     *
-     * @return int
-     */
-    public function type(): int
-    {
-        return Tokens::COMPOUND_END;
-    }
-
     /**
      * Mutate the given state
      *
@@ -29,6 +19,8 @@ final class CompoundEnd extends Token
     public function mutateState(State $state): void
     {
         $state->tree()->emerge();
+
+        $state->expectedToken = $state->tree()->inObject() ? Tokens::AFTER_OBJECT_VALUE : Tokens::AFTER_ARRAY_VALUE;
     }
 
     /**

src/Tokens/Constant.php

@@ -2,20 +2,23 @@
 
 namespace Cerbero\JsonParser\Tokens;
 
+use Cerbero\JsonParser\State;
+
 /**
- * The constant token, includes colons for convenience.
+ * The constant token.
  *
  */
 final class Constant extends Token
 {
     /**
-     * Retrieve the token type
+     * Mutate the given state
      *
-     * @return int
+     * @param State $state
+     * @return void
      */
-    public function type(): int
+    public function mutateState(State $state): void
     {
-        return $this->value == ':' ? Tokens::COLON : Tokens::SCALAR_CONST;
+        $state->expectedToken = $state->tree()->inObject() ? Tokens::AFTER_OBJECT_VALUE : Tokens::AFTER_ARRAY_VALUE;
     }
 
     /**
@@ -25,6 +28,6 @@ final class Constant extends Token
      */
     public function endsChunk(): bool
     {
-        return $this->value != ':';
+        return true;
     }
 }

src/Tokens/ScalarString.php

@@ -17,16 +17,6 @@ final class ScalarString extends Token
      */
     private bool $isKey = false;
 
-    /**
-     * Retrieve the token type
-     *
-     * @return int
-     */
-    public function type(): int
-    {
-        return Tokens::SCALAR_STRING;
-    }
-
     /**
      * Mutate the given state
      *
@@ -37,7 +27,11 @@ final class ScalarString extends Token
     {
         if ($this->isKey = $state->expectsKey) {
             $state->expectsKey = false;
+            $state->expectedToken = Tokens::COLON;
+            return;
         }
+
+        $state->expectedToken = $state->tree()->inObject() ? Tokens::AFTER_OBJECT_VALUE : Tokens::AFTER_ARRAY_VALUE;
     }
 
     /**

src/Tokens/Token.php

@@ -19,11 +19,23 @@ abstract class Token implements Stringable
     protected string $value;
 
     /**
-     * Retrieve the token type
+     * Mutate the given state
      *
-     * @return int
+     * @param State $state
+     * @return void
      */
-    abstract public function type(): int;
+    abstract public function mutateState(State $state): void;
+
+    /**
+     * Determine whether this token matches the given type
+     *
+     * @param int $type
+     * @return bool
+     */
+    public function matches(int $type): bool
+    {
+        return (Tokens::TYPES[$this->value[0]] & $type) != 0;
+    }
 
     /**
      * Set the token value
@@ -45,18 +57,7 @@ abstract class Token implements Stringable
      */
     public function isValue(): bool
     {
-        return ($this->type() | Tokens::VALUE_ANY) == Tokens::VALUE_ANY;
+        return (Tokens::TYPES[$this->value[0]] | Tokens::VALUE_ANY) == Tokens::VALUE_ANY;
-    }
-
-    /**
-     * Mutate the given state
-     *
-     * @param State $state
-     * @return void
-     */
-    public function mutateState(State $state): void
-    {
-        return;
     }
 
     /**

src/Tokens/Tokens.php

@@ -67,19 +67,19 @@ final class Tokens
      * @var array<string, bool>
      */
     public const BOUNDARIES = [
-        "\xEF" => true,
-        "\xBB" => true,
-        "\xBF" => true,
-        "\n" => true,
-        "\r" => true,
-        "\t" => true,
-        ' ' => true,
         '{' => true,
         '}' => true,
         '[' => true,
         ']' => true,
-        ':' => true,
         ',' => true,
+        ':' => true,
+        ' ' => true,
+        "\n" => true,
+        "\r" => true,
+        "\t" => true,
+        "\xEF" => true,
+        "\xBB" => true,
+        "\xBF" => true,
     ];
 
     /**
@@ -92,8 +92,8 @@ final class Tokens
         '}' => true,
         '[' => true,
         ']' => true,
-        ':' => true,
         ',' => true,
+        ':' => true,
     ];
 
     /**
@@ -107,7 +107,7 @@ final class Tokens
         self::ARRAY_BEGIN => CompoundBegin::class,
         self::OBJECT_END => CompoundEnd::class,
         self::ARRAY_END => CompoundEnd::class,
-        self::COLON => Constant::class,
+        self::COLON => Colon::class,
         self::SCALAR_CONST => Constant::class,
         self::SCALAR_STRING => ScalarString::class,
     ];