improved lexer/parser & refactoring variable parsing

phpstan-baseline.neon

@@ -20,6 +20,11 @@ parameters:
 			count: 2
 			path: performance/benchmark.php
 
+		-
+			message: "#^Strict comparison using \\=\\=\\= between false and false will always evaluate to true\\.$#"
+			count: 1
+			path: src/Parse/Parser.php
+
 		-
 			message: "#^Strict comparison using \\!\\=\\= between null and null will always evaluate to false\\.$#"
 			count: 1

src/Nodes/Variable.php

@@ -6,41 +6,69 @@
 use Keepsuit\Liquid\Contracts\CanBeRendered;
 use Keepsuit\Liquid\Contracts\HasParseTreeVisitorChildren;
 use Keepsuit\Liquid\Exceptions\SyntaxException;
-use Keepsuit\Liquid\Parse\ParseContext;
+use Keepsuit\Liquid\Parse\Expression;
 use Keepsuit\Liquid\Parse\Parser;
-use Keepsuit\Liquid\Parse\Regex;
 use Keepsuit\Liquid\Parse\TokenType;
 use Keepsuit\Liquid\Render\Context;
 use Keepsuit\Liquid\Support\Arr;
 
 class Variable implements CanBeEvaluated, CanBeRendered, HasParseTreeVisitorChildren
 {
-    const JustTagAttributes = '/\A'.Regex::TagAttributes.'\z/';
-
-    protected mixed $name = null;
-
-    public readonly ?int $lineNumber;
-
-    /**
-     * @var array<array{0:string,1:array}>
-     */
-    protected array $filters = [];
-
     /**
      * @throws SyntaxException
      */
     public function __construct(
-        public readonly string $markup,
-        public readonly ParseContext $parseContext
+        protected mixed $name,
+        /** @var array<array{0:string,1:array}> */
+        protected array $filters = [],
+        protected string $markup = '',
+        public readonly ?int $lineNumber = null
     ) {
-        $this->lineNumber = $this->parseContext->lineNumber;
+    }
 
+    public static function fromMarkup(string $markup, int $lineNumber = null): Variable
+    {
         try {
-            $this->parseVariable($markup);
+            $variable = static::fromParser(new Parser($markup), $lineNumber);
         } catch (SyntaxException $exception) {
             $exception->markupContext = sprintf('{{%s}}', $markup);
             throw $exception;
         }
+
+        $variable->markup = $markup;
+
+        return $variable;
+    }
+
+    public static function fromParser(Parser $parser, int $lineNumber = null): Variable
+    {
+        if ($parser->look(TokenType::EndOfString)) {
+            return new Variable(
+                name: null,
+                filters: [],
+                lineNumber: $lineNumber,
+            );
+        }
+
+        $markup = $parser->toString();
+
+        $name = static::parseExpression($parser->expression());
+
+        $filters = [];
+        while ($parser->consumeOrFalse(TokenType::Pipe)) {
+            $filterName = $parser->consume(TokenType::Identifier);
+            $filterArgs = $parser->consumeOrFalse(TokenType::Colon) ? static::parseFilterArgs($parser) : [];
+            $filters[] = static::parseFilterExpressions($filterName, $filterArgs);
+        }
+
+        $parser->consume(TokenType::EndOfString);
+
+        return new Variable(
+            name: $name,
+            filters: $filters,
+            markup: $markup,
+            lineNumber: $lineNumber,
+        );
     }
 
     public function name(): mixed
@@ -107,31 +135,7 @@ public function evaluate(Context $context): mixed
         return $output;
     }
 
-    /**
-     * @throws SyntaxException
-     */
-    protected function parseVariable(string $markup): void
-    {
-        $this->filters = [];
-
-        $parser = new Parser($markup);
-
-        if ($parser->look(TokenType::EndOfString)) {
-            return;
-        }
-
-        $this->name = $this->parseContext->parseExpression($parser->expression());
-
-        while ($parser->consumeOrFalse(TokenType::Pipe)) {
-            $filterName = $parser->consume(TokenType::Identifier);
-            $filterArgs = $parser->consumeOrFalse(TokenType::Colon) ? $this->parseFilterArgs($parser) : [];
-            $this->filters[] = $this->parseFilterExpressions($filterName, $filterArgs);
-        }
-
-        $parser->consume(TokenType::EndOfString);
-    }
-
-    protected function parseFilterArgs(Parser $parser): array
+    protected static function parseFilterArgs(Parser $parser): array
     {
         $filterArgs = [$parser->argument()];
         while ($parser->consumeOrFalse(TokenType::Comma)) {
@@ -142,29 +146,37 @@ protected function parseFilterArgs(Parser $parser): array
     }
 
     /**
+     * @param  array<string|array<string,string>>  $filterArgs
      * @return array{0:string, 1:array, 2:array<string,mixed>}
      */
-    protected function parseFilterExpressions(string $filterName, array $filterArgs): array
+    protected static function parseFilterExpressions(string $filterName, array $filterArgs): array
     {
         $parsedArgs = [];
         $parsedNamedArgs = [];
 
         foreach ($filterArgs as $arg) {
-            if (preg_match(self::JustTagAttributes, $arg, $matches) === 1) {
-                $parsedNamedArgs[$matches[1]] = $this->parseContext->parseExpression($matches[2]);
+            if (is_array($arg)) {
+                foreach ($arg as $key => $value) {
+                    $parsedNamedArgs[$key] = static::parseExpression($value);
+                }
             } else {
-                $parsedArgs[] = $this->parseContext->parseExpression($arg);
+                $parsedArgs[] = static::parseExpression($arg);
             }
         }
 
         return [$filterName, $parsedArgs, $parsedNamedArgs];
     }
 
-    protected function evaluateFilterExpressions(Context $context, array $filterArgs): array
+    protected static function evaluateFilterExpressions(Context $context, array $filterArgs): array
     {
         return array_map(
             fn (mixed $value) => $context->evaluate($value),
             $filterArgs
         );
     }
+
+    protected static function parseExpression(string $markup): mixed
+    {
+        return Expression::parse($markup);
+    }
 }

src/Parse/BlockParser.php

@@ -237,7 +237,7 @@ protected static function whitespaceHandler(string $token, ParseContext $parseCo
     protected static function createVariable(string $token, ParseContext $parseContext): Variable
     {
         if (preg_match(static::CONTENT_OF_VARIABLE, $token, $matches) === 1) {
-            return new Variable($matches[1], $parseContext);
+            return Variable::fromMarkup($matches[1], $parseContext->lineNumber);
         }
 
         throw SyntaxException::missingVariableTerminator($token, $parseContext);

src/Parse/Lexer.php

@@ -28,6 +28,7 @@ class Lexer
         ')' => TokenType::CloseRound,
         '?' => TokenType::QuestionMark,
         '-' => TokenType::Dash,
+        '=' => TokenType::Equals,
     ];
 
     protected const WHITESPACE_OR_NOTHING = '/\G\s*/';
@@ -39,6 +40,11 @@ public function __construct(
     ) {
     }
 
+    /**
+     * @return array<array{0:TokenType, 1:string, 2:int}>
+     *
+     * @throws SyntaxException
+     */
     public function tokenize(): array
     {
         if ($this->result instanceof Exception) {
@@ -61,12 +67,12 @@ public function tokenize(): array
             }
 
             $token = match (true) {
-                preg_match(self::COMPARISON_OPERATOR, $this->input, $matches, offset: $currentIndex) === 1 => [TokenType::Comparison, $matches[0]],
-                preg_match(self::STRING_LITERAL, $this->input, $matches, offset: $currentIndex) === 1 => [TokenType::String, $matches[0]],
-                preg_match(self::NUMBER_LITERAL, $this->input, $matches, offset: $currentIndex) === 1 => [TokenType::Number, $matches[0]],
-                preg_match(self::IDENTIFIER, $this->input, $matches, offset: $currentIndex) === 1 => [TokenType::Identifier, $matches[0]],
-                preg_match(self::DOTDOT, $this->input, $matches, offset: $currentIndex) === 1 => [TokenType::DotDot, $matches[0]],
-                array_key_exists($this->input[$currentIndex], self::SPECIAL_CHARACTERS) => [self::SPECIAL_CHARACTERS[$this->input[$currentIndex]], $this->input[$currentIndex]],
+                preg_match(self::COMPARISON_OPERATOR, $this->input, $matches, offset: $currentIndex) === 1 => [TokenType::Comparison, $matches[0], $currentIndex],
+                preg_match(self::STRING_LITERAL, $this->input, $matches, offset: $currentIndex) === 1 => [TokenType::String, $matches[0], $currentIndex],
+                preg_match(self::NUMBER_LITERAL, $this->input, $matches, offset: $currentIndex) === 1 => [TokenType::Number, $matches[0], $currentIndex],
+                preg_match(self::IDENTIFIER, $this->input, $matches, offset: $currentIndex) === 1 => [TokenType::Identifier, $matches[0], $currentIndex],
+                preg_match(self::DOTDOT, $this->input, $matches, offset: $currentIndex) === 1 => [TokenType::DotDot, $matches[0], $currentIndex],
+                array_key_exists($this->input[$currentIndex], self::SPECIAL_CHARACTERS) => [self::SPECIAL_CHARACTERS[$this->input[$currentIndex]], $this->input[$currentIndex], $currentIndex],
                 default => SyntaxException::unexpectedCharacter($this->input[$currentIndex]),
             };
 
@@ -79,7 +85,7 @@ public function tokenize(): array
             $currentIndex += strlen($token[1]);
         }
 
-        $output[] = [TokenType::EndOfString];
+        $output[] = [TokenType::EndOfString, '', $currentIndex];
         $this->result = $output;
 
         return $output;

src/Parse/Parser.php

@@ -7,7 +7,7 @@
 class Parser
 {
     /**
-     * @var array<int, array{0: TokenType, 1: string}>
+     * @var array<array{0:TokenType, 1:string, 2:int}>
      */
     protected array $tokens;
 
@@ -16,7 +16,7 @@ class Parser
     /**
      * @throws SyntaxException
      */
-    public function __construct(string $input)
+    public function __construct(protected string $input)
     {
         $this->tokens = (new Lexer($input))->tokenize();
         $this->pointer = 0;
@@ -27,6 +27,9 @@ public function jump(int $int): void
         $this->pointer = $int;
     }
 
+    /**
+     * @throws SyntaxException
+     */
     public function consume(TokenType $type = null): string
     {
         $token = $this->tokens[$this->pointer];
@@ -49,12 +52,15 @@ public function consumeOrFalse(TokenType $type): string|false
         }
     }
 
-    public function idOrFalse(string $identifier): string|false
+    /**
+     * @throws SyntaxException
+     */
+    public function id(string $identifier): string|false
     {
         $token = $this->tokens[$this->pointer];
 
         if ($token === null || $token[0] !== TokenType::Identifier) {
-            return false;
+            throw SyntaxException::unexpectedTokenType(TokenType::Identifier, $token[0]);
         }
 
         if ($token[1] !== $identifier) {
@@ -66,9 +72,18 @@ public function idOrFalse(string $identifier): string|false
         return $token[1];
     }
 
+    public function idOrFalse(string $identifier): string|false
+    {
+        try {
+            return $this->id($identifier);
+        } catch (SyntaxException) {
+            return false;
+        }
+    }
+
     public function look(TokenType $type, int $offset = 0): bool
     {
-        $token = $this->tokens[$this->pointer + $offset] ?? null;
+        $token = $this->tokens[$this->pointer + $offset];
 
         if ($token === null) {
             return false;
@@ -77,6 +92,9 @@ public function look(TokenType $type, int $offset = 0): bool
         return $token[0] === $type;
     }
 
+    /**
+     * @throws SyntaxException
+     */
     public function expression(): string
     {
         $token = $this->tokens[$this->pointer];
@@ -98,16 +116,51 @@ public function expression(): string
         };
     }
 
-    public function argument(): string
+    /**
+     * @return array<string, string>
+     */
+    public function attributes(TokenType $separator = null): array
     {
-        $output = match (true) {
-            $this->look(TokenType::Identifier) && $this->look(TokenType::Colon, 1) => $this->consume().$this->consume().' ',
-            default => ''
-        };
+        $attributes = [];
+
+        if ($this->look(TokenType::EndOfString) !== false) {
+            return $attributes;
+        }
 
-        return $output.$this->expression();
+        do {
+            $attribute = $this->consume(TokenType::Identifier);
+            $this->consume(TokenType::Colon);
+            $attributes[$attribute] = $this->expression();
+
+            $shouldContinue = match (true) {
+                $separator === null => $this->look(TokenType::EndOfString) === false,
+                default => $this->consumeOrFalse($separator) !== false
+            };
+        } while ($shouldContinue);
+
+        return $attributes;
     }
 
+    /**
+     * @return array<string, string>|string
+     *
+     * @throws SyntaxException
+     */
+    public function argument(): string|array
+    {
+        if ($this->look(TokenType::Identifier) && $this->look(TokenType::Colon, 1)) {
+            $identifier = $this->consume(TokenType::Identifier);
+            $this->consume(TokenType::Colon);
+
+            return [$identifier => $this->expression()];
+        }
+
+        return $this->expression();
+    }
+
+    /**
+     * @throws SyntaxException
+     */
     protected function variableLookups(): string
     {
         $output = match (true) {
@@ -125,4 +178,11 @@ protected function variableLookups(): string
 
         return $output.$this->variableLookups();
     }
+
+    public function toString(): string
+    {
+        $current = $this->tokens[$this->pointer];
+
+        return substr($this->input, $current[2] ?? 0);
+    }
 }

src/Parse/TokenType.php

@@ -20,6 +20,7 @@ enum TokenType
     case CloseRound;
     case QuestionMark;
     case Dash;
+    case Equals;
 
     public function toString(): string
     {
@@ -40,6 +41,7 @@ public function toString(): string
             self::CloseRound => 'CloseRound',
             self::QuestionMark => 'QuestionMark',
             self::Dash => 'Dash',
+            self::Equals => 'Equals',
         };
     }
 }