Earley now uses OrderedSet for better output stability

lark/lark.py

@@ -62,19 +62,20 @@ class LarkOptions(Serialize):
     regex: bool
     g_regex_flags: int
     keep_all_tokens: bool
-    tree_class: Any
+    tree_class: Optional[Callable[[str, List], Any]]
     parser: _ParserArgType
     lexer: _LexerArgType
     ambiguity: 'Literal["auto", "resolve", "explicit", "forest"]'
     postlex: Optional[PostLex]
     priority: 'Optional[Literal["auto", "normal", "invert"]]'
     lexer_callbacks: Dict[str, Callable[[Token], Token]]
     use_bytes: bool
+    ordered_sets: bool
     edit_terminals: Optional[Callable[[TerminalDef], TerminalDef]]
     import_paths: 'List[Union[str, Callable[[Union[None, str, PackageResource], str], Tuple[str, str]]]]'
     source_path: Optional[str]
 
-    OPTIONS_DOC = """
+    OPTIONS_DOC = r"""
     **===  General Options  ===**
 
     start
@@ -141,6 +142,8 @@ class LarkOptions(Serialize):
             Dictionary of callbacks for the lexer. May alter tokens during lexing. Use with caution.
     use_bytes
             Accept an input of type ``bytes`` instead of ``str``.
+    ordered_sets
+            Should Earley use ordered-sets to achieve stable output (~10% slower than regular sets. Default: True)
     edit_terminals
             A callback for editing the terminals before parse.
     import_paths
@@ -179,6 +182,7 @@ class LarkOptions(Serialize):
         'edit_terminals': None,
         'g_regex_flags': 0,
         'use_bytes': False,
+        'ordered_sets': True,
         'import_paths': [],
         'source_path': None,
         '_plugins': {},

lark/load_grammar.py

@@ -11,7 +11,7 @@
 from contextlib import suppress
 from typing import List, Tuple, Union, Callable, Dict, Optional, Sequence
 
-from .utils import bfs, logger, classify_bool, is_id_continue, is_id_start, bfs_all_unique, small_factors
+from .utils import bfs, logger, classify_bool, is_id_continue, is_id_start, bfs_all_unique, small_factors, OrderedSet
 from .lexer import Token, TerminalDef, PatternStr, PatternRE
 
 from .parse_tree_builder import ParseTreeBuilder
@@ -781,7 +781,7 @@ def compile(self, start, terminals_to_keep):
                     assert len({(r.alias, r.order, r.options) for r in dups}) == len(dups)
 
             # Remove duplicates
-            compiled_rules = list(set(compiled_rules))
+            compiled_rules = list(OrderedSet(compiled_rules))
 
         # Filter out unused rules
         while True:

lark/parser_frontends.py

@@ -208,7 +208,8 @@ def create_earley_parser(lexer_conf: LexerConf, parser_conf: ParserConf, options
     else:
         f = create_earley_parser__basic
 
-    return f(lexer_conf, parser_conf, resolve_ambiguity=resolve_ambiguity, debug=debug, tree_class=tree_class, **extra)
+    return f(lexer_conf, parser_conf, resolve_ambiguity=resolve_ambiguity,
+             debug=debug, tree_class=tree_class, ordered_sets=options.ordered_sets, **extra)
 
 
 

lark/parsers/earley.py

@@ -9,34 +9,37 @@
 is explained here: https://lark-parser.readthedocs.io/en/latest/_static/sppf/sppf.html
 """
 
-import typing
-
+from typing import TYPE_CHECKING, Callable, Optional, List, Any
 from collections import deque
 
 from ..lexer import Token
 from ..tree import Tree
 from ..exceptions import UnexpectedEOF, UnexpectedToken
-from ..utils import logger
+from ..utils import logger, OrderedSet
 from .grammar_analysis import GrammarAnalyzer
 from ..grammar import NonTerminal
 from .earley_common import Item
-from .earley_forest import ForestSumVisitor, SymbolNode, TokenNode, ForestToParseTree
+from .earley_forest import ForestSumVisitor, SymbolNode, StableSymbolNode, TokenNode, ForestToParseTree
 
-if typing.TYPE_CHECKING:
+if TYPE_CHECKING:
     from ..common import LexerConf, ParserConf
 
 class Parser:
     lexer_conf: 'LexerConf'
     parser_conf: 'ParserConf'
     debug: bool
 
-    def __init__(self, lexer_conf: 'LexerConf', parser_conf: 'ParserConf', term_matcher, resolve_ambiguity=True, debug=False, tree_class=Tree):
+    def __init__(self, lexer_conf: 'LexerConf', parser_conf: 'ParserConf', term_matcher: Callable,
+                 resolve_ambiguity: bool=True, debug: bool=False,
+                 tree_class: Optional[Callable[[str, List], Any]]=Tree, ordered_sets: bool=True):
         analysis = GrammarAnalyzer(parser_conf)
         self.lexer_conf = lexer_conf
         self.parser_conf = parser_conf
         self.resolve_ambiguity = resolve_ambiguity
         self.debug = debug
-        self.tree_class = tree_class
+        self.Tree = tree_class
+        self.Set = OrderedSet if ordered_sets else set
+        self.SymbolNode = StableSymbolNode if ordered_sets else SymbolNode
 
         self.FIRST = analysis.FIRST
         self.NULLABLE = analysis.NULLABLE
@@ -94,7 +97,7 @@ def predict_and_complete(self, i, to_scan, columns, transitives):
             if item.is_complete:   ### (item.s == string)
                 if item.node is None:
                     label = (item.s, item.start, i)
-                    item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, SymbolNode(*label))
+                    item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, self.SymbolNode(*label))
                     item.node.add_family(item.s, item.rule, item.start, None, None)
 
                 # create_leo_transitives(item.rule.origin, item.start)
@@ -109,7 +112,7 @@ def predict_and_complete(self, i, to_scan, columns, transitives):
 
                     new_item = Item(transitive.rule, transitive.ptr, transitive.start)
                     label = (root_transitive.s, root_transitive.start, i)
-                    new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, SymbolNode(*label))
+                    new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, self.SymbolNode(*label))
                     new_item.node.add_path(root_transitive, item.node)
                     if new_item.expect in self.TERMINALS:
                         # Add (B :: aC.B, h, y) to Q
@@ -133,7 +136,7 @@ def predict_and_complete(self, i, to_scan, columns, transitives):
                     for originator in originators:
                         new_item = originator.advance()
                         label = (new_item.s, originator.start, i)
-                        new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, SymbolNode(*label))
+                        new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, self.SymbolNode(*label))
                         new_item.node.add_family(new_item.s, new_item.rule, i, originator.node, item.node)
                         if new_item.expect in self.TERMINALS:
                             # Add (B :: aC.B, h, y) to Q
@@ -154,7 +157,7 @@ def predict_and_complete(self, i, to_scan, columns, transitives):
                 if item.expect in held_completions:
                     new_item = item.advance()
                     label = (new_item.s, item.start, i)
-                    new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, SymbolNode(*label))
+                    new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, self.SymbolNode(*label))
                     new_item.node.add_family(new_item.s, new_item.rule, new_item.start, item.node, held_completions[item.expect])
                     new_items.append(new_item)
 
@@ -190,13 +193,13 @@ def scan(i, token, to_scan):
             Earley predictor, based on the previously completed tokens.
             This ensures that at each phase of the parse we have a custom
             lexer context, allowing for more complex ambiguities."""
-            next_to_scan = set()
-            next_set = set()
+            next_to_scan = self.Set()
+            next_set = self.Set()
             columns.append(next_set)
             transitives.append({})
             node_cache = {}
 
-            for item in set(to_scan):
+            for item in self.Set(to_scan):
                 if match(item.expect, token):
                     new_item = item.advance()
                     label = (new_item.s, new_item.start, i)
@@ -209,7 +212,7 @@ def scan(i, token, to_scan):
                     # ForestSumVisitor after the basic lexer has already
                     # "used up" the terminal priorities
                     token_node = TokenNode(token, term, priority=0)
-                    new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, SymbolNode(*label))
+                    new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, self.SymbolNode(*label))
                     new_item.node.add_family(new_item.s, item.rule, new_item.start, item.node, token_node)
 
                     if new_item.expect in self.TERMINALS:
@@ -260,8 +263,8 @@ def parse(self, lexer, start):
         assert start, start
         start_symbol = NonTerminal(start)
 
-        columns = [set()]
-        to_scan = set()     # The scan buffer. 'Q' in E.Scott's paper.
+        columns = [self.Set()]
+        to_scan = self.Set()     # The scan buffer. 'Q' in E.Scott's paper.
 
         ## Predict for the start_symbol.
         # Add predicted items to the first Earley set (for the predictor) if they
@@ -296,9 +299,9 @@ def parse(self, lexer, start):
         if len(solutions) > 1:
             assert False, 'Earley should not generate multiple start symbol items!'
 
-        if self.tree_class is not None:
+        if self.Tree is not None:
             # Perform our SPPF -> AST conversion
-            transformer = ForestToParseTree(self.tree_class, self.callbacks, self.forest_sum_visitor and self.forest_sum_visitor(), self.resolve_ambiguity)
+            transformer = ForestToParseTree(self.Tree, self.callbacks, self.forest_sum_visitor and self.forest_sum_visitor(), self.resolve_ambiguity)
             return transformer.transform(solutions[0])
 
         # return the root of the SPPF

lark/parsers/earley_forest.py

@@ -7,6 +7,7 @@
 https://web.archive.org/web/20190616123959/http://www.bramvandersanden.com/post/2014/06/shared-packed-parse-forest/
 """
 
+from typing import Type, AbstractSet
 from random import randint
 from collections import deque
 from operator import attrgetter
@@ -15,7 +16,7 @@
 
 from ..parse_tree_builder import AmbiguousIntermediateExpander
 from ..visitors import Discard
-from ..utils import logger
+from ..utils import logger, OrderedSet
 from ..tree import Tree
 
 class ForestNode:
@@ -44,13 +45,14 @@ class SymbolNode(ForestNode):
         is_intermediate: True if this node is an intermediate node.
         priority: The priority of the node's symbol.
     """
+    Set: Type[AbstractSet] = set   # Overridden by StableSymbolNode
     __slots__ = ('s', 'start', 'end', '_children', 'paths', 'paths_loaded', 'priority', 'is_intermediate', '_hash')
     def __init__(self, s, start, end):
         self.s = s
         self.start = start
         self.end = end
-        self._children = set()
-        self.paths = set()
+        self._children = self.Set()
+        self.paths = self.Set()
         self.paths_loaded = False
 
         ### We use inf here as it can be safely negated without resorting to conditionals,
@@ -68,7 +70,7 @@ def add_path(self, transitive, node):
     def load_paths(self):
         for transitive, node in self.paths:
             if transitive.next_titem is not None:
-                vn = SymbolNode(transitive.next_titem.s, transitive.next_titem.start, self.end)
+                vn = type(self)(transitive.next_titem.s, transitive.next_titem.start, self.end)
                 vn.add_path(transitive.next_titem, node)
                 self.add_family(transitive.reduction.rule.origin, transitive.reduction.rule, transitive.reduction.start, transitive.reduction.node, vn)
             else:
@@ -110,6 +112,10 @@ def __repr__(self):
             symbol = self.s.name
         return "({}, {}, {}, {})".format(symbol, self.start, self.end, self.priority)
 
+class StableSymbolNode(SymbolNode):
+    "A version of SymbolNode that uses OrderedSet for output stability"
+    Set = OrderedSet
+
 class PackedNode(ForestNode):
     """
     A Packed Node represents a single derivation in a symbol node.

lark/parsers/xearley.py

@@ -1,4 +1,4 @@
-"""This module implements an experimental Earley parser with a dynamic lexer
+"""This module implements an Earley parser with a dynamic lexer
 
 The core Earley algorithm used here is based on Elizabeth Scott's implementation, here:
     https://www.sciencedirect.com/science/article/pii/S1571066108001497
@@ -14,19 +14,25 @@
 Earley's power in parsing any CFG.
 """
 
+from typing import TYPE_CHECKING, Callable, Optional, List, Any
 from collections import defaultdict
 
 from ..tree import Tree
 from ..exceptions import UnexpectedCharacters
 from ..lexer import Token
 from ..grammar import Terminal
 from .earley import Parser as BaseParser
-from .earley_forest import SymbolNode, TokenNode
+from .earley_forest import TokenNode
 
+if TYPE_CHECKING:
+    from ..common import LexerConf, ParserConf
 
 class Parser(BaseParser):
-    def __init__(self, lexer_conf, parser_conf, term_matcher, resolve_ambiguity=True, complete_lex = False, debug=False, tree_class=Tree):
-        BaseParser.__init__(self, lexer_conf, parser_conf, term_matcher, resolve_ambiguity, debug, tree_class)
+    def __init__(self, lexer_conf: 'LexerConf', parser_conf: 'ParserConf', term_matcher: Callable,
+                 resolve_ambiguity: bool=True, complete_lex: bool=False, debug: bool=False,
+                 tree_class: Optional[Callable[[str, List], Any]]=Tree, ordered_sets: bool=True):
+        BaseParser.__init__(self, lexer_conf, parser_conf, term_matcher, resolve_ambiguity,
+                            debug, tree_class, ordered_sets)
         self.ignore = [Terminal(t) for t in lexer_conf.ignore]
         self.complete_lex = complete_lex
 
@@ -49,7 +55,7 @@ def scan(i, to_scan):
             # they complete, we push all tokens into a buffer (delayed_matches), to
             # be held possibly for a later parse step when we reach the point in the
             # input stream at which they complete.
-            for item in set(to_scan):
+            for item in self.Set(to_scan):
                 m = match(item.expect, stream, i)
                 if m:
                     t = Token(item.expect.name, m.group(0), i, text_line, text_column)
@@ -81,8 +87,8 @@ def scan(i, to_scan):
                     # If we're ignoring up to the end of the file, # carry over the start symbol if it already completed.
                     delayed_matches[m.end()].extend([(item, i, None) for item in columns[i] if item.is_complete and item.s == start_symbol])
 
-            next_to_scan = set()
-            next_set = set()
+            next_to_scan = self.Set()
+            next_set = self.Set()
             columns.append(next_set)
             transitives.append({})
 
@@ -100,7 +106,7 @@ def scan(i, to_scan):
                     new_item = item.advance()
                     label = (new_item.s, new_item.start, i)
                     token_node = TokenNode(token, terminals[token.type])
-                    new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, SymbolNode(*label))
+                    new_item.node = node_cache[label] if label in node_cache else node_cache.setdefault(label, self.SymbolNode(*label))
                     new_item.node.add_family(new_item.s, item.rule, new_item.start, item.node, token_node)
                 else:
                     new_item = item

lark/utils.py

@@ -2,7 +2,7 @@
 import os
 from itertools import product
 from collections import deque
-from typing import Callable, Iterator, List, Optional, Tuple, Type, TypeVar, Union, Dict, Any, Sequence, Iterable
+from typing import Callable, Iterator, List, Optional, Tuple, Type, TypeVar, Union, Dict, Any, Sequence, Iterable, AbstractSet
 
 ###{standalone
 import sys, re
@@ -328,3 +328,30 @@ def small_factors(n: int, max_factor: int) -> List[Tuple[int, int]]:
         if a + b <= max_factor:
             return small_factors(r, max_factor) + [(a, b)]
     assert False, "Failed to factorize %s" % n
+
+
+class OrderedSet(AbstractSet[T]):
+    """A minimal OrderedSet implementation, using a dictionary.
+
+    (relies on the dictionary being ordered)
+    """
+    def __init__(self, items: Iterable[T] =()):
+        self.d = dict.fromkeys(items)
+
+    def __contains__(self, item: Any) -> bool:
+        return item in self.d
+
+    def add(self, item: T):
+        self.d[item] = None
+
+    def __iter__(self) -> Iterator[T]:
+        return iter(self.d)
+
+    def remove(self, item: T):
+        del self.d[item]
+
+    def __bool__(self):
+        return bool(self.d)
+
+    def __len__(self) -> int:
+        return len(self.d)