cucumber-expressions: Use a parser to parse Cucumber Expressions

cucumber-expressions/Makefile

@@ -1 +1,2 @@
+LANGUAGES ?= go javascript ruby java
 include default.mk

cucumber-expressions/README.md

@@ -1,9 +1,71 @@
-See [website docs](https://cucumber.io/docs/cucumber/cucumber-expressions/) for more details.
+See [website docs](https://cucumber.io/docs/cucumber/cucumber-expressions/) 
+for more details.
+
+## Grammar ##
+
+A Cucumber Expression has the following AST:
+
+```
+cucumber-expression :=  ( alternation | optional | parameter | text )*
+alternation := (?<=left-boundary) + alternative* + ( '/' + alternative* )+ + (?=right-boundary)
+left-boundary := whitespace | } | ^
+right-boundary := whitespace | { | $
+alternative: = optional | parameter | text 
+optional := '(' + option* + ')'
+option := optional | parameter | text
+parameter := '{' + name* + '}'
+name := whitespace | .
+text := whitespace | ')' | '}' | .
+```
+
+The AST is constructed from the following tokens:
+```
+escape := '\'
+token := whitespace | '(' | ')' | '{' | '}' | '/' | .
+. := any non-reserved codepoint
+```
+
+Note:
+ * While `parameter` is allowed to appear as part of `alternative` and 
+  `option` in the AST, such an AST is not a valid a Cucumber Expression.
+ * While `optional` is allowed to appear as part of `option` in the AST,
+   such an AST is not a valid a Cucumber Expression.
+ * ASTs with empty alternatives or alternatives that only
+   contain an optional are valid ASTs but invalid Cucumber Expressions.
+ * All escaped tokens (tokens starting with a backslash) are rewritten to their
+   unescaped equivalent after parsing.
+
+### Production Rules
+
+The AST can be rewritten into a regular expression by the following production
+rules:
+
+```
+cucumber-expression -> '^' + rewrite(node[0]) + ... + rewrite(node[n-1]) + '$'
+alternation -> '(?:' + rewrite(node[0]) +'|' + ...  +'|' + rewerite(node[n-1]) + ')'
+alternative -> rewrite(node[0]) + ... + rewrite(node[n-1])
+optional -> '(?:' + rewrite(node[0]) + ... + rewrite(node[n-1]) + ')?'
+parameter -> {
+     parameter_name := node[0].text + ... + node[n-1].text
+     parameter_pattern := parameter_type_registry[parameter_name] 
+    '((?:' + parameter_pattern[0] + ')|(?:' ... + ')|(?:' + parameter_pattern[n-1] +  '))'
+} 
+text -> {
+ escape_regex := escape '^', `$`, `[`, `]`, `(`, `)` `\`, `{`, `}`,  `.`, `|`, `?`, `*`, `+`
+ escape_regex(token.text)
+}
+```
 
 ## Acknowledgements
 
 The Cucumber Expression syntax is inspired by similar expression syntaxes in
-other BDD tools, such as [Turnip](https://github.com/jnicklas/turnip), [Behat](https://github.com/Behat/Behat) and [Behave](https://github.com/behave/behave).
+other BDD tools, such as [Turnip](https://github.com/jnicklas/turnip), 
+[Behat](https://github.com/Behat/Behat) and 
+[Behave](https://github.com/behave/behave).
 
 Big thanks to Jonas Nicklas, Konstantin Kudryashov and Jens Engel for
 implementing those libraries.
+
+The [Tiny-Compiler-Parser tutorial](https://blog.klipse.tech/javascript/2017/02/08/tiny-compiler-parser.html)
+by [Yehonathan Sharvit](https://github.com/viebel) inspired the design of the
+Cucumber expression parser. 

cucumber-expressions/examples.txt

@@ -2,7 +2,7 @@ I have {int} cuke(s)
 I have 22 cukes
 [22]
 ---
-I have {int} cuke(s) and some \[]^$.|?*+
+I have {int} cuke(s) and some \\[]^$.|?*+
 I have 1 cuke and some \[]^$.|?*+
 [1]
 ---
@@ -37,3 +37,7 @@ a purchase for $33
 Some ${float} of cukes at {int}° Celsius
 Some $3.50 of cukes at 42° Celsius
 [3.5,42]
+---
+I select the {int}st/nd/rd/th Cucumber
+I select the 3rd Cucumber
+[3]

cucumber-expressions/go/.rsync

@@ -2,3 +2,4 @@
 ../../.templates/github/ .github/
 ../../.templates/go/ .
 ../examples.txt examples.txt
+../testdata .

cucumber-expressions/go/ast.go

@@ -0,0 +1,147 @@
+package cucumberexpressions
+
+import (
+	"strings"
+	"unicode"
+)
+
+const escapeCharacter rune = '\\'
+const alternationCharacter rune = '/'
+const beginParameterCharacter rune = '{'
+const endParameterCharacter rune = '}'
+const beginOptionalCharacter rune = '('
+const endOptionalCharacter rune = ')'
+
+type nodeType string
+
+const (
+	textNode        nodeType = "TEXT_NODE"
+	optionalNode    nodeType = "OPTIONAL_NODE"
+	alternationNode nodeType = "ALTERNATION_NODE"
+	alternativeNode nodeType = "ALTERNATIVE_NODE"
+	parameterNode   nodeType = "PARAMETER_NODE"
+	expressionNode  nodeType = "EXPRESSION_NODE"
+)
+
+type node struct {
+	NodeType nodeType `json:"type"`
+	Start    int      `json:"start"`
+	End      int      `json:"end"`
+	Token    string   `json:"token"`
+	Nodes    []node   `json:"nodes"`
+}
+
+func (node node) text() string {
+	builder := strings.Builder{}
+	builder.WriteString(node.Token)
+
+	if node.Nodes == nil {
+		return builder.String()
+	}
+
+	for _, c := range node.Nodes {
+		builder.WriteString(c.text())
+	}
+	return builder.String()
+}
+
+type tokenType string
+
+const (
+	startOfLine    tokenType = "START_OF_LINE"
+	endOfLine      tokenType = "END_OF_LINE"
+	whiteSpace     tokenType = "WHITE_SPACE"
+	beginOptional  tokenType = "BEGIN_OPTIONAL"
+	endOptional    tokenType = "END_OPTIONAL"
+	beginParameter tokenType = "BEGIN_PARAMETER"
+	endParameter   tokenType = "END_PARAMETER"
+	alternation    tokenType = "ALTERNATION"
+	text           tokenType = "TEXT"
+)
+
+type token struct {
+	Text      string    `json:"text"`
+	TokenType tokenType `json:"type"`
+	Start     int       `json:"start"`
+	End       int       `json:"end"`
+}
+
+var nullNode = node{textNode, -1, -1, "", nil}
+
+func isEscapeCharacter(r rune) bool {
+	return r == escapeCharacter
+}
+
+func canEscape(r rune) bool {
+	if unicode.Is(unicode.White_Space, r) {
+		return true
+	}
+	switch r {
+	case escapeCharacter:
+		return true
+	case alternationCharacter:
+		return true
+	case beginParameterCharacter:
+		return true
+	case endParameterCharacter:
+		return true
+	case beginOptionalCharacter:
+		return true
+	case endOptionalCharacter:
+		return true
+	}
+	return false
+}
+
+func typeOf(r rune) (tokenType, error) {
+	if unicode.Is(unicode.White_Space, r) {
+		return whiteSpace, nil
+	}
+	switch r {
+	case alternationCharacter:
+		return alternation, nil
+	case beginParameterCharacter:
+		return beginParameter, nil
+	case endParameterCharacter:
+		return endParameter, nil
+	case beginOptionalCharacter:
+		return beginOptional, nil
+	case endOptionalCharacter:
+		return endOptional, nil
+	}
+	return text, nil
+}
+
+func symbol(tokenType tokenType) string {
+	switch tokenType {
+	case beginOptional:
+		return string(beginOptionalCharacter)
+	case endOptional:
+		return string(endOptionalCharacter)
+	case beginParameter:
+		return string(beginParameterCharacter)
+	case endParameter:
+		return string(endParameterCharacter)
+	case alternation:
+		return string(alternationCharacter)
+	}
+
+	return ""
+}
+
+func purpose(tokenType tokenType) string {
+	switch tokenType {
+	case beginOptional:
+		return "optional text"
+	case endOptional:
+		return "optional text"
+	case beginParameter:
+		return "a parameter"
+	case endParameter:
+		return "optional text"
+	case alternation:
+		return "alternation"
+	}
+
+	return ""
+}