From f424de24885b9dbf2039aa4222d1fbbdd0a495ec Mon Sep 17 00:00:00 2001 From: "M.P. Korstanje" Date: Sat, 3 Aug 2024 00:02:56 +0200 Subject: [PATCH] go: generate formatted parser The go parser is currently generated and then formatted. This however requires that go is installed when generating the parser. This added dependency makes code generation more complicated than it needs to be as this is yet another tool people need to install. This in turn makes #154 more complicated than it needs to be. By making the code generator generate formatted code we avoid this problem all together. --- go/Makefile | 1 - go/parser.go | 1 - go/parser.go.razor | 386 +++++++++++++++++++++++---------------------- 3 files changed, 195 insertions(+), 193 deletions(-) diff --git a/go/Makefile b/go/Makefile index d7d420854..6c5b6ab22 100644 --- a/go/Makefile +++ b/go/Makefile @@ -55,7 +55,6 @@ dialects_builtin.go: ../gherkin-languages.json dialects_builtin.go.jq $(GHERKIN_PARSER): $(GHERKIN_RAZOR) ../gherkin.berp berp -g ../gherkin.berp -t $< -o $@ --noBOM - gofmt -w $@ acceptance/testdata/%.tokens: ../testdata/% ../testdata/%.tokens mkdir -p $(@D) diff --git a/go/parser.go b/go/parser.go index 570e4babe..ed6dddba7 100644 --- a/go/parser.go +++ b/go/parser.go @@ -104,7 +104,6 @@ type RuleType int const ( RuleTypeNone RuleType = iota - RuleTypeEOF RuleTypeEmpty RuleTypeComment diff --git a/go/parser.go.razor b/go/parser.go.razor index 7b173db1f..2e772d8a9 100644 --- a/go/parser.go.razor +++ b/go/parser.go.razor @@ -3,35 +3,36 @@ { switch(production.Type) { - case ProductionRuleType.Start: - @:ctxt.startRule(@Raw("RuleType" + production.RuleName.Replace("#", ""))); - break; - case ProductionRuleType.End: - @:ctxt.endRule(@Raw("RuleType" + production.RuleName.Replace("#", ""))); - break; - case ProductionRuleType.Process: - @:ctxt.build(token); - break; + case ProductionRuleType.Start: + @:ctxt.startRule(@Raw("RuleType" + production.RuleName.Replace("#", ""))) + break; + case ProductionRuleType.End: + @:ctxt.endRule(@Raw("RuleType" + production.RuleName.Replace("#", ""))) + break; + case ProductionRuleType.Process: + @:ctxt.build(token) + break; } } @helper HandleParserError(IEnumerable expectedTokens, State state) { - // var stateComment = "State: @state.Id - @Raw(state.Comment)" - var expectedTokens = []string{"@Raw(string.Join("\", \"", expectedTokens))"} - if line.IsEof() { - err = &parseError{ - msg: fmt.Sprintf("unexpected end of file, expected: %s", strings.Join(expectedTokens,", ")), - loc: &Location{Line: line.LineNumber, Column: 0}, - } - } else { - err = &parseError{ - msg: fmt.Sprintf("expected: %s, got '%s'", strings.Join(expectedTokens,", "), line.LineText), - loc: &Location{Line: line.LineNumber, Column: line.Indent() + 1}, - } - } - // if (ctxt.p.stopAtFirstError) throw error; - //ctxt.addError(err) - return @state.Id, err} + // var stateComment = "State: @state.Id - @Raw(state.Comment)" + var expectedTokens = []string{"@Raw(string.Join("\", \"", expectedTokens))"} + if line.IsEof() { + err = &parseError{ + msg: fmt.Sprintf("unexpected end of file, expected: %s", strings.Join(expectedTokens, ", ")), + loc: &Location{Line: line.LineNumber, Column: 0}, + } + } else { + err = &parseError{ + msg: fmt.Sprintf("expected: %s, got '%s'", strings.Join(expectedTokens, ", "), line.LineText), + loc: &Location{Line: line.LineNumber, Column: line.Indent() + 1}, + } + } + // if (ctxt.p.stopAtFirstError) throw error; + //ctxt.addError(err) + return @state.Id, err +} @helper MatchToken(TokenType tokenType) {ctxt.match@(tokenType)(line)} @helper IsMatchToken(TokenType tokenType) @@ -44,62 +45,63 @@ package gherkin import ( - "fmt" - "strings" + "fmt" + "strings" ) type TokenType int const ( - TokenTypeNone TokenType = iota - @foreach(var rule in Model.RuleSet.TokenRules) - { @Raw("TokenType" + rule.Name.Replace("#", "")) + TokenTypeNone TokenType = iota + @foreach(var rule in Model.RuleSet.TokenRules) +{ @Raw("TokenType" + rule.Name.Replace("#", "")) } ) func tokenTypeForRule(rt RuleType) TokenType { - return TokenTypeNone + return TokenTypeNone } func (t TokenType) Name() string { - switch t { - @foreach(var rule in Model.RuleSet.TokenRules) - { case @Raw("TokenType" + rule.Name.Replace("#", "")): return "@Raw(rule.Name.Replace("#", ""))" + switch t { + @foreach(var rule in Model.RuleSet.TokenRules) +{ case @Raw("TokenType" + rule.Name.Replace("#", "")): + return "@Raw(rule.Name.Replace("#", ""))" } - } - return "" + } + return "" } func (t TokenType) RuleType() RuleType { - switch t { - @foreach(var rule in Model.RuleSet.TokenRules) - { case @Raw("TokenType" + rule.Name.Replace("#", "")): return @Raw("RuleType" + rule.Name.Replace("#", "")) + switch t { + @foreach(var rule in Model.RuleSet.TokenRules) +{ case @Raw("TokenType" + rule.Name.Replace("#", "")): + return @Raw("RuleType" + rule.Name.Replace("#", "")) } - } - return RuleTypeNone + } + return RuleTypeNone } - type RuleType int const ( - RuleTypeNone RuleType = iota - - @foreach(var rule in Model.RuleSet.Where(r => !r.TempRule)) - { @Raw("RuleType" + rule.Name.Replace("#", "")) + RuleTypeNone RuleType = iota + @foreach(var rule in Model.RuleSet.Where(r => !r.TempRule)) +{ @Raw("RuleType" + rule.Name.Replace("#", "")) } ) func (t RuleType) IsEOF() bool { - return t == RuleTypeEOF + return t == RuleTypeEOF } func (t RuleType) Name() string { - switch t { - @foreach(var rule in Model.RuleSet.Where(r => !r.TempRule)) - { case @Raw("RuleType" + rule.Name.Replace("#", "")): return "@Raw(rule.Name)" + switch t { + @foreach(var rule in Model.RuleSet.Where(r => !r.TempRule)) +{ case @Raw("RuleType" + rule.Name.Replace("#", "")): + return "@Raw(rule.Name)" } - } - return "" + } + return "" } type Location struct { @@ -108,202 +110,204 @@ type Location struct { } type parseError struct { - msg string - loc *Location + msg string + loc *Location } func (a *parseError) Error() string { - return fmt.Sprintf("(%d:%d): %s", a.loc.Line, a.loc.Column, a.msg) + return fmt.Sprintf("(%d:%d): %s", a.loc.Line, a.loc.Column, a.msg) } type parseErrors []error + func (pe parseErrors) Error() string { - var ret = []string{"Parser errors:"} - for i := range pe { - ret = append(ret, pe[i].Error()) - } - return strings.Join(ret,"\n") + var ret = []string{"Parser errors:"} + for i := range pe { + ret = append(ret, pe[i].Error()) + } + return strings.Join(ret, "\n") } func (p *parser) Parse(s Scanner, m Matcher) (err error) { - p.builder.Reset() - m.Reset() - ctxt := &parseContext{p,s,p.builder,m,nil,nil} - var state int - ctxt.startRule(@Raw("RuleType" + @Model.RuleSet.StartRule.Name)) - for { - gl, eof, err := ctxt.scan() - if err != nil { - ctxt.addError(err) - if p.stopAtFirstError { - break - } - } - state, err = ctxt.match(state, gl) - if err != nil { - ctxt.addError(err) - if p.stopAtFirstError { - break - } - } - if eof { - // done! \o/ - break - } - } - ctxt.endRule(@Raw("RuleType" + @Model.RuleSet.StartRule.Name)) - if len(ctxt.errors) > 0 { - return ctxt.errors - } - return + p.builder.Reset() + m.Reset() + ctxt := &parseContext{p, s, p.builder, m, nil, nil} + var state int + ctxt.startRule(@Raw("RuleType" + @Model.RuleSet.StartRule.Name)) + for { + gl, eof, err := ctxt.scan() + if err != nil { + ctxt.addError(err) + if p.stopAtFirstError { + break + } + } + state, err = ctxt.match(state, gl) + if err != nil { + ctxt.addError(err) + if p.stopAtFirstError { + break + } + } + if eof { + // done! \o/ + break + } + } + ctxt.endRule(@Raw("RuleType" + @Model.RuleSet.StartRule.Name)) + if len(ctxt.errors) > 0 { + return ctxt.errors + } + return } type parseContext struct { - p *parser - s Scanner - b Builder - m Matcher - queue []*scanResult - errors parseErrors + p *parser + s Scanner + b Builder + m Matcher + queue []*scanResult + errors parseErrors } func (ctxt *parseContext) addError(e error) { - ctxt.errors = append(ctxt.errors, e); - // if (p.errors.length > 10) - // throw Errors.CompositeParserException.create(p.errors); + ctxt.errors = append(ctxt.errors, e) + // if (p.errors.length > 10) + // throw Errors.CompositeParserException.create(p.errors); } type scanResult struct { - line *Line - atEof bool - err error + line *Line + atEof bool + err error } + func (ctxt *parseContext) scan() (*Line, bool, error) { - l := len(ctxt.queue) - if l > 0 { - x := ctxt.queue[0] - ctxt.queue = ctxt.queue[1:] - return x.line, x.atEof, x.err - } - return ctxt.s.Scan() + l := len(ctxt.queue) + if l > 0 { + x := ctxt.queue[0] + ctxt.queue = ctxt.queue[1:] + return x.line, x.atEof, x.err + } + return ctxt.s.Scan() } func (ctxt *parseContext) startRule(r RuleType) (bool, error) { - ok, err := ctxt.b.StartRule(r) - if err != nil { - ctxt.addError(err) - } - return ok, err + ok, err := ctxt.b.StartRule(r) + if err != nil { + ctxt.addError(err) + } + return ok, err } func (ctxt *parseContext) endRule(r RuleType) (bool, error) { - ok, err := ctxt.b.EndRule(r) - if err != nil { - ctxt.addError(err) - } - return ok, err + ok, err := ctxt.b.EndRule(r) + if err != nil { + ctxt.addError(err) + } + return ok, err } func (ctxt *parseContext) build(t *Token) (bool, error) { - ok, err := ctxt.b.Build(t) - if err != nil { - ctxt.addError(err) - } - return ok, err + ok, err := ctxt.b.Build(t) + if err != nil { + ctxt.addError(err) + } + return ok, err } - func (ctxt *parseContext) match(state int, line *Line) (newState int, err error) { - switch(state) { - @foreach(var state in Model.States.Values.Where(s => !s.IsEndState)) - { - @:case @state.Id: - @:return ctxt.matchAt@(state.Id)(line); - } - default: - return state, fmt.Errorf("Unknown state: %+v", state); - } + switch state { + @foreach(var state in Model.States.Values.Where(s => !s.IsEndState)) + { + @:case @state.Id: + @:return ctxt.matchAt@(state.Id)(line) + } + default: + return state, fmt.Errorf("Unknown state: %+v", state) + } } - @foreach(var state in Model.States.Values.Where(s => !s.IsEndState)) { - // @Raw(state.Comment) +// @Raw(state.Comment) func (ctxt *parseContext) matchAt@(state.Id)(line *Line) (newState int, err error) { - @foreach(var transition in state.Transitions) - { - @:if ok, token, err := @MatchToken(transition.TokenType); ok { - if (transition.LookAheadHint != null) - { - @:if ctxt.lookahead@(transition.LookAheadHint.Id)(line) { - } - foreach(var production in transition.Productions) - { - @CallProduction(production) - } - @:return @transition.TargetState, err; - if (transition.LookAheadHint != null) - { - @:} - } - @:} - } - @HandleParserError(state.Transitions.Select(t => "#" + t.TokenType.ToString()).Distinct(), state) + @foreach(var transition in state.Transitions) + { + @:if ok, token, err := @MatchToken(transition.TokenType); ok { + if (transition.LookAheadHint != null) + { + @:if ctxt.lookahead@(transition.LookAheadHint.Id)(line) { + foreach(var production in transition.Productions) + { + @CallProduction(production) + } + @:return @transition.TargetState, err + @:} + } else { + foreach(var production in transition.Productions) + { + @CallProduction(production) + } + @:return @transition.TargetState, err + } + @:} + } + @HandleParserError(state.Transitions.Select(t => "#" + t.TokenType.ToString()).Distinct(), state) } } type Matcher interface { - @foreach(var rule in Model.RuleSet.TokenRules) - { Match@(rule.Name.Replace("#", ""))(line *Line) (bool,*Token,error) + @foreach(var rule in Model.RuleSet.TokenRules) + { Match@(rule.Name.Replace("#", ""))(line *Line) (bool, *Token, error) } - Reset() + Reset() } @foreach(var rule in Model.RuleSet.TokenRules) { func (ctxt *parseContext) isMatch@(rule.Name.Replace("#", ""))(line *Line) bool { - ok, _, _ := ctxt.match@(rule.Name.Replace("#", ""))(line) - return ok + ok, _, _ := ctxt.match@(rule.Name.Replace("#", ""))(line) + return ok } func (ctxt *parseContext) match@(rule.Name.Replace("#", ""))(line *Line) (bool, *Token, error) { - @if (rule.Name != "#EOF") - { - @:if line.IsEof() { - @: return false, nil, nil - @:} - } - return ctxt.m.Match@(rule.Name.Replace("#", ""))(line); + @if (rule.Name != "#EOF") + { + @:if line.IsEof() { + @: return false, nil, nil + @:} + } + return ctxt.m.Match@(rule.Name.Replace("#", ""))(line) } } - @foreach(var lookAheadHint in Model.RuleSet.LookAheadHints) { func (ctxt *parseContext) lookahead@(lookAheadHint.Id)(initialLine *Line) bool { - var queue []*scanResult - var match bool - - for { - line, atEof, err := ctxt.scan(); - queue = append(queue, &scanResult{line,atEof,err}); - - if false @foreach(var tokenType in lookAheadHint.ExpectedTokens) { || @IsMatchToken(tokenType)} { - match = true; - break - } - if !(false @foreach(var tokenType in lookAheadHint.Skip) { || @IsMatchToken(tokenType)}) { - break - } - if atEof { - break - } - } - - ctxt.queue = append(ctxt.queue, queue...) - - return match; - } + var queue []*scanResult + var match bool + + for { + line, atEof, err := ctxt.scan() + queue = append(queue, &scanResult{line, atEof, err}) + + if @{false}@foreach(var tokenType in lookAheadHint.ExpectedTokens) { || @IsMatchToken(tokenType)} { + match = true + break + } + if !(@{false}@foreach(var tokenType in lookAheadHint.Skip) { || @IsMatchToken(tokenType)}) { + break + } + if atEof { + break + } + } + + ctxt.queue = append(ctxt.queue, queue...) + + return match +} }