diff --git a/Builder/GoGenPackTableCode.go b/Builder/GoCodeTemplate.go similarity index 53% rename from Builder/GoGenPackTableCode.go rename to Builder/GoCodeTemplate.go index 4a8c3c5..994de45 100644 --- a/Builder/GoGenPackTableCode.go +++ b/Builder/GoCodeTemplate.go @@ -1,41 +1,85 @@ -/* -Copyright (c) 2021 Ke Yuchang(aceking.ke@gmail.com). All rights reserved. -Use of this source code is governed by MIT license that can be found in the LICENSE file. -*/ package builder -import "fmt" +var goCodeTemplateStr string = ` +/*Generator Code , do not modify*/ +// Code header part +{{.CodeHeader}} +import "strings" +// const part +{{.ConstPart}} +{{ if .NeedPacked }} +// Terminal Size +const NTERMINALS = {{.NTerminals}} +{{end}} -// make AnalyTable -func (b *GoBuilder) buildAnalyPackTable() { - AnalyTable := ` -var StatePackAction = []int { - %s -} -var StatePackOffset = []int { - %s -} -var StackPackCheck = []int { - %s -} -` - saction := "" - soffset := "" - scheck := "" - for _, val := range b.vnode.ActionTable { - saction += fmt.Sprintf("%d,\t", val) +var IsTrace bool = false +var StateSymStack = []StateSym{} +var StackPointer = 0 +type Context struct { + StackSym []StateSym + Stackpos int +} + +var globalContext = []Context{} + +type ValType struct { + // Union part + {{.UnionPart}} +} +type StateSym struct { + Yystate int // state + + //sym val + YySymIndex int + //other + ValType +} + +{{ if .NeedPacked }} + // It is NeedPacked + {{.PackAnalyTable}} + func (s *StateSym) Action(a int) int { + if StatePackOffset[s.Yystate]+a < 0 { + return ERROR_ACTION } - for _, val := range b.vnode.OffsetTable { - soffset += fmt.Sprintf("%d,\t", val) + if StatePackOffset[s.Yystate]+a >= len(StackPackCheck) || + StackPackCheck[StatePackOffset[s.Yystate]+a] != s.Yystate { + if a > NTERMINALS { + return StackPackGotoDef[a - NTERMINALS - 1] + }else { + return StackPackActDef[s.Yystate] + } + }else{ + return StatePackAction[StatePackOffset[s.Yystate]+a] } - for _, val := range b.vnode.CheckTable { - scheck += fmt.Sprintf("%d,\t", val) +} + {{else}} + // It is not packed + var StateActionArray = [][]int{ + {{.AnalyTable}} +} +func (s *StateSym) Action(a int) int { + return StateActionArray[s.Yystate][a] +} + + {{ end }} + + func TraceShift(s *StateSym) { + if IsTrace { + fmt.Printf("Shift %s, push state %d\n", TraceTranslate(s.YySymIndex), s.Yystate) + } +} + +// Reduce function +func ReduceFunc(reduceIndex int) *StateSym { + dollarDolar := &StateSym{} + topIndex := StackPointer - 1 + switch reduceIndex { + {{.ReduceFunc}} } - b.AnalyTable = fmt.Sprintf(AnalyTable, saction, soffset, scheck) + return dollarDolar } -func (b *GoBuilder) buildPackStateFunc() { - b.StateFunc = ` // Push StateSym func PushStateSym(state *StateSym) { TraceShift(state) @@ -52,12 +96,9 @@ func PopStateSym(num int) { StackPointer -= num } -func (s *StateSym) Action(a int) int { - if StatePackOffset[s.Yystate]+a < 0|| StatePackOffset[s.Yystate]+a >= len(StackPackCheck) || StackPackCheck[StatePackOffset[s.Yystate]+a] != s.Yystate { - return 0 - }else{ - return StatePackAction[StatePackOffset[s.Yystate]+a] - } + +func init() { + ParserInit() } func PushContex() { globalContext = append(globalContext, Context{ @@ -70,10 +111,6 @@ func PopContex() { StateSymStack = globalContext[len(globalContext)-1].StackSym globalContext = globalContext[:len(globalContext)-1] } -func init() { - ParserInit() -} - func ParserInit() { StateSymStack = []StateSym{ { @@ -129,5 +166,28 @@ func fetchLookAhead(input string, val *ValType, pos *int) int { token := GetToken(input, val, pos) return translate(token) } -` +func translate(c int) int { + var conv int = 0 + switch c { + {{.Translate}} + } + return conv +} +// Trace function for translate +func TraceTranslate(c int) string { + var conv string = "" + switch c { +{{.TranslateTrace}} + } + return conv +} +// Trace function for reduce +func TraceReduce(reduceIndex, s int, look string) { + if IsTrace { + switch reduceIndex { +{{.ReduceTrace}} + } + } } +// Code Last part +{{.CodeLast}}` diff --git a/Builder/GoGenCode.go b/Builder/GoGenCode.go deleted file mode 100644 index b0b87cf..0000000 --- a/Builder/GoGenCode.go +++ /dev/null @@ -1,354 +0,0 @@ -/* -Copyright (c) 2021 Ke Yuchang(aceking.ke@gmail.com). All rights reserved. -Use of this source code is governed by MIT license that can be found in the LICENSE file. -*/ -package builder - -import ( - "fmt" - "os" - "regexp" - "strconv" - "strings" - - parser "github.com/acekingke/yaccgo/Parser" - rules "github.com/acekingke/yaccgo/Rules" - utils "github.com/acekingke/yaccgo/Utils" -) - -type GoBuilder struct { - vnode *parser.RootVistor - HeaderPart string - CodeHeader string - ConstPart string - UnionPart string - AnalyTable string - CodeLast string - StateFunc string - ReduceFunc string - Translate string - // ReduceTrace funciton - ReduceTrace string -} - -func NewGoBuilder(w *parser.Walker) *GoBuilder { - return &GoBuilder{ - HeaderPart: `/*Generator Code , do not modify*/\n import "strings"\n`, - vnode: w.VistorNode.(*parser.RootVistor), - } -} - -func (b *GoBuilder) buildConstPart() { - b.ConstPart = "// const part \n" - for _, identifier := range b.vnode.GetIdsymtabl() { - if identifier.IDTyp == parser.TERMID && - !parser.TestPrefix(identifier.Name) { - b.ConstPart += fmt.Sprintf("const %s = %d\n", identifier.Name, identifier.Value) - } - } - b.ConstPart += fmt.Sprintf("const ERROR_ACTION = %d\nconst ACCEPT_ACTION = %d\n", b.vnode.GenErrorCode(), b.vnode.GenAcceptCode()) -} - -func (b *GoBuilder) buildUionAndCode() { - str := ` -var IsTrace bool = false -var StateSymStack = []StateSym{} -var StackPointer = 0 -type Context struct { - StackSym []StateSym - Stackpos int -} - -var globalContext = []Context{} - -type ValType struct { - %s -} -type StateSym struct { - Yystate int // state - - //sym val - YySymIndex int - //other - ValType -}` - str = fmt.Sprintf(str, b.vnode.GetUion()) - b.UnionPart = str - b.CodeHeader = b.vnode.GetCode() + "\n import \"strings\"\n" - b.CodeLast = b.vnode.GetCodeCopy() -} - -// make AnalyTable -func (b *GoBuilder) buildAnalyTable() { - AnalyTable := ` -var StateActionArray = [][]int { - %s -} -` - s := "/* " - for _, sy := range b.vnode.G.Symbols { - s += fmt.Sprintf("%s\t", sy.Name) - } - s += "*/\n" - for index, row := range b.vnode.GTable { - s += fmt.Sprintf("/* %d */ {", index) - for _, val := range row { - s += fmt.Sprintf("%d,\t", val) - } - s += "},\n" - } - b.AnalyTable = fmt.Sprintf(AnalyTable, s) -} - -func (b *GoBuilder) buildStateFunc() { - b.StateFunc = ` -// Push StateSym -func PushStateSym(state *StateSym) { - TraceShift(state) - if StackPointer >= len(StateSymStack) { - StateSymStack = append(StateSymStack, *state) - } else { - StateSymStack[StackPointer] = *state - } - StackPointer++ -} - -// Pop num StateSym -func PopStateSym(num int) { - StackPointer -= num -} - -func (s *StateSym) Action(a int) int { - return StateActionArray[s.Yystate][a] -} - -func init() { - ParserInit() -} -func PushContex() { - globalContext = append(globalContext, Context{ - StackSym: StateSymStack, - Stackpos: StackPointer, - }) -} -func PopContex() { - StackPointer = globalContext[len(globalContext)-1].Stackpos - StateSymStack = globalContext[len(globalContext)-1].StackSym - globalContext = globalContext[:len(globalContext)-1] -} -func ParserInit() { - StateSymStack = []StateSym{ - { - Yystate: 0, - YySymIndex: 1, //$ - }, - } - StackPointer = 1 -} - -func Parser(input string) *ValType { - var currentPos int = 0 - var val ValType - lookAhead := fetchLookAhead(input, &val, ¤tPos) - for { - - if StackPointer == 0 { - break - } - if StackPointer > len(StateSymStack) { - break - } - s := &StateSymStack[StackPointer-1] - a := s.Action(lookAhead) - if a == ERROR_ACTION { - lines := strings.Split(input[:currentPos], "\n") - panic("Grammar parse error near :" + lines[len(lines)-1]) - } else if a == ACCEPT_ACTION { - return &s.ValType - } else { - if a > 0 { - // shift - PushStateSym(&StateSym{ - Yystate: a, - YySymIndex: lookAhead, - ValType: val, - }) - lookAhead = fetchLookAhead(input, &val, ¤tPos) - } else { - reduceIndex := -a - SymTy := ReduceFunc(reduceIndex) - s := &StateSymStack[StackPointer-1] - gotoState := s.Action(SymTy.YySymIndex) - SymTy.Yystate = gotoState - TraceReduce(reduceIndex, gotoState, TraceTranslate(lookAhead)) - PushStateSym(SymTy) - } - } - } - return nil -} -func fetchLookAhead(input string, val *ValType, pos *int) int { - token := GetToken(input, val, pos) - return translate(token) -} -` -} - -func (b *GoBuilder) buildTranslate() { - str := ` -func translate(c int) int { - var conv int = 0 - switch c { -%s - } - return conv -} -` - caseCodes := "" - for _, sy := range b.vnode.G.Symbols { - if !sy.IsNonTerminator { - caseCodes += fmt.Sprintf("\tcase %d:\n \tconv = %d\n", sy.Value, sy.ID) - } - } - b.Translate = fmt.Sprintf(str, caseCodes) - // build Trace info - traceFun := ` -func TraceShift(s *StateSym) { - if IsTrace { - fmt.Printf("Shift %%s, push state %%d\n", TraceTranslate(s.YySymIndex), s.Yystate) - } -} -func TraceTranslate(c int) string { - var conv string = "" - switch c { -%s - } - return conv -} - -` - caseCodes = "" - for _, sy := range b.vnode.G.Symbols { - caseCodes += fmt.Sprintf("\tcase %d:\n \tconv = \"%s\"\n", sy.ID, parser.RemoveTempName(sy.Name)) - } - b.Translate += fmt.Sprintf(traceFun, caseCodes) -} - -func (b *GoBuilder) buildReduceTrace() { - str := ` -func TraceReduce(reduceIndex, s int, look string) { - if IsTrace { - switch reduceIndex { -%s - } - } -} -` - - caseCode := "" - for i := 1; i < len(b.vnode.G.ProductoinRules); i++ { - caseCode += fmt.Sprintf("\t\tcase %d: \n", i) - oneRule := b.vnode.GetRules(i - 1) - leftPartString := "use Reduce:" + parser.RemoveTempName(oneRule.LeftPart.Name) - var rightPartString string = "" - for _, rightPart := range oneRule.RighPart { - rightPartString += parser.RemoveTempName(rightPart.Name) + " " - } - strTrace := fmt.Sprintf("%s -> %s", - leftPartString, rightPartString) - caseCode += fmt.Sprintf("\n\t\tfmt.Printf(\"look ahead %%s, %s, go to state %%d\\n\", look, s)\n", strTrace) - } - str = fmt.Sprintf(str, caseCode) - b.ReduceTrace = str -} - -// make ReduceFunc -func (b *GoBuilder) buildReduceFunc() { - str := ` -func ReduceFunc(reduceIndex int) *StateSym { - dollarDolar := &StateSym{} - topIndex := StackPointer - 1 - switch reduceIndex { - %s - } - return dollarDolar -} -` - caseCode := "" - for i := 1; i < len(b.vnode.G.ProductoinRules); i++ { - productionRule := b.vnode.G.ProductoinRules[i] - caseCode += fmt.Sprintf("case %d: \n", i) - rightPartlen := len(productionRule.RighPart) - caseCode += fmt.Sprintf("\tdollarDolar.YySymIndex = %d\n", productionRule.LeftPart.ID) - caseCode += fmt.Sprintf("\tDollar := StateSymStack[topIndex-%d : StackPointer]\n\t_ = Dollar\n", rightPartlen) - //fetch the action code here - caseCode += actionCodeReplace(b.vnode, i, productionRule) - caseCode += fmt.Sprintf("\tPopStateSym(%d)\n", rightPartlen) - } - b.ReduceFunc = fmt.Sprintf(str, caseCode) -} - -func actionCodeReplace(vnode *parser.RootVistor, - index int, pr *rules.ProductoinRule) string { - oneRule := vnode.GetRules(index - 1) - // generate the comments. - strComment := "\n/*\n%s*/\n" - leftPartString := fmt.Sprint("\nLineNo:", oneRule.LineNo, "\n") + parser.RemoveTempName(oneRule.LeftPart.Name) - var rightPartString string = "" - for _, rightPart := range oneRule.RighPart { - rightPartString += parser.RemoveTempName(rightPart.Name) + " " - } - strComment = fmt.Sprintf(strComment, - fmt.Sprintf("%s -> %s\n %s\n", - leftPartString, rightPartString, oneRule.ActionCode)) - - str := oneRule.ActionCode - str = strings.ReplaceAll(str, "$$", - fmt.Sprintf("dollarDolar.%s", pr.LeftPart.Tag)) - - // find the $ and digits - reg := regexp.MustCompile(`\$[0-9]+`) - str = reg.ReplaceAllStringFunc(str, func(s string) string { - index := s[1:] - i, _ := strconv.Atoi(index) - return fmt.Sprintf("Dollar[%s].%s", index, pr.RighPart[i-1].Tag) - }) - return strComment + str + "\n" -} - -func GoGenFromString(input string, file string) error { - w, err := parser.ParseAndBuild(input) - if err != nil { - return fmt.Errorf("parse error: %s", err) - } - b := NewGoBuilder(w) - b.buildConstPart() - b.buildUionAndCode() - if b.vnode.NeedPacked && utils.PackFlags { - b.buildAnalyPackTable() - b.buildPackStateFunc() - } else { - b.buildAnalyTable() - b.buildStateFunc() - } - - b.buildReduceFunc() - b.buildReduceTrace() - b.buildTranslate() - // Create file and write to it - f, err := os.Create(file) - if err != nil { - return fmt.Errorf("create file error: %s", err) - } - f.WriteString(b.CodeHeader) - f.WriteString(b.ConstPart) - f.WriteString(b.UnionPart) - f.WriteString(b.AnalyTable) - f.WriteString(b.CodeLast) - f.WriteString(b.StateFunc) - f.WriteString(b.ReduceFunc) - f.WriteString(b.Translate) - f.WriteString(b.ReduceTrace) - f.Close() - return nil -} diff --git a/Builder/GoGenCode_test.go b/Builder/GoGenCode_test.go deleted file mode 100644 index 7f6cd11..0000000 --- a/Builder/GoGenCode_test.go +++ /dev/null @@ -1,104 +0,0 @@ -/* -Copyright (c) 2021 Ke Yuchang(aceking.ke@gmail.com). All rights reserved. -Use of this source code is governed by MIT license that can be found in the LICENSE file. -*/ -package builder - -import ( - "testing" -) - -func TestCodeGen(t *testing.T) { - str := - `// language: go - - %{ - - package main - - import ( - - "fmt" - - ) - - %} - - %union { - val int - } - - %type E - %token '+' '*' '(' ')' - %left '+' - %left '*' - %token NUM - %token NUM 100 - %start E -%% - -E: - E '+' E { - $$ = $1 + $3 - } - | E '*' E { - $$ = $1 * $3 - } - | '(' E ')' { - $$ = $2 - } - | NUM { - $$ = $1 - } - -%% - const EOF = -1 - // The parser expects the lexer to return 0 on EOF. Give it a name - // for clarity. - func GetToken(input string, valTy *ValType, pos *int) int { - if *pos >= len(input) { - return -1 - } else { - *valTy = ValType{0} - loop: - if *pos >= len(input) { - return EOF - } - c := input[*pos] - *pos++ - switch c { - case '+': - fallthrough - case '(': - fallthrough - case ')': - fallthrough - - case '*': - return int(c) - - default: - if c >= '0' && c <= '9' { // is digit - valTy.val = (valTy.val)*10 + int(c) - '0' - // next is digit - if *pos < len(input) && input[*pos] >= '0' && input[*pos] <= '9' { - goto loop - } - return NUM - } - - } - return 0 - } - } -func main() { - v := Parser("1+2*31").val - fmt.Println(v) -} - - ` - err := GoGenFromString(str, "../../sample.go") - if err != nil { - t.Error(err) - } -} diff --git a/Builder/GoTemplBuilder.go b/Builder/GoTemplBuilder.go index 0acc070..9a0edd6 100644 --- a/Builder/GoTemplBuilder.go +++ b/Builder/GoTemplBuilder.go @@ -3,9 +3,14 @@ package builder import ( "fmt" "os" + "regexp" + "strconv" + "strings" "text/template" parser "github.com/acekingke/yaccgo/Parser" + rules "github.com/acekingke/yaccgo/Rules" + utils "github.com/acekingke/yaccgo/Utils" ) type TemplateBuilder struct { @@ -56,7 +61,7 @@ func NewTemplateBuilder(w *parser.Walker) *TemplateBuilder { } func (b *TemplateBuilder) buildConstPart() { - b.NeedPacked = b.vnode.NeedPacked + b.NeedPacked = b.vnode.NeedPacked && utils.PackFlags b.NTerminals = len(b.vnode.G.VtSet) b.CodeHeader = b.vnode.GetCode() b.CodeLast = b.vnode.GetCodeCopy() @@ -66,6 +71,7 @@ func (b *TemplateBuilder) buildConstPart() { b.ConstPart += fmt.Sprintf("const %s = %d\n", identifier.Name, identifier.Value) } } + b.ConstPart += fmt.Sprintf("const ERROR_ACTION = %d\nconst ACCEPT_ACTION = %d\n", b.vnode.GenErrorCode(), b.vnode.GenAcceptCode()) } func (b *TemplateBuilder) buildUionAndCode() { @@ -177,12 +183,40 @@ func (b *TemplateBuilder) buildTranslate() { } func (b *TemplateBuilder) WriteFile(f *os.File) { - templ, err := template.ParseFiles("goCode.templ") - defer f.Close() + templ, err := template.New("gotemplate").Parse(goCodeTemplateStr) if err != nil { - fmt.Println(err) + panic(err) } + defer f.Close() if err := templ.Execute(f, b); err != nil { panic(err) } } + +func actionCodeReplace(vnode *parser.RootVistor, + index int, pr *rules.ProductoinRule) string { + oneRule := vnode.GetRules(index - 1) + // generate the comments. + strComment := "\n/*\n%s*/\n" + leftPartString := fmt.Sprint("\nLineNo:", oneRule.LineNo, "\n") + parser.RemoveTempName(oneRule.LeftPart.Name) + var rightPartString string = "" + for _, rightPart := range oneRule.RighPart { + rightPartString += parser.RemoveTempName(rightPart.Name) + " " + } + strComment = fmt.Sprintf(strComment, + fmt.Sprintf("%s -> %s\n %s\n", + leftPartString, rightPartString, oneRule.ActionCode)) + + str := oneRule.ActionCode + str = strings.ReplaceAll(str, "$$", + fmt.Sprintf("dollarDolar.%s", pr.LeftPart.Tag)) + + // find the $ and digits + reg := regexp.MustCompile(`\$[0-9]+`) + str = reg.ReplaceAllStringFunc(str, func(s string) string { + index := s[1:] + i, _ := strconv.Atoi(index) + return fmt.Sprintf("Dollar[%s].%s", index, pr.RighPart[i-1].Tag) + }) + return strComment + str + "\n" +} diff --git a/Builder/goCode.templ b/Builder/goCode.templ index 8a1a322..2632d34 100644 --- a/Builder/goCode.templ +++ b/Builder/goCode.templ @@ -8,8 +8,6 @@ import "strings" // Terminal Size const NTERMINALS = {{.NTerminals}} {{end}} -const ERROR_ACTION = 110 -const ACCEPT_ACTION = 210 var IsTrace bool = false var StateSymStack = []StateSym{} @@ -38,15 +36,15 @@ type StateSym struct { // It is NeedPacked {{.PackAnalyTable}} func (s *StateSym) Action(a int) int { - if StatePackOffset[s.Yystate]+a < 0|| - StatePackOffset[s.Yystate]+a >= len(StackPackCheck) || + if StatePackOffset[s.Yystate]+a < 0 { + return ERROR_ACTION + } + if StatePackOffset[s.Yystate]+a >= len(StackPackCheck) || StackPackCheck[StatePackOffset[s.Yystate]+a] != s.Yystate { if a > NTERMINALS { return StackPackGotoDef[a - NTERMINALS - 1] - }else if a > 0{ - return StackPackActDef[s.Yystate] }else { - return ERROR_ACTION + return StackPackActDef[s.Yystate] } }else{ return StatePackAction[StatePackOffset[s.Yystate]+a] @@ -65,7 +63,7 @@ func (s *StateSym) Action(a int) int { func TraceShift(s *StateSym) { if IsTrace { - fmt.Printf("Shift %s, push state %d\n", TraceTranslate(s.YySymIndex), s.Yystate) + fmt.Printf("Shift %s, push state %d\n", TraceTranslate(s.YySymIndex), s.Yystate) } } diff --git a/Makefile b/Makefile index c73d284..3c8351d 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,13 @@ all: build test make todo build: require - go fmt $(TOP)/... + rm -rf out/*.go + touch Builder/GoCodeTemplate.go + @echo "package builder\n var goCodeTemplateStr string=\`" > Builder/GoCodeTemplate.go + @cat Builder/GoCodeTemplate.go Builder/goCode.templ > Builder/GoCodeTemplate.go.tmp + echo "\`" >> Builder/GoCodeTemplate.go.tmp + mv Builder/GoCodeTemplate.go.tmp Builder/GoCodeTemplate.go + go fmt ./... go build -o bin/yaccgo ./yaccgo/*.go require: diff --git a/UPGRADING.md b/UPGRADING.md new file mode 100644 index 0000000..b6f977c --- /dev/null +++ b/UPGRADING.md @@ -0,0 +1,4 @@ +# refactor the packtable algorithm to be more efficient and less size +1. Optimize the packtable use action default values and goto default values +2. use the template to generate the Code +3. reserve the old codes but it will deprecate them in future diff --git a/yaccgo/command.go b/yaccgo/command.go index 3218022..741fcb4 100644 --- a/yaccgo/command.go +++ b/yaccgo/command.go @@ -65,11 +65,9 @@ type genfun func(input string, file string) error func cmdGenerate(args []string) { switch args[0] { case "go": - genCommonFunc(args[1], args[2], builder.GoGenFromString) + genCommonFunc(args[1], args[2], builder.TemplateGenFromString) case "typescript": genCommonFunc(args[1], args[2], builder.TsGenFromString) - case "template": - genCommonFunc(args[1], args[2], builder.TemplateGenFromString) case "rust": fmt.Println("not support rust yet") }