From d94425fe5d823278a8331e4ba82a9de4e9324666 Mon Sep 17 00:00:00 2001 From: "Jim.Idle" Date: Tue, 11 Apr 2023 16:47:22 +0800 Subject: [PATCH 1/2] feat: Remove pointer embedding of parser rule context Another small gain, but for one good parser this goes from 90ms to 64ms (for 16 large files) so it won't make a lot of difference for poor parser, but good ones will see a nice kick here Signed-off-by: Jim.Idle --- runtime/Go/antlr/v4/atn_config.go | 30 ++-- runtime/Go/antlr/v4/atn_config_set.go | 78 ++++---- runtime/Go/antlr/v4/atn_simulator.go | 2 +- runtime/Go/antlr/v4/jcollect.go | 22 +-- runtime/Go/antlr/v4/lexer_atn_simulator.go | 120 ++++++------- runtime/Go/antlr/v4/parser_atn_simulator.go | 168 +++++++++--------- runtime/Go/antlr/v4/parser_rule_context.go | 66 ++++++- runtime/Go/antlr/v4/prediction_context.go | 1 - .../Go/antlr/v4/prediction_context_cache.go | 2 +- runtime/Go/antlr/v4/rule_context.go | 73 -------- runtime/Go/antlr/v4/statistics.go | 36 ++-- runtime/Go/antlr/v4/tree.go | 2 - .../antlr/v4/tool/templates/codegen/Go/Go.stg | 33 +++- 13 files changed, 316 insertions(+), 317 deletions(-) diff --git a/runtime/Go/antlr/v4/atn_config.go b/runtime/Go/antlr/v4/atn_config.go index 3eebcf17af..a83f25d349 100644 --- a/runtime/Go/antlr/v4/atn_config.go +++ b/runtime/Go/antlr/v4/atn_config.go @@ -40,7 +40,7 @@ func NewATNConfig5(state ATNState, alt int, context *PredictionContext, semantic if semanticContext == nil { panic("semanticContext cannot be nil") // TODO: Necessary? } - + pac := &ATNConfig{} pac.state = state pac.alt = alt @@ -83,7 +83,7 @@ func NewATNConfig(c *ATNConfig, state ATNState, context *PredictionContext, sema } func (a *ATNConfig) InitATNConfig(c *ATNConfig, state ATNState, alt int, context *PredictionContext, semanticContext SemanticContext) { - + a.state = state a.alt = alt a.context = context @@ -158,7 +158,7 @@ func (a *ATNConfig) Equals(o Collectable[*ATNConfig]) bool { // predict the same alternative, and syntactic/semantic contexts are the same. func (a *ATNConfig) PEquals(o Collectable[*ATNConfig]) bool { var other, ok = o.(*ATNConfig) - + if !ok { return false } @@ -167,22 +167,22 @@ func (a *ATNConfig) PEquals(o Collectable[*ATNConfig]) bool { } else if other == nil { return false } - + var equal bool - + if a.context == nil { equal = other.context == nil } else { equal = a.context.Equals(other.context) } - + var ( nums = a.state.GetStateNumber() == other.state.GetStateNumber() alts = a.alt == other.alt cons = a.semanticContext.Equals(other.semanticContext) sups = a.precedenceFilterSuppressed == other.precedenceFilterSuppressed ) - + return nums && alts && cons && sups && equal } @@ -206,7 +206,7 @@ func (a *ATNConfig) PHash() int { if a.context != nil { c = a.context.Hash() } - + h := murmurInit(7) h = murmurUpdate(h, a.state.GetStateNumber()) h = murmurUpdate(h, a.alt) @@ -218,19 +218,19 @@ func (a *ATNConfig) PHash() int { // String returns a string representation of the ATNConfig, usually used for debugging purposes func (a *ATNConfig) String() string { var s1, s2, s3 string - + if a.context != nil { s1 = ",[" + fmt.Sprint(a.context) + "]" } - + if a.semanticContext != SemanticContextNone { s2 = "," + fmt.Sprint(a.semanticContext) } - + if a.reachesIntoOuterContext > 0 { s3 = ",up=" + fmt.Sprint(a.reachesIntoOuterContext) } - + return fmt.Sprintf("(%v,%v%v%v%v)", a.state, a.alt, s1, s2, s3) } @@ -313,7 +313,7 @@ func (a *ATNConfig) LEquals(other Collectable[*ATNConfig]) bool { } else if a.passedThroughNonGreedyDecision != otherT.passedThroughNonGreedyDecision { return false } - + switch { case a.lexerActionExecutor == nil && otherT.lexerActionExecutor == nil: return true @@ -324,12 +324,12 @@ func (a *ATNConfig) LEquals(other Collectable[*ATNConfig]) bool { default: return false // One but not both, are nil } - + return a.PEquals(otherT) } func checkNonGreedyDecision(source *ATNConfig, target ATNState) bool { var ds, ok = target.(DecisionState) - + return source.passedThroughNonGreedyDecision || (ok && ds.getNonGreedy()) } diff --git a/runtime/Go/antlr/v4/atn_config_set.go b/runtime/Go/antlr/v4/atn_config_set.go index ab8d915860..52dbaf8064 100644 --- a/runtime/Go/antlr/v4/atn_config_set.go +++ b/runtime/Go/antlr/v4/atn_config_set.go @@ -13,42 +13,42 @@ import ( // graph-structured stack. type ATNConfigSet struct { cachedHash int - + // configLookup is used to determine whether two ATNConfigSets are equal. We // need all configurations with the same (s, i, _, semctx) to be equal. A key // effectively doubles the number of objects associated with ATNConfigs. All // keys are hashed by (s, i, _, pi), not including the context. Wiped out when // read-only because a set becomes a DFA state. configLookup *JStore[*ATNConfig, Comparator[*ATNConfig]] - + // configs is the added elements that did not match an existing key in configLookup configs []*ATNConfig - + // TODO: These fields make me pretty uncomfortable, but it is nice to pack up // info together because it saves re-computation. Can we track conflicts as they // are added to save scanning configs later? conflictingAlts *BitSet - + // dipsIntoOuterContext is used by parsers and lexers. In a lexer, it indicates // we hit a pred while computing a closure operation. Do not make a DFA state // from the ATNConfigSet in this case. TODO: How is this used by parsers? dipsIntoOuterContext bool - + // fullCtx is whether it is part of a full context LL prediction. Used to // determine how to merge $. It is a wildcard with SLL, but not for an LL // context merge. fullCtx bool - + // Used in parser and lexer. In lexer, it indicates we hit a pred // while computing a closure operation. Don't make a DFA state from this set. hasSemanticContext bool - + // readOnly is whether it is read-only. Do not // allow any code to manipulate the set if true because DFA states will point at // sets and those must not change. It not, protect other fields; conflictingAlts // in particular, which is assigned after readOnly. readOnly bool - + // TODO: These fields make me pretty uncomfortable, but it is nice to pack up // info together because it saves re-computation. Can we track conflicts as they // are added to save scanning configs later? @@ -83,17 +83,17 @@ func (b *ATNConfigSet) Add(config *ATNConfig, mergeCache *JPCMap) bool { if b.readOnly { panic("set is read-only") } - + if config.GetSemanticContext() != SemanticContextNone { b.hasSemanticContext = true } - + if config.GetReachesIntoOuterContext() > 0 { b.dipsIntoOuterContext = true } - + existing, present := b.configLookup.Put(config) - + // The config was not already in the set // if !present { @@ -101,52 +101,52 @@ func (b *ATNConfigSet) Add(config *ATNConfig, mergeCache *JPCMap) bool { b.configs = append(b.configs, config) // Track order here return true } - + // Merge a previous (s, i, pi, _) with it and save the result rootIsWildcard := !b.fullCtx merged := merge(existing.GetContext(), config.GetContext(), rootIsWildcard, mergeCache) - + // No need to check for existing.context because config.context is in the cache, // since the only way to create new graphs is the "call rule" and here. We cache // at both places. existing.SetReachesIntoOuterContext(intMax(existing.GetReachesIntoOuterContext(), config.GetReachesIntoOuterContext())) - + // Preserve the precedence filter suppression during the merge if config.getPrecedenceFilterSuppressed() { existing.setPrecedenceFilterSuppressed(true) } - + // Replace the context because there is no need to do alt mapping existing.SetContext(merged) - + return true } // GetStates returns the set of states represented by all configurations in this config set func (b *ATNConfigSet) GetStates() *JStore[ATNState, Comparator[ATNState]] { - + // states uses the standard comparator and Hash() provided by the ATNState instance // states := NewJStore[ATNState, Comparator[ATNState]](aStateEqInst, ATNStateCollection, "ATNConfigSet.GetStates()") - + for i := 0; i < len(b.configs); i++ { states.Put(b.configs[i].GetState()) } - + return states } func (b *ATNConfigSet) GetPredicates() []SemanticContext { predicates := make([]SemanticContext, 0) - + for i := 0; i < len(b.configs); i++ { c := b.configs[i].GetSemanticContext() - + if c != SemanticContextNone { predicates = append(predicates, c) } } - + return predicates } @@ -154,12 +154,12 @@ func (b *ATNConfigSet) OptimizeConfigs(interpreter *BaseATNSimulator) { if b.readOnly { panic("set is read-only") } - + // Empty indicate no optimization is possible if b.configLookup == nil || b.configLookup.Len() == 0 { return } - + for i := 0; i < len(b.configs); i++ { config := b.configs[i] config.SetContext(interpreter.getCachedContext(config.GetContext())) @@ -170,7 +170,7 @@ func (b *ATNConfigSet) AddAll(coll []*ATNConfig) bool { for i := 0; i < len(coll); i++ { b.Add(coll[i], nil) } - + return false } @@ -185,7 +185,7 @@ func (b *ATNConfigSet) Compare(bs *ATNConfigSet) bool { return false } } - + return true } @@ -195,7 +195,7 @@ func (b *ATNConfigSet) Equals(other Collectable[ATNConfig]) bool { } else if _, ok := other.(*ATNConfigSet); !ok { return false } - + other2 := other.(*ATNConfigSet) var eca bool switch { @@ -218,10 +218,10 @@ func (b *ATNConfigSet) Hash() int { if b.cachedHash == -1 { b.cachedHash = b.hashCodeConfigs() } - + return b.cachedHash } - + return b.hashCodeConfigs() } @@ -257,35 +257,35 @@ func (b *ATNConfigSet) Clear() { } func (b *ATNConfigSet) String() string { - + s := "[" - + for i, c := range b.configs { s += c.String() - + if i != len(b.configs)-1 { s += ", " } } - + s += "]" - + if b.hasSemanticContext { s += ",hasSemanticContext=" + fmt.Sprint(b.hasSemanticContext) } - + if b.uniqueAlt != ATNInvalidAltNumber { s += ",uniqueAlt=" + fmt.Sprint(b.uniqueAlt) } - + if b.conflictingAlts != nil { s += ",conflictingAlts=" + b.conflictingAlts.String() } - + if b.dipsIntoOuterContext { s += ",dipsIntoOuterContext" } - + return s } diff --git a/runtime/Go/antlr/v4/atn_simulator.go b/runtime/Go/antlr/v4/atn_simulator.go index 6bf6516be4..afe6c9f809 100644 --- a/runtime/Go/antlr/v4/atn_simulator.go +++ b/runtime/Go/antlr/v4/atn_simulator.go @@ -22,7 +22,7 @@ func (b *BaseATNSimulator) getCachedContext(context *PredictionContext) *Predict if b.sharedContextCache == nil { return context } - + //visited := NewJMap[*PredictionContext, *PredictionContext, Comparator[*PredictionContext]](pContextEqInst, PredictionVisitedCollection, "Visit map in getCachedContext()") visited := NewVisitRecord() return getCachedBasePredictionContext(context, b.sharedContextCache, visited) diff --git a/runtime/Go/antlr/v4/jcollect.go b/runtime/Go/antlr/v4/jcollect.go index f1f9555565..ceccd96d25 100644 --- a/runtime/Go/antlr/v4/jcollect.go +++ b/runtime/Go/antlr/v4/jcollect.go @@ -122,11 +122,11 @@ type JStore[T any, C Comparator[T]] struct { } func NewJStore[T any, C Comparator[T]](comparator Comparator[T], cType CollectionSource, desc string) *JStore[T, C] { - + if comparator == nil { panic("comparator cannot be nil") } - + s := &JStore[T, C]{ store: make(map[int][]T, 1), comparator: comparator, @@ -136,7 +136,7 @@ func NewJStore[T any, C Comparator[T]](comparator Comparator[T], cType Collectio Source: cType, Description: desc, } - + // Track where we created it from if we are being asked to do so if runtimeConfig.statsTraceStacks { s.stats.CreateStack = debug.Stack() @@ -158,12 +158,12 @@ func NewJStore[T any, C Comparator[T]](comparator Comparator[T], cType Collectio // // If the given value is not present in the store, then the value is added to the store and returned as v and exists is set to false. func (s *JStore[T, C]) Put(value T) (v T, exists bool) { - + if collectStats { s.stats.Puts++ } kh := s.comparator.Hash1(value) - + var hClash bool for _, v1 := range s.store[kh] { hClash = true @@ -182,7 +182,7 @@ func (s *JStore[T, C]) Put(value T) (v T, exists bool) { s.stats.PutHashConflicts++ } s.store[kh] = append(s.store[kh], value) - + if collectStats { if len(s.store[kh]) > s.stats.MaxSlotSize { s.stats.MaxSlotSize = len(s.store[kh]) @@ -243,7 +243,7 @@ func (s *JStore[T, C]) SortedSlice(less func(i, j T) bool) []T { sort.Slice(vs, func(i, j int) bool { return less(vs[i], vs[j]) }) - + return vs } @@ -303,7 +303,7 @@ func (m *JMap[K, V, C]) Put(key K, val V) (V, bool) { m.stats.Puts++ } kh := m.comparator.Hash1(key) - + var hClash bool for _, e := range m.store[kh] { hClash = true @@ -443,7 +443,7 @@ func (pcm *JPCMap) Get(k1, k2 *PredictionContext) (*PredictionContext, bool) { } func (pcm *JPCMap) Put(k1, k2, v *PredictionContext) { - + if collectStats { pcm.stats.Puts++ } @@ -472,7 +472,7 @@ func (pcm *JPCMap) Put(k1, k2, v *PredictionContext) { } else { m2 = NewJMap[*PredictionContext, *PredictionContext, *ObjEqComparator[*PredictionContext]](pContextEqInst, PredictionContextCacheCollection, "map entry") } - + m2.Put(k2, v) pcm.store.Put(k1, m2) pcm.size++ @@ -515,7 +515,7 @@ func (pcm *JPCMap2) Get(k1, k2 *PredictionContext) (*PredictionContext, bool) { if collectStats { pcm.stats.Gets++ } - + h := dHash(k1, k2) var hClash bool for _, e := range pcm.store[h] { diff --git a/runtime/Go/antlr/v4/lexer_atn_simulator.go b/runtime/Go/antlr/v4/lexer_atn_simulator.go index cb56aa0f3d..fe938b0259 100644 --- a/runtime/Go/antlr/v4/lexer_atn_simulator.go +++ b/runtime/Go/antlr/v4/lexer_atn_simulator.go @@ -14,13 +14,13 @@ import ( var ( LexerATNSimulatorMinDFAEdge = 0 LexerATNSimulatorMaxDFAEdge = 127 // forces unicode to stay in ATN - + LexerATNSimulatorMatchCalls = 0 ) type ILexerATNSimulator interface { IATNSimulator - + reset() Match(input CharStream, mode int) int GetCharPositionInLine() int @@ -31,7 +31,7 @@ type ILexerATNSimulator interface { type LexerATNSimulator struct { BaseATNSimulator - + recog Lexer predictionMode int mergeCache *JPCMap2 @@ -50,29 +50,29 @@ func NewLexerATNSimulator(recog Lexer, atn *ATN, decisionToDFA []*DFA, sharedCon sharedContextCache: sharedContextCache, }, } - + l.decisionToDFA = decisionToDFA l.recog = recog - + // The current token's starting index into the character stream. // Shared across DFA to ATN simulation in case the ATN fails and the // DFA did not have a previous accept state. In l case, we use the // ATN-generated exception object. l.startIndex = -1 - + // line number 1..n within the input l.Line = 1 - + // The index of the character relative to the beginning of the line // 0..n-1 l.CharPositionInLine = 0 - + l.mode = LexerDefaultMode - + // Used during DFA/ATN exec to record the most recent accept configuration // info l.prevAccept = NewSimState() - + return l } @@ -87,25 +87,25 @@ func (l *LexerATNSimulator) Match(input CharStream, mode int) int { l.MatchCalls++ l.mode = mode mark := input.Mark() - + defer func() { input.Release(mark) }() - + l.startIndex = input.Index() l.prevAccept.reset() - + dfa := l.decisionToDFA[mode] - + var s0 *DFAState l.atn.stateMu.RLock() s0 = dfa.getS0() l.atn.stateMu.RUnlock() - + if s0 == nil { return l.MatchATN(input) } - + return l.execATN(input, s0) } @@ -119,7 +119,7 @@ func (l *LexerATNSimulator) reset() { func (l *LexerATNSimulator) MatchATN(input CharStream) int { startState := l.atn.modeToStartState[l.mode] - + if runtimeConfig.lexerATNSimulatorDebug { fmt.Println("MatchATN mode " + strconv.Itoa(l.mode) + " start: " + startState.String()) } @@ -127,11 +127,11 @@ func (l *LexerATNSimulator) MatchATN(input CharStream) int { s0Closure := l.computeStartState(input, startState) suppressEdge := s0Closure.hasSemanticContext s0Closure.hasSemanticContext = false - + next := l.addDFAState(s0Closure, suppressEdge) - + predict := l.execATN(input, next) - + if runtimeConfig.lexerATNSimulatorDebug { fmt.Println("DFA after MatchATN: " + l.decisionToDFA[oldMode].ToLexerString()) } @@ -139,7 +139,7 @@ func (l *LexerATNSimulator) MatchATN(input CharStream) int { } func (l *LexerATNSimulator) execATN(input CharStream, ds0 *DFAState) int { - + if runtimeConfig.lexerATNSimulatorDebug { fmt.Println("start state closure=" + ds0.configs.String()) } @@ -149,12 +149,12 @@ func (l *LexerATNSimulator) execATN(input CharStream, ds0 *DFAState) int { } t := input.LA(1) s := ds0 // s is current/from DFA state - + for { // while more work if runtimeConfig.lexerATNSimulatorDebug { fmt.Println("execATN loop starting closure: " + s.configs.String()) } - + // As we move src->trg, src->trg, we keep track of the previous trg to // avoid looking up the DFA state again, which is expensive. // If the previous target was already part of the DFA, we might @@ -196,7 +196,7 @@ func (l *LexerATNSimulator) execATN(input CharStream, ds0 *DFAState) int { t = input.LA(1) s = target // flip current DFA target becomes new src/from state } - + return l.failOrAccept(l.prevAccept, input, s.configs, t) } @@ -213,7 +213,7 @@ func (l *LexerATNSimulator) getExistingTargetState(s *DFAState, t int) *DFAState if t < LexerATNSimulatorMinDFAEdge || t > LexerATNSimulatorMaxDFAEdge { return nil } - + l.atn.edgeMu.RLock() defer l.atn.edgeMu.RUnlock() if s.getEdges() == nil { @@ -234,11 +234,11 @@ func (l *LexerATNSimulator) getExistingTargetState(s *DFAState, t int) *DFAState // returns ATNSimulatorError. func (l *LexerATNSimulator) computeTargetState(input CharStream, s *DFAState, t int) *DFAState { reach := NewOrderedATNConfigSet() - + // if we don't find an existing DFA state // Fill reach starting from closure, following t transitions l.getReachableConfigSet(input, s.configs, reach, t) - + if len(reach.configs) == 0 { // we got nowhere on t from s if !reach.hasSemanticContext { // we got nowhere on t, don't panic out l knowledge it'd @@ -258,12 +258,12 @@ func (l *LexerATNSimulator) failOrAccept(prevAccept *SimState, input CharStream, l.accept(input, lexerActionExecutor, l.startIndex, prevAccept.index, prevAccept.line, prevAccept.column) return prevAccept.dfaState.prediction } - + // if no accept and EOF is first char, return EOF if t == TokenEOF && input.Index() == l.startIndex { return TokenEOF } - + panic(NewLexerNoViableAltException(l.recog, input, l.startIndex, reach)) } @@ -275,18 +275,18 @@ func (l *LexerATNSimulator) getReachableConfigSet(input CharStream, closure *ATN // l is used to Skip processing for configs which have a lower priority // than a runtimeConfig that already reached an accept state for the same rule SkipAlt := ATNInvalidAltNumber - + for _, cfg := range closure.configs { currentAltReachedAcceptState := cfg.GetAlt() == SkipAlt if currentAltReachedAcceptState && cfg.passedThroughNonGreedyDecision { continue } - + if runtimeConfig.lexerATNSimulatorDebug { - + fmt.Printf("testing %s at %s\n", l.GetTokenName(t), cfg.String()) } - + for _, trans := range cfg.GetState().GetTransitions() { target := l.getReachableTarget(trans, t) if target != nil { @@ -324,7 +324,7 @@ func (l *LexerATNSimulator) getReachableTarget(trans Transition, t int) ATNState if trans.Matches(t, 0, LexerMaxCharValue) { return trans.getTarget() } - + return nil } @@ -335,7 +335,7 @@ func (l *LexerATNSimulator) computeStartState(input CharStream, p ATNState) *ATN cfg := NewLexerATNConfig6(target, i+1, BasePredictionContextEMPTY) l.closure(input, cfg, configs, false, false, false) } - + return configs } @@ -348,14 +348,14 @@ func (l *LexerATNSimulator) computeStartState(input CharStream, p ATNState) *ATN // The func returns true if an accept state is reached. func (l *LexerATNSimulator) closure(input CharStream, config *ATNConfig, configs *ATNConfigSet, currentAltReachedAcceptState, speculative, treatEOFAsEpsilon bool) bool { - + if runtimeConfig.lexerATNSimulatorDebug { fmt.Println("closure(" + config.String() + ")") } - + _, ok := config.state.(*RuleStopState) if ok { - + if runtimeConfig.lexerATNSimulatorDebug { if l.recog != nil { fmt.Printf("closure at %s rule stop %s\n", l.recog.GetRuleNames()[config.state.GetRuleIndex()], config) @@ -363,13 +363,13 @@ func (l *LexerATNSimulator) closure(input CharStream, config *ATNConfig, configs fmt.Printf("closure at rule stop %s\n", config) } } - + if config.context == nil || config.context.hasEmptyPath() { if config.context == nil || config.context.isEmpty() { configs.Add(config, nil) return true } - + configs.Add(NewLexerATNConfig2(config, config.state, BasePredictionContextEMPTY), nil) currentAltReachedAcceptState = true } @@ -405,15 +405,15 @@ func (l *LexerATNSimulator) closure(input CharStream, config *ATNConfig, configs // side-effect: can alter configs.hasSemanticContext func (l *LexerATNSimulator) getEpsilonTarget(input CharStream, config *ATNConfig, trans Transition, configs *ATNConfigSet, speculative, treatEOFAsEpsilon bool) *ATNConfig { - + var cfg *ATNConfig - + if trans.getSerializationType() == TransitionRULE { - + rt := trans.(*RuleTransition) newContext := SingletonBasePredictionContextCreate(config.context, rt.followState.GetStateNumber()) cfg = NewLexerATNConfig2(config, trans.getTarget(), newContext) - + } else if trans.getSerializationType() == TransitionPRECEDENCE { panic("Precedence predicates are not supported in lexers.") } else if trans.getSerializationType() == TransitionPREDICATE { @@ -426,17 +426,17 @@ func (l *LexerATNSimulator) getEpsilonTarget(input CharStream, config *ATNConfig // semantically it's not used that often. One of the key elements to // l predicate mechanism is not adding DFA states that see // predicates immediately afterwards in the ATN. For example, - + // a : ID {p1}? | ID {p2}? - + // should create the start state for rule 'a' (to save start state // competition), but should not create target of ID state. The // collection of ATN states the following ID references includes // states reached by traversing predicates. Since l is when we // test them, we cannot cash the DFA state target of ID. - + pt := trans.(*PredicateTransition) - + if runtimeConfig.lexerATNSimulatorDebug { fmt.Println("EVAL rule " + strconv.Itoa(trans.(*PredicateTransition).ruleIndex) + ":" + strconv.Itoa(pt.predIndex)) } @@ -502,14 +502,14 @@ func (l *LexerATNSimulator) evaluatePredicate(input CharStream, ruleIndex, predI savedLine := l.Line index := input.Index() marker := input.Mark() - + defer func() { l.CharPositionInLine = savedcolumn l.Line = savedLine input.Seek(index) input.Release(marker) }() - + l.Consume(input) return l.recog.Sempred(nil, ruleIndex, predIndex) } @@ -537,7 +537,7 @@ func (l *LexerATNSimulator) addDFAEdge(from *DFAState, tk int, to *DFAState, cfg suppressEdge := cfgs.hasSemanticContext cfgs.hasSemanticContext = false to = l.addDFAState(cfgs, true) - + if suppressEdge { return to } @@ -557,7 +557,7 @@ func (l *LexerATNSimulator) addDFAEdge(from *DFAState, tk int, to *DFAState, cfg from.setEdges(make([]*DFAState, LexerATNSimulatorMaxDFAEdge-LexerATNSimulatorMinDFAEdge+1)) } from.setIthEdge(tk-LexerATNSimulatorMinDFAEdge, to) // connect - + return to } @@ -566,13 +566,13 @@ func (l *LexerATNSimulator) addDFAEdge(from *DFAState, tk int, to *DFAState, cfg // configuration containing an ATN rule stop state. Later, when // traversing the DFA, we will know which rule to accept. func (l *LexerATNSimulator) addDFAState(configs *ATNConfigSet, suppressEdge bool) *DFAState { - + proposed := NewDFAState(-1, configs) var firstConfigWithRuleStopState *ATNConfig - + for _, cfg := range configs.configs { _, ok := cfg.GetState().(*RuleStopState) - + if ok { firstConfigWithRuleStopState = cfg break @@ -584,17 +584,17 @@ func (l *LexerATNSimulator) addDFAState(configs *ATNConfigSet, suppressEdge bool proposed.setPrediction(l.atn.ruleToTokenType[firstConfigWithRuleStopState.GetState().GetRuleIndex()]) } dfa := l.decisionToDFA[l.mode] - + l.atn.stateMu.Lock() defer l.atn.stateMu.Unlock() existing, present := dfa.Get(proposed) if present { - + // This state was already present, so just return it. // proposed = existing } else { - + // We need to add the new state // proposed.stateNumber = dfa.Len() @@ -642,13 +642,13 @@ func (l *LexerATNSimulator) GetTokenName(tt int) string { if tt == -1 { return "EOF" } - + var sb strings.Builder sb.Grow(6) sb.WriteByte('\'') sb.WriteRune(rune(tt)) sb.WriteByte('\'') - + return sb.String() } diff --git a/runtime/Go/antlr/v4/parser_atn_simulator.go b/runtime/Go/antlr/v4/parser_atn_simulator.go index 04668fdceb..a377a07423 100644 --- a/runtime/Go/antlr/v4/parser_atn_simulator.go +++ b/runtime/Go/antlr/v4/parser_atn_simulator.go @@ -36,7 +36,7 @@ func (c *ClosureBusy) Put(config *ATNConfig) (*ATNConfig, bool) { type ParserATNSimulator struct { BaseATNSimulator - + parser Parser predictionMode int input TokenStream @@ -48,14 +48,14 @@ type ParserATNSimulator struct { //goland:noinspection GoUnusedExportedFunction func NewParserATNSimulator(parser Parser, atn *ATN, decisionToDFA []*DFA, sharedContextCache *PredictionContextCache) *ParserATNSimulator { - + p := &ParserATNSimulator{ BaseATNSimulator: BaseATNSimulator{ atn: atn, sharedContextCache: sharedContextCache, }, } - + p.parser = parser p.decisionToDFA = decisionToDFA // SLL, LL, or LL + exact ambig detection?// @@ -74,7 +74,7 @@ func NewParserATNSimulator(parser Parser, atn *ATN, decisionToDFA []*DFA, shared // also be examined during cache lookup. // p.mergeCache = nil - + return p } @@ -100,12 +100,12 @@ func (p *ParserATNSimulator) AdaptivePredict(parser *BaseParser, input TokenStre p.input = input p.startIndex = input.Index() p.outerContext = outerContext - + dfa := p.decisionToDFA[decision] p.dfa = dfa m := input.Mark() index := input.Index() - + defer func() { p.dfa = nil p.mergeCache = nil // whack cache after each prediction @@ -116,11 +116,11 @@ func (p *ParserATNSimulator) AdaptivePredict(parser *BaseParser, input TokenStre // possible. However, it can only have a limited effect. The real solution is to encourage grammar // authors to think more carefully about their grammar and to use the new antlr.stats tag to inspect // what is happening at runtime, along with using the error listener to report ambiguities. - + input.Seek(index) input.Release(m) }() - + // Now we are certain to have a specific decision's DFA // But, do we still need an initial state? var s0 *DFAState @@ -136,7 +136,7 @@ func (p *ParserATNSimulator) AdaptivePredict(parser *BaseParser, input TokenStre s0 = dfa.getS0() } p.atn.stateMu.RUnlock() - + if s0 == nil { if outerContext == nil { outerContext = ParserRuleContextEmpty @@ -148,7 +148,7 @@ func (p *ParserATNSimulator) AdaptivePredict(parser *BaseParser, input TokenStre } fullCtx := false s0Closure := p.computeStartState(dfa.atnStartState, ParserRuleContextEmpty, fullCtx) - + p.atn.stateMu.Lock() if dfa.getPrecedenceDfa() { // If p is a precedence DFA, we use applyPrecedenceFilter @@ -169,14 +169,14 @@ func (p *ParserATNSimulator) AdaptivePredict(parser *BaseParser, input TokenStre } p.atn.stateMu.Unlock() } - + alt, re := p.execATN(dfa, s0, input, index, outerContext) parser.SetError(re) if runtimeConfig.parserATNSimulatorDebug { fmt.Println("DFA after predictATN: " + dfa.String(p.parser.GetLiteralNames(), nil)) } return alt - + } // execATN performs ATN simulation to compute a predicted alternative based @@ -214,16 +214,16 @@ func (p *ParserATNSimulator) AdaptivePredict(parser *BaseParser, input TokenStre // //goland:noinspection GoBoolExpressions func (p *ParserATNSimulator) execATN(dfa *DFA, s0 *DFAState, input TokenStream, startIndex int, outerContext ParserRuleContext) (int, RecognitionException) { - + if runtimeConfig.parserATNSimulatorDebug || runtimeConfig.parserATNSimulatorTraceATNSim { fmt.Println("execATN decision " + strconv.Itoa(dfa.decision) + ", DFA state " + s0.String() + ", LA(1)==" + p.getLookaheadName(input) + " line " + strconv.Itoa(input.LT(1).GetLine()) + ":" + strconv.Itoa(input.LT(1).GetColumn())) } - + previousD := s0 - + if runtimeConfig.parserATNSimulatorDebug { fmt.Println("s0 = " + s0.String()) } @@ -292,7 +292,7 @@ func (p *ParserATNSimulator) execATN(dfa *DFA, s0 *DFAState, input TokenStream, stopIndex := input.Index() input.Seek(startIndex) alts := p.evalSemanticContext(D.predicates, outerContext, true) - + switch alts.length() { case 0: return ATNInvalidAltNumber, p.noViableAlt(input, outerContext, D.configs, startIndex) @@ -305,7 +305,7 @@ func (p *ParserATNSimulator) execATN(dfa *DFA, s0 *DFAState, input TokenStream, } } previousD = D - + if t != TokenEOF { input.Consume() t = input.LA(1) @@ -327,7 +327,7 @@ func (p *ParserATNSimulator) getExistingTargetState(previousD *DFAState, t int) if t+1 < 0 { return nil } - + p.atn.edgeMu.RLock() defer p.atn.edgeMu.RUnlock() edges := previousD.getEdges() @@ -351,16 +351,16 @@ func (p *ParserATNSimulator) getExistingTargetState(previousD *DFAState, t int) //goland:noinspection GoBoolExpressions func (p *ParserATNSimulator) computeTargetState(dfa *DFA, previousD *DFAState, t int) *DFAState { reach := p.computeReachSet(previousD.configs, t, false) - + if reach == nil { p.addDFAEdge(dfa, previousD, t, ATNSimulatorError) return ATNSimulatorError } // create new target state we'll add to DFA after it's complete D := NewDFAState(-1, reach) - + predictedAlt := p.getUniqueAlt(reach) - + if runtimeConfig.parserATNSimulatorDebug { altSubSets := PredictionModegetConflictingAltSubsets(reach) fmt.Println("SLL altSubSets=" + fmt.Sprint(altSubSets) + @@ -418,11 +418,11 @@ func (p *ParserATNSimulator) predicateDFAState(dfaState *DFAState, decisionState // //goland:noinspection GoBoolExpressions func (p *ParserATNSimulator) execATNWithFullContext(dfa *DFA, D *DFAState, s0 *ATNConfigSet, input TokenStream, startIndex int, outerContext ParserRuleContext) (int, RecognitionException) { - + if runtimeConfig.parserATNSimulatorDebug || runtimeConfig.parserATNSimulatorTraceATNSim { fmt.Println("execATNWithFullContext " + s0.String()) } - + fullCtx := true foundExactAmbig := false var reach *ATNConfigSet @@ -430,7 +430,7 @@ func (p *ParserATNSimulator) execATNWithFullContext(dfa *DFA, D *DFAState, s0 *A input.Seek(startIndex) t := input.LA(1) predictedAlt := -1 - + for { // for more work reach = p.computeReachSet(previous, t, fullCtx) if reach == nil { @@ -494,7 +494,7 @@ func (p *ParserATNSimulator) execATNWithFullContext(dfa *DFA, D *DFAState, s0 *A } // We do not check predicates here because we have checked them // on-the-fly when doing full context prediction. - + // // In non-exact ambiguity detection mode, we might actually be able to // detect an exact ambiguity, but I'm not going to spend the cycles @@ -518,9 +518,9 @@ func (p *ParserATNSimulator) execATNWithFullContext(dfa *DFA, D *DFAState, s0 *A // looking for input because no amount of further lookahead will alter // the fact that we should predict alternative 1. We just can't say for // sure that there is an ambiguity without looking further. - + p.ReportAmbiguity(dfa, D, startIndex, input.Index(), foundExactAmbig, reach.Alts(), reach) - + return predictedAlt, nil } @@ -530,7 +530,7 @@ func (p *ParserATNSimulator) computeReachSet(closure *ATNConfigSet, t int, fullC p.mergeCache = NewJPCMap(ReachSetCollection, "Merge cache for computeReachSet()") } intermediate := NewATNConfigSet(fullCtx) - + // Configurations already in a rule stop state indicate reaching the end // of the decision rule (local context) or end of the start rule (full // context). Once reached, these configurations are never updated by a @@ -540,15 +540,15 @@ func (p *ParserATNSimulator) computeReachSet(closure *ATNConfigSet, t int, fullC // For full-context reach operations, separate handling is required to // ensure that the alternative Matching the longest overall sequence is // chosen when multiple such configurations can Match the input. - + var skippedStopStates []*ATNConfig - + // First figure out where we can reach on input t for _, c := range closure.configs { if runtimeConfig.parserATNSimulatorDebug { fmt.Println("testing " + p.GetTokenName(t) + " at " + c.String()) } - + if _, ok := c.GetState().(*RuleStopState); ok { if fullCtx || t == TokenEOF { skippedStopStates = append(skippedStopStates, c) @@ -558,7 +558,7 @@ func (p *ParserATNSimulator) computeReachSet(closure *ATNConfigSet, t int, fullC } continue } - + for _, trans := range c.GetState().GetTransitions() { target := p.getReachableTarget(trans, t) if target != nil { @@ -570,10 +570,10 @@ func (p *ParserATNSimulator) computeReachSet(closure *ATNConfigSet, t int, fullC } } } - + // Now figure out where the reach operation can take us... var reach *ATNConfigSet - + // This block optimizes the reach operation for intermediate sets which // trivially indicate a termination state for the overall // AdaptivePredict operation. @@ -641,15 +641,15 @@ func (p *ParserATNSimulator) computeReachSet(closure *ATNConfigSet, t int, fullC reach.Add(skippedStopStates[l], p.mergeCache) } } - + if runtimeConfig.parserATNSimulatorTraceATNSim { fmt.Println("computeReachSet " + closure.String() + " -> " + reach.String()) } - + if len(reach.configs) == 0 { return nil } - + return reach } @@ -700,7 +700,7 @@ func (p *ParserATNSimulator) computeStartState(a ATNState, ctx RuleContext, full fmt.Println("computeStartState from ATN state " + a.String() + " initialContext=" + initialContext.String()) } - + for i := 0; i < len(a.GetTransitions()); i++ { target := a.GetTransitions()[i].getTarget() c := NewATNConfig6(target, i+1, initialContext) @@ -754,10 +754,10 @@ func (p *ParserATNSimulator) computeStartState(a ATNState, ctx RuleContext, full // for a precedence [DFA] at a particular precedence level (determined by // calling [Parser].getPrecedence). func (p *ParserATNSimulator) applyPrecedenceFilter(configs *ATNConfigSet) *ATNConfigSet { - + statesFromAlt1 := make(map[int]*PredictionContext) configSet := NewATNConfigSet(configs.fullCtx) - + for _, config := range configs.configs { // handle alt 1 first if config.GetAlt() != 1 { @@ -776,7 +776,7 @@ func (p *ParserATNSimulator) applyPrecedenceFilter(configs *ATNConfigSet) *ATNCo } } for _, config := range configs.configs { - + if config.GetAlt() == 1 { // already handled continue @@ -800,13 +800,13 @@ func (p *ParserATNSimulator) getReachableTarget(trans Transition, ttype int) ATN if trans.Matches(ttype, 0, p.atn.maxTokenType) { return trans.getTarget() } - + return nil } //goland:noinspection GoBoolExpressions func (p *ParserATNSimulator) getPredsForAmbigAlts(ambigAlts *BitSet, configs *ATNConfigSet, nalts int) []SemanticContext { - + altToPred := make([]SemanticContext, nalts+1) for _, c := range configs.configs { if ambigAlts.contains(c.GetAlt()) { @@ -906,10 +906,10 @@ func (p *ParserATNSimulator) getSynValidOrSemInvalidAltThatFinishedDecisionEntry func (p *ParserATNSimulator) GetAltThatFinishedDecisionEntryRule(configs *ATNConfigSet) int { alts := NewIntervalSet() - + for _, c := range configs.configs { _, ok := c.GetState().(*RuleStopState) - + if c.GetReachesIntoOuterContext() > 0 || (ok && c.GetContext().hasEmptyPath()) { alts.addOne(c.GetAlt()) } @@ -917,7 +917,7 @@ func (p *ParserATNSimulator) GetAltThatFinishedDecisionEntryRule(configs *ATNCon if alts.length() == 0 { return ATNInvalidAltNumber } - + return alts.first() } @@ -937,7 +937,7 @@ type ATNConfigSetPair struct { func (p *ParserATNSimulator) splitAccordingToSemanticValidity(configs *ATNConfigSet, outerContext ParserRuleContext) []*ATNConfigSet { succeeded := NewATNConfigSet(configs.fullCtx) failed := NewATNConfigSet(configs.fullCtx) - + for _, c := range configs.configs { if c.GetSemanticContext() != SemanticContextNone { predicateEvaluationResult := c.GetSemanticContext().evaluate(p.parser, outerContext) @@ -971,7 +971,7 @@ func (p *ParserATNSimulator) evalSemanticContext(predPredictions []*PredPredicti } continue } - + predicateEvaluationResult := pair.pred.evaluate(p.parser, outerContext) if runtimeConfig.parserATNSimulatorDebug || runtimeConfig.parserATNSimulatorDFADebug { fmt.Println("eval pred " + pair.String() + "=" + fmt.Sprint(predicateEvaluationResult)) @@ -1000,7 +1000,7 @@ func (p *ParserATNSimulator) closureCheckingStopState(config *ATNConfig, configs if runtimeConfig.parserATNSimulatorTraceATNSim { fmt.Println("closure(" + config.String() + ")") } - + if _, ok := config.GetState().(*RuleStopState); ok { // We hit rule end. If we have context info, use it // run thru all possible stack tops in ctx @@ -1022,7 +1022,7 @@ func (p *ParserATNSimulator) closureCheckingStopState(config *ATNConfig, configs } returnState := p.atn.states[config.GetContext().getReturnState(i)] newContext := config.GetContext().GetParent(i) // "pop" return state - + c := NewATNConfig5(returnState, config.GetAlt(), newContext, config.GetSemanticContext()) // While we have context to pop back from, we may have // gotten that context AFTER having falling off a rule. @@ -1060,42 +1060,42 @@ func (p *ParserATNSimulator) closureWork(config *ATNConfig, configs *ATNConfigSe if i == 0 && p.canDropLoopEntryEdgeInLeftRecursiveRule(config) { continue } - + t := state.GetTransitions()[i] _, ok := t.(*ActionTransition) continueCollecting := collectPredicates && !ok c := p.getEpsilonTarget(config, t, continueCollecting, depth == 0, fullCtx, treatEOFAsEpsilon) if c != nil { newDepth := depth - + if _, ok := config.GetState().(*RuleStopState); ok { // target fell off end of rule mark resulting c as having dipped into outer context // We can't get here if incoming config was rule stop and we had context // track how far we dip into outer context. Might // come in handy and we avoid evaluating context dependent // preds if this is > 0. - + if p.dfa != nil && p.dfa.getPrecedenceDfa() { if t.(*EpsilonTransition).outermostPrecedenceReturn == p.dfa.atnStartState.GetRuleIndex() { c.setPrecedenceFilterSuppressed(true) } } - + c.SetReachesIntoOuterContext(c.GetReachesIntoOuterContext() + 1) - + _, present := closureBusy.Put(c) if present { // avoid infinite recursion for right-recursive rules continue } - + configs.dipsIntoOuterContext = true // TODO: can remove? only care when we add to set per middle of this method newDepth-- if runtimeConfig.parserATNSimulatorDebug { fmt.Println("dips into outer ctx: " + c.String()) } } else { - + if !t.getIsEpsilon() { _, present := closureBusy.Put(c) if present { @@ -1120,9 +1120,9 @@ func (p *ParserATNSimulator) canDropLoopEntryEdgeInLeftRecursiveRule(config *ATN if !runtimeConfig.lRLoopEntryBranchOpt { return false } - + _p := config.GetState() - + // First check to see if we are in StarLoopEntryState generated during // left-recursion elimination. For efficiency, also check if // the context has an empty stack case. If so, it would mean @@ -1139,7 +1139,7 @@ func (p *ParserATNSimulator) canDropLoopEntryEdgeInLeftRecursiveRule(config *ATN config.GetContext().hasEmptyPath() { return false } - + // Require all return states to return back to the same rule // that p is in. numCtxs := config.GetContext().length() @@ -1153,38 +1153,38 @@ func (p *ParserATNSimulator) canDropLoopEntryEdgeInLeftRecursiveRule(config *ATN decisionStartState := x.(BlockStartState) blockEndStateNum := decisionStartState.getEndState().stateNumber blockEndState := p.atn.states[blockEndStateNum].(*BlockEndState) - + // Verify that the top of each stack context leads to loop entry/exit // state through epsilon edges and w/o leaving rule. - + for i := 0; i < numCtxs; i++ { // for each stack context returnStateNumber := config.GetContext().getReturnState(i) returnState := p.atn.states[returnStateNumber] - + // all states must have single outgoing epsilon edge if len(returnState.GetTransitions()) != 1 || !returnState.GetTransitions()[0].getIsEpsilon() { return false } - + // Look for prefix op case like 'not expr', (' type ')' expr returnStateTarget := returnState.GetTransitions()[0].getTarget() if returnState.GetStateType() == ATNStateBlockEnd && returnStateTarget == _p { continue } - + // Look for 'expr op expr' or case where expr's return state is block end // of (...)* internal block; the block end points to loop back // which points to p but we don't need to check that if returnState == blockEndState { continue } - + // Look for ternary expr ? expr : expr. The return state points at block end, // which points at loop entry state if returnStateTarget == blockEndState { continue } - + // Look for complex prefix 'between expr and expr' case where 2nd expr's // return state points at block end state of (...)* internal block if returnStateTarget.GetStateType() == ATNStateBlockEnd && @@ -1193,11 +1193,11 @@ func (p *ParserATNSimulator) canDropLoopEntryEdgeInLeftRecursiveRule(config *ATN returnStateTarget.GetTransitions()[0].getTarget() == _p { continue } - + // anything else ain't conforming return false } - + return true } @@ -1207,7 +1207,7 @@ func (p *ParserATNSimulator) getRuleName(index int) string { } var sb strings.Builder sb.Grow(32) - + sb.WriteString("') @@ -1215,7 +1215,7 @@ func (p *ParserATNSimulator) getRuleName(index int) string { } func (p *ParserATNSimulator) getEpsilonTarget(config *ATNConfig, t Transition, collectPredicates, inContext, fullCtx, treatEOFAsEpsilon bool) *ATNConfig { - + switch t.getSerializationType() { case TransitionRULE: return p.ruleTransition(config, t.(*RuleTransition)) @@ -1252,7 +1252,7 @@ func (p *ParserATNSimulator) actionTransition(config *ATNConfig, t *ActionTransi //goland:noinspection GoBoolExpressions func (p *ParserATNSimulator) precedenceTransition(config *ATNConfig, pt *PrecedencePredicateTransition, collectPredicates, inContext, fullCtx bool) *ATNConfig { - + if runtimeConfig.parserATNSimulatorDebug { fmt.Println("PRED (collectPredicates=" + fmt.Sprint(collectPredicates) + ") " + strconv.Itoa(pt.precedence) + ">=_p, ctx dependent=true") @@ -1289,7 +1289,7 @@ func (p *ParserATNSimulator) precedenceTransition(config *ATNConfig, //goland:noinspection GoBoolExpressions func (p *ParserATNSimulator) predTransition(config *ATNConfig, pt *PredicateTransition, collectPredicates, inContext, fullCtx bool) *ATNConfig { - + if runtimeConfig.parserATNSimulatorDebug { fmt.Println("PRED (collectPredicates=" + fmt.Sprint(collectPredicates) + ") " + strconv.Itoa(pt.ruleIndex) + ":" + strconv.Itoa(pt.predIndex) + ", ctx dependent=" + fmt.Sprint(pt.isCtxDependent)) @@ -1403,15 +1403,15 @@ func (p *ParserATNSimulator) GetTokenName(t int) string { if t == TokenEOF { return "EOF" } - + if p.parser != nil && p.parser.GetLiteralNames() != nil && t < len(p.parser.GetLiteralNames()) { return p.parser.GetLiteralNames()[t] + "<" + strconv.Itoa(t) + ">" } - + if p.parser != nil && p.parser.GetLiteralNames() != nil && t < len(p.parser.GetSymbolicNames()) { return p.parser.GetSymbolicNames()[t] + "<" + strconv.Itoa(t) + ">" } - + return strconv.Itoa(t) } @@ -1423,9 +1423,9 @@ func (p *ParserATNSimulator) getLookaheadName(input TokenStream) string { // it out for clarity now that alg. works well. We can leave this // "dead" code for a bit. func (p *ParserATNSimulator) dumpDeadEndConfigs(_ *NoViableAltException) { - + panic("Not implemented") - + // fmt.Println("dead end configs: ") // var decs = nvae.deadEndConfigs // @@ -1507,13 +1507,13 @@ func (p *ParserATNSimulator) addDFAEdge(dfa *DFA, from *DFAState, t int, to *DFA } from.setIthEdge(t+1, to) // connect p.atn.edgeMu.Unlock() - + if runtimeConfig.parserATNSimulatorDebug { var names []string if p.parser != nil { names = p.parser.GetLiteralNames() } - + fmt.Println("DFA=\n" + dfa.String(names, nil)) } return to @@ -1532,7 +1532,7 @@ func (p *ParserATNSimulator) addDFAState(dfa *DFA, d *DFAState) *DFAState { if d == ATNSimulatorError { return d } - + existing, present := dfa.Get(d) if present { if runtimeConfig.parserATNSimulatorTraceATNSim { @@ -1540,7 +1540,7 @@ func (p *ParserATNSimulator) addDFAState(dfa *DFA, d *DFAState) *DFAState { } return existing } - + // The state will be added if not already there or we will be given back the existing state struct // if it is present. // @@ -1551,11 +1551,11 @@ func (p *ParserATNSimulator) addDFAState(dfa *DFA, d *DFAState) *DFAState { d.configs.configLookup = nil } dfa.Put(d) - + if runtimeConfig.parserATNSimulatorTraceATNSim { fmt.Println("addDFAState new " + d.String()) } - + return d } diff --git a/runtime/Go/antlr/v4/parser_rule_context.go b/runtime/Go/antlr/v4/parser_rule_context.go index 78b3b1c9f7..c249bc1385 100644 --- a/runtime/Go/antlr/v4/parser_rule_context.go +++ b/runtime/Go/antlr/v4/parser_rule_context.go @@ -31,7 +31,9 @@ type ParserRuleContext interface { } type BaseParserRuleContext struct { - *BaseRuleContext + parentCtx RuleContext + invokingState int + RuleIndex int start, stop Token exception RecognitionException @@ -40,8 +42,22 @@ type BaseParserRuleContext struct { func NewBaseParserRuleContext(parent ParserRuleContext, invokingStateNumber int) *BaseParserRuleContext { prc := new(BaseParserRuleContext) + InitBaseParserRuleContext(prc, parent, invokingStateNumber) + return prc +} + +func InitBaseParserRuleContext(prc *BaseParserRuleContext, parent ParserRuleContext, invokingStateNumber int) { + // What context invoked b rule? + prc.parentCtx = parent - prc.BaseRuleContext = NewBaseRuleContext(parent, invokingStateNumber) + // What state invoked the rule associated with b context? + // The "return address" is the followState of invokingState + // If parent is nil, b should be -1. + if parent == nil { + prc.invokingState = -1 + } else { + prc.invokingState = invokingStateNumber + } prc.RuleIndex = -1 // * If we are debugging or building a parse tree for a Visitor, @@ -56,8 +72,6 @@ func NewBaseParserRuleContext(parent ParserRuleContext, invokingStateNumber int) // The exception that forced prc rule to return. If the rule successfully // completed, prc is {@code nil}. prc.exception = nil - - return prc } func (prc *BaseParserRuleContext) SetException(e RecognitionException) { @@ -340,6 +354,50 @@ func (prc *BaseParserRuleContext) String(ruleNames []string, stop RuleContext) s return s } +func (prc *BaseParserRuleContext) SetParent(v Tree) { + if v == nil { + prc.parentCtx = nil + } else { + prc.parentCtx = v.(RuleContext) + } +} + +func (prc *BaseParserRuleContext) GetInvokingState() int { + return prc.invokingState +} + +func (prc *BaseParserRuleContext) SetInvokingState(t int) { + prc.invokingState = t +} + +func (prc *BaseParserRuleContext) GetRuleIndex() int { + return prc.RuleIndex +} + +func (prc *BaseParserRuleContext) GetAltNumber() int { + return ATNInvalidAltNumber +} + +func (prc *BaseParserRuleContext) SetAltNumber(_ int) {} + +// IsEmpty returns true if the context of b is empty. +// +// A context is empty if there is no invoking state, meaning nobody calls +// current context. +func (prc *BaseParserRuleContext) IsEmpty() bool { + return prc.invokingState == -1 +} + +// GetParent returns the combined text of all child nodes. This method only considers +// tokens which have been added to the parse tree. +// +// Since tokens on hidden channels (e.g. whitespace or comments) are not +// added to the parse trees, they will not appear in the output of this +// method. +func (prc *BaseParserRuleContext) GetParent() Tree { + return prc.parentCtx +} + var ParserRuleContextEmpty = NewBaseParserRuleContext(nil, -1) type InterpreterRuleContext interface { diff --git a/runtime/Go/antlr/v4/prediction_context.go b/runtime/Go/antlr/v4/prediction_context.go index 8830da7501..c1b80cc1f0 100644 --- a/runtime/Go/antlr/v4/prediction_context.go +++ b/runtime/Go/antlr/v4/prediction_context.go @@ -677,7 +677,6 @@ func combineCommonParents(parents *[]*PredictionContext) { } } -//func getCachedBasePredictionContext(context *PredictionContext, contextCache *PredictionContextCache, visited *JMap[*PredictionContext, *PredictionContext, Comparator[*PredictionContext]]) *PredictionContext { func getCachedBasePredictionContext(context *PredictionContext, contextCache *PredictionContextCache, visited *VisitRecord) *PredictionContext { if context.isEmpty() { return context diff --git a/runtime/Go/antlr/v4/prediction_context_cache.go b/runtime/Go/antlr/v4/prediction_context_cache.go index 5ea527ac96..25dfb11e8f 100644 --- a/runtime/Go/antlr/v4/prediction_context_cache.go +++ b/runtime/Go/antlr/v4/prediction_context_cache.go @@ -26,7 +26,7 @@ func (p *PredictionContextCache) add(ctx *PredictionContext) *PredictionContext if ctx.isEmpty() { return BasePredictionContextEMPTY } - + // Put will return the existing entry if it is present (note this is done via Equals, not whether it is // the same pointer), otherwise it will add the new entry and return that. // diff --git a/runtime/Go/antlr/v4/rule_context.go b/runtime/Go/antlr/v4/rule_context.go index 73771db8f8..f2ad04793e 100644 --- a/runtime/Go/antlr/v4/rule_context.go +++ b/runtime/Go/antlr/v4/rule_context.go @@ -38,76 +38,3 @@ type RuleContext interface { String([]string, RuleContext) string } - -type BaseRuleContext struct { - parentCtx RuleContext - invokingState int - RuleIndex int -} - -func NewBaseRuleContext(parent RuleContext, invokingState int) *BaseRuleContext { - - rn := new(BaseRuleContext) - - // What context invoked b rule? - rn.parentCtx = parent - - // What state invoked the rule associated with b context? - // The "return address" is the followState of invokingState - // If parent is nil, b should be -1. - if parent == nil { - rn.invokingState = -1 - } else { - rn.invokingState = invokingState - } - - return rn -} - -func (b *BaseRuleContext) GetBaseRuleContext() *BaseRuleContext { - return b -} - -func (b *BaseRuleContext) SetParent(v Tree) { - if v == nil { - b.parentCtx = nil - } else { - b.parentCtx = v.(RuleContext) - } -} - -func (b *BaseRuleContext) GetInvokingState() int { - return b.invokingState -} - -func (b *BaseRuleContext) SetInvokingState(t int) { - b.invokingState = t -} - -func (b *BaseRuleContext) GetRuleIndex() int { - return b.RuleIndex -} - -func (b *BaseRuleContext) GetAltNumber() int { - return ATNInvalidAltNumber -} - -func (b *BaseRuleContext) SetAltNumber(_ int) {} - -// IsEmpty returns true if the context of b is empty. -// -// A context is empty if there is no invoking state, meaning nobody calls -// current context. -func (b *BaseRuleContext) IsEmpty() bool { - return b.invokingState == -1 -} - -// GetParent returns the combined text of all child nodes. This method only considers -// tokens which have been added to the parse tree. -// -// Since tokens on hidden channels (e.g. whitespace or comments) are not -// added to the parse trees, they will not appear in the output of this -// method. -func (b *BaseRuleContext) GetParent() Tree { - return b.parentCtx -} diff --git a/runtime/Go/antlr/v4/statistics.go b/runtime/Go/antlr/v4/statistics.go index 4c038f0e2c..70c0673a0f 100644 --- a/runtime/Go/antlr/v4/statistics.go +++ b/runtime/Go/antlr/v4/statistics.go @@ -24,7 +24,7 @@ const collectStats = true // It is exported so that it can be used by others to look for things that are not already looked for in the // runtime statistics. type goRunStats struct { - + // jStats is a slice of all the [JStatRec] records that have been created, which is one for EVERY collection created // during a run. It is exported so that it can be used by others to look for things that are not already looked for // within this package. @@ -97,7 +97,7 @@ func WithTopN(topN int) statsOption { // // [Jim Idle]: https:://github.com/jim-idle func (s *goRunStats) Analyze() { - + // Look for anything that looks strange and record it in our local maps etc for the report to present it // s.CollectionAnomalies() @@ -106,17 +106,17 @@ func (s *goRunStats) Analyze() { // TopNCollections looks through all the statistical records and gathers the top ten collections by size. func (s *goRunStats) TopNCollections() { - + // Let's sort the stat records by MaxSize // sort.Slice(s.jStats, func(i, j int) bool { return s.jStats[i].MaxSize > s.jStats[j].MaxSize }) - + for i := 0; i < len(s.jStats) && i < s.topN; i++ { s.topNByMax = append(s.topNByMax, s.jStats[i]) } - + // Sort by the number of times used // sort.Slice(s.jStats, func(i, j int) bool { @@ -131,7 +131,7 @@ func (s *goRunStats) TopNCollections() { // path, which should represent a directory. Generated files will be prefixed with the given prefix and will be // given a type name such as `anomalies` and a time stamp such as `2021-09-01T12:34:56` and a .md suffix. func (s *goRunStats) Report(dir string, prefix string) error { - + isDir, err := isDirectory(dir) switch { case err != nil: @@ -140,7 +140,7 @@ func (s *goRunStats) Report(dir string, prefix string) error { return fmt.Errorf("output directory `%s` is not a directory", dir) } s.reportCollections(dir, prefix) - + // Clean out any old data in case the user forgets // s.Reset() @@ -170,7 +170,7 @@ background-color: black; ++++`) _ = f.Close() - + fname := filepath.Join(dir, prefix+"_"+"_"+collectionsFile+"_"+".adoc") // If the file doesn't exist, create it, or append to the file f, err = os.OpenFile(fname, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644) @@ -184,9 +184,9 @@ background-color: black; } }(f) _, _ = f.WriteString("= Collections for " + prefix + "\n\n") - + _, _ = f.WriteString("== Summary\n") - + if s.unusedCollections != nil { _, _ = f.WriteString("=== Unused Collections\n") _, _ = f.WriteString("Unused collections incur a penalty for allocation that makes them a candidate for either\n") @@ -194,18 +194,18 @@ background-color: black; _, _ = f.WriteString(" consider removing it. If you are using a collection that is used, but not very often,\n") _, _ = f.WriteString(" you should consider using lazy initialization to defer the allocation until it is\n") _, _ = f.WriteString(" actually needed.\n\n") - + _, _ = f.WriteString("\n.Unused collections\n") _, _ = f.WriteString(`[cols="<3,>1"]` + "\n\n") _, _ = f.WriteString("|===\n") _, _ = f.WriteString("| Type | Count\n") - + for k, v := range s.unusedCollections { _, _ = f.WriteString("| " + CollectionDescriptors[k].SybolicName + " | " + strconv.Itoa(v) + "\n") } f.WriteString("|===\n\n") } - + _, _ = f.WriteString("\n.Summary of Collections\n") _, _ = f.WriteString(`[cols="<3,>1"]` + "\n\n") _, _ = f.WriteString("|===\n") @@ -215,7 +215,7 @@ background-color: black; } _, _ = f.WriteString("| Total | " + strconv.Itoa(len(s.jStats)) + "\n") _, _ = f.WriteString("|===\n\n") - + _, _ = f.WriteString("\n.Summary of Top " + strconv.Itoa(s.topN) + " Collections by MaxSize\n") _, _ = f.WriteString(`[cols="<1,<3,>1,>1,>1,>1"]` + "\n\n") _, _ = f.WriteString("|===\n") @@ -230,7 +230,7 @@ background-color: black; _, _ = f.WriteString("\n") } _, _ = f.WriteString("|===\n\n") - + _, _ = f.WriteString("\n.Summary of Top " + strconv.Itoa(s.topN) + " Collections by Access\n") _, _ = f.WriteString(`[cols="<1,<3,>1,>1,>1,>1,>1"]` + "\n\n") _, _ = f.WriteString("|===\n") @@ -261,11 +261,11 @@ func (s *goRunStats) CollectionAnomalies() { defer s.jStatsLock.RUnlock() s.counts = make(map[CollectionSource]int, len(s.jStats)) for _, c := range s.jStats { - + // Accumlate raw counts // s.counts[c.Source]++ - + // Look for allocated but unused collections and count them if c.MaxSize == 0 && c.Puts == 0 { if s.unusedCollections == nil { @@ -277,5 +277,5 @@ func (s *goRunStats) CollectionAnomalies() { fmt.Println("Collection ", c.Description, "accumulated a max size of ", c.MaxSize, " - this is probably too large and indicates a poorly formed grammar") } } - + } diff --git a/runtime/Go/antlr/v4/tree.go b/runtime/Go/antlr/v4/tree.go index 6c17164381..c288420fb2 100644 --- a/runtime/Go/antlr/v4/tree.go +++ b/runtime/Go/antlr/v4/tree.go @@ -33,9 +33,7 @@ type ParseTree interface { type RuleNode interface { ParseTree - GetRuleContext() RuleContext - GetBaseRuleContext() *BaseRuleContext } type TerminalNode interface { diff --git a/tool/resources/org/antlr/v4/tool/templates/codegen/Go/Go.stg b/tool/resources/org/antlr/v4/tool/templates/codegen/Go/Go.stg index c11d2b5316..53d444e295 100644 --- a/tool/resources/org/antlr/v4/tool/templates/codegen/Go/Go.stg +++ b/tool/resources/org/antlr/v4/tool/templates/codegen/Go/Go.stg @@ -1126,7 +1126,7 @@ Set()}; separator="\n\n"> } type struct { - **antlr.BaseParserRuleContext + *antlr.BaseParserRuleContext parser antlr.Parser @@ -1135,17 +1135,34 @@ type struct { func NewEmpty() * { var p = new() - p.BaseParserRuleContext = Newantlr.NewBaseParserRuleContext(nil, -1) + + p. = New(nil, -1) // Jim super + + antlr.InitBaseParserRuleContext(&p.BaseParserRuleContext, nil, -1) + p.RuleIndex = RULE_ return p } +func InitEmpty(p *) { + + p. = New(nil, -1) // Jim super + + antlr.InitBaseParserRuleContext(&p.BaseParserRuleContext, nil, -1) + + p.RuleIndex = RULE_ +} + func (*) Is() {} func New(parser antlr.Parser, parent antlr.ParserRuleContext, invokingState int }>) * { var p = new() - p.BaseParserRuleContext = Newantlr.NewBaseParserRuleContext(parent, invokingState) + + p. = New(parent, invokingState) + + antlr.InitBaseParserRuleContext(&p.BaseParserRuleContext, parent, invokingState) + p.parser = parser p.RuleIndex = RULE_ @@ -1225,8 +1242,8 @@ func (s *) GetParser() antlr.Parser { return s.parser } -func (s *) CopyFrom(ctx *) { - s.BaseParserRuleContext.CopyFrom(ctx.BaseParserRuleContext) +func (s *) CopyAll(ctx *) { + s.CopyFrom(&ctx.BaseParserRuleContext) = ctx.}; separator="\n"> } @@ -1252,7 +1269,7 @@ func (s *) ToStringTree(ruleNames []string, recog antlr.Reco AltLabelStructDecl(struct, attrs, getters, dispatchMethods, tokenDecls, tokenTypeDecls, tokenListDecls, ruleContextDecls, ruleContextListDecls, attributeDecls) ::= << type struct { - *Context + Context @@ -1261,9 +1278,9 @@ type struct { func New(parser antlr.Parser, ctx antlr.ParserRuleContext) * { var p = new() - p.Context = NewEmptyContext() + InitEmptyContext(&p.Context) p.parser = parser - p.CopyFrom(ctx.(*Context)) + p.CopyAll(ctx.(*Context)) return p } From 16b7a525b72a8e1fccc22c491ff03c49a873d67c Mon Sep 17 00:00:00 2001 From: "Jim.Idle" Date: Tue, 11 Apr 2023 16:50:10 +0800 Subject: [PATCH 2/2] feat: Corrects the test template when looking at parse tree correctness o Uses reflection to check that the tree structure is all correct o Disables test that uses superClass as this basically doesn't work/isn't a concept in Go. Signed-off-by: Jim.Idle --- .../runtime/descriptors/ParseTrees/AltNum.txt | 2 ++ .../antlr/v4/test/runtime/helpers/Test.go.stg | 19 ++++++++++++++----- 2 files changed, 16 insertions(+), 5 deletions(-) diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/AltNum.txt b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/AltNum.txt index fa81083bf5..324a1bf51f 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/AltNum.txt +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/descriptors/ParseTrees/AltNum.txt @@ -35,3 +35,5 @@ xyz """(a:3 x (b:2 y) z) """ +[skip] +Go diff --git a/runtime-testsuite/resources/org/antlr/v4/test/runtime/helpers/Test.go.stg b/runtime-testsuite/resources/org/antlr/v4/test/runtime/helpers/Test.go.stg index 0def6a57ad..18331fef45 100644 --- a/runtime-testsuite/resources/org/antlr/v4/test/runtime/helpers/Test.go.stg +++ b/runtime-testsuite/resources/org/antlr/v4/test/runtime/helpers/Test.go.stg @@ -1,12 +1,15 @@ package main import ( - "test/parser" - "github.com/antlr/antlr4/runtime/Go/antlr/v4" "fmt" - "os" + "github.com/antlr/antlr4/runtime/Go/antlr/v4" + "os" + "test/parser" ) + +import "reflect" + type TreeShapeListener struct { *parser.BaseListener } @@ -18,8 +21,14 @@ func NewTreeShapeListener() *TreeShapeListener { func (this *TreeShapeListener) EnterEveryRule(ctx antlr.ParserRuleContext) { for i := 0; i\