diff --git a/experimental/token/cursor.go b/experimental/token/cursor.go index 6b0230c1..f3d090a5 100644 --- a/experimental/token/cursor.go +++ b/experimental/token/cursor.go @@ -59,7 +59,7 @@ func NewCursorAt(tok Token) *Cursor { return &Cursor{ withContext: tok.withContext, - idx: tok.ID().index(), // Convert to 0-based index. + idx: tok.ID().naturalIndex(), // Convert to 0-based index. } } @@ -178,7 +178,7 @@ func (c *Cursor) PrevSkippable() Token { if c.IsSynthetic() { c.idx-- } else { - c.idx = tok.ID().index() + c.idx = tok.ID().naturalIndex() } return tok } diff --git a/experimental/token/raw.go b/experimental/token/raw.go index 1dd43630..dacea18f 100644 --- a/experimental/token/raw.go +++ b/experimental/token/raw.go @@ -54,24 +54,38 @@ func (t ID) String() string { if t == 0 { return "Token()" } - return fmt.Sprintf("Token(%d)", t.index()) + if t < 0 { + return fmt.Sprintf("Token(%d)", t.syntheticIndex()) + } + return fmt.Sprintf("Token(%d)", t.naturalIndex()) } -// index returns the index of this token in the stream. -func (t ID) index() int { +// naturalIndex returns the index of this token in the natural stream. +func (t ID) naturalIndex() int { if t.IsZero() { - panic("protocompile/token: called index on zero token") + panic("protocompile/token: called naturalIndex on zero token") } if t < 0 { - // Need to invert the bits, because synthetic tokens are - // stored as negative numbers. - return ^int(t) + panic("protocompile/token: called naturalIndex on synthetic token") } // Need to subtract off one, because the zeroth // ID is used as a "missing" sentinel. return int(t) - 1 } +// syntheticIndex returns the index of this token in the synthetic stream. +func (t ID) syntheticIndex() int { + if t.IsZero() { + panic("protocompile/token: called syntheticIndex on zero token") + } + if t > 0 { + panic("protocompile/token: called syntheticIndex on natural token") + } + // Need to invert the bits, because synthetic tokens are + // stored as negative numbers. + return ^int(t) +} + // Constants for extracting the parts of tokenImpl.kindAndOffset. const ( kindMask = 0b111 diff --git a/experimental/token/token.go b/experimental/token/token.go index ff842d40..2ef9167a 100644 --- a/experimental/token/token.go +++ b/experimental/token/token.go @@ -325,7 +325,7 @@ func (t Token) Children() *Cursor { start, _ := t.StartEnd() return &Cursor{ withContext: t.withContext, - idx: start.id.index() + 1, // Skip the start! + idx: start.id.naturalIndex() + 1, // Skip the start! } } @@ -497,12 +497,12 @@ func (t Token) nat() *nat { if t.IsSynthetic() { return nil } - return &t.Context().Stream().nats[t.id.index()] + return &t.Context().Stream().nats[t.id.naturalIndex()] } func (t Token) synth() *synth { if !t.IsSynthetic() { return nil } - return &t.Context().Stream().synths[t.id.index()] + return &t.Context().Stream().synths[t.id.syntheticIndex()] }